Compare commits

...
Author SHA1 Message Date
whyour c9f1b2d4bb fix: preserve legacy SQLite addons loaded from build directories 2026-10-23 14:33:13 +08:00
whyour 5fdfd9077d fix: clean SQLite compiler artifacts before publishing repaired dependencies 2026-10-23 14:15:08 +08:00
whyour ce2e396b3e fix: remove legacy SQLite compiler artifacts from migration bundles 2026-10-23 14:04:42 +08:00
whyour 49cec13fcd test: classify subprocess crashes without exposing credentials 2026-10-23 13:50:37 +08:00
whyour 81de8f7094 fix: accept ARM1176-compatible ELF architecture attributes 2026-10-23 13:10:42 +08:00
whyour dce4f60c47 fix: restore locked build tools for the ARMv6 migration bridge 2026-10-23 12:16:28 +08:00
whyour 1623e89a71 fix: rebuild legacy migration SQLite for ARMv6 on the builder 2026-10-23 11:54:52 +08:00
whyour 1a49e81ffd fix: prevent second-hop migration from returning to a legacy backend 2026-10-23 11:08:20 +08:00
whyour 656b9419d6 test: budget startup for minimum-Node concurrent cache workers 2026-10-23 10:51:16 +08:00
whyour 8d2f67e263 Merge QEMU Node probe timeout fix into migration review 2026-10-23 10:40:27 +08:00
whyour 2a32f528f1 fix: initialize sample environment in migration candidate tests 2026-10-23 10:37:30 +08:00
whyour 384ce170c7 feat: bridge legacy panel upgrades in two verified steps 2026-10-23 10:33:00 +08:00
whyour cc001ecebd fix: allow emulated architectures time for Node URL probes 2026-10-23 01:53:28 +08:00
whyour 04c8a63518 Merge Alpine Node runtime fix into migration review branch 2026-10-23 01:49:18 +08:00
whyour 8976447b1e fix: pin Alpine amd64 Node runtime and verify URL recovery 2026-10-23 01:43:56 +08:00
whyour 0a586fbd1d fix: preserve newest-first execution instance ordering 2026-10-23 01:42:58 +08:00
whyour 15d54866a8 feat: migrate SQLite runtime and verify complete upgrade bundles
Use MikroORM's official Node SQLite dialect with bounded database isolation, preserve legacy schema and data for rollback, and require Node 22.17 or newer. Update Debian runtimes and retain ARMv6 musl compatibility without a SQLite addon.

Complete short archive writes, stop orphaned SQL workers after owner death, preserve buffered startup requests, and prestart the archived backend on every platform. Cover real package installation, migration, failure recovery, and legacy database compatibility.

Keep authentication storage failures distinct from expired sessions with a pinned Keyv runtime, propagated cache errors, and retryable HTTP 503 responses. Preserve genuine 401 failures and verify legacy cache bytes, initialization safety, and session recovery.
2026-10-22 13:47:26 +08:00
134 changed files with 8527 additions and 3449 deletions
+26 -12
View File
@@ -550,10 +550,14 @@ jobs:
cache-from: type=registry,ref=whyour/qinglong:cache-debian-python3.10 cache-from: type=registry,ref=whyour/qinglong:cache-debian-python3.10
cache-to: type=registry,ref=whyour/qinglong:cache-debian-python3.10,mode=max cache-to: type=registry,ref=whyour/qinglong:cache-debian-python3.10,mode=max
# One unmodified amd64 package for migration acceptance only. This does not # Unmodified amd64 packages for glibc and musl migration acceptance. This does not
# publish bundles or advance development/formal channel indexes. # publish bundles or advance development/formal channel indexes.
build-npm-migration-bundle: build-npm-migration-bundle:
needs: build-debian needs: [build-debian, build-alpine]
strategy:
fail-fast: false
matrix:
libc: [glibc, musl]
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 20 timeout-minutes: 20
steps: steps:
@@ -561,7 +565,7 @@ jobs:
- uses: docker/setup-buildx-action@v4 - uses: docker/setup-buildx-action@v4
- name: Export the official platform package from this run's image digest - name: Export the official platform package from this run's image digest
env: env:
IMAGE_DIGEST: ${{ needs.build-debian.outputs.image-digest }} IMAGE_DIGEST: ${{ matrix.libc == 'musl' && needs.build-alpine.outputs.image-digest || needs.build-debian.outputs.image-digest }}
run: | run: |
set -euo pipefail set -euo pipefail
[[ "$IMAGE_DIGEST" =~ ^sha256:[a-f0-9]{64}$ ]] [[ "$IMAGE_DIGEST" =~ ^sha256:[a-f0-9]{64}$ ]]
@@ -569,20 +573,21 @@ jobs:
docker buildx build --platform linux/amd64 --build-arg "BASE_IMAGE=$image" --file docker/Dockerfile.upgrade-bundle --output "type=local,dest=$RUNNER_TEMP/npm-migration-bundle" . docker buildx build --platform linux/amd64 --build-arg "BASE_IMAGE=$image" --file docker/Dockerfile.upgrade-bundle --output "type=local,dest=$RUNNER_TEMP/npm-migration-bundle" .
- uses: actions/upload-artifact@v6 - uses: actions/upload-artifact@v6
with: with:
name: npm-migration-bundle-${{ github.sha }} name: npm-migration-bundle-${{ matrix.libc }}-${{ github.sha }}
path: ${{ runner.temp }}/npm-migration-bundle/* path: ${{ runner.temp }}/npm-migration-bundle/*
if-no-files-found: error if-no-files-found: error
retention-days: 3 retention-days: 3
verify-npm-package: verify-npm-package:
needs: [build-static, build-npm-migration-bundle] needs: [build-static, build-npm-migration-bundle, build-alpine]
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 30 timeout-minutes: 30
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
node: ['20.18.1', '24'] node: ['20.18.1', '22.17.0', '24', 'alpine']
container: node:${{ matrix.node }}-bookworm container:
image: ${{ matrix.node == 'alpine' && format('ghcr.io/{0}@{1}', github.repository, needs.build-alpine.outputs.image-digest) || format('node:{0}-bookworm', matrix.node) }}
steps: steps:
- uses: actions/checkout@v6 - uses: actions/checkout@v6
- uses: actions/download-artifact@v6 - uses: actions/download-artifact@v6
@@ -591,22 +596,30 @@ jobs:
path: ${{ runner.temp }}/npm-panel/ path: ${{ runner.temp }}/npm-panel/
- uses: actions/download-artifact@v6 - uses: actions/download-artifact@v6
with: with:
name: npm-migration-bundle-${{ github.sha }} name: npm-migration-bundle-${{ matrix.node == 'alpine' && 'musl' || 'glibc' }}-${{ github.sha }}
path: ${{ runner.temp }}/npm-migration-bundle/ path: ${{ runner.temp }}/npm-migration-bundle/
- name: Install isolated service test tools - name: Install isolated service test tools
run: | run: |
apt-get update if [ -f /etc/alpine-release ]; then
apt-get install -y --no-install-recommends python3 openssl curl jq nginx procps apk add --no-cache python3 openssl curl jq nginx procps make g++
else
apt-get update
apt-get install -y --no-install-recommends python3 openssl curl jq nginx procps
fi
# The published legacy launcher resets PATH to standard directories. # The published legacy launcher resets PATH to standard directories.
# /usr/local is confined to this disposable job container; PM2_HOME # /usr/local is confined to this disposable job container; PM2_HOME
# remains private to each verification run. # remains private to each verification run.
npm install --global --prefix /usr/local --no-audit --no-fund pm2@6.0.14 npm install --global --prefix /usr/local --no-audit --no-fund pm2@6.0.14
- name: Install and upgrade the exact npm archive - name: Install and upgrade the exact npm archive
if: matrix.node != '20.18.1'
env: env:
QL_PANEL_PACKAGE_TEST_ISOLATED: '1' QL_PANEL_PACKAGE_TEST_ISOLATED: '1'
QL_PANEL_TEST_PM2: /usr/local/bin/pm2 QL_PANEL_TEST_PM2: /usr/local/bin/pm2
QL_PANEL_TEST_BUNDLE_DIR: ${{ runner.temp }}/npm-migration-bundle QL_PANEL_TEST_BUNDLE_DIR: ${{ runner.temp }}/npm-migration-bundle
run: node scripts/verify-panel-package.cjs verify "$RUNNER_TEMP/npm-panel" "$RUNNER_TEMP/npm-panel-results" run: node scripts/verify-panel-package.cjs verify "$RUNNER_TEMP/npm-panel" "$RUNNER_TEMP/npm-panel-results"
- name: Reject an unsupported runtime while preserving the old installation
if: matrix.node == '20.18.1'
run: node scripts/verify-panel-runtime-floor.cjs "$RUNNER_TEMP/npm-panel" "$RUNNER_TEMP/npm-panel-results"
- uses: actions/upload-artifact@v6 - uses: actions/upload-artifact@v6
with: with:
name: qinglong-npm-verification-${{ matrix.node }}-${{ github.sha }} name: qinglong-npm-verification-${{ matrix.node }}-${{ github.sha }}
@@ -701,7 +714,8 @@ jobs:
if [ "$BUNDLE_LIBC" = glibc ]; then digest="$DEBIAN_DIGEST"; fi if [ "$BUNDLE_LIBC" = glibc ]; then digest="$DEBIAN_DIGEST"; fi
test -n "$digest" test -n "$digest"
image="ghcr.io/${GITHUB_REPOSITORY}@${digest}" image="ghcr.io/${GITHUB_REPOSITORY}@${digest}"
docker buildx build --platform "$BUNDLE_PLATFORM" --build-arg "BASE_IMAGE=$image" --file docker/Dockerfile.upgrade-bundle --output type=local,dest=output/upgrade . legacy=$(node -p "require('./docker/migration-bridge-images.json')['$BUNDLE_LIBC']")
docker buildx build --platform "$BUNDLE_PLATFORM" --build-arg "BASE_IMAGE=$image" --build-arg "LEGACY_IMAGE=$legacy" --file docker/Dockerfile.upgrade-bridge --output type=local,dest=output/upgrade .
- uses: actions/upload-artifact@v6 - uses: actions/upload-artifact@v6
with: with:
name: upgrade-${{ matrix.name }}-${{ github.sha }} name: upgrade-${{ matrix.name }}-${{ github.sha }}
@@ -721,7 +735,7 @@ jobs:
- uses: actions/checkout@v6 - uses: actions/checkout@v6
- uses: actions/setup-node@v6 - uses: actions/setup-node@v6
with: with:
node-version: '20' node-version: '24'
- uses: pnpm/action-setup@v6 - uses: pnpm/action-setup@v6
with: with:
version: '8.3.1' version: '8.3.1'
+2 -2
View File
@@ -31,7 +31,7 @@ jobs:
version: '8.3.1' version: '8.3.1'
- uses: actions/setup-node@v6 - uses: actions/setup-node@v6
with: with:
node-version: '20' node-version: '22'
cache: pnpm cache: pnpm
- run: pnpm install --frozen-lockfile - run: pnpm install --frozen-lockfile
- run: pnpm build:front && pnpm build:back && pnpm build:info - run: pnpm build:front && pnpm build:back && pnpm build:info
@@ -120,7 +120,7 @@ jobs:
run: | run: |
docker run --rm --network none --entrypoint node \ docker run --rm --network none --entrypoint node \
qinglong-readiness:${{ matrix.image }} \ qinglong-readiness:${{ matrix.image }} \
-e 'const assert=require("node:assert/strict"); const {createRequire}=require("node:module"); for(const [name,want] of Object.entries({"@grpc/grpc-js":"1.14.5",compression:"1.8.2",multer:"2.4.0",sequelize:"6.37.8"}))assert.equal(require(name+"/package.json").version,want); const expressRequire=createRequire(require.resolve("express")); assert.equal(expressRequire("proxy-addr/package.json").version,"2.0.8"); const sockRequire=createRequire(require.resolve("sockjs")); assert.equal(sockRequire("websocket-driver/package.json").version,"0.7.5"); require("sqlite3"); require("nodemailer"); require("undici"); console.log("Minimal production patches and runtime imports verified")' -e 'const assert=require("node:assert/strict"); const {createRequire}=require("node:module"); for(const [name,want] of Object.entries({"@grpc/grpc-js":"1.14.5",compression:"1.8.2",multer:"2.4.0"}))assert.equal(require(name+"/package.json").version,want); const expressRequire=createRequire(require.resolve("express")); assert.equal(expressRequire("proxy-addr/package.json").version,"2.0.8"); const sockRequire=createRequire(require.resolve("sockjs")); assert.equal(sockRequire("websocket-driver/package.json").version,"0.7.5"); assert.equal(typeof require("@mikro-orm/sql").NodeSqliteDialect,"function"); const db=new (require("node:sqlite").DatabaseSync)(":memory:"); assert.equal(db.prepare("SELECT 42 AS value").get().value,42); db.close(); require("nodemailer"); require("undici"); console.log("Minimal production patches and runtime imports verified")'
- name: Export native Debian image for isolated dev validation - name: Export native Debian image for isolated dev validation
if: matrix.image == 'debian' if: matrix.image == 'debian'
run: | run: |
@@ -0,0 +1,201 @@
name: SQLite migration candidate
on:
pull_request:
branches: [develop]
workflow_dispatch:
permissions:
contents: read
concurrency:
group: sqlite-migration-candidate-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
SOURCE_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
jobs:
build-static:
if: ${{ (github.event_name == 'pull_request' && github.head_ref == 'codex/sqlite-runtime-migration' && github.event.pull_request.head.repo.full_name == github.repository) || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/codex/sqlite-runtime-migration') }}
runs-on: ubuntu-latest
timeout-minutes: 25
steps:
- uses: actions/checkout@v6
with:
ref: ${{ env.SOURCE_COMMIT }}
- uses: pnpm/action-setup@v6
with:
version: '8.3.1'
- uses: actions/setup-node@v6
with:
node-version: '22.17.0'
cache: pnpm
- run: pnpm install --frozen-lockfile
- run: cp .env.example .env
- run: npm test
- run: npm run build:front && npm run build:back && npm run build:info
- uses: actions/upload-artifact@v6
with:
name: migration-static-${{ env.SOURCE_COMMIT }}
path: static/
include-hidden-files: true
if-no-files-found: error
retention-days: 7
test-node24:
needs: build-static
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v6
with:
ref: ${{ env.SOURCE_COMMIT }}
- uses: pnpm/action-setup@v6
with:
version: '8.3.1'
- uses: actions/setup-node@v6
with:
node-version: '24'
cache: pnpm
- run: pnpm install --frozen-lockfile
- run: cp .env.example .env
- run: npm test
build-images:
needs: build-static
runs-on: ubuntu-22.04
timeout-minutes: 90
permissions:
contents: read
packages: write
strategy:
fail-fast: false
matrix:
include:
- libc: musl
file: docker/Dockerfile
platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x,linux/386
- libc: glibc
file: docker/Dockerfile.debian
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
steps:
- uses: actions/checkout@v6
with:
ref: ${{ env.SOURCE_COMMIT }}
- uses: actions/download-artifact@v6
with:
name: migration-static-${{ env.SOURCE_COMMIT }}
path: static/
- uses: docker/setup-qemu-action@v4
with:
cache-image: false
- uses: docker/setup-buildx-action@v4
- uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ github.token }}
- uses: docker/build-push-action@v7
id: image
with:
context: .
file: ${{ matrix.file }}
platforms: ${{ matrix.platforms }}
build-args: |
QL_BRANCH=codex/sqlite-runtime-migration
SOURCE_COMMIT=${{ env.SOURCE_COMMIT }}
push: true
tags: ghcr.io/${{ github.repository }}:migration-candidate-${{ env.SOURCE_COMMIT }}-${{ matrix.libc }}
cache-from: type=registry,ref=whyour/qinglong:cache-${{ matrix.libc == 'musl' && 'alpine' || 'debian' }}
- name: Record the immutable candidate image
env:
IMAGE_DIGEST: ${{ steps.image.outputs.digest }}
BUNDLE_LIBC: ${{ matrix.libc }}
run: |
[[ "$IMAGE_DIGEST" =~ ^sha256:[a-f0-9]{64}$ ]]
printf '%s\n' "ghcr.io/$GITHUB_REPOSITORY@$IMAGE_DIGEST" > "$BUNDLE_LIBC-image.txt"
- uses: actions/upload-artifact@v6
with:
name: migration-${{ matrix.libc }}-image-${{ env.SOURCE_COMMIT }}
path: ${{ matrix.libc }}-image.txt
retention-days: 7
build-bundles:
needs: build-images
runs-on: ubuntu-22.04
timeout-minutes: 90
strategy:
fail-fast: false
matrix:
include:
- { platform: linux/amd64, libc: musl, name: x64-musl }
- { platform: linux/arm/v6, libc: musl, name: armv6-musl }
- { platform: linux/arm/v7, libc: musl, name: armv7-musl }
- { platform: linux/arm64, libc: musl, name: arm64-musl }
- { platform: linux/ppc64le, libc: musl, name: ppc64-musl }
- { platform: linux/s390x, libc: musl, name: s390x-musl }
- { platform: linux/386, libc: musl, name: ia32-musl }
- { platform: linux/amd64, libc: glibc, name: x64-glibc }
- { platform: linux/arm/v7, libc: glibc, name: armv7-glibc }
- { platform: linux/arm64, libc: glibc, name: arm64-glibc }
- { platform: linux/ppc64le, libc: glibc, name: ppc64-glibc }
- { platform: linux/s390x, libc: glibc, name: s390x-glibc }
steps:
- uses: actions/checkout@v6
with:
ref: ${{ env.SOURCE_COMMIT }}
- uses: actions/download-artifact@v6
with:
name: migration-${{ matrix.libc }}-image-${{ env.SOURCE_COMMIT }}
- uses: docker/setup-qemu-action@v4
with:
cache-image: false
- uses: docker/setup-buildx-action@v4
- name: Build and verify both online hops using the unchanged legacy client
env:
BUNDLE_PLATFORM: ${{ matrix.platform }}
BUNDLE_LIBC: ${{ matrix.libc }}
run: |
set -euo pipefail
image=$(cat "$BUNDLE_LIBC-image.txt")
legacy=$(node -p "require('./docker/migration-bridge-images.json')['$BUNDLE_LIBC']")
docker buildx build --platform "$BUNDLE_PLATFORM" --build-arg "BASE_IMAGE=$image" --build-arg "LEGACY_IMAGE=$legacy" --file docker/Dockerfile.upgrade-bridge --output type=local,dest=output/upgrade .
- uses: actions/upload-artifact@v6
with:
name: migration-bundles-${{ matrix.name }}-${{ env.SOURCE_COMMIT }}
path: output/upgrade/*
if-no-files-found: error
retention-days: 7
verify-index:
needs: [build-bundles, test-node24]
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v6
with:
ref: ${{ env.SOURCE_COMMIT }}
- uses: actions/setup-node@v6
with:
node-version: '24'
- uses: pnpm/action-setup@v6
with:
version: '8.3.1'
- run: pnpm install --prod --frozen-lockfile && npm --prefix cli ci
- uses: actions/download-artifact@v6
with:
pattern: migration-bundles-*-${{ env.SOURCE_COMMIT }}
path: output/upgrade/
merge-multiple: true
- run: node scripts/build-bundle-installer.cjs output/upgrade/install-bundle.cjs
- name: Validate every platform without publishing a channel
run: |
node -e "const fs=require('fs'),{releaseIndex}=require('./scripts/publish-upgrade-bundles.cjs'); const index=releaseIndex('output/upgrade',process.env.SOURCE_COMMIT,process.env.GITHUB_REPOSITORY,{migration:true}); fs.writeFileSync('output/upgrade/candidate-index.json',JSON.stringify(index));"
- uses: actions/upload-artifact@v6
with:
name: migration-candidate-index-${{ env.SOURCE_COMMIT }}
path: |
output/upgrade/candidate-index.json
output/upgrade/install-bundle.cjs
retention-days: 7
+1 -1
View File
@@ -18,7 +18,7 @@ jobs:
version: '8.3.1' version: '8.3.1'
- uses: actions/setup-node@v6 - uses: actions/setup-node@v6
with: with:
node-version: '20' node-version: '24'
cache: pnpm cache: pnpm
- run: pnpm install --frozen-lockfile - run: pnpm install --frozen-lockfile
- run: cp .env.example .env - run: cp .env.example .env
+2
View File
@@ -1,5 +1,7 @@
# Node.js 原生部署与 Python venv # Node.js 原生部署与 Python venv
面板要求 Node.js >=22.17,推荐 Node.js 24,并要求该 Node 构建提供 `node:sqlite`。应用升级保留现有 Node;旧 Node 20 环境需先升级运行时。远程管理 CLI 的最低版本要求独立于面板。
设置 `QL_DIR` 为安装目录,`QL_DATA_DIR` 为以 `/data` 结尾的绝对持久化数据路径。 设置 `QL_DIR` 为安装目录,`QL_DATA_DIR` 为以 `/data` 结尾的绝对持久化数据路径。
如需隔离 Python,请在启动进程的环境中显式设置 `PYTHON_VENV_DIR`(绝对路径或相对 `QL_DIR` 的路径),例如 `export PYTHON_VENV_DIR=/srv/qinglong/.venv`。未设置时保留现有 Python prefix 行为。 如需隔离 Python,请在启动进程的环境中显式设置 `PYTHON_VENV_DIR`(绝对路径或相对 `QL_DIR` 的路径),例如 `export PYTHON_VENV_DIR=/srv/qinglong/.venv`。未设置时保留现有 Python prefix 行为。
+21 -14
View File
@@ -5,10 +5,7 @@ import CronService from '../services/cron';
import CronViewService from '../services/cronView'; import CronViewService from '../services/cronView';
import { celebrate, Joi } from 'celebrate'; import { celebrate, Joi } from 'celebrate';
import { commonCronSchema } from '../validation/schedule'; import { commonCronSchema } from '../validation/schedule';
import { import { RunningInstanceModel, InstanceStatus } from '../data/runningInstance';
RunningInstanceModel,
InstanceStatus,
} from '../data/runningInstance';
import { t } from '../shared/i18n'; import { t } from '../shared/i18n';
import cronClient from '../schedule/client'; import cronClient from '../schedule/client';
@@ -19,8 +16,17 @@ export default (app: Router) => {
route.use(async (req, res, next) => { route.use(async (req, res, next) => {
// Keep stop/status callbacks available even when the scheduler is down. // Keep stop/status callbacks available even when the scheduler is down.
if (['POST', 'PUT', 'DELETE'].includes(req.method) && if (
['/', '/run', '/enable', '/disable', '/views/enable', '/views/disable'].includes(req.path)) { ['POST', 'PUT', 'DELETE'].includes(req.method) &&
[
'/',
'/run',
'/enable',
'/disable',
'/views/enable',
'/views/disable',
].includes(req.path)
) {
try { try {
await cronClient.readiness.ensureReady(); await cronClient.readiness.ensureReady();
} catch (error) { } catch (error) {
@@ -497,13 +503,10 @@ export default (app: Router) => {
}), }),
async (req: Request<{ id: number }>, res: Response, next: NextFunction) => { async (req: Request<{ id: number }>, res: Response, next: NextFunction) => {
try { try {
const instances = await RunningInstanceModel.findAll({ const instances = await RunningInstanceModel.find(
where: { { cron_id: req.params.id },
cron_id: req.params.id, { orderBy: [{ started_at: 'desc' }] },
}, );
order: [['started_at', 'DESC']],
raw: true,
});
return res.send({ code: 200, data: instances }); return res.send({ code: 200, data: instances });
} catch (e) { } catch (e) {
return next(e); return next(e);
@@ -519,7 +522,11 @@ export default (app: Router) => {
instanceId: Joi.number().required(), instanceId: Joi.number().required(),
}), }),
}), }),
async (req: Request<{ id: number; instanceId: number }>, res: Response, next: NextFunction) => { async (
req: Request<{ id: number; instanceId: number }>,
res: Response,
next: NextFunction,
) => {
try { try {
const cronService = Container.get(CronService); const cronService = Container.get(CronService);
const data = await cronService.stopInstance(req.params.instanceId); const data = await cronService.stopInstance(req.params.instanceId);
+93 -157
View File
@@ -1,38 +1,32 @@
import { statistics } from '../data/statistics';
import { Query, QuerySpec, orderBy } from '../data/queries';
import { Router, Request, Response, NextFunction } from 'express'; import { Router, Request, Response, NextFunction } from 'express';
import { Container } from 'typedi'; import { Container } from 'typedi';
import { fn, col, where, Op } from 'sequelize';
import { CrontabModel } from '../data/cron'; import { CrontabModel } from '../data/cron';
import { CrontabStatModel } from '../data/cronStats'; import { CrontabStatModel } from '../data/cronStats';
import { import { RunningInstanceModel, InstanceStatus } from '../data/runningInstance';
RunningInstanceModel,
InstanceStatus,
} from '../data/runningInstance';
import dayjs from 'dayjs'; import dayjs from 'dayjs';
import os from 'os'; import os from 'os';
import { isEmpty } from 'lodash'; import { isEmpty } from 'lodash';
import { t, tf } from '../shared/i18n'; import { t, tf } from '../shared/i18n';
const route = Router(); const route = Router();
export default (app: Router) => { export default (app: Router) => {
app.use('/dashboard', route); app.use('/dashboard', route);
route.post('/record', async (req: Request, res: Response) => { route.post('/record', async (req: Request, res: Response) => {
try { try {
const { ref_id, code, elapsed } = req.body; const { ref_id, code, elapsed } = req.body;
if (!ref_id) return res.send({ code: 400, message: 'ref_id required' }); if (!ref_id) return res.send({ code: 400, message: 'ref_id required' });
const today = dayjs().format('YYYY-MM-DD'); const today = dayjs().format('YYYY-MM-DD');
const isSuccess = code === 0 ? 1 : 0; const isSuccess = code === 0 ? 1 : 0;
const isFail = code !== 0 ? 1 : 0; const isFail = code !== 0 ? 1 : 0;
const elapsedMs = (Number(elapsed) || 0) * 1000; const elapsedMs = (Number(elapsed) || 0) * 1000;
const existing = await CrontabStatModel.findOne(
const existing = await CrontabStatModel.findOne({ { ref_id: Number(ref_id), date: today },
where: { ref_id: Number(ref_id), date: today }, {},
}); );
if (existing) { if (existing) {
await CrontabStatModel.update( await CrontabStatModel.nativeUpdate(
{ id: existing.id },
{ {
run_count: (existing.run_count || 0) + 1, run_count: (existing.run_count || 0) + 1,
success_count: (existing.success_count || 0) + isSuccess, success_count: (existing.success_count || 0) + isSuccess,
@@ -40,10 +34,9 @@ export default (app: Router) => {
total_time: (existing.total_time || 0) + elapsedMs, total_time: (existing.total_time || 0) + elapsedMs,
max_time: Math.max(existing.max_time || 0, elapsedMs), max_time: Math.max(existing.max_time || 0, elapsedMs),
}, },
{ where: { id: existing.id } },
); );
} else { } else {
await CrontabStatModel.create({ await CrontabStatModel.insert({
ref_id: Number(ref_id), ref_id: Number(ref_id),
date: today, date: today,
run_count: 1, run_count: 1,
@@ -53,41 +46,27 @@ export default (app: Router) => {
max_time: elapsedMs, max_time: elapsedMs,
}); });
} }
res.send({ code: 200 }); res.send({ code: 200 });
} catch (e) { } catch (e) {
res.send({ code: 500 }); res.send({ code: 500 });
} }
}); });
route.get( route.get(
'/overview', '/overview',
async (req: Request, res: Response, next: NextFunction) => { async (req: Request, res: Response, next: NextFunction) => {
try { try {
const today = dayjs().format('YYYY-MM-DD'); const today = dayjs().format('YYYY-MM-DD');
const [total, enabled, disabled, stats] = await Promise.all([ const [total, enabled, disabled, stats] = await Promise.all([
CrontabModel.count(), CrontabModel.count({}),
CrontabModel.count({ where: { isDisabled: 0 } }), CrontabModel.count({ isDisabled: 0 }),
CrontabModel.count({ where: { isDisabled: 1 } }), CrontabModel.count({ isDisabled: 1 }),
CrontabStatModel.findOne({ statistics.totals(today),
attributes: [
[fn('SUM', col('run_count')), 'total_runs'],
[fn('SUM', col('success_count')), 'total_success'],
[fn('SUM', col('fail_count')), 'total_fail'],
[fn('SUM', col('total_time')), 'total_time'],
],
where: { date: today },
raw: true,
}),
]); ]);
const row = stats as any; const row = stats as any;
const totalRuns = Number(row?.total_runs) || 0; const totalRuns = Number(row?.total_runs) || 0;
const totalSuccess = Number(row?.total_success) || 0; const totalSuccess = Number(row?.total_success) || 0;
const totalFail = Number(row?.total_fail) || 0; const totalFail = Number(row?.total_fail) || 0;
const totalTime = Number(row?.total_time) || 0; const totalTime = Number(row?.total_time) || 0;
res.send({ res.send({
code: 200, code: 200,
data: { data: {
@@ -97,7 +76,10 @@ export default (app: Router) => {
todayRuns: totalRuns, todayRuns: totalRuns,
todaySuccess: totalSuccess, todaySuccess: totalSuccess,
todayFail: totalFail, todayFail: totalFail,
successRate: totalRuns > 0 ? ((totalSuccess / totalRuns) * 100).toFixed(1) : '0', successRate:
totalRuns > 0
? ((totalSuccess / totalRuns) * 100).toFixed(1)
: '0',
avgTime: totalRuns > 0 ? Math.round(totalTime / totalRuns) : 0, avgTime: totalRuns > 0 ? Math.round(totalTime / totalRuns) : 0,
}, },
}); });
@@ -106,25 +88,25 @@ export default (app: Router) => {
} }
}, },
); );
for (const isSuccess of [false, true]) { for (const isSuccess of [false, true]) {
route.get( route.get(
isSuccess ? '/successes' : '/failures', isSuccess ? '/successes' : '/failures',
async (req: Request, res: Response, next: NextFunction) => { async (req: Request, res: Response, next: NextFunction) => {
try { try {
const countColumn = isSuccess ? 'success_count' : 'fail_count'; const countColumn = isSuccess ? 'success_count' : 'fail_count';
const rows = (await CrontabStatModel.findAll({ const rows = (await statistics.results(
attributes: ['ref_id', [fn('SUM', col(countColumn)), 'result_count']], dayjs().format('YYYY-MM-DD'),
where: { date: dayjs().format('YYYY-MM-DD'), [countColumn]: { [Op.gt]: 0 } }, countColumn,
group: ['ref_id'], )) as any[];
order: [[fn('SUM', col(countColumn)), 'DESC'], ['ref_id', 'ASC']], const crons =
raw: true, rows.length > 0
})) as any[]; ? await CrontabModel.find(
const crons = rows.length > 0 ? await CrontabModel.findAll({ { id: { ['$in']: rows.map((row) => Number(row.ref_id)) } },
attributes: ['id', 'name', 'command'], {
where: { id: { [Op.in]: rows.map((row) => Number(row.ref_id)) } }, fields: ['id', 'name', 'command'],
raw: true, },
}) : []; )
: [];
const cronMap = new Map(crons.map((cron) => [cron.id, cron])); const cronMap = new Map(crons.map((cron) => [cron.id, cron]));
res.send({ res.send({
code: 200, code: 200,
@@ -135,7 +117,9 @@ export default (app: Router) => {
id, id,
name: cron?.name || cron?.command || tf('任务#%s', id), name: cron?.name || cron?.command || tf('任务#%s', id),
command: cron?.command || '', command: cron?.command || '',
[isSuccess ? 'successCount' : 'failCount']: Number(row.result_count), [isSuccess ? 'successCount' : 'failCount']: Number(
row.result_count,
),
deleted: !cron, deleted: !cron,
}; };
}), }),
@@ -146,7 +130,6 @@ export default (app: Router) => {
}, },
); );
} }
route.get( route.get(
'/trend', '/trend',
async (req: Request, res: Response, next: NextFunction) => { async (req: Request, res: Response, next: NextFunction) => {
@@ -156,22 +139,7 @@ export default (app: Router) => {
for (let i = days - 1; i >= 0; i--) { for (let i = days - 1; i >= 0; i--) {
dates.push(dayjs().subtract(i, 'day').format('YYYY-MM-DD')); dates.push(dayjs().subtract(i, 'day').format('YYYY-MM-DD'));
} }
const rows = (await statistics.trend(dates)) as any[];
const rows = (await CrontabStatModel.findAll({
attributes: [
'date',
[fn('SUM', col('run_count')), 'total_runs'],
[fn('SUM', col('success_count')), 'total_success'],
[fn('SUM', col('fail_count')), 'total_fail'],
],
where: {
date: { [Op.in]: dates },
},
group: ['date'],
order: [['date', 'ASC']],
raw: true,
})) as any[];
const dataMap: Record<string, any> = {}; const dataMap: Record<string, any> = {};
rows.forEach((r: any) => { rows.forEach((r: any) => {
dataMap[r.date] = { dataMap[r.date] = {
@@ -180,89 +148,52 @@ export default (app: Router) => {
fail: Number(r.total_fail) || 0, fail: Number(r.total_fail) || 0,
}; };
}); });
const data = dates.map((d) => ({ const data = dates.map((d) => ({
date: dayjs(d).format('MM-DD'), date: dayjs(d).format('MM-DD'),
...(dataMap[d] || { total: 0, success: 0, fail: 0 }), ...(dataMap[d] || { total: 0, success: 0, fail: 0 }),
})); }));
res.send({ code: 200, data }); res.send({ code: 200, data });
} catch (e) { } catch (e) {
next(e); next(e);
} }
}, },
); );
route.get( route.get(
'/top-time', '/top-time',
async (req: Request, res: Response, next: NextFunction) => { async (req: Request, res: Response, next: NextFunction) => {
try { try {
const today = dayjs().format('YYYY-MM-DD'); const today = dayjs().format('YYYY-MM-DD');
const rows = (await statistics.ranking(today, 'total_time')) as any[];
const rows = (await CrontabStatModel.findAll({
attributes: [
'ref_id',
[fn('SUM', col('total_time')), 'total_time'],
[fn('SUM', col('run_count')), 'run_count'],
[fn('MAX', col('max_time')), 'max_time'],
],
where: { date: today, run_count: { [Op.gt]: 0 } },
group: ['ref_id'],
order: [[fn('SUM', col('total_time')), 'DESC']],
limit: 5,
raw: true,
})) as any[];
const ids = rows.map((r) => Number(r.ref_id)); const ids = rows.map((r) => Number(r.ref_id));
const crons = await CrontabModel.findAll({ const crons = await CrontabModel.find({ id: { ['$in']: ids } }, {});
where: { id: { [Op.in]: ids } },
raw: true,
});
const nameMap: Record<number, string> = {}; const nameMap: Record<number, string> = {};
crons.forEach((c: any) => { nameMap[c.id] = c.name || c.command; }); crons.forEach((c: any) => {
nameMap[c.id] = c.name || c.command;
});
const data = rows.map((r: any, i) => ({ const data = rows.map((r: any, i) => ({
rank: i + 1, rank: i + 1,
name: nameMap[Number(r.ref_id)] || tf('任务#%s', r.ref_id), name: nameMap[Number(r.ref_id)] || tf('任务#%s', r.ref_id),
avgTime: Math.round(Number(r.total_time) / Number(r.run_count)), avgTime: Math.round(Number(r.total_time) / Number(r.run_count)),
maxTime: Number(r.max_time), maxTime: Number(r.max_time),
})); }));
res.send({ code: 200, data }); res.send({ code: 200, data });
} catch (e) { } catch (e) {
next(e); next(e);
} }
}, },
); );
route.get( route.get(
'/top-count', '/top-count',
async (req: Request, res: Response, next: NextFunction) => { async (req: Request, res: Response, next: NextFunction) => {
try { try {
const today = dayjs().format('YYYY-MM-DD'); const today = dayjs().format('YYYY-MM-DD');
const rows = (await statistics.ranking(today, 'run_count')) as any[];
const rows = (await CrontabStatModel.findAll({
attributes: [
'ref_id',
[fn('SUM', col('run_count')), 'run_count'],
[fn('SUM', col('total_time')), 'total_time'],
[fn('SUM', col('success_count')), 'success_count'],
],
where: { date: today, run_count: { [Op.gt]: 0 } },
group: ['ref_id'],
order: [[fn('SUM', col('run_count')), 'DESC']],
limit: 5,
raw: true,
})) as any[];
const ids = rows.map((r) => Number(r.ref_id)); const ids = rows.map((r) => Number(r.ref_id));
const crons = await CrontabModel.findAll({ const crons = await CrontabModel.find({ id: { ['$in']: ids } }, {});
where: { id: { [Op.in]: ids } },
raw: true,
});
const nameMap: Record<number, string> = {}; const nameMap: Record<number, string> = {};
crons.forEach((c: any) => { nameMap[c.id] = c.name || c.command; }); crons.forEach((c: any) => {
nameMap[c.id] = c.name || c.command;
});
const data = rows.map((r: any, i) => ({ const data = rows.map((r: any, i) => ({
rank: i + 1, rank: i + 1,
name: nameMap[Number(r.ref_id)] || tf('任务#%s', r.ref_id), name: nameMap[Number(r.ref_id)] || tf('任务#%s', r.ref_id),
@@ -270,48 +201,42 @@ export default (app: Router) => {
avgTime: Math.round(Number(r.total_time) / Number(r.run_count)), avgTime: Math.round(Number(r.total_time) / Number(r.run_count)),
successRate: successRate:
Number(r.run_count) > 0 Number(r.run_count) > 0
? ((Number(r.success_count) / Number(r.run_count)) * 100).toFixed(1) ? ((Number(r.success_count) / Number(r.run_count)) * 100).toFixed(
1,
)
: '0', : '0',
})); }));
res.send({ code: 200, data }); res.send({ code: 200, data });
} catch (e) { } catch (e) {
next(e); next(e);
} }
}, },
); );
route.get( route.get(
'/runtime', '/runtime',
async (req: Request, res: Response, next: NextFunction) => { async (req: Request, res: Response, next: NextFunction) => {
try { try {
const runningInstances = await RunningInstanceModel.findAll({ const runningInstances = await RunningInstanceModel.find(
where: { {
status: InstanceStatus.running, status: InstanceStatus.running,
}, },
raw: true, {},
}); );
const queuedCrons = await CrontabModel.find(
const queuedCrons = await CrontabModel.findAll({ {
where: {
status: 3, // queued status: 3, // queued
}, },
raw: true, {},
}); );
// Fetch cron names for running instances // Fetch cron names for running instances
const cronIds = [ const cronIds = [
...new Set(runningInstances.map((i: any) => i.cron_id)), ...new Set(runningInstances.map((i: any) => i.cron_id)),
]; ];
const crons = const crons =
cronIds.length > 0 cronIds.length > 0
? await CrontabModel.findAll({ ? await CrontabModel.find({ id: cronIds }, {})
where: { id: cronIds },
raw: true,
})
: []; : [];
const cronMap = new Map(crons.map((c: any) => [c.id, c])); const cronMap = new Map(crons.map((c: any) => [c.id, c]));
const now = dayjs().unix(); const now = dayjs().unix();
const running = runningInstances.map((inst: any) => { const running = runningInstances.map((inst: any) => {
const cron = cronMap.get(inst.cron_id); const cron = cronMap.get(inst.cron_id);
@@ -324,19 +249,18 @@ export default (app: Router) => {
logPath: inst.log_path, logPath: inst.log_path,
}; };
}); });
const dayAgo = dayjs().subtract(24, 'hour').unix(); const dayAgo = dayjs().subtract(24, 'hour').unix();
const idleTasks = await CrontabModel.findAll({ const idleTasks = await CrontabModel.find(
where: { {
isDisabled: 0, isDisabled: 0,
status: 1, status: 1,
last_execution_time: { [Op.lt]: dayAgo }, last_execution_time: { ['$lt']: dayAgo },
}, },
order: [['last_execution_time', 'ASC']], {
limit: 5, orderBy: orderBy([['last_execution_time', 'ASC']]),
raw: true, limit: 5,
}); },
);
res.send({ res.send({
code: 200, code: 200,
data: { data: {
@@ -357,33 +281,45 @@ export default (app: Router) => {
} }
}, },
); );
route.get( route.get(
'/labels', '/labels',
async (req: Request, res: Response, next: NextFunction) => { async (req: Request, res: Response, next: NextFunction) => {
try { try {
const today = dayjs().format('YYYY-MM-DD'); const today = dayjs().format('YYYY-MM-DD');
const [crons, stats] = (await Promise.all([ const [crons, stats] = await Promise.all([
CrontabModel.findAll({ where: { isDisabled: 0 }, raw: true }), CrontabModel.find({ isDisabled: 0 }, {}),
CrontabStatModel.findAll({ where: { date: today }, raw: true }), CrontabStatModel.find({ date: today }, {}),
])); ]);
const statMap: Record<number, any> = {}; const statMap: Record<number, any> = {};
stats.forEach((s: any) => { statMap[s.ref_id] = s; }); stats.forEach((s: any) => {
statMap[s.ref_id] = s;
const labelMap: Record<string, { count: number; runs: number; success: number; totalTime: number }> = {}; });
const labelMap: Record<
string,
{
count: number;
runs: number;
success: number;
totalTime: number;
}
> = {};
crons.forEach((c) => { crons.forEach((c) => {
let rawLabels = c.labels; let rawLabels = c.labels;
if (typeof rawLabels === 'string') rawLabels = JSON.parse(rawLabels); if (typeof rawLabels === 'string') rawLabels = JSON.parse(rawLabels);
const labels: string[] = Array.isArray(rawLabels) const labels: string[] = Array.isArray(rawLabels)
? [...new Set((rawLabels as string[]).filter((l: string) => !isEmpty(l)))] ? [
...new Set(
(rawLabels as string[]).filter((l: string) => !isEmpty(l)),
),
]
: []; : [];
if (labels.length === 0) { if (labels.length === 0) {
labels.push(t('未分类')); labels.push(t('未分类'));
} }
const st = statMap[c.id!]; const st = statMap[c.id!];
labels.forEach((label: string) => { labels.forEach((label: string) => {
if (!labelMap[label]) labelMap[label] = { count: 0, runs: 0, success: 0, totalTime: 0 }; if (!labelMap[label])
labelMap[label] = { count: 0, runs: 0, success: 0, totalTime: 0 };
labelMap[label].count += 1; labelMap[label].count += 1;
if (st) { if (st) {
labelMap[label].runs += Number(st.run_count) || 0; labelMap[label].runs += Number(st.run_count) || 0;
@@ -392,24 +328,22 @@ export default (app: Router) => {
} }
}); });
}); });
const data = Object.entries(labelMap) const data = Object.entries(labelMap)
.map(([label, v]) => ({ .map(([label, v]) => ({
label, label,
count: v.count, count: v.count,
todayRuns: v.runs, todayRuns: v.runs,
successRate: v.runs > 0 ? ((v.success / v.runs) * 100).toFixed(1) : '0', successRate:
v.runs > 0 ? ((v.success / v.runs) * 100).toFixed(1) : '0',
avgTime: v.runs > 0 ? Math.round(v.totalTime / v.runs) : 0, avgTime: v.runs > 0 ? Math.round(v.totalTime / v.runs) : 0,
})) }))
.sort((a, b) => b.todayRuns - a.todayRuns); .sort((a, b) => b.todayRuns - a.todayRuns);
res.send({ code: 200, data }); res.send({ code: 200, data });
} catch (e) { } catch (e) {
next(e); next(e);
} }
}, },
); );
route.get( route.get(
'/system', '/system',
async (req: Request, res: Response, next: NextFunction) => { async (req: Request, res: Response, next: NextFunction) => {
@@ -422,7 +356,9 @@ export default (app: Router) => {
uptime: Math.floor(process.uptime()), uptime: Math.floor(process.uptime()),
memTotal: os.totalmem(), memTotal: os.totalmem(),
memFree: os.freemem(), memFree: os.freemem(),
memUsagePercent: ((1 - os.freemem() / os.totalmem()) * 100).toFixed(1), memUsagePercent: ((1 - os.freemem() / os.totalmem()) * 100).toFixed(
1,
),
heapUsed: Math.round(memUsage.heapUsed / 1024 / 1024), heapUsed: Math.round(memUsage.heapUsed / 1024 / 1024),
heapTotal: Math.round(memUsage.heapTotal / 1024 / 1024), heapTotal: Math.round(memUsage.heapTotal / 1024 / 1024),
loadAvg: os.loadavg().map((v) => Number(v.toFixed(2))), loadAvg: os.loadavg().map((v) => Number(v.toFixed(2))),
+2 -5
View File
@@ -1,19 +1,16 @@
import { errStack } from '../shared/errors'; import { errStack } from '../shared/errors';
// Also terminate if the primary disappears without forwarding a signal. // Also terminate if the primary disappears without forwarding a signal.
const onParentDisconnect = () => process.exit(1); const onParentDisconnect = () => process.exit(1);
process.once('disconnect', onParentDisconnect); process.once('disconnect', onParentDisconnect);
async function initializeDatabase() { async function initializeDatabase() {
const { sequelize } = await import('../data'); const { database } = await import('../data');
try { try {
const { default: loadDatabase } = await import('../loaders/db'); const { default: loadDatabase } = await import('../loaders/db');
await loadDatabase(); await loadDatabase();
} finally { } finally {
await sequelize.close(); await database.close();
} }
} }
initializeDatabase() initializeDatabase()
.catch((error) => { .catch((error) => {
console.error(`[boot] Database initialization failed:\n${errStack(error)}`); console.error(`[boot] Database initialization failed:\n${errStack(error)}`);
+21 -19
View File
@@ -3,11 +3,12 @@
import 'reflect-metadata'; import 'reflect-metadata';
import dns from 'dns'; import dns from 'dns';
import http from 'http'; import http from 'http';
// The first scheduler connection can outlive its short RPC deadline. Let the // The first scheduler connection can outlive its short RPC deadline. Let the
// normal single-flight recovery finish within the supervised startup budget. // normal single-flight recovery finish within the supervised startup budget.
async function waitForUpgradeScheduler( async function waitForUpgradeScheduler(
readiness: { recover(): Promise<boolean> }, readiness: {
recover(): Promise<boolean>;
},
timeoutMs: number, timeoutMs: number,
retryMs = 250, retryMs = 250,
) { ) {
@@ -22,13 +23,12 @@ async function waitForUpgradeScheduler(
} while (Date.now() < deadline); } while (Date.now() < deadline);
throw new Error('Upgrade scheduler readiness failed'); throw new Error('Upgrade scheduler readiness failed');
} }
async function probeUpgrade() { async function probeUpgrade() {
if (!process.send || process.env.QL_UPGRADE_PROBE !== '1') if (!process.send || process.env.QL_UPGRADE_PROBE !== '1')
throw new Error('Upgrade probe requires its isolated supervisor'); throw new Error('Upgrade probe requires its isolated supervisor');
dns.setDefaultResultOrder('ipv4first'); dns.setDefaultResultOrder('ipv4first');
const { default: config } = await import('../config'); const { default: config } = await import('../config');
const { sequelize } = await import('../data'); const { database } = await import('../data');
const { default: loadDatabase } = await import('../loaders/db'); const { default: loadDatabase } = await import('../loaders/db');
const { Container } = await import('typedi'); const { Container } = await import('typedi');
const { GrpcServerService } = await import('../services/grpc'); const { GrpcServerService } = await import('../services/grpc');
@@ -43,17 +43,17 @@ async function probeUpgrade() {
try { try {
await loadDatabase(); await loadDatabase();
// Exercise an actual on-disk transaction as well as schema initialization. // Exercise an actual on-disk transaction as well as schema initialization.
await sequelize.transaction(async (transaction) => { await database.transaction(async () => {
await sequelize.query('CREATE TABLE upgrade_probe (value INTEGER)', { await database.query('CREATE TABLE upgrade_probe (value INTEGER)');
transaction, await database.query('INSERT INTO upgrade_probe VALUES (42)');
}); const rows = await database.query('SELECT value FROM upgrade_probe');
await sequelize.query('INSERT INTO upgrade_probe VALUES (42)', { if (
transaction, (
}); rows[0] as {
const [rows] = await sequelize.query('SELECT value FROM upgrade_probe', { value: number;
transaction, }
}); )?.value !== 42
if ((rows[0] as { value: number })?.value !== 42) )
throw new Error('Upgrade database readback failed'); throw new Error('Upgrade database readback failed');
}); });
inject(); inject();
@@ -64,8 +64,11 @@ async function probeUpgrade() {
cronClient.readiness, cronClient.readiness,
process.arch === 'arm' && process.arch === 'arm' &&
Number( Number(
(process.config.variables as { arm_version?: string | number }) (
.arm_version, process.config.variables as {
arm_version?: string | number;
}
).arm_version,
) === 6 ) === 6
? 60000 ? 60000
: 10000, : 10000,
@@ -119,10 +122,9 @@ async function probeUpgrade() {
} finally { } finally {
await server.shutdown(); await server.shutdown();
grpc.getServer().forceShutdown(); grpc.getServer().forceShutdown();
await sequelize.close(); await database.close();
} }
} }
process.once('disconnect', () => process.exit(1)); process.once('disconnect', () => process.exit(1));
probeUpgrade() probeUpgrade()
.then(() => process.exit(0)) .then(() => process.exit(0))
+28
View File
@@ -0,0 +1,28 @@
import { EventEmitter } from 'events';
import { database } from '.';
import { DatabaseClient } from './databaseClient';
// Keep the existing Keyv serialization and namespace. Only its SQLite transport
// changes; old cached authentication records remain readable in either version.
export class CacheStore extends EventEmitter {
readonly opts = { dialect: 'sqlite' };
namespace?: string;
constructor(private client: () => DatabaseClient = () => database) {
super();
}
get(key: string): Promise<string | undefined> {
return this.client().request('cache', 'get', key);
}
async set(key: string, value: string) {
return this.client().request('cache', 'set', key, value);
}
async delete(key: string): Promise<boolean> {
return this.client().request('cache', 'delete', key);
}
async clear(): Promise<void> {
throw new Error('Clearing all authentication namespaces is not supported');
}
async disconnect(): Promise<void> {
/* The shared database client owns the worker. */
}
}
+3 -36
View File
@@ -1,5 +1,4 @@
import { sequelize } from '.'; import { Repository, RecordRow } from './repository';
import { DataTypes, Model, ModelDefined } from 'sequelize';
export class Crontab { export class Crontab {
name?: string; name?: string;
@@ -62,37 +61,5 @@ export enum CrontabStatus {
'queued' = 3, 'queued' = 3,
} }
export interface CronInstance extends Model<Crontab, Crontab>, Crontab {} export type CronInstance = RecordRow<Crontab>;
export const CrontabModel = sequelize.define<CronInstance>('Crontab', { export const CrontabModel = new Repository<Crontab>('Crontab');
name: {
unique: 'compositeIndex',
type: DataTypes.STRING,
},
command: {
unique: 'compositeIndex',
type: DataTypes.STRING,
},
schedule: {
unique: 'compositeIndex',
type: DataTypes.STRING,
},
timestamp: DataTypes.STRING,
saved: DataTypes.BOOLEAN,
status: DataTypes.NUMBER,
isSystem: DataTypes.NUMBER,
pid: DataTypes.NUMBER,
isDisabled: DataTypes.NUMBER,
isPinned: DataTypes.NUMBER,
log_path: DataTypes.STRING,
queued_token: DataTypes.STRING,
labels: DataTypes.JSON,
last_running_time: DataTypes.NUMBER,
last_execution_time: DataTypes.NUMBER,
sub_id: { type: DataTypes.NUMBER, allowNull: true },
extra_schedules: DataTypes.JSON,
task_before: DataTypes.STRING,
task_after: DataTypes.STRING,
log_name: DataTypes.STRING,
allow_multiple_instances: DataTypes.NUMBER,
work_dir: DataTypes.STRING,
});
+3 -48
View File
@@ -1,5 +1,4 @@
import { DataTypes, Model } from 'sequelize'; import { Repository, RecordRow } from './repository';
import { sequelize } from '.';
export class CrontabStat { export class CrontabStat {
id?: number; id?: number;
@@ -23,49 +22,5 @@ export class CrontabStat {
} }
} }
export interface CrontabStatInstance extends Model<CrontabStat, CrontabStat>, CrontabStat {} export type CrontabStatInstance = RecordRow<CrontabStat>;
export const CrontabStatModel = new Repository<CrontabStat>('CrontabStat');
export const CrontabStatModel = sequelize.define<CrontabStatInstance>(
'CrontabStat',
{
id: {
type: DataTypes.INTEGER,
primaryKey: true,
autoIncrement: true,
},
ref_id: {
type: DataTypes.NUMBER,
allowNull: false,
},
date: {
type: DataTypes.STRING,
allowNull: false,
},
run_count: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
success_count: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
fail_count: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
total_time: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
max_time: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
},
{
indexes: [
{ unique: true, fields: ['ref_id', 'date'] },
{ fields: ['date'] },
],
},
);
+3 -23
View File
@@ -1,5 +1,4 @@
import { sequelize } from '.'; import { Repository, RecordRow } from './repository';
import { DataTypes, Model } from 'sequelize';
export enum CronViewType { export enum CronViewType {
'系统' = 1, '系统' = 1,
@@ -39,24 +38,5 @@ export class CrontabView {
} }
} }
export interface CronViewInstance export type CronViewInstance = RecordRow<CrontabView>;
extends Model<CrontabView, CrontabView>, export const CrontabViewModel = new Repository<CrontabView>('CrontabView');
CrontabView {}
export const CrontabViewModel = sequelize.define<CronViewInstance>(
'CrontabView',
{
name: {
unique: 'name',
type: DataTypes.STRING,
},
position: DataTypes.NUMBER,
isDisabled: DataTypes.NUMBER,
filters: DataTypes.JSON,
sorts: DataTypes.JSON,
filterRelation: {
type: DataTypes.STRING,
allowNull: true,
},
type: DataTypes.NUMBER,
},
);
+325
View File
@@ -0,0 +1,325 @@
import { AsyncLocalStorage } from 'async_hooks';
import { ChildProcess, fork } from 'child_process';
import { join } from 'path';
import { setTimeout, clearTimeout } from 'timers';
import { DatabaseError } from './databaseErrors';
type Scope = { active: boolean; child: boolean; token: number };
type Reply = {
id: number;
value?: any;
error?: { code: string; message: string };
};
type Limits = { timeoutMs?: number; queueLimit?: number };
// One worker per backend process; SQLite itself coordinates independent processes.
// A transaction owns this client's queue until its callback and commit finish.
export class DatabaseClient {
private worker?: ChildProcess;
private stopped: Promise<void> = Promise.resolve();
private failed?: Error;
private closed = false;
private nextId = 0;
private queued = 0;
private tail: Promise<void> = Promise.resolve();
private pending = new Map<
number,
{
resolve(value: any): void;
reject(error: Error): void;
timer: NodeJS.Timeout;
}
>();
private scope = new AsyncLocalStorage<Scope>();
private readonly timeoutMs: number;
private readonly queueLimit: number;
constructor(
public readonly storage: string,
private cacheStorage?: string,
limits: Limits = {},
) {
this.timeoutMs =
limits.timeoutMs ?? (process.arch === 'arm' ? 300000 : 60000);
this.queueLimit = limits.queueLimit ?? 256;
if (
!Number.isSafeInteger(this.timeoutMs) ||
this.timeoutMs < 1 ||
!Number.isSafeInteger(this.queueLimit) ||
this.queueLimit < 1
)
throw new Error('Invalid database worker limits');
}
private abort(error: Error) {
this.failed ||= error;
for (const request of this.pending.values()) {
clearTimeout(request.timer);
request.reject(this.failed);
}
this.pending.clear();
this.unref();
this.worker?.kill('SIGKILL');
}
private unref() {
this.worker?.unref();
this.worker?.channel?.unref();
}
private start() {
if (this.worker) return this.worker;
const [major, minor] = process.versions.node.split('.').map(Number);
if (major < 22 || (major === 22 && minor < 17))
throw new DatabaseError(
'Qinglong requires Node >=22.17; keep the previous installation until Node is upgraded',
'UNSUPPORTED_NODE',
);
// ts-node's CLI places its executable in execArgv, rather than using -r.
// Source workers register their own compiler; compiled workers need none.
const preloads = __filename.endsWith('.ts')
? ['--require', require.resolve('ts-node/register/transpile-only')]
: [];
const armv6 =
process.arch === 'arm' &&
Number(
(process.config.variables as { arm_version?: string | number })
.arm_version,
) === 6;
const worker = fork(require.resolve('./databaseProcess'), [], {
serialization: 'advanced',
stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
execArgv: [
...preloads,
...(armv6 ? ['--jitless'] : []),
'--max-old-space-size=128',
],
env: {
...process.env,
NODE_OPTIONS: '',
...(__filename.endsWith('.ts')
? { TS_NODE_PROJECT: join(__dirname, 'tsconfig.worker.json') }
: {}),
},
});
this.stopped = new Promise((resolve) =>
worker.once('close', () => resolve()),
);
const onOwnerExit = () => worker.kill('SIGKILL');
process.once('exit', onOwnerExit);
worker.once('close', () => process.removeListener('exit', onOwnerExit));
this.worker = worker;
worker.on('message', (reply: Reply & { type?: string }) => {
if (reply.type === 'fatal' && reply.error) {
this.abort(new DatabaseError(reply.error.message, reply.error.code));
return;
}
const request = this.pending.get(reply.id);
if (!request) return;
this.pending.delete(reply.id);
clearTimeout(request.timer);
if (!this.pending.size) this.unref();
if (reply.error)
request.reject(
new DatabaseError(reply.error.message, reply.error.code),
);
else request.resolve(reply.value);
});
const fail = () =>
this.abort(
new DatabaseError(
'Database worker stopped; restart the service before retrying a write',
'DATABASE_WORKER_EXIT',
),
);
worker.on('error', fail);
worker.on('exit', () => {
if (!this.closed) fail();
});
worker.send({
storage: this.storage,
cacheStorage: this.cacheStorage,
ownerPid: process.pid,
});
this.unref();
return worker;
}
private send(
operation: string,
args: any[] = [],
token?: number,
): Promise<any> {
if (this.failed) return Promise.reject(this.failed);
if (this.closed)
return Promise.reject(
new DatabaseError('Database is closed', 'DATABASE_CLOSED'),
);
return new Promise((resolve, reject) => {
try {
const worker = this.start();
const id = ++this.nextId;
const timer = setTimeout(
() =>
this.abort(
new DatabaseError(
'Database worker timed out; writes are never automatically replayed',
'DATABASE_WORKER_TIMEOUT',
),
),
this.timeoutMs,
);
timer.unref();
this.pending.set(id, { resolve, reject, timer });
worker.ref();
worker.channel?.ref();
try {
worker.send({ id, operation, args, token }, (error) => {
if (error)
this.abort(
new DatabaseError(
'Database IPC disconnected',
'DATABASE_WORKER_EXIT',
),
);
});
} catch (error) {
this.pending.delete(id);
clearTimeout(timer);
if (!this.pending.size) this.unref();
reject(error);
}
} catch (error) {
reject(error);
}
});
}
private exclusive<T>(
operation: () => Promise<T>,
shutdown = false,
): Promise<T> {
if (!shutdown && this.queued >= this.queueLimit)
return Promise.reject(
new DatabaseError('Database request queue is full', 'DATABASE_BUSY'),
);
this.queued++;
const previous = this.tail;
let release!: () => void;
this.tail = new Promise<void>((resolve) => {
release = resolve;
});
return previous.then(operation).finally(() => {
this.queued--;
release();
});
}
request<T = any>(operation: string, ...args: any[]): Promise<T> {
const scope = this.scope.getStore();
if (scope) {
if (!scope.active || scope.child)
return Promise.reject(
new DatabaseError(
'Transaction scope is no longer available',
'TRANSACTION_SCOPE',
),
);
if (this.pending.size >= this.queueLimit)
return Promise.reject(
new DatabaseError('Database request queue is full', 'DATABASE_BUSY'),
);
return this.send(operation, args, scope.token);
}
return this.exclusive(() => this.send(operation, args));
}
async transaction<T>(operation: () => Promise<T>): Promise<T> {
const parent = this.scope.getStore();
if (parent && (!parent.active || parent.child))
throw new DatabaseError(
'Concurrent or expired nested transaction',
'TRANSACTION_SCOPE',
);
const run = async () => {
const token = ++this.nextId;
if (parent) parent.child = true;
const scope: Scope = { active: true, child: false, token };
let began = false;
let timer: NodeJS.Timeout | undefined;
try {
await this.send('begin', [], token);
began = true;
const deadline = new Promise<never>((_, reject) => {
timer = setTimeout(
() =>
reject(
new DatabaseError(
'Database transaction timed out',
'DATABASE_TRANSACTION_TIMEOUT',
),
),
this.timeoutMs,
);
});
const value = await Promise.race([
this.scope.run(scope, operation),
deadline,
]);
scope.active = false;
await this.send('commit', [], token);
return value;
} catch (error) {
scope.active = false;
if (began) {
try {
await this.send('rollback', [], token);
} catch {
this.abort(
new DatabaseError(
'Transaction rollback failed; restart the service',
'DATABASE_ROLLBACK_FAILED',
),
);
}
}
throw error;
} finally {
clearTimeout(timer);
if (parent) parent.child = false;
}
};
return parent ? run() : this.exclusive(run);
}
query<T = Record<string, unknown>>(
sql: string,
params: unknown[] = [],
): Promise<T[]> {
return this.request('query', sql, params);
}
get healthy() {
return !this.failed && !this.closed;
}
async close() {
if (this.scope.getStore())
throw new DatabaseError(
'Cannot close within a transaction',
'TRANSACTION_SCOPE',
);
await this.exclusive(async () => {
if (this.closed) return;
try {
if (this.worker && !this.failed) await this.send('close');
} finally {
this.closed = true;
this.worker?.ref();
this.worker?.kill('SIGKILL');
await this.stopped;
}
}, true);
}
}
+36
View File
@@ -0,0 +1,36 @@
// Error details (SQL, bindings, credentials) stay inside the database worker.
export class DatabaseError extends Error {
constructor(message: string, public readonly code: string) {
super(message);
this.name = 'DatabaseError';
}
}
export function databaseStartupFailure(value: unknown) {
const error = (value && typeof value === 'object' ? value : {}) as Error & { code?: string };
const missing = /Cannot find (?:module|package) ['"]([^'"]+)['"]/.exec(
error.message || '',
)?.[1];
const importer = /Require stack:\r?\n-\s+([^\r\n]+)/.exec(
error.message || '',
)?.[1];
const dependency =
/MODULE_NOT_FOUND/.test(error.code || '') &&
Boolean(
missing &&
((!missing.startsWith('.') &&
!missing.startsWith('/') &&
!missing.startsWith('node:')) ||
/node_modules[/\\]/.test(missing) ||
/node_modules[/\\]/.test(importer || '')),
);
return dependency
? {
code: 'MODULE_NOT_FOUND',
message: "Cannot find module '@mikro-orm/sql'",
}
: {
code: 'DATABASE_WORKER_EXIT',
message: 'Database process failed to start',
};
}
+61
View File
@@ -0,0 +1,61 @@
import { databaseStartupFailure } from './databaseErrors';
import { Worker } from 'worker_threads';
// IPC disconnect cannot run while DatabaseSync is executing SQL. Start this
// independent, allocation-light monitor before opening SQLite so a SIGKILLed
// backend cannot leave an orphan holding the old service's database write lock.
async function monitorOwner(ownerPid: number) {
if (!Number.isSafeInteger(ownerPid) || ownerPid < 1)
throw new Error('Invalid database owner');
const monitor = new Worker(
`
const {parentPort,workerData}=require('node:worker_threads');
process.on('uncaughtException',()=>process.kill(process.pid,'SIGKILL'));
const check=()=>{
if(process.ppid!==workerData.ownerPid)process.kill(process.pid,'SIGKILL');
};
check();
setInterval(check,500);
parentPort.postMessage('ready');
`,
{
eval: true,
workerData: { ownerPid },
// V8's process-wide --jitless remains active on ARMv6. Do not load the
// source worker's ts-node compiler into this tiny, plain-JavaScript monitor.
execArgv: [],
resourceLimits: {
maxOldGenerationSizeMb: 8,
maxYoungGenerationSizeMb: 2,
stackSizeMb: 1,
},
},
);
monitor.once('exit', () => process.exit(1));
await new Promise<void>((resolve, reject) => {
monitor.once('message', () => resolve());
monitor.once('error', reject);
});
monitor.unref();
}
// The bootstrap catches dependency import errors before the ORM module loads.
// The advanced IPC serializer preserves Date, Buffer and undefined values.
process.once('disconnect', () => process.exit(1));
process.once('message', async (settings) => {
const pending: unknown[] = [];
const buffer = (request: unknown) => pending.push(request);
// Keep a listener throughout asynchronous monitor startup. Node may flush
// several buffered IPC messages while delivering the initial settings.
process.on('message', buffer);
try {
await monitorOwner((settings as { ownerPid: number }).ownerPid);
require('./databaseWorker').startDatabaseWorker(settings);
process.removeListener('message', buffer);
for (const request of pending) process.emit('message', request, undefined);
} catch (error) {
process.send?.({ type: 'fatal', error: databaseStartupFailure(error) });
} finally {
process.removeListener('message', buffer);
}
});
+280
View File
@@ -0,0 +1,280 @@
import { mkdirSync } from 'fs';
import { dirname } from 'path';
import { EntitySchema, Type } from '@mikro-orm/core';
import { MikroORM, SqliteDriver, NodeSqliteDialect, raw } from '@mikro-orm/sql';
import { entityDefinitions } from './schema';
// Sequelize DATE readers expect TEXT. Numeric epoch dates would break rollback.
class LegacyDateType extends Type<Date, string> {
convertToDatabaseValue(value: Date) {
return value == null
? value
: value.toISOString().replace('T', ' ').replace('Z', ' +00:00');
}
convertToJSValue(value: string | Date) {
return value == null
? value
: value instanceof Date
? value
: new Date(value);
}
getColumnType() {
return 'DATETIME';
}
compareAsType() {
return 'string' as const;
}
get runtimeType() {
return 'Date';
}
}
const entities = Object.values(entityDefinitions).map(
(definition) =>
new EntitySchema({
...definition,
properties: Object.fromEntries(
Object.entries(definition.properties).map(([name, field]) => [
name,
{
...field,
type:
field.type === 'legacyDate' ? new LegacyDateType() : field.type,
},
]),
),
} as any),
);
let orm: Awaited<ReturnType<typeof MikroORM.init<SqliteDriver>>>;
let cache: any;
const transactions: number[] = [];
let ready: Promise<void>;
let workerData: { storage: string; cacheStorage?: string; ownerPid: number };
async function initialize() {
if (workerData.storage !== ':memory:')
mkdirSync(dirname(workerData.storage), { recursive: true });
orm = await MikroORM.init<SqliteDriver>({
driver: SqliteDriver,
driverOptions: new NodeSqliteDialect(workerData.storage),
dbName: workerData.storage,
entities,
debug: false,
allowGlobalContext: false,
discovery: { warnWhenNoEntities: false },
});
await orm.em.getConnection().execute('PRAGMA busy_timeout=10000');
}
function filter(name: keyof typeof entityDefinitions, value: any): any {
if (value === undefined) throw new Error('Undefined database predicate');
if (value == null || typeof value !== 'object' || value instanceof Date)
return value;
if (Array.isArray(value)) return value.map((part) => filter(name, part));
const properties = entityDefinitions[name].properties;
return Object.fromEntries(
Object.entries(value).map(([key, item]: [string, any]) => {
if (item === undefined) throw new Error('Undefined database predicate');
// LIKE searches JSON's serialized text, matching the pre-migration behavior.
if (
key in properties &&
(properties as any)[key].type === 'json' &&
item &&
typeof item === 'object' &&
'$like' in item
)
return [raw(`cast("${key}" as text)`), item];
return [key, filter(name, item)];
}),
);
}
function values(
name: keyof typeof entityDefinitions,
payload: Record<string, any>,
insert = false,
) {
const properties = entityDefinitions[name].properties;
const data = Object.fromEntries(
Object.entries(payload).filter(
([key, value]) => key in properties && value !== undefined,
),
);
for (const [key, value] of Object.entries(data)) {
if ((properties as any)[key].type === 'json' && value !== null)
data[key] = JSON.parse(JSON.stringify(value));
}
const now = new Date();
if (insert && !data.createdAt) data.createdAt = now;
data.updatedAt = now;
return data;
}
function plain(name: keyof typeof entityDefinitions, row: any) {
if (!row) return null;
return Object.fromEntries(
Object.keys(entityDefinitions[name].properties)
.filter((key) => row[key] !== undefined)
.map((key) => [key, row[key]]),
);
}
async function execute(
operation: string,
args: any[],
token?: number,
): Promise<any> {
await ready;
// Reject queued work after owner death; the monitor interrupts running SQL.
if (process.ppid !== workerData.ownerPid) process.exit(1);
const connection = orm.em.getConnection();
if (operation === 'begin') {
await connection.execute(
transactions.length ? `SAVEPOINT "ql_${token}"` : 'BEGIN IMMEDIATE',
);
transactions.push(token!);
return;
}
if (token !== transactions[transactions.length - 1])
throw new Error('Database transaction ownership mismatch');
if (operation === 'commit' || operation === 'rollback') {
if (!transactions.length) throw new Error('No active database transaction');
const nested = transactions.length > 1;
if (operation === 'rollback' && nested)
await connection.execute(`ROLLBACK TO SAVEPOINT "ql_${token}"`);
await connection.execute(
nested ? `RELEASE SAVEPOINT "ql_${token}"` : operation.toUpperCase(),
);
transactions.pop();
return;
}
if (operation === 'close') {
if (transactions.length)
throw new Error('Cannot close an active transaction');
cache?.close();
await orm.close();
return;
}
if (operation === 'query') return connection.execute(args[0], args[1]);
if (operation === 'cache') {
if (!workerData.cacheStorage)
throw new Error('No cache database configured');
if (!cache) {
const { DatabaseSync } = require('node:sqlite');
mkdirSync(dirname(workerData.cacheStorage), { recursive: true });
cache = new DatabaseSync(workerData.cacheStorage);
cache.exec(
'PRAGMA busy_timeout=10000;CREATE TABLE IF NOT EXISTS keyv(key VARCHAR(255) PRIMARY KEY,value TEXT)',
);
}
const [action, key, value] = args;
switch (action) {
case 'get':
return cache.prepare('SELECT value FROM keyv WHERE key=?').get(key)
?.value;
case 'set':
cache
.prepare(
'INSERT INTO keyv(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value',
)
.run(key, value);
return true;
case 'delete':
return (
cache.prepare('DELETE FROM keyv WHERE key=?').run(key).changes > 0
);
default:
throw new Error('Unsupported cache operation');
}
}
const [name, condition, options = {}] = args as [
keyof typeof entityDefinitions,
any,
any,
];
if (!(name in entityDefinitions)) throw new Error('Unknown entity');
const em = orm.em.fork();
const entityName = entities.find((entity) => entity.meta.className === name)!;
if (operation !== 'insert' && condition == null)
throw new Error('Missing database predicate');
const query = operation === 'insert' ? {} : filter(name, condition);
const readOptions = {
...options,
orderBy: options.orderBy?.map((order: Record<string, string>) =>
Object.fromEntries(
Object.entries(order).map(([field, direction]) => [
name === 'Env' && field === 'isPinned'
? raw('COALESCE("isPinned",0)')
: field,
direction,
]),
),
),
};
switch (operation) {
case 'find':
return (await em.find(entityName, query, readOptions)).map((row) =>
plain(name, row),
);
case 'findOne':
return plain(name, await em.findOne(entityName, query, readOptions));
case 'count':
return em.count(entityName, query);
case 'insert': {
const id = await em.insert(entityName, values(name, condition, true));
return plain(name, await em.findOneOrFail(entityName, { id }));
}
case 'nativeUpdate':
return em.nativeUpdate(entityName, query, values(name, options));
case 'nativeDelete':
return em.nativeDelete(entityName, query);
default:
throw new Error('Unsupported database operation');
}
}
export function startDatabaseWorker(settings: typeof workerData) {
workerData = settings;
ready = initialize();
void ready.catch(() => {});
let queue = Promise.resolve();
process.on('message', ({ id, operation, args, token }) => {
// Continue after a failed statement so the owner can still roll back.
queue = queue.then(async () => {
try {
process.send?.({
id,
value: await execute(operation, args, token),
});
} catch (error: any) {
const code =
error.name === 'UniqueConstraintViolationException'
? 'UNIQUE_CONSTRAINT'
: error.name === 'TableNotFoundException'
? 'DATABASE_SCHEMA_MISSING'
: (
{
5: 'SQLITE_BUSY',
6: 'SQLITE_LOCKED',
8: 'SQLITE_READONLY',
10: 'SQLITE_IOERR',
11: 'SQLITE_CORRUPT',
13: 'SQLITE_FULL',
14: 'SQLITE_CANTOPEN',
26: 'SQLITE_NOTADB',
} as Record<number, string>
)[Number(error.errcode) & 255] || 'DATABASE_QUERY_FAILED';
process.send?.({
id,
error: {
code,
message:
code === 'UNIQUE_CONSTRAINT'
? 'Resource already exists'
: `Database operation failed (${code})`,
},
});
}
});
});
}
+3 -16
View File
@@ -1,5 +1,4 @@
import { sequelize } from '.'; import { Repository, RecordRow } from './repository';
import { DataTypes, Model, ModelDefined } from 'sequelize';
export class Dependence { export class Dependence {
timestamp?: string; timestamp?: string;
@@ -47,17 +46,5 @@ export enum versionDependenceCommandTypes {
'=', '=',
} }
export interface DependenceInstance export type DependenceInstance = RecordRow<Dependence>;
extends Model<Dependence, Dependence>, export const DependenceModel = new Repository<Dependence>('Dependence');
Dependence {}
export const DependenceModel = sequelize.define<DependenceInstance>(
'Dependence',
{
name: DataTypes.STRING,
type: DataTypes.NUMBER,
timestamp: DataTypes.STRING,
status: DataTypes.NUMBER,
log: DataTypes.JSON,
remark: DataTypes.STRING,
},
);
+3 -13
View File
@@ -1,5 +1,4 @@
import { DataTypes, Model } from 'sequelize'; import { Repository, RecordRow } from './repository';
import { sequelize } from '.';
export class Env { export class Env {
value?: string; value?: string;
@@ -38,14 +37,5 @@ export const initPosition = 4500000000000000;
export const stepPosition = 10000000000; export const stepPosition = 10000000000;
export const minPosition = 100; export const minPosition = 100;
export interface EnvInstance extends Model<Env, Env>, Env {} export type EnvInstance = RecordRow<Env>;
export const EnvModel = sequelize.define<EnvInstance>('Env', { export const EnvModel = new Repository<Env>('Env');
value: { type: DataTypes.STRING, unique: 'compositeIndex' },
timestamp: DataTypes.STRING,
status: DataTypes.NUMBER,
position: DataTypes.NUMBER,
name: { type: DataTypes.STRING, unique: 'compositeIndex' },
remarks: DataTypes.STRING,
isPinned: DataTypes.NUMBER,
labels: DataTypes.JSON,
});
+5 -18
View File
@@ -1,23 +1,10 @@
import { Sequelize, Transaction } from 'sequelize';
import config from '../config/index'; import config from '../config/index';
import { join } from 'path'; import { join } from 'path';
import { DatabaseClient } from './databaseClient';
export const sequelize = new Sequelize({ export const database = new DatabaseClient(
dialect: 'sqlite', join(config.dbPath, 'database.sqlite'),
storage: join(config.dbPath, 'database.sqlite'), join(config.dbPath, 'keyv.sqlite'),
logging: false, );
retry: {
max: 10,
match: ['SQLITE_BUSY: database is locked'],
},
pool: {
max: 5,
min: 2,
idle: 30000,
acquire: 30000,
evict: 10000,
},
transactionType: Transaction.TYPES.IMMEDIATE,
});
export type ResponseType<T> = { code: number; data?: T; message?: string }; export type ResponseType<T> = { code: number; data?: T; message?: string };
+11 -11
View File
@@ -1,5 +1,4 @@
import { sequelize } from '.'; import { Repository, RecordRow } from './repository';
import { DataTypes, Model, ModelDefined } from 'sequelize';
export class App { export class App {
name: string; name: string;
@@ -24,13 +23,14 @@ export interface AppToken {
expiration: number; expiration: number;
} }
export type AppScope = 'envs' | 'crons' | 'configs' | 'scripts' | 'logs' | 'system' | 'dashboard'; export type AppScope =
| 'envs'
| 'crons'
| 'configs'
| 'scripts'
| 'logs'
| 'system'
| 'dashboard';
export interface AppInstance extends Model<App, App>, App {} export type AppInstance = RecordRow<App>;
export const AppModel = sequelize.define<AppInstance>('App', { export const AppModel = new Repository<App>('App');
name: { type: DataTypes.STRING, unique: 'name' },
scopes: DataTypes.JSON,
client_id: DataTypes.STRING,
client_secret: DataTypes.STRING,
tokens: DataTypes.JSON,
});
+25
View File
@@ -0,0 +1,25 @@
export type Query = Record<string, any>;
export type QuerySpec = {
where: Query;
orderBy?: Record<string, string>[];
offset?: number;
limit?: number;
};
export function orderBy(order: string[][]): Record<string, string>[] {
return order.map(([field, direction]) => {
if (!/^(asc|desc)$/i.test(direction))
throw new Error('Invalid sort direction');
return { [field]: direction.toLowerCase() };
});
}
export function textSearch(fields: string[], text: string): Query {
return {
$or: fields.flatMap((field) =>
[...new Set([text, encodeURI(text)])].map((value) => ({
[field]: { $like: `%${value}%` },
})),
),
};
}
+54
View File
@@ -0,0 +1,54 @@
import { database } from '.';
import { DatabaseClient } from './databaseClient';
export type RecordRow<T> = T & { id: number; createdAt: Date; updatedAt: Date };
export type Query = Record<string, any>;
export type ReadOptions = {
fields?: string[];
orderBy?: Record<string, string>[];
limit?: number;
offset?: number;
};
// A small IPC boundary with MikroORM's native query/operation names. No ORM
// entities or request identity maps escape the single database worker.
export class Repository<T> {
constructor(
private name: string,
private client: () => DatabaseClient = () => database,
) {}
find(where: Query = {}, options: ReadOptions = {}): Promise<RecordRow<T>[]> {
return this.client().request('find', this.name, where, options);
}
findOne(
where: Query,
options: ReadOptions = {},
): Promise<RecordRow<T> | null> {
return this.client().request('findOne', this.name, where, options);
}
count(where: Query = {}): Promise<number> {
return this.client().request('count', this.name, where);
}
insert(data: Partial<T>): Promise<RecordRow<T>> {
return this.client().request('insert', this.name, data);
}
nativeUpdate(where: Query, data: Partial<T>): Promise<number> {
return this.client().request('nativeUpdate', this.name, where, data);
}
nativeDelete(where: Query): Promise<number> {
return this.client().request('nativeDelete', this.name, where);
}
async insertMany(data: Partial<T>[]): Promise<RecordRow<T>[]> {
return this.client().transaction(async () => {
const rows = [];
for (const item of data) rows.push(await this.insert(item));
return rows;
});
}
async getOrCreate(where: Query): Promise<RecordRow<T>> {
return this.client().transaction(
async () =>
(await this.findOne(where)) || this.insert(where as Partial<T>),
);
}
}
+3 -38
View File
@@ -1,5 +1,4 @@
import { sequelize } from '.'; import { Repository, RecordRow } from './repository';
import { DataTypes, Model } from 'sequelize';
export enum InstanceStatus { export enum InstanceStatus {
'running' = 0, 'running' = 0,
@@ -41,41 +40,7 @@ export class RunningInstance {
} }
} }
export interface RunningInstanceModel export type RunningInstanceModel = RecordRow<RunningInstanceAttributes>;
extends Model<RunningInstanceAttributes, RunningInstanceAttributes>, export const RunningInstanceModel = new Repository<RunningInstanceAttributes>(
RunningInstanceAttributes {}
export const RunningInstanceModel = sequelize.define<RunningInstanceModel>(
'RunningInstance', 'RunningInstance',
{
cron_id: {
type: DataTypes.NUMBER,
allowNull: false,
},
pid: {
type: DataTypes.NUMBER,
allowNull: true,
},
log_path: {
type: DataTypes.STRING,
allowNull: true,
},
started_at: {
type: DataTypes.NUMBER,
allowNull: false,
},
finished_at: {
type: DataTypes.NUMBER,
allowNull: true,
},
status: {
type: DataTypes.NUMBER,
allowNull: false,
defaultValue: InstanceStatus.running,
},
exit_code: {
type: DataTypes.NUMBER,
allowNull: true,
},
},
); );
+178
View File
@@ -0,0 +1,178 @@
// Persisted SQLite names and types are part of the rollback contract.
type Field = {
type: string;
fieldName: string;
nullable?: boolean;
primary?: boolean;
autoincrement?: boolean;
};
function entity<Name extends string>(
name: Name,
tableName: string,
groups: Record<string, string[]>,
required: string[] = [],
) {
const properties: Record<string, Field> = {
id: { type: 'number', fieldName: 'id', primary: true, autoincrement: true },
};
for (const [type, fields] of Object.entries(groups))
for (const field of fields)
properties[field] = {
type,
fieldName: field,
nullable: !required.includes(field),
};
for (const field of ['createdAt', 'updatedAt'])
properties[field] = {
type: 'legacyDate',
fieldName: field,
nullable: false,
};
return { name, tableName, properties };
}
export const entityDefinitions = {
App: entity(
'App',
'Apps',
{
string: ['name', 'client_id', 'client_secret'],
json: ['scopes', 'tokens'],
},
[],
),
Auth: entity('Auth', 'Auths', { string: ['ip', 'type'], json: ['info'] }, []),
CrontabStat: entity(
'CrontabStat',
'CrontabStats',
{
number: [
'ref_id',
'run_count',
'success_count',
'fail_count',
'total_time',
'max_time',
],
string: ['date'],
},
['ref_id', 'date'],
),
CrontabView: entity(
'CrontabView',
'CrontabViews',
{
string: ['name', 'filterRelation'],
number: ['position', 'isDisabled', 'type'],
json: ['filters', 'sorts'],
},
[],
),
Crontab: entity(
'Crontab',
'Crontabs',
{
string: [
'name',
'command',
'schedule',
'timestamp',
'log_path',
'queued_token',
'task_before',
'task_after',
'log_name',
'work_dir',
],
boolean: ['saved'],
number: [
'status',
'isSystem',
'pid',
'isDisabled',
'isPinned',
'last_running_time',
'last_execution_time',
'sub_id',
'allow_multiple_instances',
],
json: ['labels', 'extra_schedules'],
},
[],
),
Dependence: entity(
'Dependence',
'Dependences',
{
string: ['name', 'timestamp', 'remark'],
number: ['type', 'status'],
json: ['log'],
},
[],
),
Env: entity(
'Env',
'Envs',
{
string: ['value', 'timestamp', 'name', 'remarks'],
number: ['status', 'position', 'isPinned'],
json: ['labels'],
},
[],
),
RunningInstance: entity(
'RunningInstance',
'RunningInstances',
{
number: [
'cron_id',
'pid',
'started_at',
'finished_at',
'status',
'exit_code',
],
string: ['log_path'],
},
['cron_id', 'started_at', 'status'],
),
Subscription: entity(
'Subscription',
'Subscriptions',
{
string: [
'name',
'url',
'schedule',
'type',
'whitelist',
'blacklist',
'dependences',
'extensions',
'sub_before',
'sub_after',
'branch',
'pull_type',
'log_path',
'schedule_type',
'alias',
'proxy',
],
json: ['interval_schedule', 'pull_option'],
number: ['status', 'pid', 'is_disabled', 'autoAddCron', 'autoDelCron'],
},
[],
),
};
export const initialSchema = [
'CREATE TABLE IF NOT EXISTS `Apps` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `scopes` JSON, `client_id` VARCHAR(255), `client_secret` VARCHAR(255), `tokens` JSON, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`name`))',
'CREATE TABLE IF NOT EXISTS `Auths` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `ip` VARCHAR(255), `type` VARCHAR(255), `info` JSON, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL)',
'CREATE TABLE IF NOT EXISTS `CrontabStats` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `ref_id` NUMBER NOT NULL, `date` VARCHAR(255) NOT NULL, `run_count` NUMBER DEFAULT 0, `success_count` NUMBER DEFAULT 0, `fail_count` NUMBER DEFAULT 0, `total_time` NUMBER DEFAULT 0, `max_time` NUMBER DEFAULT 0, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL)',
'CREATE TABLE IF NOT EXISTS `CrontabViews` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `position` NUMBER, `isDisabled` NUMBER, `filters` JSON, `sorts` JSON, `filterRelation` VARCHAR(255), `type` NUMBER, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`name`))',
'CREATE TABLE IF NOT EXISTS `Crontabs` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `command` VARCHAR(255), `schedule` VARCHAR(255), `timestamp` VARCHAR(255), `saved` TINYINT(1), `status` NUMBER, `isSystem` NUMBER, `pid` NUMBER, `isDisabled` NUMBER, `isPinned` NUMBER, `log_path` VARCHAR(255), `queued_token` VARCHAR(255), `labels` JSON, `last_running_time` NUMBER, `last_execution_time` NUMBER, `sub_id` NUMBER, `extra_schedules` JSON, `task_before` VARCHAR(255), `task_after` VARCHAR(255), `log_name` VARCHAR(255), `allow_multiple_instances` NUMBER, `work_dir` VARCHAR(255), `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`name`, `command`, `schedule`))',
'CREATE TABLE IF NOT EXISTS `Dependences` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `type` NUMBER, `timestamp` VARCHAR(255), `status` NUMBER, `log` JSON, `remark` VARCHAR(255), `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL)',
'CREATE TABLE IF NOT EXISTS `Envs` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `value` VARCHAR(255), `timestamp` VARCHAR(255), `status` NUMBER, `position` NUMBER, `name` VARCHAR(255), `remarks` VARCHAR(255), `isPinned` NUMBER, `labels` JSON, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`value`, `name`))',
'CREATE TABLE IF NOT EXISTS `RunningInstances` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `cron_id` NUMBER NOT NULL, `pid` NUMBER, `log_path` VARCHAR(255), `started_at` NUMBER NOT NULL, `finished_at` NUMBER, `status` NUMBER NOT NULL DEFAULT 0, `exit_code` NUMBER, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL)',
'CREATE TABLE IF NOT EXISTS `Subscriptions` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `url` VARCHAR(255), `schedule` VARCHAR(255), `interval_schedule` JSON, `type` VARCHAR(255), `whitelist` VARCHAR(255), `blacklist` VARCHAR(255), `status` NUMBER, `dependences` VARCHAR(255), `extensions` VARCHAR(255), `sub_before` VARCHAR(255), `sub_after` VARCHAR(255), `branch` VARCHAR(255), `pull_type` VARCHAR(255), `pull_option` JSON, `pid` NUMBER, `is_disabled` NUMBER, `log_path` VARCHAR(255), `schedule_type` VARCHAR(255), `alias` VARCHAR(255), `proxy` VARCHAR(255), `autoAddCron` NUMBER, `autoDelCron` NUMBER, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`name`, `url`, `schedule`, `interval_schedule`), UNIQUE (`alias`))',
'CREATE INDEX IF NOT EXISTS `crontab_stats_date` ON `CrontabStats` (`date`)',
'CREATE UNIQUE INDEX IF NOT EXISTS `crontab_stats_ref_id_date` ON `CrontabStats` (`ref_id`, `date`)',
];
+45
View File
@@ -0,0 +1,45 @@
import { database } from '.';
// Aggregate SQL stays explicit and bound; public input is never used as an SQL
// identifier. These queries preserve the dashboard response names.
export const statistics = {
async totals(date: string) {
const rows = await database.query(
`SELECT SUM(run_count) AS total_runs,
SUM(success_count) AS total_success, SUM(fail_count) AS total_fail,
SUM(total_time) AS total_time FROM CrontabStats WHERE date=?`,
[date],
);
return rows[0];
},
results(date: string, column: 'success_count' | 'fail_count') {
if (!['success_count', 'fail_count'].includes(column))
throw new Error('Invalid statistic');
return database.query(
`SELECT ref_id,SUM(${column}) AS result_count FROM CrontabStats
WHERE date=? AND ${column}>0 GROUP BY ref_id ORDER BY result_count DESC,ref_id ASC`,
[date],
);
},
trend(dates: string[]) {
if (!dates.length) return Promise.resolve([]);
return database.query(
`SELECT date,SUM(run_count) AS total_runs,
SUM(success_count) AS total_success,SUM(fail_count) AS total_fail FROM CrontabStats
WHERE date IN (${dates
.map(() => '?')
.join(',')}) GROUP BY date ORDER BY date ASC`,
dates,
);
},
ranking(date: string, metric: 'total_time' | 'run_count') {
if (!['total_time', 'run_count'].includes(metric))
throw new Error('Invalid statistic');
return database.query(
`SELECT ref_id,SUM(total_time) AS total_time,SUM(run_count) AS run_count,
SUM(success_count) AS success_count,MAX(max_time) AS max_time FROM CrontabStats
WHERE date=? AND run_count>0 GROUP BY ref_id ORDER BY SUM(${metric}) DESC LIMIT 5`,
[date],
);
},
};
+3 -45
View File
@@ -1,5 +1,4 @@
import { sequelize } from '.'; import { Repository, RecordRow } from './repository';
import { DataTypes, Model, ModelDefined } from 'sequelize';
import { SimpleIntervalSchedule } from 'toad-scheduler'; import { SimpleIntervalSchedule } from 'toad-scheduler';
type SimpleIntervalScheduleUnit = keyof SimpleIntervalSchedule; type SimpleIntervalScheduleUnit = keyof SimpleIntervalSchedule;
@@ -70,46 +69,5 @@ export enum SubscriptionStatus {
'queued', 'queued',
} }
export interface SubscriptionInstance export type SubscriptionInstance = RecordRow<Subscription>;
extends Model<Subscription, Subscription>, export const SubscriptionModel = new Repository<Subscription>('Subscription');
Subscription {}
export const SubscriptionModel = sequelize.define<SubscriptionInstance>(
'Subscription',
{
name: {
unique: 'compositeIndex',
type: DataTypes.STRING,
},
url: {
unique: 'compositeIndex',
type: DataTypes.STRING,
},
schedule: {
unique: 'compositeIndex',
type: DataTypes.STRING,
},
interval_schedule: {
unique: 'compositeIndex',
type: DataTypes.JSON,
},
type: DataTypes.STRING,
whitelist: DataTypes.STRING,
blacklist: DataTypes.STRING,
status: DataTypes.NUMBER,
dependences: DataTypes.STRING,
extensions: DataTypes.STRING,
sub_before: DataTypes.STRING,
sub_after: DataTypes.STRING,
branch: DataTypes.STRING,
pull_type: DataTypes.STRING,
pull_option: DataTypes.JSON,
pid: DataTypes.NUMBER,
is_disabled: DataTypes.NUMBER,
log_path: DataTypes.STRING,
schedule_type: DataTypes.STRING,
alias: { type: DataTypes.STRING, unique: 'alias' },
proxy: { type: DataTypes.STRING, allowNull: true },
autoAddCron: { type: DataTypes.NUMBER, allowNull: true },
autoDelCron: { type: DataTypes.NUMBER, allowNull: true },
},
);
+3 -13
View File
@@ -1,5 +1,4 @@
import { sequelize } from '.'; import { Repository, RecordRow } from './repository';
import { DataTypes, Model, ModelDefined } from 'sequelize';
import { NotificationInfo } from './notify'; import { NotificationInfo } from './notify';
export class SystemInfo { export class SystemInfo {
@@ -106,14 +105,5 @@ export type SystemModelInfo = SystemConfigInfo &
Partial<NotifyLogInfo> & Partial<NotifyLogInfo> &
Partial<AuthInfo>; Partial<AuthInfo>;
export interface SystemInstance export type SystemInstance = RecordRow<SystemInfo>;
extends Model<SystemInfo, SystemInfo>, export const SystemModel = new Repository<SystemInfo>('Auth');
SystemInfo {}
export const SystemModel = sequelize.define<SystemInstance>('Auth', {
ip: DataTypes.STRING,
type: DataTypes.STRING,
info: {
type: DataTypes.JSON,
allowNull: true,
},
});
+11
View File
@@ -0,0 +1,11 @@
{
"extends": "../tsconfig.json",
"compilerOptions": {
"module": "node20",
"moduleResolution": "node16",
"rootDir": "..",
"outDir": "../../static/build"
},
"include": ["./databaseWorker.ts", "./schema.ts"],
"exclude": ["node_modules"]
}
+6 -22
View File
@@ -1,30 +1,14 @@
import Logger from './logger'; import Logger from './logger';
import { EnvModel } from '../data/env'; import { database } from '../data';
import { CrontabModel } from '../data/cron'; import { initialSchema } from '../data/schema';
import { DependenceModel } from '../data/dependence';
import { AppModel } from '../data/open';
import { SystemModel } from '../data/system';
import { SubscriptionModel } from '../data/subscription';
import { CrontabViewModel } from '../data/cronView';
import { CrontabStatModel } from '../data/cronStats';
import { RunningInstanceModel } from '../data/runningInstance';
import { sequelize } from '../data';
import { migrateSchema } from '../shared/schemaMigrations'; import { migrateSchema } from '../shared/schemaMigrations';
export default async () => { export default async () => {
try { try {
await CrontabModel.sync(); await database.transaction(async () => {
await DependenceModel.sync(); for (const sql of initialSchema) await database.query(sql);
await AppModel.sync(); await migrateSchema(database);
await SystemModel.sync(); });
await EnvModel.sync();
await SubscriptionModel.sync();
await CrontabViewModel.sync();
await CrontabStatModel.sync();
await RunningInstanceModel.sync();
await migrateSchema(sequelize);
Logger.info('[boot] DB loaded'); Logger.info('[boot] DB loaded');
} catch (error) { } catch (error) {
Logger.error('[boot] DB load failed', error); Logger.error('[boot] DB load failed', error);
+85 -99
View File
@@ -16,7 +16,7 @@ import { t } from '../shared/i18n';
import { AppScope } from '../data/open'; import { AppScope } from '../data/open';
import uniqueConstraintError from '../middlewares/uniqueConstraintError'; import uniqueConstraintError from '../middlewares/uniqueConstraintError';
import protectedPathCase from '../middlewares/protectedPathCase'; import protectedPathCase from '../middlewares/protectedPathCase';
import { withAuthAvailability } from '../middlewares/authAvailability';
function resolveTrustProxy(value = process.env.QL_TRUST_PROXY) { function resolveTrustProxy(value = process.env.QL_TRUST_PROXY) {
const setting = value?.trim(); const setting = value?.trim();
if (!setting) { if (!setting) {
@@ -30,24 +30,20 @@ function resolveTrustProxy(value = process.env.QL_TRUST_PROXY) {
} }
return setting; return setting;
} }
export default ({ app }: { app: Application }) => { export default ({ app }: { app: Application }) => {
// Security: Enable strict routing to prevent case-insensitive path bypass // Security: Enable strict routing to prevent case-insensitive path bypass
app.set('case sensitive routing', true); app.set('case sensitive routing', true);
app.set('strict routing', true); app.set('strict routing', true);
app.set('trust proxy', resolveTrustProxy()); app.set('trust proxy', resolveTrustProxy());
app.use(cors()); app.use(cors());
// Security: Reject case variations under protected API namespaces before // Security: Reject case variations under protected API namespaces before
// authentication checks can interpret the request differently from routing. // authentication checks can interpret the request differently from routing.
app.use(protectedPathCase); app.use(protectedPathCase);
// Rewrite URLs to strip baseUrl prefix if configured // Rewrite URLs to strip baseUrl prefix if configured
// This allows the rest of the app to work without baseUrl awareness // This allows the rest of the app to work without baseUrl awareness
if (config.baseUrl) { if (config.baseUrl) {
app.use(rewrite(`${config.baseUrl}/*`, '/$1')); app.use(rewrite(`${config.baseUrl}/*`, '/$1'));
} }
app.get(`${config.api.prefix}/env.js`, serveEnv); app.get(`${config.api.prefix}/env.js`, serveEnv);
app.use( app.use(
`${config.api.prefix}/static`, `${config.api.prefix}/static`,
@@ -58,7 +54,6 @@ export default ({ app }: { app: Application }) => {
}, },
}), }),
); );
const credentialPaths = ['/api', '/open'].flatMap((prefix) => const credentialPaths = ['/api', '/open'].flatMap((prefix) =>
['/user/login', '/user/init', '/user/two-factor/login'].map( ['/user/login', '/user/init', '/user/two-factor/login'].map(
(route) => `${prefix}${route}`, (route) => `${prefix}${route}`,
@@ -69,13 +64,10 @@ export default ({ app }: { app: Application }) => {
bodyParser.json({ limit: '16kb' }), bodyParser.json({ limit: '16kb' }),
bodyParser.urlencoded({ limit: '16kb', extended: false }), bodyParser.urlencoded({ limit: '16kb', extended: false }),
); );
app.use(bodyParser.json({ limit: '50mb' })); app.use(bodyParser.json({ limit: '50mb' }));
app.use(bodyParser.urlencoded({ limit: '50mb', extended: true })); app.use(bodyParser.urlencoded({ limit: '50mb', extended: true }));
const frontendPath = path.join(config.rootPath, 'static/dist'); const frontendPath = path.join(config.rootPath, 'static/dist');
app.use(express.static(frontendPath)); app.use(express.static(frontendPath));
app.use( app.use(
expressjwt({ expressjwt({
secret: config.jwt.secret, secret: config.jwt.secret,
@@ -84,7 +76,6 @@ export default ({ app }: { app: Application }) => {
path: [...config.apiWhiteList, /^(\/(?!api\/).*)$/i], path: [...config.apiWhiteList, /^(\/(?!api\/).*)$/i],
}), }),
); );
app.use((req: Request, res, next) => { app.use((req: Request, res, next) => {
if (!req.headers) { if (!req.headers) {
req.platform = 'desktop'; req.platform = 'desktop';
@@ -94,91 +85,84 @@ export default ({ app }: { app: Application }) => {
} }
return next(); return next();
}); });
app.use(
app.use(async (req: Request, res, next) => { withAuthAvailability(async (req: Request, res, next) => {
const pathLower = req.path.toLowerCase(); const pathLower = req.path.toLowerCase();
if (!['/open/', '/api/'].some((x) => pathLower.startsWith(x))) { if (!['/open/', '/api/'].some((x) => pathLower.startsWith(x))) {
return next();
}
const headerToken = getToken(req);
if (pathLower.startsWith('/open/')) {
const apps = await shareStore.getApps();
const doc = apps?.filter((x) =>
x.tokens?.find((y) => y.value === headerToken),
)?.[0];
if (doc && doc.tokens && doc.tokens.length > 0) {
const currentToken = doc.tokens.find((x) => x.value === headerToken);
const keyMatch = pathLower.match(/\/open\/([a-z]+)\/*/);
const key = keyMatch && keyMatch[1];
if (!doc.scopes.includes(key as AppScope)) {
const err = new UnauthorizedError('credentials_bad_scheme', {
message: t('暂无权限'),
});
return next(err);
}
if (
!currentToken ||
currentToken.expiration < Math.round(Date.now() / 1000)
) {
const err = new UnauthorizedError('invalid_token', {
message: t('Token 已失效'),
});
return next(err);
}
return next(); return next();
} }
} const headerToken = getToken(req);
if (pathLower.startsWith('/open/')) {
const originPath = `${req.baseUrl}${pathLower === '/' ? '' : pathLower}`; const apps = await shareStore.getApps();
if ( const doc = apps?.filter((x) =>
!headerToken && x.tokens?.find((y) => y.value === headerToken),
originPath && )?.[0];
config.apiWhiteList.includes(originPath) if (doc && doc.tokens && doc.tokens.length > 0) {
) { const currentToken = doc.tokens.find((x) => x.value === headerToken);
return next(); const keyMatch = pathLower.match(/\/open\/([a-z]+)\/*/);
} const key = keyMatch && keyMatch[1];
if (!doc.scopes.includes(key as AppScope)) {
const authInfo = await shareStore.getAuthInfo(); const err = new UnauthorizedError('credentials_bad_scheme', {
if (isValidToken(authInfo, headerToken, req.platform, config.jwt.secret)) { message: t('暂无权限'),
return next(); });
} return next(err);
}
const errorCode = headerToken ? 'invalid_token' : 'credentials_required'; if (
const errorMessage = headerToken ? t('Token 已失效') : t('请先登录'); !currentToken ||
const err = new UnauthorizedError(errorCode, { message: errorMessage }); currentToken.expiration < Math.round(Date.now() / 1000)
next(err); ) {
}); const err = new UnauthorizedError('invalid_token', {
message: t('Token 已失效'),
app.use(async (req, res, next) => { });
const pathLower = req.path.toLowerCase(); return next(err);
if ( }
![ return next();
'/api/user/init', }
'/api/user/notification/init', }
'/open/user/init', const originPath = `${req.baseUrl}${pathLower === '/' ? '' : pathLower}`;
'/open/user/notification/init', if (
].includes(pathLower) !headerToken &&
) { originPath &&
return next(); config.apiWhiteList.includes(originPath)
} ) {
const authInfo = (await shareStore.getAuthInfo()) || ({} as AuthInfo); return next();
}
let isInitialized = !isDefaultAuthInfo(authInfo); const authInfo = await shareStore.getAuthInfo();
if (
if (isInitialized) { isValidToken(authInfo, headerToken, req.platform, config.jwt.secret)
return res.send({ code: 450, message: t('未知错误') }); ) {
} else { return next();
return next(); }
} const errorCode = headerToken ? 'invalid_token' : 'credentials_required';
}); const errorMessage = headerToken ? t('Token 已失效') : t('请先登录');
const err = new UnauthorizedError(errorCode, { message: errorMessage });
next(err);
}),
);
app.use(
withAuthAvailability(async (req, res, next) => {
const pathLower = req.path.toLowerCase();
if (
![
'/api/user/init',
'/api/user/notification/init',
'/open/user/init',
'/open/user/notification/init',
].includes(pathLower)
) {
return next();
}
const authInfo = (await shareStore.getAuthInfo()) || ({} as AuthInfo);
let isInitialized = !isDefaultAuthInfo(authInfo);
if (isInitialized) {
return res.send({ code: 450, message: t('未知错误') });
} else {
return next();
}
}),
);
app.use(rewrite('/open/*', '/api/$1')); app.use(rewrite('/open/*', '/api/$1'));
app.use(config.api.prefix, routes()); app.use(config.api.prefix, routes());
app.get('*', (req, res, next) => { app.get('*', (req, res, next) => {
// Don't serve index.html for API routes // Don't serve index.html for API routes
if ( if (
@@ -196,13 +180,13 @@ export default ({ app }: { app: Application }) => {
} }
}); });
}); });
app.use(validationErrors); app.use(validationErrors);
app.use(uniqueConstraintError); app.use(uniqueConstraintError);
app.use( app.use(
( (
err: Error & { status: number }, err: Error & {
status: number;
},
req: Request, req: Request,
res: Response, res: Response,
next: NextFunction, next: NextFunction,
@@ -216,15 +200,16 @@ export default ({ app }: { app: Application }) => {
return next(err); return next(err);
}, },
); );
app.use( app.use(
( (
err: Error & { errors: any[] }, err: Error & {
errors: any[];
},
req: Request, req: Request,
res: Response, res: Response,
next: NextFunction, next: NextFunction,
) => { ) => {
if (err.name.includes('Sequelize')) { if (err.name === 'DatabaseError') {
return res return res
.status(500) .status(500)
.send({ .send({
@@ -237,10 +222,11 @@ export default ({ app }: { app: Application }) => {
return next(err); return next(err);
}, },
); );
app.use( app.use(
( (
err: Error & { status: number }, err: Error & {
status: number;
},
req: Request, req: Request,
res: Response, res: Response,
next: NextFunction, next: NextFunction,
+65 -82
View File
@@ -1,3 +1,5 @@
import { textSearch } from '../data/queries';
import { Query, QuerySpec, orderBy } from '../data/queries';
import DependenceService from '../services/dependence'; import DependenceService from '../services/dependence';
import { exec } from 'child_process'; import { exec } from 'child_process';
import { Container } from 'typedi'; import { Container } from 'typedi';
@@ -5,7 +7,6 @@ import { Crontab, CrontabModel, CrontabStatus } from '../data/cron';
import CronService from '../services/cron'; import CronService from '../services/cron';
import EnvService from '../services/env'; import EnvService from '../services/env';
import { DependenceModel, DependenceStatus } from '../data/dependence'; import { DependenceModel, DependenceStatus } from '../data/dependence';
import { Op } from 'sequelize';
import config from '../config'; import config from '../config';
import { CrontabViewModel, CronViewType } from '../data/cronView'; import { CrontabViewModel, CronViewType } from '../data/cronView';
import { initPosition } from '../data/env'; import { initPosition } from '../data/env';
@@ -20,7 +21,6 @@ import Logger from './logger';
import cronClient from '../schedule/client'; import cronClient from '../schedule/client';
import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance'; import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
import { setLang, systemLang } from '../shared/i18n'; import { setLang, systemLang } from '../shared/i18n';
export default async () => { export default async () => {
const cronService = Container.get(CronService); const cronService = Container.get(CronService);
const envService = Container.get(EnvService); const envService = Container.get(EnvService);
@@ -28,17 +28,14 @@ export default async () => {
const systemService = Container.get(SystemService); const systemService = Container.get(SystemService);
const userService = Container.get(UserService); const userService = Container.get(UserService);
const openService = Container.get(OpenService); const openService = Container.get(OpenService);
// 初始化增加系统配置 // 初始化增加系统配置
await openService.initializeSystemApp(); await openService.initializeSystemApp();
const [systemConfig] = await SystemModel.findOrCreate({ const systemConfig = await SystemModel.getOrCreate({
where: { type: AuthDataType.systemConfig }, type: AuthDataType.systemConfig,
}); });
await SystemModel.findOrCreate({ await SystemModel.getOrCreate({ type: AuthDataType.notification });
where: { type: AuthDataType.notification }, const authConfig = await SystemModel.getOrCreate({
}); type: AuthDataType.authConfig,
const [authConfig] = await SystemModel.findOrCreate({
where: { type: AuthDataType.authConfig },
}); });
if (!authConfig?.info || isDemoEnv()) { if (!authConfig?.info || isDemoEnv()) {
let authInfo = { let authInfo = {
@@ -54,36 +51,34 @@ export default async () => {
} catch (error) { } catch (error) {
Logger.warn('Failed to read auth config file, using default credentials'); Logger.warn('Failed to read auth config file, using default credentials');
} }
await SystemModel.upsert({ await SystemModel.nativeUpdate(
id: authConfig?.id, { id: authConfig.id },
info: authInfo, {
type: AuthDataType.authConfig, info: authInfo,
}); type: AuthDataType.authConfig,
},
);
} }
const installDependencies = async () => { const installDependencies = async () => {
const docs = await DependenceModel.findAll({ const docs = await DependenceModel.find(
where: {}, {},
order: [ {
['type', 'DESC'], orderBy: orderBy([
['createdAt', 'DESC'], ['type', 'DESC'],
], ['createdAt', 'DESC'],
raw: true, ]),
}); },
);
await DependenceModel.update( await DependenceModel.nativeUpdate(
{ id: docs.map((x) => x.id!) },
{ status: DependenceStatus.queued, log: [] }, { status: DependenceStatus.queued, log: [] },
{ where: { id: docs.map((x) => x.id!) } },
); );
setTimeout(async () => { setTimeout(async () => {
await dependenceService.installDependenceOneByOne(docs); await dependenceService.installDependenceOneByOne(docs);
const bootAfterLoader = await import('./bootAfter'); const bootAfterLoader = await import('./bootAfter');
bootAfterLoader.default(); bootAfterLoader.default();
}, 5000); }, 5000);
}; };
// 初始化更新 linux/python/nodejs 镜像源配置 // 初始化更新 linux/python/nodejs 镜像源配置
if (systemConfig.info?.pythonMirror) { if (systemConfig.info?.pythonMirror) {
systemService.updatePythonMirror({ systemService.updatePythonMirror({
@@ -106,43 +101,40 @@ export default async () => {
nodeMirror: systemConfig.info?.nodeMirror, nodeMirror: systemConfig.info?.nodeMirror,
}); });
} }
// 初始化新增默认全部任务视图 // 初始化新增默认全部任务视图
CrontabViewModel.findAll({ CrontabViewModel.find({ type: CronViewType.系统, name: '全部任务' }, {}).then(
where: { type: CronViewType.系统, name: '全部任务' }, (docs) => {
raw: true, if (docs.length === 0) {
}).then((docs) => { CrontabViewModel.insert({
if (docs.length === 0) { name: '全部任务',
CrontabViewModel.create({ type: CronViewType.系统,
name: '全部任务', position: initPosition / 2,
type: CronViewType.系统, });
position: initPosition / 2, }
}); },
} );
});
// 初始化更新所有任务状态为空闲 // 初始化更新所有任务状态为空闲
await CrontabModel.update({ status: CrontabStatus.idle }, { where: {} }); await CrontabModel.nativeUpdate({}, { status: CrontabStatus.idle });
// 清空所有运行中的实例记录(服务重启后进程已不存在) // 清空所有运行中的实例记录(服务重启后进程已不存在)
await RunningInstanceModel.update( await RunningInstanceModel.nativeUpdate(
{ status: InstanceStatus.running },
{ {
status: InstanceStatus.stopped, status: InstanceStatus.stopped,
finished_at: Math.floor(Date.now() / 1000), finished_at: Math.floor(Date.now() / 1000),
// The previous process's exit code cannot be recovered after a restart. // The previous process's exit code cannot be recovered after a restart.
}, },
{ where: { status: InstanceStatus.running } },
); );
// 初始化时执行一次所有的 ql repo 任务 // 初始化时执行一次所有的 ql repo 任务
CrontabModel.findAll({ CrontabModel.find(
where: { {
isDisabled: { [Op.ne]: 1 }, isDisabled: { ['$ne']: 1 },
command: { $or: [
[Op.or]: [{ [Op.like]: `%ql repo%` }, { [Op.like]: `%ql raw%` }], ...textSearch(['command'], 'ql repo').$or,
}, ...textSearch(['command'], 'ql raw').$or,
],
}, },
}).then((docs) => { {},
).then((docs) => {
for (let i = 0; i < docs.length; i++) { for (let i = 0; i < docs.length; i++) {
const doc = docs[i]; const doc = docs[i];
if (doc) { if (doc) {
@@ -150,87 +142,78 @@ export default async () => {
} }
} }
}); });
// 更新2.11.3以前的脚本路径 // 更新2.11.3以前的脚本路径
CrontabModel.findAll({ CrontabModel.find(
where: { {
command: { $or: ['scripts', 'config', 'log', 'db'].flatMap(
[Op.or]: [ (folder) =>
{ [Op.like]: `%\/${config.rootPath}\/scripts\/%` }, textSearch(['command'], `/${config.rootPath}/${folder}/`).$or,
{ [Op.like]: `%\/${config.rootPath}\/config\/%` }, ),
{ [Op.like]: `%\/${config.rootPath}\/log\/%` },
{ [Op.like]: `%\/${config.rootPath}\/db\/%` },
],
},
}, },
}).then(async (docs) => { {},
).then(async (docs) => {
for (let i = 0; i < docs.length; i++) { for (let i = 0; i < docs.length; i++) {
const doc = docs[i]; const doc = docs[i];
if (doc) { if (doc) {
if (doc.command.includes(`${config.rootPath}/scripts/`)) { if (doc.command.includes(`${config.rootPath}/scripts/`)) {
await CrontabModel.update( await CrontabModel.nativeUpdate(
{ id: doc.id },
{ command: doc.command.replace(`${config.rootPath}/scripts/`, '') }, { command: doc.command.replace(`${config.rootPath}/scripts/`, '') },
{ where: { id: doc.id } },
); );
} }
if (doc.command.includes(`${config.rootPath}/log/`)) { if (doc.command.includes(`${config.rootPath}/log/`)) {
await CrontabModel.update( await CrontabModel.nativeUpdate(
{ id: doc.id },
{ {
command: `${config.dataPath}/log/${doc.command.replace( command: `${config.dataPath}/log/${doc.command.replace(
`${config.rootPath}/log/`, `${config.rootPath}/log/`,
'', '',
)}`, )}`,
}, },
{ where: { id: doc.id } },
); );
} }
if (doc.command.includes(`${config.rootPath}/config/`)) { if (doc.command.includes(`${config.rootPath}/config/`)) {
await CrontabModel.update( await CrontabModel.nativeUpdate(
{ id: doc.id },
{ {
command: `${config.dataPath}/config/${doc.command.replace( command: `${config.dataPath}/config/${doc.command.replace(
`${config.rootPath}/config/`, `${config.rootPath}/config/`,
'', '',
)}`, )}`,
}, },
{ where: { id: doc.id } },
); );
} }
if (doc.command.includes(`${config.rootPath}/db/`)) { if (doc.command.includes(`${config.rootPath}/db/`)) {
await CrontabModel.update( await CrontabModel.nativeUpdate(
{ id: doc.id },
{ {
command: `${config.dataPath}/db/${doc.command.replace( command: `${config.dataPath}/db/${doc.command.replace(
`${config.rootPath}/db/`, `${config.rootPath}/db/`,
'', '',
)}`, )}`,
}, },
{ where: { id: doc.id } },
); );
} }
} }
} }
}); });
// 初始化语言(必须在 autosave_crontab 之前) // 初始化语言(必须在 autosave_crontab 之前)
const lang = systemConfig.info?.lang || systemLang(); const lang = systemConfig.info?.lang || systemLang();
setLang(lang); setLang(lang);
// 确保 lang_env.sh 存在 // 确保 lang_env.sh 存在
try { try {
const langEnvExist = await fileExist(config.langEnvFile); const langEnvExist = await fileExist(config.langEnvFile);
if (!langEnvExist) { if (!langEnvExist) {
await writeFile(config.langEnvFile, `export QL_LANG='${lang}'\n`); await writeFile(config.langEnvFile, `export QL_LANG='${lang}'\n`);
} }
} catch { } } catch {}
// 初始化保存一次ck和定时任务数据 // 初始化保存一次ck和定时任务数据
cronClient.readiness.configure(async () => { cronClient.readiness.configure(async () => {
await cronService.autosave_crontab(true); await cronService.autosave_crontab(true);
await systemService.restoreCronConcurrency(); await systemService.restoreCronConcurrency();
}); });
await cronClient.readiness.recover(); await cronClient.readiness.recover();
await envService.set_envs(); await envService.set_envs();
const authInfo = await userService.getAuthInfo(); const authInfo = await userService.getAuthInfo();
await shareStore.updateAuthInfo(authInfo); await shareStore.updateAuthInfo(authInfo);
await openService.refreshApps(); await openService.refreshApps();
+1 -8
View File
@@ -9,20 +9,17 @@ import { join } from 'path';
import { t } from '../shared/i18n'; import { t } from '../shared/i18n';
import Logger from './logger'; import Logger from './logger';
import { assertSubscriptionAlias } from '../shared/subscriptionPath'; import { assertSubscriptionAlias } from '../shared/subscriptionPath';
export default async () => { export default async () => {
const systemService = Container.get(SystemService); const systemService = Container.get(SystemService);
const scheduleService = Container.get(ScheduleService); const scheduleService = Container.get(ScheduleService);
const subscriptionService = Container.get(SubscriptionService); const subscriptionService = Container.get(SubscriptionService);
const sshKeyService = Container.get(SshKeyService); const sshKeyService = Container.get(SshKeyService);
// 生成内置token // 生成内置token
let tokenCommand = `ts-node-transpile-only ${join( let tokenCommand = `ts-node-transpile-only ${join(
config.rootPath, config.rootPath,
'back/token.ts', 'back/token.ts',
)}`; )}`;
const tokenFile = join(config.rootPath, 'static/build/token.js'); const tokenFile = join(config.rootPath, 'static/build/token.js');
if (await fileExist(tokenFile)) { if (await fileExist(tokenFile)) {
tokenCommand = `node ${tokenFile}`; tokenCommand = `node ${tokenFile}`;
} }
@@ -41,7 +38,6 @@ export default async () => {
}, },
true, true,
); );
// 运行删除日志任务 // 运行删除日志任务
const data = await systemService.getSystemConfig(); const data = await systemService.getSystemConfig();
if (data && data.info) { if (data && data.info) {
@@ -61,20 +57,17 @@ export default async () => {
true, true,
); );
} }
systemService.updateTimezone(data.info); systemService.updateTimezone(data.info);
// Apply global SSH key if configured // Apply global SSH key if configured
if (data.info.globalSshKey) { if (data.info.globalSshKey) {
await sshKeyService.addGlobalSSHKey(data.info.globalSshKey, 'global'); await sshKeyService.addGlobalSSHKey(data.info.globalSshKey, 'global');
} }
} }
await subscriptionService.setSshConfig(); await subscriptionService.setSshConfig();
const subs = await subscriptionService.list(); const subs = await subscriptionService.list();
for (const sub of subs) { for (const sub of subs) {
try { try {
const doc = sub.get({ plain: true }); const doc = sub;
assertSubscriptionAlias(doc.alias); assertSubscriptionAlias(doc.alias);
await subscriptionService.handleTask(doc, !sub.is_disabled); await subscriptionService.handleTask(doc, !sub.is_disabled);
} catch (error) { } catch (error) {
+18
View File
@@ -0,0 +1,18 @@
import type { RequestHandler } from 'express';
import { t } from '../shared/i18n';
// Express 4 does not forward rejected async middleware promises itself.
export function withAuthAvailability(handler: RequestHandler): RequestHandler {
return (req, res, next) => {
void Promise.resolve()
.then(() => handler(req, res, next))
.catch(() => {
res.setHeader('Retry-After', '1');
next(
Object.assign(new Error(t('认证服务暂时不可用,请稍后重试')), {
status: 503,
}),
);
});
};
}
+3 -4
View File
@@ -1,14 +1,13 @@
import { ErrorRequestHandler } from 'express'; import { ErrorRequestHandler } from 'express';
import { UniqueConstraintError } from 'sequelize'; import { DatabaseError } from '../data/databaseErrors';
import { t } from '../shared/i18n'; import { t } from '../shared/i18n';
// Handle the database constraint itself so concurrent writes are covered too. // Handle the database constraint itself so concurrent writes are covered too.
const uniqueConstraintError: ErrorRequestHandler = (err, req, res, next) => { const uniqueConstraintError: ErrorRequestHandler = (err, req, res, next) => {
if (!(err instanceof UniqueConstraintError)) return next(err); if (!(err instanceof DatabaseError) || err.code !== 'UNIQUE_CONSTRAINT')
return next(err);
// Constraint details can contain environment values or application credentials. // Constraint details can contain environment values or application credentials.
res res
.status(409) .status(409)
.json({ code: 409, message: t('资源已存在,请检查重复的名称或值') }); .json({ code: 409, message: t('资源已存在,请检查重复的名称或值') });
}; };
export default uniqueConstraintError; export default uniqueConstraintError;
+1 -39
View File
@@ -39,10 +39,7 @@ import {
RunCronsRequest, RunCronsRequest,
} from '../protos/api'; } from '../protos/api';
import { NotificationInfo } from '../data/notify'; import { NotificationInfo } from '../data/notify';
import { Model } from 'sequelize';
Container.set('logger', LoggerInstance); Container.set('logger', LoggerInstance);
export const getEnvs = async ( export const getEnvs = async (
call: ServerUnaryCall<GetEnvsRequest, EnvsResponse>, call: ServerUnaryCall<GetEnvsRequest, EnvsResponse>,
callback: sendUnaryData<EnvsResponse>, callback: sendUnaryData<EnvsResponse>,
@@ -62,7 +59,6 @@ export const getEnvs = async (
}); });
} }
}; };
export const createEnv = async ( export const createEnv = async (
call: ServerUnaryCall<CreateEnvRequest, EnvsResponse>, call: ServerUnaryCall<CreateEnvRequest, EnvsResponse>,
callback: sendUnaryData<EnvsResponse>, callback: sendUnaryData<EnvsResponse>,
@@ -75,7 +71,6 @@ export const createEnv = async (
callback(e); callback(e);
} }
}; };
export const updateEnv = async ( export const updateEnv = async (
call: ServerUnaryCall<UpdateEnvRequest, EnvResponse>, call: ServerUnaryCall<UpdateEnvRequest, EnvResponse>,
callback: sendUnaryData<EnvResponse>, callback: sendUnaryData<EnvResponse>,
@@ -88,7 +83,6 @@ export const updateEnv = async (
message: 'id parameter is required', message: 'id parameter is required',
}); });
} }
const envService = Container.get(EnvService); const envService = Container.get(EnvService);
const data = await envService.update( const data = await envService.update(
pick(call.request.env, ['id', 'name', 'value', 'remarks']) as EnvItem, pick(call.request.env, ['id', 'name', 'value', 'remarks']) as EnvItem,
@@ -98,7 +92,6 @@ export const updateEnv = async (
callback(e); callback(e);
} }
}; };
export const deleteEnvs = async ( export const deleteEnvs = async (
call: ServerUnaryCall<DeleteEnvsRequest, Response>, call: ServerUnaryCall<DeleteEnvsRequest, Response>,
callback: sendUnaryData<Response>, callback: sendUnaryData<Response>,
@@ -110,7 +103,6 @@ export const deleteEnvs = async (
message: 'ids parameter is required', message: 'ids parameter is required',
}); });
} }
const envService = Container.get(EnvService); const envService = Container.get(EnvService);
await envService.remove(call.request.ids); await envService.remove(call.request.ids);
callback(null, { code: 200 }); callback(null, { code: 200 });
@@ -118,7 +110,6 @@ export const deleteEnvs = async (
callback(e); callback(e);
} }
}; };
export const moveEnv = async ( export const moveEnv = async (
call: ServerUnaryCall<MoveEnvRequest, EnvResponse>, call: ServerUnaryCall<MoveEnvRequest, EnvResponse>,
callback: sendUnaryData<EnvResponse>, callback: sendUnaryData<EnvResponse>,
@@ -131,7 +122,6 @@ export const moveEnv = async (
message: 'id parameter is required', message: 'id parameter is required',
}); });
} }
const envService = Container.get(EnvService); const envService = Container.get(EnvService);
const data = await envService.move(call.request.id, { const data = await envService.move(call.request.id, {
fromIndex: call.request.fromIndex, fromIndex: call.request.fromIndex,
@@ -142,7 +132,6 @@ export const moveEnv = async (
callback(e); callback(e);
} }
}; };
export const disableEnvs = async ( export const disableEnvs = async (
call: ServerUnaryCall<DisableEnvsRequest, Response>, call: ServerUnaryCall<DisableEnvsRequest, Response>,
callback: sendUnaryData<Response>, callback: sendUnaryData<Response>,
@@ -154,7 +143,6 @@ export const disableEnvs = async (
message: 'ids parameter is required', message: 'ids parameter is required',
}); });
} }
const envService = Container.get(EnvService); const envService = Container.get(EnvService);
await envService.disabled(call.request.ids); await envService.disabled(call.request.ids);
callback(null, { code: 200 }); callback(null, { code: 200 });
@@ -162,7 +150,6 @@ export const disableEnvs = async (
callback(e); callback(e);
} }
}; };
export const enableEnvs = async ( export const enableEnvs = async (
call: ServerUnaryCall<EnableEnvsRequest, Response>, call: ServerUnaryCall<EnableEnvsRequest, Response>,
callback: sendUnaryData<Response>, callback: sendUnaryData<Response>,
@@ -174,7 +161,6 @@ export const enableEnvs = async (
message: 'ids parameter is required', message: 'ids parameter is required',
}); });
} }
const envService = Container.get(EnvService); const envService = Container.get(EnvService);
await envService.enabled(call.request.ids); await envService.enabled(call.request.ids);
callback(null, { code: 200 }); callback(null, { code: 200 });
@@ -182,7 +168,6 @@ export const enableEnvs = async (
callback(e); callback(e);
} }
}; };
export const updateEnvNames = async ( export const updateEnvNames = async (
call: ServerUnaryCall<UpdateEnvNamesRequest, Response>, call: ServerUnaryCall<UpdateEnvNamesRequest, Response>,
callback: sendUnaryData<Response>, callback: sendUnaryData<Response>,
@@ -194,7 +179,6 @@ export const updateEnvNames = async (
message: 'ids parameter is required', message: 'ids parameter is required',
}); });
} }
const envService = Container.get(EnvService); const envService = Container.get(EnvService);
await envService.updateNames({ await envService.updateNames({
ids: call.request.ids, ids: call.request.ids,
@@ -205,7 +189,6 @@ export const updateEnvNames = async (
callback(e); callback(e);
} }
}; };
export const getEnvById = async ( export const getEnvById = async (
call: ServerUnaryCall<GetEnvByIdRequest, EnvResponse>, call: ServerUnaryCall<GetEnvByIdRequest, EnvResponse>,
callback: sendUnaryData<EnvResponse>, callback: sendUnaryData<EnvResponse>,
@@ -218,7 +201,6 @@ export const getEnvById = async (
message: 'id parameter is required', message: 'id parameter is required',
}); });
} }
const envService = Container.get(EnvService); const envService = Container.get(EnvService);
const data = await envService.getDb({ id: call.request.id }); const data = await envService.getDb({ id: call.request.id });
callback(null, { callback(null, {
@@ -229,7 +211,6 @@ export const getEnvById = async (
callback(e); callback(e);
} }
}; };
export const systemNotify = async ( export const systemNotify = async (
call: ServerUnaryCall<SystemNotifyRequest, Response>, call: ServerUnaryCall<SystemNotifyRequest, Response>,
callback: sendUnaryData<Response>, callback: sendUnaryData<Response>,
@@ -247,7 +228,6 @@ export const systemNotify = async (
callback(e); callback(e);
} }
}; };
export const recordNotifyLog = async ( export const recordNotifyLog = async (
call: ServerUnaryCall<RecordNotifyLogRequest, Response>, call: ServerUnaryCall<RecordNotifyLogRequest, Response>,
callback: sendUnaryData<Response>, callback: sendUnaryData<Response>,
@@ -267,12 +247,9 @@ export const recordNotifyLog = async (
callback(error); callback(error);
} }
}; };
const normalizeCronData = (data: CronItem | null): CronItem | undefined => { const normalizeCronData = (data: CronItem | null): CronItem | undefined => {
if (!data) return undefined; if (!data) return undefined;
// create() returns a Sequelize instance; spreading it omits attribute getters. const cron = data;
const cron =
data instanceof Model ? (data.get({ plain: true }) as CronItem) : data;
return { return {
...cron, ...cron,
labels: cron.labels ?? [], labels: cron.labels ?? [],
@@ -283,7 +260,6 @@ const normalizeCronData = (data: CronItem | null): CronItem | undefined => {
task_after: cron.task_after ?? undefined, task_after: cron.task_after ?? undefined,
}; };
}; };
export const getCronDetail = async ( export const getCronDetail = async (
call: ServerUnaryCall<CronDetailRequest, CronDetailResponse>, call: ServerUnaryCall<CronDetailRequest, CronDetailResponse>,
callback: sendUnaryData<CronDetailResponse>, callback: sendUnaryData<CronDetailResponse>,
@@ -305,7 +281,6 @@ export const getCronDetail = async (
callback(e); callback(e);
} }
}; };
export const createCron = async ( export const createCron = async (
call: ServerUnaryCall<CreateCronRequest, CronResponse>, call: ServerUnaryCall<CreateCronRequest, CronResponse>,
callback: sendUnaryData<CronResponse>, callback: sendUnaryData<CronResponse>,
@@ -318,7 +293,6 @@ export const createCron = async (
callback(e); callback(e);
} }
}; };
export const updateCron = async ( export const updateCron = async (
call: ServerUnaryCall<UpdateCronRequest, CronResponse>, call: ServerUnaryCall<UpdateCronRequest, CronResponse>,
callback: sendUnaryData<CronResponse>, callback: sendUnaryData<CronResponse>,
@@ -326,7 +300,6 @@ export const updateCron = async (
try { try {
const cronService = Container.get(CronService); const cronService = Container.get(CronService);
const { id, ...fields } = call.request; const { id, ...fields } = call.request;
const updateRequest = { const updateRequest = {
id, id,
...Object.entries(fields).reduce((acc: any, [key, value]) => { ...Object.entries(fields).reduce((acc: any, [key, value]) => {
@@ -336,14 +309,12 @@ export const updateCron = async (
return acc; return acc;
}, {}), }, {}),
} as UpdateCronRequest; } as UpdateCronRequest;
const data = (await cronService.update(updateRequest)) as CronItem; const data = (await cronService.update(updateRequest)) as CronItem;
callback(null, { code: 200, data: normalizeCronData(data) }); callback(null, { code: 200, data: normalizeCronData(data) });
} catch (e: any) { } catch (e: any) {
callback(e); callback(e);
} }
}; };
export const deleteCrons = async ( export const deleteCrons = async (
call: ServerUnaryCall<DeleteCronsRequest, Response>, call: ServerUnaryCall<DeleteCronsRequest, Response>,
callback: sendUnaryData<Response>, callback: sendUnaryData<Response>,
@@ -356,7 +327,6 @@ export const deleteCrons = async (
callback(e); callback(e);
} }
}; };
export const getCrons = async ( export const getCrons = async (
call: ServerUnaryCall<GetCronsRequest, CronsResponse>, call: ServerUnaryCall<GetCronsRequest, CronsResponse>,
callback: sendUnaryData<CronsResponse>, callback: sendUnaryData<CronsResponse>,
@@ -384,7 +354,6 @@ export const getCrons = async (
}); });
} }
}; };
export const getCronById = async ( export const getCronById = async (
call: ServerUnaryCall<GetCronByIdRequest, CronResponse>, call: ServerUnaryCall<GetCronByIdRequest, CronResponse>,
callback: sendUnaryData<CronResponse>, callback: sendUnaryData<CronResponse>,
@@ -397,7 +366,6 @@ export const getCronById = async (
message: 'id parameter is required', message: 'id parameter is required',
}); });
} }
const cronService = Container.get(CronService); const cronService = Container.get(CronService);
const data = (await cronService.getDb({ id: call.request.id })) as CronItem; const data = (await cronService.getDb({ id: call.request.id })) as CronItem;
callback(null, { code: 200, data: normalizeCronData(data) }); callback(null, { code: 200, data: normalizeCronData(data) });
@@ -409,7 +377,6 @@ export const getCronById = async (
}); });
} }
}; };
export const enableCrons = async ( export const enableCrons = async (
call: ServerUnaryCall<EnableCronsRequest, Response>, call: ServerUnaryCall<EnableCronsRequest, Response>,
callback: sendUnaryData<Response>, callback: sendUnaryData<Response>,
@@ -421,7 +388,6 @@ export const enableCrons = async (
message: 'ids parameter is required', message: 'ids parameter is required',
}); });
} }
const cronService = Container.get(CronService); const cronService = Container.get(CronService);
await cronService.enabled(call.request.ids); await cronService.enabled(call.request.ids);
callback(null, { code: 200 }); callback(null, { code: 200 });
@@ -429,7 +395,6 @@ export const enableCrons = async (
callback(e); callback(e);
} }
}; };
export const disableCrons = async ( export const disableCrons = async (
call: ServerUnaryCall<DisableCronsRequest, Response>, call: ServerUnaryCall<DisableCronsRequest, Response>,
callback: sendUnaryData<Response>, callback: sendUnaryData<Response>,
@@ -441,7 +406,6 @@ export const disableCrons = async (
message: 'ids parameter is required', message: 'ids parameter is required',
}); });
} }
const cronService = Container.get(CronService); const cronService = Container.get(CronService);
await cronService.disabled(call.request.ids); await cronService.disabled(call.request.ids);
callback(null, { code: 200 }); callback(null, { code: 200 });
@@ -449,7 +413,6 @@ export const disableCrons = async (
callback(e); callback(e);
} }
}; };
export const runCrons = async ( export const runCrons = async (
call: ServerUnaryCall<RunCronsRequest, Response>, call: ServerUnaryCall<RunCronsRequest, Response>,
callback: sendUnaryData<Response>, callback: sendUnaryData<Response>,
@@ -461,7 +424,6 @@ export const runCrons = async (
message: 'ids parameter is required', message: 'ids parameter is required',
}); });
} }
const cronService = Container.get(CronService); const cronService = Container.get(CronService);
await cronService.run(call.request.ids); await cronService.run(call.request.ids);
callback(null, { code: 200 }); callback(null, { code: 200 });
+2 -1
View File
@@ -4,6 +4,7 @@ import path from 'path';
import { request } from 'undici'; import { request } from 'undici';
import { HealthCheckRequest, HealthCheckResponse } from '../protos/health'; import { HealthCheckRequest, HealthCheckResponse } from '../protos/health';
import config from '../config'; import config from '../config';
import { database } from '../data';
function formatError(error: unknown): string { function formatError(error: unknown): string {
if (!(error instanceof Error)) { if (!(error instanceof Error)) {
@@ -54,7 +55,7 @@ const check = async (
switch (call.request.service) { switch (call.request.service) {
// Local scheduler liveness only: never call HTTP from this probe. // Local scheduler liveness only: never call HTTP from this probe.
case 'scheduler': case 'scheduler':
return callback(null, { status: 1 }); return callback(null, { status: database.healthy ? 1 : 2 });
case 'cron': { case 'cron': {
const healthUrl = `http://localhost:${config.port}${ const healthUrl = `http://localhost:${config.port}${
config.baseUrl || '' config.baseUrl || ''
+233 -283
View File
File diff suppressed because it is too large Load Diff
+19 -31
View File
@@ -1,3 +1,4 @@
import { Query, QuerySpec, orderBy } from '../data/queries';
import { Service, Inject } from 'typedi'; import { Service, Inject } from 'typedi';
import winston from 'winston'; import winston from 'winston';
import { CrontabView, CrontabViewModel } from '../data/cronView'; import { CrontabView, CrontabViewModel } from '../data/cronView';
@@ -7,12 +8,12 @@ import {
minPosition, minPosition,
stepPosition, stepPosition,
} from '../data/env'; } from '../data/env';
import { FindOptions } from 'sequelize';
@Service() @Service()
export default class CronViewService { export default class CronViewService {
constructor(@Inject('logger') private logger: winston.Logger) {} constructor(
@Inject('logger')
private logger: winston.Logger,
) {}
public async create(payload: CrontabView): Promise<CrontabView> { public async create(payload: CrontabView): Promise<CrontabView> {
let position = initPosition; let position = initPosition;
const views = await this.list(); const views = await this.list();
@@ -22,61 +23,51 @@ export default class CronViewService {
position = position / 2; position = position / 2;
const tab = new CrontabView({ ...payload, position }); const tab = new CrontabView({ ...payload, position });
const doc = await this.insert(tab); const doc = await this.insert(tab);
await this.checkPosition(tab.position!); await this.checkPosition(tab.position!);
return doc; return doc;
} }
public async insert(payload: CrontabView): Promise<CrontabView> { public async insert(payload: CrontabView): Promise<CrontabView> {
return await CrontabViewModel.create(payload, { returning: true }); return await CrontabViewModel.insert(payload);
} }
public async update(payload: CrontabView): Promise<CrontabView> { public async update(payload: CrontabView): Promise<CrontabView> {
const doc = await this.getDb({ id: payload.id }); const doc = await this.getDb({ id: payload.id });
const tab = new CrontabView({ ...doc, ...payload }); const tab = new CrontabView({ ...doc, ...payload });
const newDoc = await this.updateDb(tab); const newDoc = await this.updateDb(tab);
return newDoc; return newDoc;
} }
public async updateDb(payload: CrontabView): Promise<CrontabView> { public async updateDb(payload: CrontabView): Promise<CrontabView> {
await CrontabViewModel.update(payload, { where: { id: payload.id } }); await CrontabViewModel.nativeUpdate({ id: payload.id }, payload);
return await this.getDb({ id: payload.id }); return await this.getDb({ id: payload.id });
} }
public async remove(ids: number[]) { public async remove(ids: number[]) {
await CrontabViewModel.destroy({ where: { id: ids } }); await CrontabViewModel.nativeDelete({ id: ids });
} }
public async list(): Promise<CrontabView[]> { public async list(): Promise<CrontabView[]> {
try { try {
const result = await CrontabViewModel.findAll({ const result = await CrontabViewModel.find(
where: {}, {},
order: [['position', 'DESC']], {
}); orderBy: orderBy([['position', 'DESC']]),
},
);
return result; return result;
} catch (error) { } catch (error) {
throw error; throw error;
} }
} }
public async getDb(query: QuerySpec['where']): Promise<CrontabView> {
public async getDb( const doc: any = await CrontabViewModel.findOne({ ...query }, {});
query: FindOptions<CrontabView>['where'],
): Promise<CrontabView> {
const doc: any = await CrontabViewModel.findOne({ where: { ...query } });
if (!doc) { if (!doc) {
throw new Error(`CronView ${JSON.stringify(query)} not found`); throw new Error(`CronView ${JSON.stringify(query)} not found`);
} }
return doc.get({ plain: true }); return doc;
} }
public async disabled(ids: number[]) { public async disabled(ids: number[]) {
await CrontabViewModel.update({ isDisabled: 1 }, { where: { id: ids } }); await CrontabViewModel.nativeUpdate({ id: ids }, { isDisabled: 1 });
} }
public async enabled(ids: number[]) { public async enabled(ids: number[]) {
await CrontabViewModel.update({ isDisabled: 0 }, { where: { id: ids } }); await CrontabViewModel.nativeUpdate({ id: ids }, { isDisabled: 0 });
} }
private async checkPosition(position: number) { private async checkPosition(position: number) {
const precisionPosition = parseFloat(position.toPrecision(16)); const precisionPosition = parseFloat(position.toPrecision(16));
if (precisionPosition < minPosition || precisionPosition > maxPosition) { if (precisionPosition < minPosition || precisionPosition > maxPosition) {
@@ -88,11 +79,9 @@ export default class CronViewService {
} }
} }
} }
private getPrecisionPosition(position: number): number { private getPrecisionPosition(position: number): number {
return parseFloat(position.toPrecision(16)); return parseFloat(position.toPrecision(16));
} }
public async move({ public async move({
id, id,
fromIndex, fromIndex,
@@ -118,7 +107,6 @@ export default class CronViewService {
id, id,
position: this.getPrecisionPosition(targetPosition), position: this.getPrecisionPosition(targetPosition),
}); });
await this.checkPosition(targetPosition); await this.checkPosition(targetPosition);
return newDoc; return newDoc;
} }
+48 -93
View File
@@ -1,3 +1,5 @@
import { textSearch } from '../data/queries';
import { Query, QuerySpec, orderBy } from '../data/queries';
import { Service, Inject } from 'typedi'; import { Service, Inject } from 'typedi';
import winston from 'winston'; import winston from 'winston';
import config from '../config'; import config from '../config';
@@ -10,7 +12,6 @@ import {
} from '../data/dependence'; } from '../data/dependence';
import { spawn } from 'cross-spawn'; import { spawn } from 'cross-spawn';
import SockService from './sock'; import SockService from './sock';
import { FindOptions, Op } from 'sequelize';
import { import {
fileExist, fileExist,
getPid, getPid,
@@ -27,14 +28,13 @@ import taskLimit from '../shared/pLimit';
import { detectOS } from '../config/util'; import { detectOS } from '../config/util';
import { LINUX_DEPENDENCE_COMMAND } from '../config/const'; import { LINUX_DEPENDENCE_COMMAND } from '../config/const';
import { t, tf } from '../shared/i18n'; import { t, tf } from '../shared/i18n';
@Service() @Service()
export default class DependenceService { export default class DependenceService {
constructor( constructor(
@Inject('logger') private logger: winston.Logger, @Inject('logger')
private logger: winston.Logger,
private sockService: SockService, private sockService: SockService,
) {} ) {}
public async create(payloads: Dependence[]): Promise<Dependence[]> { public async create(payloads: Dependence[]): Promise<Dependence[]> {
const tabs = payloads.map((x) => { const tabs = payloads.map((x) => {
getInstallCommand(x.type, x.name); getInstallCommand(x.type, x.name);
@@ -45,14 +45,14 @@ export default class DependenceService {
this.installDependenceOneByOne(docs); this.installDependenceOneByOne(docs);
return docs; return docs;
} }
public async insert(payloads: Dependence[]): Promise<Dependence[]> { public async insert(payloads: Dependence[]): Promise<Dependence[]> {
const docs = await DependenceModel.bulkCreate(payloads); const docs = await DependenceModel.insertMany(payloads);
return docs; return docs;
} }
public async update( public async update(
payload: Dependence & { id: string }, payload: Dependence & {
id: string;
},
): Promise<Dependence> { ): Promise<Dependence> {
const { id, ...other } = payload; const { id, ...other } = payload;
const doc = await this.getDb({ id }); const doc = await this.getDb({ id });
@@ -66,14 +66,12 @@ export default class DependenceService {
this.installDependenceOneByOne([newDoc]); this.installDependenceOneByOne([newDoc]);
return newDoc; return newDoc;
} }
private async updateDb(payload: Dependence): Promise<Dependence> { private async updateDb(payload: Dependence): Promise<Dependence> {
await DependenceModel.update(payload, { where: { id: payload.id } }); await DependenceModel.nativeUpdate({ id: payload.id }, payload);
return await this.getDb({ id: payload.id }); return await this.getDb({ id: payload.id });
} }
public async remove(ids: number[], force = false): Promise<Dependence[]> { public async remove(ids: number[], force = false): Promise<Dependence[]> {
const docs = await DependenceModel.findAll({ where: { id: ids } }); const docs = await DependenceModel.find({ id: ids }, {});
for (const doc of docs) { for (const doc of docs) {
taskLimit.removeQueuedDependency(doc); taskLimit.removeQueuedDependency(doc);
} }
@@ -88,22 +86,18 @@ export default class DependenceService {
); );
const unInstalledDeps = docs.filter((x) => !installedDeps.includes(x)); const unInstalledDeps = docs.filter((x) => !installedDeps.includes(x));
await this.removeDb(unInstalledDeps.map((x) => x.id!)); await this.removeDb(unInstalledDeps.map((x) => x.id!));
if (installedDeps.length) { if (installedDeps.length) {
await DependenceModel.update( await DependenceModel.nativeUpdate(
{ id: installedDeps.map((x) => x.id!) },
{ status: DependenceStatus.queued, log: [] }, { status: DependenceStatus.queued, log: [] },
{ where: { id: installedDeps.map((x) => x.id!) } },
); );
this.installDependenceOneByOne(installedDeps, false, force); this.installDependenceOneByOne(installedDeps, false, force);
} }
return docs; return docs;
} }
public async removeDb(ids: number[]) { public async removeDb(ids: number[]) {
await DependenceModel.destroy({ where: { id: ids } }); await DependenceModel.nativeDelete({ id: ids });
} }
public async dependencies( public async dependencies(
{ {
searchValue, searchValue,
@@ -130,13 +124,7 @@ export default class DependenceService {
condition.status = status.split(',').map(Number); condition.status = status.split(',').map(Number);
} }
if (searchValue) { if (searchValue) {
const encodeText = encodeURI(searchValue); Object.assign(condition, textSearch(['name'], searchValue));
condition.name = {
[Op.or]: [
{ [Op.like]: `%${searchValue}%` },
{ [Op.like]: `%${encodeText}%` },
],
};
} }
try { try {
return await this.find(condition, sort); return await this.find(condition, sort);
@@ -144,7 +132,6 @@ export default class DependenceService {
throw error; throw error;
} }
} }
private async refreshInstalledStatuses(type?: DependenceTypes) { private async refreshInstalledStatuses(type?: DependenceTypes) {
const cacheDependenceTypes = [ const cacheDependenceTypes = [
DependenceTypes.nodejs, DependenceTypes.nodejs,
@@ -153,13 +140,13 @@ export default class DependenceService {
if (type !== undefined && !cacheDependenceTypes.includes(type)) { if (type !== undefined && !cacheDependenceTypes.includes(type)) {
return; return;
} }
const docs = await DependenceModel.find(
const docs = await DependenceModel.findAll({ {
where: {
status: DependenceStatus.installed, status: DependenceStatus.installed,
type: type === undefined ? { [Op.in]: cacheDependenceTypes } : type, type: type === undefined ? { ['$in']: cacheDependenceTypes } : type,
}, },
}); {},
);
const checks = await concurrentRun( const checks = await concurrentRun(
docs docs
.filter( .filter(
@@ -175,15 +162,13 @@ export default class DependenceService {
const missingIds = (checks || []).filter( const missingIds = (checks || []).filter(
(id): id is number => id !== undefined, (id): id is number => id !== undefined,
); );
if (missingIds.length) { if (missingIds.length) {
await DependenceModel.update( await DependenceModel.nativeUpdate(
{ id: missingIds },
{ status: DependenceStatus.installFailed }, { status: DependenceStatus.installFailed },
{ where: { id: missingIds } },
); );
} }
} }
private async isDependenceInstalled(dependency: Dependence) { private async isDependenceInstalled(dependency: Dependence) {
let depName = dependency.name.trim(); let depName = dependency.name.trim();
const depVersionStr = versionDependenceCommandTypes[dependency.type]; const depVersionStr = versionDependenceCommandTypes[dependency.type];
@@ -201,7 +186,6 @@ export default class DependenceService {
depVersion = parsedVersion; depVersion = parsedVersion;
} }
} }
const depInfo = ( const depInfo = (
await promiseExecSuccess(getGetCommand(dependency.type, depName)) await promiseExecSuccess(getGetCommand(dependency.type, depName))
) )
@@ -211,12 +195,10 @@ export default class DependenceService {
(dependency.type === DependenceTypes.nodejs && (dependency.type === DependenceTypes.nodejs &&
depInfo.split(' ')?.[0] === depName) || depInfo.split(' ')?.[0] === depName) ||
dependency.type === DependenceTypes.python3; dependency.type === DependenceTypes.python3;
return Boolean( return Boolean(
depInfo && nameMatches && (!depVersion || depInfo.includes(depVersion)), depInfo && nameMatches && (!depVersion || depInfo.includes(depVersion)),
); );
} }
public installDependenceOneByOne( public installDependenceOneByOne(
docs: Dependence[], docs: Dependence[],
isInstall: boolean = true, isInstall: boolean = true,
@@ -225,32 +207,27 @@ export default class DependenceService {
docs.forEach((dep) => { docs.forEach((dep) => {
this.installOrUninstallDependency(dep, isInstall, force); this.installOrUninstallDependency(dep, isInstall, force);
}); });
return taskLimit.waitDependencyQueueDone(); return taskLimit.waitDependencyQueueDone();
} }
public async reInstall(ids: number[]): Promise<Dependence[]> { public async reInstall(ids: number[]): Promise<Dependence[]> {
await DependenceModel.update( await DependenceModel.nativeUpdate(
{ id: ids },
{ status: DependenceStatus.queued, log: [] }, { status: DependenceStatus.queued, log: [] },
{ where: { id: ids } },
); );
const docs = await DependenceModel.find({ id: ids }, {});
const docs = await DependenceModel.findAll({ where: { id: ids } });
for (const doc of docs) { for (const doc of docs) {
taskLimit.removeQueuedDependency(doc); taskLimit.removeQueuedDependency(doc);
} }
this.installDependenceOneByOne(docs, true, true); this.installDependenceOneByOne(docs, true, true);
return docs; return docs;
} }
public async cancel(ids: number[]) { public async cancel(ids: number[]) {
const docs = await DependenceModel.findAll({ where: { id: ids } }); const docs = await DependenceModel.find({ id: ids }, {});
for (const doc of docs) { for (const doc of docs) {
taskLimit.removeQueuedDependency(doc); taskLimit.removeQueuedDependency(doc);
let depInstallCommand = getInstallCommand(doc.type, doc.name); let depInstallCommand = getInstallCommand(doc.type, doc.name);
let depUnInstallCommand = getUninstallCommand(doc.type, doc.name); let depUnInstallCommand = getUninstallCommand(doc.type, doc.name);
const isLinuxDependence = doc.type === DependenceTypes.linux; const isLinuxDependence = doc.type === DependenceTypes.linux;
if (isLinuxDependence) { if (isLinuxDependence) {
const osType = await detectOS(); const osType = await detectOS();
if (!osType) { if (!osType) {
@@ -268,44 +245,37 @@ export default class DependenceService {
pid && (await killTask(pid)); pid && (await killTask(pid));
} }
} }
await DependenceModel.update( await DependenceModel.nativeUpdate(
{ id: ids },
{ status: DependenceStatus.cancelled }, { status: DependenceStatus.cancelled },
{ where: { id: ids } },
); );
} }
private async find(query: any, sort: any = []): Promise<Dependence[]> { private async find(query: any, sort: any = []): Promise<Dependence[]> {
const docs = await DependenceModel.findAll({ const docs = await DependenceModel.find(
where: { ...query }, { ...query },
order: [...sort, ['createdAt', 'DESC']], {
}); orderBy: orderBy([...sort, ['createdAt', 'DESC']]),
},
);
return docs; return docs;
} }
public async getDb(query: QuerySpec['where']): Promise<Dependence> {
public async getDb( const doc: any = await DependenceModel.findOne({ ...query }, {});
query: FindOptions<Dependence>['where'],
): Promise<Dependence> {
const doc: any = await DependenceModel.findOne({ where: { ...query } });
if (!doc) { if (!doc) {
throw new Error(`Dependency ${JSON.stringify(query)} not found`); throw new Error(`Dependency ${JSON.stringify(query)} not found`);
} }
return doc.get({ plain: true }); return doc;
} }
private async updateLog(ids: number[], log: string): Promise<void> { private async updateLog(ids: number[], log: string): Promise<void> {
taskLimit.updateDepLog(async () => { taskLimit.updateDepLog(async () => {
const docs = await DependenceModel.findAll({ where: { id: ids } }); const docs = await DependenceModel.find({ id: ids }, {});
for (const doc of docs) { for (const doc of docs) {
const newLog = doc?.log ? [...doc.log, log] : [log]; const newLog = doc?.log ? [...doc.log, log] : [log];
await DependenceModel.update( await DependenceModel.nativeUpdate({ id: doc.id }, { log: newLog });
{ log: newLog },
{ where: { id: doc.id } },
);
} }
return null; return null;
}); });
} }
public installOrUninstallDependency( public installOrUninstallDependency(
dependency: Dependence, dependency: Dependence,
isInstall: boolean = true, isInstall: boolean = true,
@@ -322,14 +292,14 @@ export default class DependenceService {
if (!isInstall && force) { if (!isInstall && force) {
return this.removeDb([dependency.id!]).then(() => null); return this.removeDb([dependency.id!]).then(() => null);
} }
return DependenceModel.update( return DependenceModel.nativeUpdate(
{ id: dependency.id },
{ {
status: isInstall status: isInstall
? DependenceStatus.installFailed ? DependenceStatus.installFailed
: DependenceStatus.removeFailed, : DependenceStatus.removeFailed,
log: [String(error)], log: [String(error)],
}, },
{ where: { id: dependency.id } },
).then(() => null); ).then(() => null);
} }
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
@@ -350,9 +320,9 @@ export default class DependenceService {
taskLimit.removeQueuedDependency(dependency); taskLimit.removeQueuedDependency(dependency);
if (isLinuxDependence) { if (isLinuxDependence) {
if (!osType) { if (!osType) {
await DependenceModel.update( await DependenceModel.nativeUpdate(
{ id: depIds },
{ status: DependenceStatus.installFailed }, { status: DependenceStatus.installFailed },
{ where: { id: depIds } },
); );
const startTime = dayjs(); const startTime = dayjs();
const message = tf( const message = tf(
@@ -375,12 +345,10 @@ export default class DependenceService {
} }
linuxCommand = LINUX_DEPENDENCE_COMMAND[osType]; linuxCommand = LINUX_DEPENDENCE_COMMAND[osType];
} }
const status = isInstall const status = isInstall
? DependenceStatus.installing ? DependenceStatus.installing
: DependenceStatus.removing; : DependenceStatus.removing;
await DependenceModel.update({ status }, { where: { id: depIds } }); await DependenceModel.nativeUpdate({ id: depIds }, { status });
let command = validatedCommand; let command = validatedCommand;
if (isLinuxDependence) { if (isLinuxDependence) {
command = isInstall command = isInstall
@@ -388,7 +356,6 @@ export default class DependenceService {
: `${linuxCommand.uninstall} ${depName.trim()}`; : `${linuxCommand.uninstall} ${depName.trim()}`;
} }
const startTime = dayjs(); const startTime = dayjs();
const message = tf( const message = tf(
'开始%s依赖 %s,开始时间 %s\n\n', '开始%s依赖 %s,开始时间 %s\n\n',
actionText, actionText,
@@ -402,7 +369,6 @@ export default class DependenceService {
status, status,
}); });
this.updateLog(depIds, message); this.updateLog(depIds, message);
// 判断是否已经安装过依赖 // 判断是否已经安装过依赖
// pip must resolve optional dependencies even if the base distribution // pip must resolve optional dependencies even if the base distribution
// is already present; its version alone cannot prove extras are ready. // is already present; its version alone cannot prove extras are ready.
@@ -416,7 +382,6 @@ export default class DependenceService {
if (isLinuxDependence) { if (isLinuxDependence) {
getCommand = `${linuxCommand.info} ${depName}`; getCommand = `${linuxCommand.info} ${depName}`;
} }
let depVersion = ''; let depVersion = '';
if (!isPythonDependence && depName.includes(depVersionStr)) { if (!isPythonDependence && depName.includes(depVersionStr)) {
const symbolRegx = new RegExp( const symbolRegx = new RegExp(
@@ -431,7 +396,6 @@ export default class DependenceService {
const depInfo = (await promiseExecSuccess(getCommand)) const depInfo = (await promiseExecSuccess(getCommand))
.replace(/\s{2,}/, ' ') .replace(/\s{2,}/, ' ')
.replace(/\s+$/, ''); .replace(/\s+$/, '');
if ( if (
depInfo && depInfo &&
((isNodeDependence && depInfo.split(' ')?.[0] === depName) || ((isNodeDependence && depInfo.split(' ')?.[0] === depName) ||
@@ -456,9 +420,9 @@ export default class DependenceService {
status: DependenceStatus.installed, status: DependenceStatus.installed,
}); });
this.updateLog(depIds, _message); this.updateLog(depIds, _message);
await DependenceModel.update( await DependenceModel.nativeUpdate(
{ id: depIds },
{ status: DependenceStatus.installed }, { status: DependenceStatus.installed },
{ where: { id: depIds } },
); );
return resolve(null); return resolve(null);
} }
@@ -472,7 +436,6 @@ export default class DependenceService {
const cp = spawn(`${proxyStr} ${command}`, { const cp = spawn(`${proxyStr} ${command}`, {
shell: '/bin/bash', shell: '/bin/bash',
}); });
cp.stdout.on('data', async (data) => { cp.stdout.on('data', async (data) => {
this.sockService.sendMessage({ this.sockService.sendMessage({
type: socketMessageType, type: socketMessageType,
@@ -482,7 +445,6 @@ export default class DependenceService {
}); });
this.updateLog(depIds, data.toString()); this.updateLog(depIds, data.toString());
}); });
cp.stderr.on('data', async (data) => { cp.stderr.on('data', async (data) => {
this.sockService.sendMessage({ this.sockService.sendMessage({
type: socketMessageType, type: socketMessageType,
@@ -492,7 +454,6 @@ export default class DependenceService {
}); });
this.updateLog(depIds, data.toString()); this.updateLog(depIds, data.toString());
}); });
cp.on('error', async (err) => { cp.on('error', async (err) => {
this.sockService.sendMessage({ this.sockService.sendMessage({
type: socketMessageType, type: socketMessageType,
@@ -502,12 +463,10 @@ export default class DependenceService {
}); });
this.updateLog(depIds, JSON.stringify(err)); this.updateLog(depIds, JSON.stringify(err));
}); });
cp.on('exit', async (code) => { cp.on('exit', async (code) => {
const endTime = dayjs(); const endTime = dayjs();
const isSucceed = code === 0; const isSucceed = code === 0;
const resultText = isSucceed ? t('成功') : t('失败'); const resultText = isSucceed ? t('成功') : t('失败');
const message = const message =
'\n' + '\n' +
tf( tf(
@@ -531,24 +490,20 @@ export default class DependenceService {
status: exitStatus, status: exitStatus,
}); });
this.updateLog(depIds, message); this.updateLog(depIds, message);
const docs = await DependenceModel.find({ id: depIds }, {});
const docs = await DependenceModel.findAll({ where: { id: depIds } });
const _docIds = docs const _docIds = docs
.filter((x) => x.status !== DependenceStatus.cancelled) .filter((x) => x.status !== DependenceStatus.cancelled)
.map((x) => x.id!); .map((x) => x.id!);
if (_docIds.length > 0) { if (_docIds.length > 0) {
await DependenceModel.update( await DependenceModel.nativeUpdate(
{ id: _docIds },
{ status: exitStatus }, { status: exitStatus },
{ where: { id: _docIds } },
); );
} }
// 如果删除依赖成功或者强制删除 // 如果删除依赖成功或者强制删除
if ((isSucceed || force) && !isInstall) { if ((isSucceed || force) && !isInstall) {
this.removeDb(depIds); this.removeDb(depIds);
} }
resolve(null); resolve(null);
}); });
}); });
+41 -85
View File
@@ -1,5 +1,6 @@
import { textSearch } from '../data/queries';
import { Query, QuerySpec, orderBy } from '../data/queries';
import groupBy from 'lodash/groupBy'; import groupBy from 'lodash/groupBy';
import { FindOptions, Op } from 'sequelize';
import { Inject, Service } from 'typedi'; import { Inject, Service } from 'typedi';
import winston from 'winston'; import winston from 'winston';
import config from '../config'; import config from '../config';
@@ -13,12 +14,13 @@ import {
stepPosition, stepPosition,
} from '../data/env'; } from '../data/env';
import { writeFileWithLock } from '../shared/utils'; import { writeFileWithLock } from '../shared/utils';
import { sequelize } from '../data'; import { database } from '../data';
@Service() @Service()
export default class EnvService { export default class EnvService {
constructor(@Inject('logger') private logger: winston.Logger) { } constructor(
@Inject('logger')
private logger: winston.Logger,
) {}
public async create(payloads: Env[]): Promise<Env[]> { public async create(payloads: Env[]): Promise<Env[]> {
const envs = await this.envs(); const envs = await this.envs();
let position = initPosition; let position = initPosition;
@@ -39,16 +41,14 @@ export default class EnvService {
await this.checkPosition(tabs[tabs.length - 1].position!); await this.checkPosition(tabs[tabs.length - 1].position!);
return docs; return docs;
} }
public async insert(payloads: Env[]): Promise<Env[]> { public async insert(payloads: Env[]): Promise<Env[]> {
const result: Env[] = []; const result: Env[] = [];
for (const env of payloads) { for (const env of payloads) {
const doc = await EnvModel.create(env, { returning: true }); const doc = await EnvModel.insert(env);
result.push(doc); result.push(doc);
} }
return result; return result;
} }
public async update(payload: Env): Promise<Env> { public async update(payload: Env): Promise<Env> {
const doc = await this.getDb({ id: payload.id }); const doc = await this.getDb({ id: payload.id });
const tab = new Env({ ...doc, ...payload }); const tab = new Env({ ...doc, ...payload });
@@ -56,17 +56,14 @@ export default class EnvService {
await this.set_envs(); await this.set_envs();
return newDoc; return newDoc;
} }
private async updateDb(payload: Env): Promise<Env> { private async updateDb(payload: Env): Promise<Env> {
await EnvModel.update({ ...payload }, { where: { id: payload.id } }); await EnvModel.nativeUpdate({ id: payload.id }, { ...payload });
return await this.getDb({ id: payload.id }); return await this.getDb({ id: payload.id });
} }
public async remove(ids: number[]) { public async remove(ids: number[]) {
await EnvModel.destroy({ where: { id: ids } }); await EnvModel.nativeDelete({ id: ids });
await this.set_envs(); await this.set_envs();
} }
public async move( public async move(
id: number, id: number,
{ {
@@ -89,16 +86,13 @@ export default class EnvService {
? (envs[toIndex].position! + envs[toIndex - 1].position!) / 2 ? (envs[toIndex].position! + envs[toIndex - 1].position!) / 2
: (envs[toIndex].position! + envs[toIndex + 1].position!) / 2; : (envs[toIndex].position! + envs[toIndex + 1].position!) / 2;
} }
const newDoc = await this.update({ const newDoc = await this.update({
id, id,
position: this.getPrecisionPosition(targetPosition), position: this.getPrecisionPosition(targetPosition),
}); });
await this.checkPosition(targetPosition, envs[toIndex].position!); await this.checkPosition(targetPosition, envs[toIndex].position!);
return newDoc; return newDoc;
} }
private async checkPosition(position: number, edge: number = 0) { private async checkPosition(position: number, edge: number = 0) {
const precisionPosition = this.getPrecisionPosition(position); const precisionPosition = this.getPrecisionPosition(position);
if ( if (
@@ -114,40 +108,20 @@ export default class EnvService {
} }
} }
} }
private getPrecisionPosition(position: number): number { private getPrecisionPosition(position: number): number {
return Math.trunc(parseFloat(position.toPrecision(16))); return Math.trunc(parseFloat(position.toPrecision(16)));
} }
public async envs(searchText: string = '', query: any = {}): Promise<Env[]> { public async envs(searchText: string = '', query: any = {}): Promise<Env[]> {
let condition = { ...query }; let condition = { ...query };
if (searchText) { if (searchText) {
const encodeText = encodeURI(searchText);
const reg = {
[Op.or]: [
{ [Op.like]: `%${searchText}%` },
{ [Op.like]: `%${encodeText}%` },
],
};
condition = { condition = {
...condition, ...condition,
[Op.or]: [ ...textSearch(['name', 'value', 'remarks'], searchText),
{
name: reg,
},
{
value: reg,
},
{
remarks: reg,
},
],
}; };
} }
try { try {
const result = await this.find(condition, [ const result = await this.find(condition, [
[sequelize.literal('COALESCE(`isPinned`, 0)'), 'DESC'], ['isPinned', 'DESC'],
['position', 'DESC'], ['position', 'DESC'],
['createdAt', 'ASC'], ['createdAt', 'ASC'],
]); ]);
@@ -156,90 +130,73 @@ export default class EnvService {
throw error; throw error;
} }
} }
private async find(query: any, sort: any = []): Promise<Env[]> { private async find(query: any, sort: any = []): Promise<Env[]> {
const docs = await EnvModel.findAll({ const docs = await EnvModel.find(
where: { ...query }, { ...query },
order: [...sort], {
}); orderBy: orderBy([...sort]),
return docs.map((x) => x.get({ plain: true })); },
);
return docs.map((x) => x);
} }
public async getDb(query: QuerySpec['where']): Promise<Env> {
public async getDb(query: FindOptions<Env>['where']): Promise<Env> { const doc: any = await EnvModel.findOne({ ...query }, {});
const doc: any = await EnvModel.findOne({ where: { ...query } });
if (!doc) { if (!doc) {
throw new Error(`Env ${JSON.stringify(query)} not found`); throw new Error(`Env ${JSON.stringify(query)} not found`);
} }
return doc.get({ plain: true }); return doc;
} }
public async disabled(ids: number[]) { public async disabled(ids: number[]) {
await EnvModel.update( await EnvModel.nativeUpdate({ id: ids }, { status: EnvStatus.disabled });
{ status: EnvStatus.disabled },
{ where: { id: ids } },
);
await this.set_envs(); await this.set_envs();
} }
public async enabled(ids: number[]) { public async enabled(ids: number[]) {
await EnvModel.update({ status: EnvStatus.normal }, { where: { id: ids } }); await EnvModel.nativeUpdate({ id: ids }, { status: EnvStatus.normal });
await this.set_envs(); await this.set_envs();
} }
public async updateNames({ ids, name }: { ids: number[]; name: string }) { public async updateNames({ ids, name }: { ids: number[]; name: string }) {
await EnvModel.update({ name }, { where: { id: ids } }); await EnvModel.nativeUpdate({ id: ids }, { name });
await this.set_envs(); await this.set_envs();
} }
public async pin(ids: number[]) { public async pin(ids: number[]) {
await EnvModel.update({ isPinned: 1 }, { where: { id: ids } }); await EnvModel.nativeUpdate({ id: ids }, { isPinned: 1 });
} }
public async unPin(ids: number[]) { public async unPin(ids: number[]) {
await EnvModel.update({ isPinned: 0 }, { where: { id: ids } }); await EnvModel.nativeUpdate({ id: ids }, { isPinned: 0 });
} }
public async addLabels(ids: number[], labels: string[]) { public async addLabels(ids: number[], labels: string[]) {
await sequelize.transaction(async (transaction) => { await database.transaction(async () => {
const docs = await EnvModel.findAll({ const docs = await EnvModel.find({ id: ids }, {});
where: { id: ids },
transaction,
});
for (const doc of docs) { for (const doc of docs) {
const env = doc.get({ plain: true }); const env = doc;
await EnvModel.update( await EnvModel.nativeUpdate(
{ id: env.id },
{ labels: Array.from(new Set([...(env.labels || []), ...labels])) }, { labels: Array.from(new Set([...(env.labels || []), ...labels])) },
{ where: { id: env.id }, transaction },
); );
} }
}); });
return await this.find({ id: ids }); return await this.find({ id: ids });
} }
public async removeLabels(ids: number[], labels: string[]) { public async removeLabels(ids: number[], labels: string[]) {
await sequelize.transaction(async (transaction) => { await database.transaction(async () => {
const docs = await EnvModel.findAll({ const docs = await EnvModel.find({ id: ids }, {});
where: { id: ids },
transaction,
});
for (const doc of docs) { for (const doc of docs) {
const env = doc.get({ plain: true }); const env = doc;
await EnvModel.update( await EnvModel.nativeUpdate(
{ id: env.id },
{ {
labels: (env.labels || []).filter( labels: (env.labels || []).filter(
(label: string) => !labels.includes(label), (label: string) => !labels.includes(label),
), ),
}, },
{ where: { id: env.id }, transaction },
); );
} }
}); });
return await this.find({ id: ids }); return await this.find({ id: ids });
} }
public async set_envs() { public async set_envs() {
const envs = await this.envs('', { const envs = await this.envs('', {
name: { [Op.not]: null }, name: { ['$ne']: null },
status: EnvStatus.normal, status: EnvStatus.normal,
}); });
const groups = groupBy(envs, 'name'); const groups = groupBy(envs, 'name');
@@ -249,7 +206,6 @@ export default class EnvService {
for (const key in groups) { for (const key in groups) {
if (Object.prototype.hasOwnProperty.call(groups, key)) { if (Object.prototype.hasOwnProperty.call(groups, key)) {
const group = groups[key]; const group = groups[key];
// 忽略不符合bash要求的环境变量名称 // 忽略不符合bash要求的环境变量名称
if (/^[a-zA-Z_][0-9a-zA-Z_]*$/.test(key)) { if (/^[a-zA-Z_][0-9a-zA-Z_]*$/.test(key)) {
let value = group let value = group
@@ -262,9 +218,9 @@ export default class EnvService {
.map((x) => x.value) .map((x) => x.value)
.join('&') .join('&')
.replace(/\\/g, '\\\\')}`; .replace(/\\/g, '\\\\')}`;
js_env_string += `process.env[${JSON.stringify(key)}]=${JSON.stringify( js_env_string += `process.env[${JSON.stringify(
group.map((x) => x.value).join('&'), key,
)};\n`; )}]=${JSON.stringify(group.map((x) => x.value).join('&'))};\n`;
py_env_string += `os.environ['${key}']='''${_env_value.replace( py_env_string += `os.environ['${key}']='''${_env_value.replace(
/\'/g, /\'/g,
"\\'", "\\'",
+4
View File
@@ -1,3 +1,4 @@
import { database } from '../data';
import { Service } from 'typedi'; import { Service } from 'typedi';
import Logger from '../loaders/logger'; import Logger from '../loaders/logger';
import cronClient from '../schedule/client'; import cronClient from '../schedule/client';
@@ -10,6 +11,7 @@ interface HealthStatus {
services: { services: {
http: boolean; http: boolean;
grpc: boolean; grpc: boolean;
database: boolean;
}; };
metrics: { metrics: {
uptime: number; uptime: number;
@@ -33,6 +35,7 @@ export class HealthService {
services: { services: {
http: true, http: true,
grpc: true, grpc: true,
database: database.healthy,
}, },
metrics: { metrics: {
uptime: Math.floor((Date.now() - this.startTime) / 1000), uptime: Math.floor((Date.now() - this.startTime) / 1000),
@@ -66,6 +69,7 @@ export class HealthService {
Logger.error('gRPC server check failed:', err); Logger.error('gRPC server check failed:', err);
} }
if (!status.services.database) status.status = 'error';
return status; return status;
} }
} }
+21 -49
View File
@@ -1,9 +1,9 @@
import { textSearch } from '../data/queries';
import { Service, Inject } from 'typedi'; import { Service, Inject } from 'typedi';
import winston from 'winston'; import winston from 'winston';
import { createRandomString } from '../config/util'; import { createRandomString } from '../config/util';
import { App, AppModel } from '../data/open'; import { App, AppModel } from '../data/open';
import { v4 as uuidV4 } from 'uuid'; import { v4 as uuidV4 } from 'uuid';
import sequelize, { Op } from 'sequelize';
import { shareStore } from '../shared/store'; import { shareStore } from '../shared/store';
import { t } from '../shared/i18n'; import { t } from '../shared/i18n';
import { lock } from 'proper-lockfile'; import { lock } from 'proper-lockfile';
@@ -11,18 +11,19 @@ import path from 'path';
import os from 'os'; import os from 'os';
import { writeFile, rename, rm, stat, chmod } from 'fs/promises'; import { writeFile, rename, rm, stat, chmod } from 'fs/promises';
import config from '../config'; import config from '../config';
@Service() @Service()
export default class OpenService { export default class OpenService {
constructor(@Inject('logger') private logger: winston.Logger) {} constructor(
@Inject('logger')
private logger: winston.Logger,
) {}
// Every database mutation and full authentication-cache publication shares // Every database mutation and full authentication-cache publication shares
// this cross-process lock. Lock a separate marker, not token.json itself. // this cross-process lock. Lock a separate marker, not token.json itself.
private async withAppsLock<T>(operation: () => Promise<T>): Promise<T> { private async withAppsLock<T>(operation: () => Promise<T>): Promise<T> {
const target = path.join(config.configPath, '.apps-state'); const target = path.join(config.configPath, '.apps-state');
const release = await lock(target, { const release = await lock(target, {
realpath: false, realpath: false,
stale: 10_000, stale: 10000,
retries: { retries: 10, factor: 2, minTimeout: 100, maxTimeout: 3000 }, retries: { retries: 10, factor: 2, minTimeout: 100, maxTimeout: 3000 },
}); });
try { try {
@@ -31,11 +32,10 @@ export default class OpenService {
await release(); await release();
} }
} }
public async initializeSystemApp(): Promise<void> { public async initializeSystemApp(): Promise<void> {
await this.withAppsLock(async () => { await this.withAppsLock(async () => {
const doc = await AppModel.findOne({ where: { name: 'system' } }); const doc = await AppModel.findOne({ name: 'system' }, {});
const app = doc?.get({ plain: true }); const app = doc;
if (!app) { if (!app) {
await this.insert({ await this.insert({
name: 'system', name: 'system',
@@ -44,26 +44,23 @@ export default class OpenService {
client_secret: createRandomString(24, 24), client_secret: createRandomString(24, 24),
}); });
} else if (!app.scopes.includes('dashboard')) { } else if (!app.scopes.includes('dashboard')) {
await AppModel.update( await AppModel.nativeUpdate(
{ id: app.id },
{ scopes: [...app.scopes, 'dashboard'] }, { scopes: [...app.scopes, 'dashboard'] },
{ where: { id: app.id } },
); );
} }
await shareStore.updateApps(await this.find({})); await shareStore.updateApps(await this.find({}));
}); });
} }
public async refreshApps(): Promise<void> { public async refreshApps(): Promise<void> {
await this.withAppsLock(async () => { await this.withAppsLock(async () => {
await shareStore.updateApps(await this.find({})); await shareStore.updateApps(await this.find({}));
}); });
} }
public async findApps(): Promise<App[] | null> { public async findApps(): Promise<App[] | null> {
const docs = await this.find({}); const docs = await this.find({});
return docs; return docs;
} }
public async create(payload: App): Promise<App> { public async create(payload: App): Promise<App> {
return this.withAppsLock(async () => { return this.withAppsLock(async () => {
const tab = { ...payload }; const tab = { ...payload };
@@ -75,12 +72,10 @@ export default class OpenService {
return { ...doc, tokens: [] }; return { ...doc, tokens: [] };
}); });
} }
private async insert(payload: App): Promise<App> { private async insert(payload: App): Promise<App> {
const doc = await AppModel.create(payload, { returning: true }); const doc = await AppModel.insert(payload);
return doc.get({ plain: true }); return doc;
} }
public async update(payload: App): Promise<App> { public async update(payload: App): Promise<App> {
const newDoc = await this.updateDb({ const newDoc = await this.updateDb({
name: payload.name, name: payload.name,
@@ -89,32 +84,28 @@ export default class OpenService {
} as App); } as App);
return { ...newDoc, tokens: [] }; return { ...newDoc, tokens: [] };
} }
private async updateDb(payload: Partial<App>): Promise<App> { private async updateDb(payload: Partial<App>): Promise<App> {
return this.withAppsLock(async () => { return this.withAppsLock(async () => {
await AppModel.update(payload, { where: { id: payload.id } }); await AppModel.nativeUpdate({ id: payload.id }, payload);
const apps = await this.find({}); const apps = await this.find({});
await shareStore.updateApps(apps); await shareStore.updateApps(apps);
return apps?.find((x) => x.id === payload.id) as App; return apps?.find((x) => x.id === payload.id) as App;
}); });
} }
public async getDb(query: Record<string, any>): Promise<App> { public async getDb(query: Record<string, any>): Promise<App> {
const doc = await AppModel.findOne({ where: query }); const doc = await AppModel.findOne(query, {});
if (!doc) { if (!doc) {
throw new Error(`App ${JSON.stringify(query)} not found`); throw new Error(`App ${JSON.stringify(query)} not found`);
} }
return doc.get({ plain: true }); return doc;
} }
public async remove(ids: number[]) { public async remove(ids: number[]) {
return this.withAppsLock(async () => { return this.withAppsLock(async () => {
await AppModel.destroy({ where: { id: ids } }); await AppModel.nativeDelete({ id: ids });
const apps = await this.find({}); const apps = await this.find({});
await shareStore.updateApps(apps); await shareStore.updateApps(apps);
}); });
} }
public async resetSecret(id: number): Promise<App> { public async resetSecret(id: number): Promise<App> {
const tab: Partial<App> = { const tab: Partial<App> = {
client_secret: createRandomString(24, 24), client_secret: createRandomString(24, 24),
@@ -130,7 +121,6 @@ export default class OpenService {
const newDoc = await this.updateDb(tab); const newDoc = await this.updateDb(tab);
return newDoc; return newDoc;
} }
public async list( public async list(
searchText: string = '', searchText: string = '',
sort: any = {}, sort: any = {},
@@ -138,18 +128,7 @@ export default class OpenService {
): Promise<App[]> { ): Promise<App[]> {
let condition = { ...query }; let condition = { ...query };
if (searchText) { if (searchText) {
const encodeText = encodeURI(searchText); condition = { ...condition, ...textSearch(['name'], searchText) };
const reg = {
[Op.or]: [
{ [Op.like]: `%${searchText}%` },
{ [Op.like]: `%${encodeText}%` },
],
};
condition = {
...condition,
name: reg,
};
} }
try { try {
const result = await this.find(condition); const result = await this.find(condition);
@@ -160,19 +139,16 @@ export default class OpenService {
throw error; throw error;
} }
} }
private async find(query: Record<string, any>, sort?: any): Promise<App[]> { private async find(query: Record<string, any>, sort?: any): Promise<App[]> {
const docs = await AppModel.findAll({ where: { ...query } }); const docs = await AppModel.find({ ...query }, {});
return docs.map((x) => x.get({ plain: true })); return docs.map((x) => x);
} }
public async authToken(credentials: { public async authToken(credentials: {
client_id: string; client_id: string;
client_secret: string; client_secret: string;
}): Promise<any> { }): Promise<any> {
return this.withAppsLock(() => this.issueToken(credentials)); return this.withAppsLock(() => this.issueToken(credentials));
} }
// Caller holds withAppsLock through database update and cache publication. // Caller holds withAppsLock through database update and cache publication.
private async issueToken({ private async issueToken({
client_id, client_id,
@@ -183,7 +159,7 @@ export default class OpenService {
}): Promise<any> { }): Promise<any> {
let token = uuidV4(); let token = uuidV4();
const expiration = Math.round(Date.now() / 1000) + 2592000; // 2592000 30天 const expiration = Math.round(Date.now() / 1000) + 2592000; // 2592000 30天
const doc = await AppModel.findOne({ where: { client_id, client_secret } }); const doc = await AppModel.findOne({ client_id, client_secret }, {});
if (doc) { if (doc) {
const timestamp = Math.round(Date.now() / 1000); const timestamp = Math.round(Date.now() / 1000);
const invalidTokens = (doc.tokens || []).filter( const invalidTokens = (doc.tokens || []).filter(
@@ -199,10 +175,7 @@ export default class OpenService {
} else { } else {
tokens = [...invalidTokens, { value: token, expiration }]; tokens = [...invalidTokens, { value: token, expiration }];
} }
await AppModel.update( await AppModel.nativeUpdate({ client_id, client_secret }, { tokens });
{ tokens },
{ where: { client_id, client_secret } },
);
const apps = await this.find({}); const apps = await this.find({});
await shareStore.updateApps(apps); await shareStore.updateApps(apps);
return { return {
@@ -217,7 +190,6 @@ export default class OpenService {
return { code: 400, message: t('client_id 或 client_seret 有误') }; return { code: 400, message: t('client_id 或 client_seret 有误') };
} }
} }
public async generateSystemToken(renew = false): Promise<{ public async generateSystemToken(renew = false): Promise<{
value: string; value: string;
expiration: number; expiration: number;
+25 -41
View File
@@ -1,11 +1,10 @@
import fs from 'fs/promises'; import fs from 'fs/promises';
import path from 'path'; import path from 'path';
import dayjs from 'dayjs'; import dayjs from 'dayjs';
import { Op } from 'sequelize';
import { Service } from 'typedi'; import { Service } from 'typedi';
import config from '../config'; import config from '../config';
import { CrontabStatModel } from '../data/cronStats'; import { CrontabStatModel } from '../data/cronStats';
import { sequelize } from '../data'; import { database } from '../data';
import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance'; import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
import { AuthDataType, SystemModel } from '../data/system'; import { AuthDataType, SystemModel } from '../data/system';
import { import {
@@ -15,54 +14,50 @@ import {
normalizeRetentionPolicy, normalizeRetentionPolicy,
RetentionPolicy, RetentionPolicy,
} from '../shared/retention'; } from '../shared/retention';
export interface StorageCleanupRequest extends RetentionPolicy { export interface StorageCleanupRequest extends RetentionPolicy {
dependenceCacheTypes?: DependenceCacheType[]; dependenceCacheTypes?: DependenceCacheType[];
compactDatabase?: boolean; compactDatabase?: boolean;
} }
function runningInstanceWhere(days: number) { function runningInstanceWhere(days: number) {
const cutoff = dayjs().subtract(days, 'day').unix(); const cutoff = dayjs().subtract(days, 'day').unix();
return { return {
status: { [Op.ne]: InstanceStatus.running }, status: { ['$ne']: InstanceStatus.running },
[Op.or]: [ ['$or']: [
{ finished_at: { [Op.lt]: cutoff } }, { finished_at: { ['$lt']: cutoff } },
{ {
finished_at: { [Op.is]: null }, finished_at: { ['$eq']: null },
started_at: { [Op.lt]: cutoff }, started_at: { ['$lt']: cutoff },
}, },
], ],
}; };
} }
function cronStatWhere(days: number) { function cronStatWhere(days: number) {
return { return {
date: { [Op.lt]: dayjs().subtract(days, 'day').format('YYYY-MM-DD') }, date: { ['$lt']: dayjs().subtract(days, 'day').format('YYYY-MM-DD') },
}; };
} }
@Service() @Service()
export default class RetentionService { export default class RetentionService {
public async updatePolicy(policy: Partial<RetentionPolicy>) { public async updatePolicy(policy: Partial<RetentionPolicy>) {
const normalized = normalizeRetentionPolicy(policy); const normalized = normalizeRetentionPolicy(policy);
const systemConfig = await SystemModel.findOne({ const systemConfig = await SystemModel.findOne(
where: { type: AuthDataType.systemConfig }, { type: AuthDataType.systemConfig },
}); {},
);
if (!systemConfig) { if (!systemConfig) {
throw new Error('System config not found'); throw new Error('System config not found');
} }
await SystemModel.update( await SystemModel.nativeUpdate(
{ id: systemConfig.id },
{ {
info: { info: {
...systemConfig.info, ...systemConfig.info,
...normalized, ...normalized,
}, },
}, },
{ where: { id: systemConfig.id } },
); );
return normalized; return normalized;
} }
public async preview(request: StorageCleanupRequest) { public async preview(request: StorageCleanupRequest) {
const policy = normalizeRetentionPolicy(request); const policy = normalizeRetentionPolicy(request);
const dependenceCacheTypes = (request.dependenceCacheTypes || []).filter( const dependenceCacheTypes = (request.dependenceCacheTypes || []).filter(
@@ -70,14 +65,12 @@ export default class RetentionService {
); );
const [runningInstances, cronStats, dependenceCaches] = await Promise.all([ const [runningInstances, cronStats, dependenceCaches] = await Promise.all([
policy.runningInstanceRetentionDays > 0 policy.runningInstanceRetentionDays > 0
? RunningInstanceModel.count({ ? RunningInstanceModel.count(
where: runningInstanceWhere(policy.runningInstanceRetentionDays), runningInstanceWhere(policy.runningInstanceRetentionDays),
}) )
: 0, : 0,
policy.cronStatRetentionDays > 0 policy.cronStatRetentionDays > 0
? CrontabStatModel.count({ ? CrontabStatModel.count(cronStatWhere(policy.cronStatRetentionDays))
where: cronStatWhere(policy.cronStatRetentionDays),
})
: 0, : 0,
Promise.all( Promise.all(
dependenceCacheTypes.map(async (type) => ({ dependenceCacheTypes.map(async (type) => ({
@@ -88,7 +81,6 @@ export default class RetentionService {
})), })),
), ),
]); ]);
return { return {
policy, policy,
runningInstances, runningInstances,
@@ -101,29 +93,23 @@ export default class RetentionService {
compactDatabase: Boolean(request.compactDatabase), compactDatabase: Boolean(request.compactDatabase),
}; };
} }
public async cleanup(request: StorageCleanupRequest) { public async cleanup(request: StorageCleanupRequest) {
const preview = await this.preview(request); const preview = await this.preview(request);
const deleted = await sequelize.transaction(async (transaction) => { const deleted = await database.transaction(async () => {
const runningInstances = const runningInstances =
preview.policy.runningInstanceRetentionDays > 0 preview.policy.runningInstanceRetentionDays > 0
? await RunningInstanceModel.destroy({ ? await RunningInstanceModel.nativeDelete(
where: runningInstanceWhere( runningInstanceWhere(preview.policy.runningInstanceRetentionDays),
preview.policy.runningInstanceRetentionDays, )
),
transaction,
})
: 0; : 0;
const cronStats = const cronStats =
preview.policy.cronStatRetentionDays > 0 preview.policy.cronStatRetentionDays > 0
? await CrontabStatModel.destroy({ ? await CrontabStatModel.nativeDelete(
where: cronStatWhere(preview.policy.cronStatRetentionDays), cronStatWhere(preview.policy.cronStatRetentionDays),
transaction, )
})
: 0; : 0;
return { runningInstances, cronStats }; return { runningInstances, cronStats };
}); });
const dependenceCaches = []; const dependenceCaches = [];
for (const cache of preview.dependenceCaches) { for (const cache of preview.dependenceCaches) {
await fs.rm(path.join(config.dependenceCachePath, cache.type), { await fs.rm(path.join(config.dependenceCachePath, cache.type), {
@@ -132,14 +118,12 @@ export default class RetentionService {
}); });
dependenceCaches.push(cache); dependenceCaches.push(cache);
} }
if ( if (
request.compactDatabase && request.compactDatabase &&
(deleted.runningInstances || deleted.cronStats) (deleted.runningInstances || deleted.cronStats)
) { ) {
await sequelize.query('VACUUM'); await database.query('VACUUM');
} }
return { return {
deleted: { deleted: {
...deleted, ...deleted,
+70 -101
View File
@@ -1,3 +1,5 @@
import { textSearch } from '../data/queries';
import { Query, QuerySpec, orderBy } from '../data/queries';
import { Service, Inject } from 'typedi'; import { Service, Inject } from 'typedi';
import winston from 'winston'; import winston from 'winston';
import config from '../config'; import config from '../config';
@@ -19,7 +21,6 @@ import {
rmPath, rmPath,
} from '../config/util'; } from '../config/util';
import fs from 'fs/promises'; import fs from 'fs/promises';
import { FindOptions, Op } from 'sequelize';
import path from 'path'; import path from 'path';
import ScheduleService, { TaskCallbacks } from './schedule'; import ScheduleService, { TaskCallbacks } from './schedule';
import { SimpleIntervalSchedule } from 'toad-scheduler'; import { SimpleIntervalSchedule } from 'toad-scheduler';
@@ -42,17 +43,16 @@ import CrontabService from './cron';
import taskLimit from '../shared/pLimit'; import taskLimit from '../shared/pLimit';
import { logStreamManager } from '../shared/logStreamManager'; import { logStreamManager } from '../shared/logStreamManager';
import { LogReadOptions, readLogChunk } from '../shared/logReader'; import { LogReadOptions, readLogChunk } from '../shared/logReader';
@Service() @Service()
export default class SubscriptionService { export default class SubscriptionService {
constructor( constructor(
@Inject('logger') private logger: winston.Logger, @Inject('logger')
private logger: winston.Logger,
private scheduleService: ScheduleService, private scheduleService: ScheduleService,
private sockService: SockService, private sockService: SockService,
private sshKeyService: SshKeyService, private sshKeyService: SshKeyService,
private crontabService: CrontabService, private crontabService: CrontabService,
) {} ) {}
public async list( public async list(
searchText?: string, searchText?: string,
ids?: string, ids?: string,
@@ -60,37 +60,23 @@ export default class SubscriptionService {
let query = {}; let query = {};
const subIds = JSON.parse(ids || '[]'); const subIds = JSON.parse(ids || '[]');
if (searchText) { if (searchText) {
const reg = { query = textSearch(['name', 'url'], searchText);
[Op.or]: [
{ [Op.like]: `%${searchText}%` },
{ [Op.like]: `%${encodeURI(searchText)}%` },
],
};
query = {
[Op.or]: [
{
name: reg,
},
{
url: reg,
},
],
};
} }
try { try {
const result = await SubscriptionModel.findAll({ const result = await SubscriptionModel.find(
where: { ...query, ...(ids ? { id: subIds } : undefined) }, { ...query, ...(ids ? { id: subIds } : undefined) },
order: [ {
['is_disabled', 'ASC'], orderBy: orderBy([
['createdAt', 'DESC'], ['is_disabled', 'ASC'],
], ['createdAt', 'DESC'],
}); ]),
},
);
return result; return result;
} catch (error) { } catch (error) {
throw error; throw error;
} }
} }
public async handleTask( public async handleTask(
doc: Subscription, doc: Subscription,
needCreate = true, needCreate = true,
@@ -101,7 +87,6 @@ export default class SubscriptionService {
const { url } = formatUrl(doc); const { url } = formatUrl(doc);
doc.command = formatCommand(doc, url as string); doc.command = formatCommand(doc, url as string);
} }
if (doc.schedule_type === 'crontab') { if (doc.schedule_type === 'crontab') {
this.scheduleService.cancelCronTask(doc as any); this.scheduleService.cancelCronTask(doc as any);
needCreate && needCreate &&
@@ -122,34 +107,34 @@ export default class SubscriptionService {
)); ));
} }
} }
public async setSshConfig() { public async setSshConfig() {
const docs = await SubscriptionModel.findAll(); const docs = await SubscriptionModel.find({}, {});
await this.sshKeyService.setSshConfig(docs); await this.sshKeyService.setSshConfig(docs);
} }
private taskCallbacks(doc: Subscription): TaskCallbacks { private taskCallbacks(doc: Subscription): TaskCallbacks {
return { return {
onBefore: async (startTime) => { onBefore: async (startTime) => {
const logTime = startTime.format('YYYY-MM-DD-HH-mm-ss'); const logTime = startTime.format('YYYY-MM-DD-HH-mm-ss');
const logPath = `${doc.alias}/${logTime}.log`; const logPath = `${doc.alias}/${logTime}.log`;
await SubscriptionModel.update( await SubscriptionModel.nativeUpdate(
{ id: doc.id },
{ {
status: SubscriptionStatus.running, status: SubscriptionStatus.running,
log_path: logPath, log_path: logPath,
}, },
{ where: { id: doc.id } },
); );
const absolutePath = await handleLogPath( const absolutePath = await handleLogPath(
logPath as string, logPath as string,
tf('## 开始执行... %s\n', startTime.format('YYYY-MM-DD HH:mm:ss')), tf('## 开始执行... %s\n', startTime.format('YYYY-MM-DD HH:mm:ss')),
); );
// 执行sub_before // 执行sub_before
let beforeStr = ''; let beforeStr = '';
try { try {
if (doc.sub_before) { if (doc.sub_before) {
await logStreamManager.write(absolutePath, `\n## ${t('执行before命令...')}\n\n`); await logStreamManager.write(
absolutePath,
`\n## ${t('执行before命令...')}\n\n`,
);
beforeStr = await promiseExec(doc.sub_before); beforeStr = await promiseExec(doc.sub_before);
} }
} catch (error: any) { } catch (error: any) {
@@ -161,11 +146,11 @@ export default class SubscriptionService {
} }
}, },
onStart: async (cp: ChildProcessWithoutNullStreams, startTime) => { onStart: async (cp: ChildProcessWithoutNullStreams, startTime) => {
await SubscriptionModel.update( await SubscriptionModel.nativeUpdate(
{ id: doc.id },
{ {
pid: cp.pid, pid: cp.pid,
}, },
{ where: { id: doc.id } },
); );
}, },
onEnd: async (cp, endTime, diff) => { onEnd: async (cp, endTime, diff) => {
@@ -173,7 +158,6 @@ export default class SubscriptionService {
try { try {
const sub = await this.getDb({ id: doc.id }); const sub = await this.getDb({ id: doc.id });
absolutePath = await handleLogPath(sub.log_path as string); absolutePath = await handleLogPath(sub.log_path as string);
// 执行 sub_after // 执行 sub_after
let afterStr = ''; let afterStr = '';
try { try {
@@ -186,12 +170,12 @@ export default class SubscriptionService {
} }
} catch (error: any) { } catch (error: any) {
afterStr = afterStr =
(error.stderr && error.stderr.toString()) || JSON.stringify(error); (error.stderr && error.stderr.toString()) ||
JSON.stringify(error);
} }
if (afterStr) { if (afterStr) {
await logStreamManager.write(absolutePath, `${afterStr}\n`); await logStreamManager.write(absolutePath, `${afterStr}\n`);
} }
await logStreamManager.write( await logStreamManager.write(
absolutePath, absolutePath,
'\n' + '\n' +
@@ -206,10 +190,10 @@ export default class SubscriptionService {
try { try {
if (absolutePath) await logStreamManager.closeStream(absolutePath); if (absolutePath) await logStreamManager.closeStream(absolutePath);
} finally { } finally {
await SubscriptionModel.update( await SubscriptionModel.nativeUpdate({ id: doc.id }, {
{ status: SubscriptionStatus.idle, pid: null } as any, status: SubscriptionStatus.idle,
{ where: { id: doc.id } }, pid: null,
); } as any);
this.sockService.sendMessage({ this.sockService.sendMessage({
type: 'runSubscriptionEnd', type: 'runSubscriptionEnd',
message: t('订阅执行完成'), message: t('订阅执行完成'),
@@ -230,22 +214,19 @@ export default class SubscriptionService {
}, },
}; };
} }
public async create(payload: Subscription): Promise<Subscription> { public async create(payload: Subscription): Promise<Subscription> {
assertSubscriptionAlias(payload.alias); assertSubscriptionAlias(payload.alias);
return withSubscriptionMutation(async () => { return withSubscriptionMutation(async () => {
const tab = new Subscription(payload); const tab = new Subscription(payload);
const doc = await this.insert(tab); const doc = await this.insert(tab);
await this.handleTask(doc.get({ plain: true })); await this.handleTask(doc);
await this.setSshConfig(); await this.setSshConfig();
return doc; return doc;
}); });
} }
public async insert(payload: Subscription): Promise<SubscriptionInstance> { public async insert(payload: Subscription): Promise<SubscriptionInstance> {
return await SubscriptionModel.create(payload, { returning: true }); return await SubscriptionModel.insert(payload);
} }
public async update(payload: Subscription): Promise<Subscription> { public async update(payload: Subscription): Promise<Subscription> {
assertSubscriptionAlias(payload.alias); assertSubscriptionAlias(payload.alias);
return withSubscriptionMutation(async () => { return withSubscriptionMutation(async () => {
@@ -258,7 +239,6 @@ export default class SubscriptionService {
return newDoc; return newDoc;
}); });
} }
private async removeSshConfigForSubscription(doc: Subscription) { private async removeSshConfigForSubscription(doc: Subscription) {
if (doc.type !== 'private-repo' || doc.pull_type !== 'ssh-key') return; if (doc.type !== 'private-repo' || doc.pull_type !== 'ssh-key') return;
// Invalid historical rows must remain deletable without deriving unsafe paths. // Invalid historical rows must remain deletable without deriving unsafe paths.
@@ -269,12 +249,10 @@ export default class SubscriptionService {
} }
await this.sshKeyService.removeSSHKey(doc.alias, ''); await this.sshKeyService.removeSSHKey(doc.alias, '');
} }
public async updateDb(payload: Subscription): Promise<Subscription> { public async updateDb(payload: Subscription): Promise<Subscription> {
await SubscriptionModel.update(payload, { where: { id: payload.id } }); await SubscriptionModel.nativeUpdate({ id: payload.id }, payload);
return await this.getDb({ id: payload.id }); return await this.getDb({ id: payload.id });
} }
public async status({ public async status({
ids, ids,
status, status,
@@ -300,16 +278,16 @@ export default class SubscriptionService {
if (last_running_time > 0) { if (last_running_time > 0) {
options.last_running_time = last_running_time; options.last_running_time = last_running_time;
} }
return await SubscriptionModel.nativeUpdate({ id: ids }, { ...options });
return await SubscriptionModel.update(
{ ...options },
{ where: { id: ids } },
);
} }
public async remove(
public async remove(ids: number[], query: { force?: boolean }) { ids: number[],
query: {
force?: boolean;
},
) {
return withSubscriptionMutation(async () => { return withSubscriptionMutation(async () => {
const docs = await SubscriptionModel.findAll({ where: { id: ids } }); const docs = await SubscriptionModel.find({ id: ids }, {});
const storagePaths: string[] = []; const storagePaths: string[] = [];
const retainedPaths: string[] = []; const retainedPaths: string[] = [];
const collectPaths = ( const collectPaths = (
@@ -321,7 +299,9 @@ export default class SubscriptionService {
// A URL accepted by one engine can produce an invalid name in another. // A URL accepted by one engine can produce an invalid name in another.
// Validate each candidate independently, retaining the safe candidate. // Validate each candidate independently, retaining the safe candidate.
if (!Object.values(names).every(isSubscriptionStorageName)) continue; if (!Object.values(names).every(isSubscriptionStorageName)) continue;
const resolve = deleting ? resolveSubscriptionStoragePath : path.resolve; const resolve = deleting
? resolveSubscriptionStoragePath
: path.resolve;
const scriptPath = resolve(config.scriptPath, names.script); const scriptPath = resolve(config.scriptPath, names.script);
if (scriptPath) targets.push(scriptPath); if (scriptPath) targets.push(scriptPath);
if (names.repo) { if (names.repo) {
@@ -329,7 +309,10 @@ export default class SubscriptionService {
if (repoPath) targets.push(repoPath); if (repoPath) targets.push(repoPath);
} }
if (names.raw) { if (names.raw) {
const rawPath = resolve(path.join(config.dataPath, 'raw'), names.raw); const rawPath = resolve(
path.join(config.dataPath, 'raw'),
names.raw,
);
if (rawPath) targets.push(rawPath); if (rawPath) targets.push(rawPath);
} }
} }
@@ -342,70 +325,65 @@ export default class SubscriptionService {
resolveSubscriptionStoragePath(config.repoPath, doc.alias); resolveSubscriptionStoragePath(config.repoPath, doc.alias);
collectPaths(doc, storagePaths, true); collectPaths(doc, storagePaths, true);
} }
const remaining = await SubscriptionModel.findAll({ where: {} }); const remaining = await SubscriptionModel.find({}, {});
for (const doc of remaining) { for (const doc of remaining) {
if (!ids.includes(doc.id!)) collectPaths(doc, retainedPaths); if (!ids.includes(doc.id!)) collectPaths(doc, retainedPaths);
} }
} }
for (const doc of docs) { for (const doc of docs) {
await this.handleTask(doc.get({ plain: true }), false); await this.handleTask(doc, false);
} }
await SubscriptionModel.destroy({ where: { id: ids } }); await SubscriptionModel.nativeDelete({ id: ids });
for (const doc of docs) { for (const doc of docs) {
await this.removeSshConfigForSubscription(doc); await this.removeSshConfigForSubscription(doc);
} }
await this.setSshConfig(); await this.setSshConfig();
if (query?.force === true) { if (query?.force === true) {
const crons = await CrontabModel.findAll({ where: { sub_id: ids } }); const crons = await CrontabModel.find({ sub_id: ids }, {});
if (crons?.length) { if (crons?.length) {
await this.crontabService.remove(crons.map((x) => x.id!)); await this.crontabService.remove(crons.map((x) => x.id!));
} }
for (const storagePath of new Set(storagePaths)) { for (const storagePath of new Set(storagePaths)) {
// rm is recursive: an ancestor must also be retained if a surviving // rm is recursive: an ancestor must also be retained if a surviving
// subscription uses a nested branch, and vice versa. // subscription uses a nested branch, and vice versa.
const shared = retainedPaths.some((retained) => const shared = retainedPaths.some(
retained === storagePath || (retained) =>
retained.startsWith(`${storagePath}${path.sep}`) || retained === storagePath ||
storagePath.startsWith(`${retained}${path.sep}`), retained.startsWith(`${storagePath}${path.sep}`) ||
storagePath.startsWith(`${retained}${path.sep}`),
); );
if (!shared) await rmPath(storagePath); if (!shared) await rmPath(storagePath);
} }
} }
}); });
} }
public async getDb(query: QuerySpec['where']): Promise<Subscription> {
public async getDb( const doc = await SubscriptionModel.findOne({ ...query }, {});
query: FindOptions<Subscription>['where'],
): Promise<Subscription> {
const doc = await SubscriptionModel.findOne({ where: { ...query } });
if (!doc) { if (!doc) {
throw new Error(`Subscription ${JSON.stringify(query)} not found`); throw new Error(`Subscription ${JSON.stringify(query)} not found`);
} }
return doc.get({ plain: true }); return doc;
} }
public async run(ids: number[]) { public async run(ids: number[]) {
await SubscriptionModel.update( await SubscriptionModel.nativeUpdate(
{ id: ids },
{ status: SubscriptionStatus.queued }, { status: SubscriptionStatus.queued },
{ where: { id: ids } },
); );
ids.forEach((id) => { ids.forEach((id) => {
this.runSingle(id); this.runSingle(id);
}); });
} }
public async stop(ids: number[]) { public async stop(ids: number[]) {
const docs = await SubscriptionModel.findAll({ where: { id: ids } }); const docs = await SubscriptionModel.find({ id: ids }, {});
let failure: unknown; let failure: unknown;
for (const doc of docs) { for (const doc of docs) {
try { try {
if (doc.pid) { if (doc.pid) {
await killTask(doc.pid, true); await killTask(doc.pid, true);
} }
await SubscriptionModel.update( await SubscriptionModel.nativeUpdate(
{ id: doc.id, pid: doc.pid ?? null },
{ status: SubscriptionStatus.idle, pid: null } as any, { status: SubscriptionStatus.idle, pid: null } as any,
{ where: { id: doc.id, pid: doc.pid ?? null } },
); );
} catch (error) { } catch (error) {
this.logger.error(error); this.logger.error(error);
@@ -414,15 +392,12 @@ export default class SubscriptionService {
} }
if (failure) throw failure; if (failure) throw failure;
} }
private async runSingle(subscriptionId: number) { private async runSingle(subscriptionId: number) {
const subscription = await this.getDb({ id: subscriptionId }); const subscription = await this.getDb({ id: subscriptionId });
if (subscription.status !== SubscriptionStatus.queued) { if (subscription.status !== SubscriptionStatus.queued) {
return; return;
} }
const command = formatCommand(subscription); const command = formatCommand(subscription);
this.scheduleService.runTask(command, this.taskCallbacks(subscription), { this.scheduleService.runTask(command, this.taskCallbacks(subscription), {
name: subscription.name, name: subscription.name,
schedule: subscription.schedule, schedule: subscription.schedule,
@@ -431,25 +406,22 @@ export default class SubscriptionService {
runOrigin: 'subscription', runOrigin: 'subscription',
}); });
} }
public async disabled(ids: number[]) { public async disabled(ids: number[]) {
await SubscriptionModel.update({ is_disabled: 1 }, { where: { id: ids } }); await SubscriptionModel.nativeUpdate({ id: ids }, { is_disabled: 1 });
const docs = await SubscriptionModel.findAll({ where: { id: ids } }); const docs = await SubscriptionModel.find({ id: ids }, {});
await this.setSshConfig(); await this.setSshConfig();
for (const doc of docs) { for (const doc of docs) {
await this.handleTask(doc.get({ plain: true }), false); await this.handleTask(doc, false);
} }
} }
public async enabled(ids: number[]) { public async enabled(ids: number[]) {
await SubscriptionModel.update({ is_disabled: 0 }, { where: { id: ids } }); await SubscriptionModel.nativeUpdate({ id: ids }, { is_disabled: 0 });
const docs = await SubscriptionModel.findAll({ where: { id: ids } }); const docs = await SubscriptionModel.find({ id: ids }, {});
await this.setSshConfig(); await this.setSshConfig();
for (const doc of docs) { for (const doc of docs) {
await this.handleTask(doc.get({ plain: true })); await this.handleTask(doc);
} }
} }
public async log(id: number, options: LogReadOptions = {}) { public async log(id: number, options: LogReadOptions = {}) {
const doc = await this.getDb({ id }); const doc = await this.getDb({ id });
if (!doc || !doc.log_path) { if (!doc || !doc.log_path) {
@@ -461,17 +433,14 @@ export default class SubscriptionService {
truncated: false, truncated: false,
}; };
} }
const absolutePath = await handleLogPath(doc.log_path as string); const absolutePath = await handleLogPath(doc.log_path as string);
return await readLogChunk(absolutePath, options); return await readLogChunk(absolutePath, options);
} }
public async logs(id: number) { public async logs(id: number) {
const doc = await this.getDb({ id }); const doc = await this.getDb({ id });
if (!doc) { if (!doc) {
return []; return [];
} }
if (doc.log_path) { if (doc.log_path) {
const relativeDir = path.dirname(`${doc.log_path}`); const relativeDir = path.dirname(`${doc.log_path}`);
const dir = path.dirname(resolveLogPath(config.logPath, doc.log_path)); const dir = path.dirname(resolveLogPath(config.logPath, doc.log_path));
+33 -67
View File
@@ -1,3 +1,4 @@
import { Query, QuerySpec, orderBy } from '../data/queries';
import { persistNotificationAudit } from '../shared/notificationAudit'; import { persistNotificationAudit } from '../shared/notificationAudit';
import { spawn } from 'cross-spawn'; import { spawn } from 'cross-spawn';
import { execFile } from 'child_process'; import { execFile } from 'child_process';
@@ -46,18 +47,16 @@ import os from 'os';
import dayjs from 'dayjs'; import dayjs from 'dayjs';
import { t, setLang } from '../shared/i18n'; import { t, setLang } from '../shared/i18n';
import { updateLinuxMirrorFile } from '../config/util'; import { updateLinuxMirrorFile } from '../config/util';
@Service() @Service()
export default class SystemService { export default class SystemService {
@Inject((type) => NotificationService) @Inject((type) => NotificationService)
private notificationService!: NotificationService; private notificationService!: NotificationService;
constructor( constructor(
@Inject('logger') private logger: winston.Logger, @Inject('logger')
private logger: winston.Logger,
private scheduleService: ScheduleService, private scheduleService: ScheduleService,
private sockService: SockService, private sockService: SockService,
) {} ) {}
public async getSystemConfig() { public async getSystemConfig() {
const doc = await this.getDb({ type: AuthDataType.systemConfig }); const doc = await this.getDb({ type: AuthDataType.systemConfig });
return { return {
@@ -65,22 +64,19 @@ export default class SystemService {
info: { ...doc.info, timezone: doc.info?.timezone || 'Asia/Shanghai' }, info: { ...doc.info, timezone: doc.info?.timezone || 'Asia/Shanghai' },
}; };
} }
private async updateAuthDb(payload: SystemInfo): Promise<SystemInfo> { private async updateAuthDb(payload: SystemInfo): Promise<SystemInfo> {
const { id, ...others } = payload; const { id, ...others } = payload;
await SystemModel.update(others, { where: { id } }); await SystemModel.nativeUpdate({ id }, others);
const doc = await this.getDb({ id }); const doc = await this.getDb({ id });
return doc; return doc;
} }
public async getDb(query: any): Promise<SystemInfo> { public async getDb(query: any): Promise<SystemInfo> {
const doc = await SystemModel.findOne({ where: query }); const doc = await SystemModel.findOne(query, {});
if (!doc) { if (!doc) {
throw new Error(`System ${JSON.stringify(query)} not found`); throw new Error(`System ${JSON.stringify(query)} not found`);
} }
return doc.get({ plain: true }); return doc;
} }
public async updateNotificationMode(notificationInfo: NotificationInfo) { public async updateNotificationMode(notificationInfo: NotificationInfo) {
const code = Math.random().toString().slice(-6); const code = Math.random().toString().slice(-6);
const isSuccess = await this.notificationService.testNotify( const isSuccess = await this.notificationService.testNotify(
@@ -98,7 +94,6 @@ export default class SystemService {
return { code: 400, message: t('通知发送失败,请检查参数') }; return { code: 400, message: t('通知发送失败,请检查参数') };
} }
} }
public async updateLogRemoveFrequency(info: SystemModelInfo) { public async updateLogRemoveFrequency(info: SystemModelInfo) {
const oDoc = await this.getSystemConfig(); const oDoc = await this.getSystemConfig();
const result = await this.updateAuthDb({ const result = await this.updateAuthDb({
@@ -125,7 +120,6 @@ export default class SystemService {
} }
return { code: 200, data: info }; return { code: 200, data: info };
} }
public async updateCronConcurrency(info: SystemModelInfo) { public async updateCronConcurrency(info: SystemModelInfo) {
const limit = info.cronConcurrency ?? 0; const limit = info.cronConcurrency ?? 0;
if (!Number.isInteger(limit) || limit < 0 || limit > 2147483647) { if (!Number.isInteger(limit) || limit < 0 || limit > 2147483647) {
@@ -142,7 +136,6 @@ export default class SystemService {
return { code: 200, data: info }; return { code: 200, data: info };
}); });
} }
public async restoreCronConcurrency() { public async restoreCronConcurrency() {
await withSchedulerMutation(async () => { await withSchedulerMutation(async () => {
const doc = await this.getSystemConfig(); const doc = await this.getSystemConfig();
@@ -151,7 +144,6 @@ export default class SystemService {
await cronClient.setConcurrency(limit); await cronClient.setConcurrency(limit);
}); });
} }
public async updateDependenceProxy(info: SystemModelInfo) { public async updateDependenceProxy(info: SystemModelInfo) {
const oDoc = await this.getSystemConfig(); const oDoc = await this.getSystemConfig();
await this.updateAuthDb({ await this.updateAuthDb({
@@ -170,7 +162,6 @@ export default class SystemService {
} }
return { code: 200, data: info }; return { code: 200, data: info };
} }
public async updateNodeMirror(info: SystemModelInfo, res?: Response) { public async updateNodeMirror(info: SystemModelInfo, res?: Response) {
const oDoc = await this.getSystemConfig(); const oDoc = await this.getSystemConfig();
await this.updateAuthDb({ await this.updateAuthDb({
@@ -182,12 +173,13 @@ export default class SystemService {
cmd = `pnpm config set registry ${shellQuote(info.nodeMirror)}`; cmd = `pnpm config set registry ${shellQuote(info.nodeMirror)}`;
} }
let command = `cd && ${cmd}`; let command = `cd && ${cmd}`;
const docs = await DependenceModel.findAll({ const docs = await DependenceModel.find(
where: { {
type: DependenceTypes.nodejs, type: DependenceTypes.nodejs,
status: DependenceStatus.installed, status: DependenceStatus.installed,
}, },
}); {},
);
if (docs.length > 0) { if (docs.length > 0) {
command += ` && pnpm i -g`; command += ` && pnpm i -g`;
} }
@@ -219,7 +211,6 @@ export default class SystemService {
}, },
); );
} }
public async updatePythonMirror(info: SystemModelInfo) { public async updatePythonMirror(info: SystemModelInfo) {
const oDoc = await this.getSystemConfig(); const oDoc = await this.getSystemConfig();
await this.updateAuthDb({ await this.updateAuthDb({
@@ -233,7 +224,6 @@ export default class SystemService {
await promiseExec(cmd); await promiseExec(cmd);
return { code: 200, data: info }; return { code: 200, data: info };
} }
public async updateLinuxMirror( public async updateLinuxMirror(
info: SystemModelInfo, info: SystemModelInfo,
res?: Response, res?: Response,
@@ -280,13 +270,11 @@ export default class SystemService {
}, },
); );
} }
public async checkUpdate() { public async checkUpdate() {
const controller = new AbortController(); const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 15000); const timeout = setTimeout(() => controller.abort(), 15000);
try { try {
const currentVersionContent = await parseVersion(config.versionFile); const currentVersionContent = await parseVersion(config.versionFile);
const { body, statusCode } = await request( const { body, statusCode } = await request(
`${config.lastVersionFile}?t=${Date.now()}`, `${config.lastVersionFile}?t=${Date.now()}`,
{ signal: controller.signal }, { signal: controller.signal },
@@ -301,7 +289,6 @@ export default class SystemService {
) { ) {
throw new Error('Invalid version response'); throw new Error('Invalid version response');
} }
return { return {
code: 200, code: 200,
data: { data: {
@@ -323,7 +310,6 @@ export default class SystemService {
clearTimeout(timeout); clearTimeout(timeout);
} }
} }
private checkHasNewVersion(curVersion: string, lastVersion: string) { private checkHasNewVersion(curVersion: string, lastVersion: string) {
const curArr = curVersion.split('.').map((x) => parseInt(x, 10)); const curArr = curVersion.split('.').map((x) => parseInt(x, 10));
const lastArr = lastVersion.split('.').map((x) => parseInt(x, 10)); const lastArr = lastVersion.split('.').map((x) => parseInt(x, 10));
@@ -342,25 +328,21 @@ export default class SystemService {
} }
return false; return false;
} }
public async updateSystem() { public async updateSystem() {
return this.startUpgradeJob('download'); return this.startUpgradeJob('download');
} }
private upgradeJobs() { private upgradeJobs() {
return require(path.join( return require(path.join(
config.rootPath, config.rootPath,
'cli/src/internal/maintenance/panelUpgrade.cjs', 'cli/src/internal/maintenance/panelUpgrade.cjs',
)); ));
} }
public async getUpgradeStatus() { public async getUpgradeStatus() {
return { return {
code: 200, code: 200,
data: await this.upgradeJobs().getStatus(config.tmpPath), data: await this.upgradeJobs().getStatus(config.tmpPath),
}; };
} }
private async startUpgradeJob(action: 'download' | 'apply') { private async startUpgradeJob(action: 'download' | 'apply') {
try { try {
const data = await this.upgradeJobs().startJob( const data = await this.upgradeJobs().startJob(
@@ -383,7 +365,6 @@ export default class SystemService {
return { code: 400, message: t('升级任务启动失败,请检查日志后重试') }; return { code: 400, message: t('升级任务启动失败,请检查日志后重试') };
} }
} }
public async reloadSystem(target?: 'system' | 'data') { public async reloadSystem(target?: 'system' | 'data') {
if (target && !['system', 'data'].includes(target)) { if (target && !['system', 'data'].includes(target)) {
return { code: 400, message: t('参数错误') }; return { code: 400, message: t('参数错误') };
@@ -401,7 +382,6 @@ export default class SystemService {
}); });
return { code: 200 }; return { code: 200 };
} }
public async notify({ public async notify({
title, title,
content, content,
@@ -418,35 +398,38 @@ export default class SystemService {
if (notificationInfo && typeString) { if (notificationInfo && typeString) {
notificationInfo.type = typeString; notificationInfo.type = typeString;
} }
const isSuccess = await this.notificationService.notify( const isSuccess = await this.notificationService.notify(
title, title,
content, content,
notificationInfo, notificationInfo,
); );
if (isSuccess) { if (isSuccess) {
return { code: 200, message: t('通知发送成功') }; return { code: 200, message: t('通知发送成功') };
} else { } else {
return { code: 400, message: t('通知发送失败,请检查系统设置/通知配置') }; return { code: 400, message: t('通知发送失败,请检查系统设置/通知配置') };
} }
} }
public async recordNotifyLog(info: NotifyLogInfo): Promise<void> { public async recordNotifyLog(info: NotifyLogInfo): Promise<void> {
await persistNotificationAudit(info); await persistNotificationAudit(info);
} }
public async getNotifyLog(): Promise<Array<NotifyLogInfo>> { public async getNotifyLog(): Promise<Array<NotifyLogInfo>> {
const docs = await SystemModel.findAll({ const docs = await SystemModel.find(
where: { type: AuthDataType.notifyLog }, { type: AuthDataType.notifyLog },
order: [['id', 'DESC']], {
limit: 200, orderBy: orderBy([['id', 'DESC']]),
}); limit: 200,
},
);
return docs.map((x) => ({ ...x.info, id: x.id })); return docs.map((x) => ({ ...x.info, id: x.id }));
} }
public async run( public async run(
{ command, logPath }: { command: string; logPath?: string }, {
command,
logPath,
}: {
command: string;
logPath?: string;
},
callback: TaskCallbacks, callback: TaskCallbacks,
) { ) {
if (!command.startsWith(TASK_COMMAND)) { if (!command.startsWith(TASK_COMMAND)) {
@@ -463,17 +446,14 @@ export default class SystemService {
}, },
); );
} }
public async stop({ command, pid }: { command: string; pid: number }) { public async stop({ command, pid }: { command: string; pid: number }) {
if (!pid && !command) { if (!pid && !command) {
return { code: 400, message: t('参数错误') }; return { code: 400, message: t('参数错误') };
} }
if (pid) { if (pid) {
await killTask(pid); await killTask(pid);
return { code: 200 }; return { code: 200 };
} }
if (!command.startsWith(TASK_COMMAND)) { if (!command.startsWith(TASK_COMMAND)) {
command = `${TASK_COMMAND} ${command}`; command = `${TASK_COMMAND} ${command}`;
} }
@@ -485,7 +465,6 @@ export default class SystemService {
return { code: 400, message: t('任务未找到') }; return { code: 400, message: t('任务未找到') };
} }
} }
public async exportData(res: Response, type?: string[]) { public async exportData(res: Response, type?: string[]) {
try { try {
let dataDirs = ['db', 'upload']; let dataDirs = ['db', 'upload'];
@@ -527,7 +506,6 @@ export default class SystemService {
return res.send({ code: 400, message: error.message }); return res.send({ code: 400, message: error.message });
} }
} }
public async importData() { public async importData() {
try { try {
await promiseExec(`rm -rf ${path.join(config.tmpPath, 'data')}`); await promiseExec(`rm -rf ${path.join(config.tmpPath, 'data')}`);
@@ -539,7 +517,6 @@ export default class SystemService {
return { code: 400, message: error.message }; return { code: 400, message: error.message };
} }
} }
public async getSystemLog( public async getSystemLog(
res: Response, res: Response,
query: { query: {
@@ -559,7 +536,6 @@ export default class SystemService {
.reverse() .reverse()
.filter((x) => x.title.endsWith('.log')) .filter((x) => x.title.endsWith('.log'))
.filter((x) => x.createTime >= startTime && x.createTime <= endTime); .filter((x) => x.createTime >= startTime && x.createTime <= endTime);
const limit = Math.min( const limit = Math.min(
Math.max(Number(query.limit) || 1024 * 1024, 1), Math.max(Number(query.limit) || 1024 * 1024, 1),
1024 * 1024, 1024 * 1024,
@@ -567,7 +543,10 @@ export default class SystemService {
const total = logs.reduce((size, log) => size + (log.size || 0), 0); const total = logs.reduce((size, log) => size + (log.size || 0), 0);
let remaining = limit; let remaining = limit;
const selected: Array< const selected: Array<
(typeof logs)[number] & { start: number; length: number } (typeof logs)[number] & {
start: number;
length: number;
}
> = []; > = [];
for (let index = logs.length - 1; index >= 0 && remaining > 0; index--) { for (let index = logs.length - 1; index >= 0 && remaining > 0; index--) {
const log = logs[index]; const log = logs[index];
@@ -578,7 +557,6 @@ export default class SystemService {
remaining -= length; remaining -= length;
} }
} }
const contentLength = selected.reduce((size, log) => size + log.length, 0); const contentLength = selected.reduce((size, log) => size + log.length, 0);
res.set({ res.set({
'Content-Length': contentLength, 'Content-Length': contentLength,
@@ -590,7 +568,6 @@ export default class SystemService {
res.end(); res.end();
return; return;
} }
const currentLog = fileNames.shift(); const currentLog = fileNames.shift();
if (currentLog) { if (currentLog) {
const currentFileStream = fs.createReadStream( const currentFileStream = fs.createReadStream(
@@ -607,7 +584,6 @@ export default class SystemService {
} }
})(res, selected); })(res, selected);
} }
public async deleteSystemLog() { public async deleteSystemLog() {
const result = await readDirs(config.systemLogPath, config.systemLogPath); const result = await readDirs(config.systemLogPath, config.systemLogPath);
const logs = result.reverse().filter((x) => x.title.endsWith('.log')); const logs = result.reverse().filter((x) => x.title.endsWith('.log'));
@@ -625,7 +601,6 @@ export default class SystemService {
} }
} }
} }
public async updateTimezone(info: SystemModelInfo) { public async updateTimezone(info: SystemModelInfo) {
if (!info.timezone) { if (!info.timezone) {
info.timezone = 'Asia/Shanghai'; info.timezone = 'Asia/Shanghai';
@@ -642,7 +617,6 @@ export default class SystemService {
return { code: 400, message: t('设置时区失败') }; return { code: 400, message: t('设置时区失败') };
} }
} }
public async updateLanguage(info: SystemModelInfo) { public async updateLanguage(info: SystemModelInfo) {
const oDoc = await this.getSystemConfig(); const oDoc = await this.getSystemConfig();
const lang = info.lang || 'zh'; const lang = info.lang || 'zh';
@@ -662,7 +636,6 @@ export default class SystemService {
setLang(lang); setLang(lang);
return { code: 200, data: { lang } }; return { code: 200, data: { lang } };
} }
public async updatePanelTitle(info: SystemModelInfo) { public async updatePanelTitle(info: SystemModelInfo) {
const oDoc = await this.getSystemConfig(); const oDoc = await this.getSystemConfig();
const panelTitle = info.panelTitle?.trim() || ''; const panelTitle = info.panelTitle?.trim() || '';
@@ -672,43 +645,36 @@ export default class SystemService {
}); });
return { code: 200, data: { panelTitle } }; return { code: 200, data: { panelTitle } };
} }
public async updateGlobalSshKey(info: SystemModelInfo) { public async updateGlobalSshKey(info: SystemModelInfo) {
const oDoc = await this.getSystemConfig(); const oDoc = await this.getSystemConfig();
const result = await this.updateAuthDb({ const result = await this.updateAuthDb({
...oDoc, ...oDoc,
info: { ...oDoc.info, ...info }, info: { ...oDoc.info, ...info },
}); });
// Apply the global SSH key // Apply the global SSH key
const SshKeyService = require('./sshKey').default; const SshKeyService = require('./sshKey').default;
const Container = require('typedi').Container; const Container = require('typedi').Container;
const sshKeyService = Container.get(SshKeyService); const sshKeyService = Container.get(SshKeyService);
if (info.globalSshKey) { if (info.globalSshKey) {
await sshKeyService.addGlobalSSHKey(info.globalSshKey, 'global'); await sshKeyService.addGlobalSSHKey(info.globalSshKey, 'global');
} else { } else {
await sshKeyService.removeGlobalSSHKey('global'); await sshKeyService.removeGlobalSSHKey('global');
} }
return { code: 200, data: result }; return { code: 200, data: result };
} }
public async cleanDependence(type: 'node' | 'python3') { public async cleanDependence(type: 'node' | 'python3') {
if (!type || !['node', 'python3'].includes(type)) { if (!type || !['node', 'python3'].includes(type)) {
return { code: 400, message: t('参数错误') }; return { code: 400, message: t('参数错误') };
} }
const finalPath = path.join(config.dependenceCachePath, type); const finalPath = path.join(config.dependenceCachePath, type);
await fs.promises.rm(finalPath, { recursive: true, force: true }); await fs.promises.rm(finalPath, { recursive: true, force: true });
await DependenceModel.update( await DependenceModel.nativeUpdate(
{ status: DependenceStatus.installFailed },
{ {
where: { type:
type: type === 'node' ? DependenceTypes.nodejs : DependenceTypes.python3,
type === 'node' ? DependenceTypes.nodejs : DependenceTypes.python3, status: DependenceStatus.installed,
status: DependenceStatus.installed,
},
}, },
{ status: DependenceStatus.installFailed },
); );
return { code: 200 }; return { code: 200 };
} }
+20 -71
View File
@@ -34,27 +34,27 @@ import {
} from '../shared/password'; } from '../shared/password';
import { serializeAuthMutation } from '../shared/authMutation'; import { serializeAuthMutation } from '../shared/authMutation';
import { LoginThrottle } from '../shared/loginThrottle'; import { LoginThrottle } from '../shared/loginThrottle';
@Service() @Service()
export default class UserService { export default class UserService {
private loginThrottle = new LoginThrottle(); private loginThrottle = new LoginThrottle();
@Inject((type) => NotificationService) @Inject((type) => NotificationService)
private notificationService!: NotificationService; private notificationService!: NotificationService;
constructor( constructor(
@Inject('logger') private logger: winston.Logger, @Inject('logger')
private logger: winston.Logger,
private scheduleService: ScheduleService, private scheduleService: ScheduleService,
private sockService: SockService, private sockService: SockService,
) {} ) {}
@serializeAuthMutation @serializeAuthMutation
public async login( public async login(
payloads: { username: string; password: string }, payloads: {
username: string;
password: string;
},
req: Request, req: Request,
): Promise<any> { ): Promise<any> {
return this.authenticate(payloads, req); return this.authenticate(payloads, req);
} }
private async authenticate( private async authenticate(
payloads: { payloads: {
username: string; username: string;
@@ -89,7 +89,6 @@ export default class UserService {
platform, platform,
blockedIps = [], blockedIps = [],
} = content; } = content;
if ( if (
ip && ip &&
blockedIps.some((blockedIp) => normalizeClientIp(blockedIp) === ip) blockedIps.some((blockedIp) => normalizeClientIp(blockedIp) === ip)
@@ -107,7 +106,6 @@ export default class UserService {
this.getLoginLog(); this.getLoginLog();
return { code: 403, message: t('该 IP 已被列入黑名单') }; return { code: 403, message: t('该 IP 已被列入黑名单') };
} }
const waitTime = this.loginThrottle.retryAfter(ip, timestamp); const waitTime = this.loginThrottle.retryAfter(ip, timestamp);
if (waitTime > 0) { if (waitTime > 0) {
return { return {
@@ -116,10 +114,8 @@ export default class UserService {
data: waitTime, data: waitTime,
}; };
} }
const passwordMatches = const passwordMatches =
username === cUsername && (await verifyPassword(password, cPassword)); username === cUsername && (await verifyPassword(password, cPassword));
if (passwordMatches && twoFactorActivated && needTwoFactor) { if (passwordMatches && twoFactorActivated && needTwoFactor) {
await this.updateAuthInfo(content, { await this.updateAuthInfo(content, {
isTwoFactorChecking: true, isTwoFactorChecking: true,
@@ -130,7 +126,6 @@ export default class UserService {
message: '', message: '',
}; };
} }
if (passwordMatches) { if (passwordMatches) {
this.loginThrottle.reset(ip); this.loginThrottle.reset(ip);
const data = createRandomString(50, 100); const data = createRandomString(50, 100);
@@ -139,7 +134,6 @@ export default class UserService {
expiresIn: config.jwt.expiresIn || expiration, expiresIn: config.jwt.expiresIn || expiration,
algorithm: 'HS384', algorithm: 'HS384',
}); });
const tokenInfo: TokenInfo = { const tokenInfo: TokenInfo = {
value: token, value: token,
timestamp, timestamp,
@@ -147,13 +141,11 @@ export default class UserService {
address, address,
platform: req.platform, platform: req.platform,
}; };
const updatedTokens = this.addTokenToList( const updatedTokens = this.addTokenToList(
tokens, tokens,
req.platform, req.platform,
tokenInfo, tokenInfo,
); );
await this.updateAuthInfo(content, { await this.updateAuthInfo(content, {
password: isPasswordHash(cPassword) password: isPasswordHash(cPassword)
? cPassword ? cPassword
@@ -252,16 +244,13 @@ export default class UserService {
} }
} }
} }
@serializeAuthMutation @serializeAuthMutation
public async logout(platform: string, tokenValue: string): Promise<any> { public async logout(platform: string, tokenValue: string): Promise<any> {
if (!platform || !tokenValue) { if (!platform || !tokenValue) {
this.logger.warn('Invalid logout parameters - empty platform or token'); this.logger.warn('Invalid logout parameters - empty platform or token');
return; return;
} }
const authInfo = await this.getAuthInfo(); const authInfo = await this.getAuthInfo();
// Verify the token exists before attempting to remove it // Verify the token exists before attempting to remove it
const tokenExists = this.findTokenInList( const tokenExists = this.findTokenInList(
authInfo.tokens, authInfo.tokens,
@@ -275,45 +264,36 @@ export default class UserService {
); );
return; return;
} }
const updatedTokens = this.removeTokenFromList( const updatedTokens = this.removeTokenFromList(
authInfo.tokens, authInfo.tokens,
platform, platform,
tokenValue, tokenValue,
); );
await this.updateAuthInfo(authInfo, { await this.updateAuthInfo(authInfo, {
token: authInfo.token === tokenValue ? '' : authInfo.token, token: authInfo.token === tokenValue ? '' : authInfo.token,
tokens: updatedTokens, tokens: updatedTokens,
}); });
} }
public async getLoginLog(): Promise<Array<SystemModelInfo | undefined>> { public async getLoginLog(): Promise<Array<SystemModelInfo | undefined>> {
const docs = await SystemModel.findAll({ const docs = await SystemModel.find({ type: AuthDataType.loginLog }, {});
where: { type: AuthDataType.loginLog },
});
if (docs && docs.length > 0) { if (docs && docs.length > 0) {
const result = docs.sort( const result = docs.sort(
(a, b) => b.info!.timestamp! - a.info!.timestamp!, (a, b) => b.info!.timestamp! - a.info!.timestamp!,
); );
if (result.length > 100) { if (result.length > 100) {
const ids = result.slice(100).map((x) => x.id!); const ids = result.slice(100).map((x) => x.id!);
await SystemModel.destroy({ await SystemModel.nativeDelete({ id: ids });
where: { id: ids },
});
} }
return result.map((x) => x.info); return result.map((x) => x.info);
} }
return []; return [];
} }
public async getIpBlacklist(): Promise<string[]> { public async getIpBlacklist(): Promise<string[]> {
const authInfo = await this.getAuthInfo(); const authInfo = await this.getAuthInfo();
return uniq((authInfo.blockedIps || []).map(normalizeClientIp)).filter( return uniq((authInfo.blockedIps || []).map(normalizeClientIp)).filter(
Boolean, Boolean,
); );
} }
@serializeAuthMutation @serializeAuthMutation
public async blockIp(ip: string): Promise<string[]> { public async blockIp(ip: string): Promise<string[]> {
const authInfo = await this.getAuthInfo(); const authInfo = await this.getAuthInfo();
@@ -324,7 +304,6 @@ export default class UserService {
await this.updateAuthInfo(authInfo, { blockedIps }); await this.updateAuthInfo(authInfo, { blockedIps });
return blockedIps; return blockedIps;
} }
@serializeAuthMutation @serializeAuthMutation
public async unblockIp(ip: string): Promise<string[]> { public async unblockIp(ip: string): Promise<string[]> {
const authInfo = await this.getAuthInfo(); const authInfo = await this.getAuthInfo();
@@ -335,12 +314,10 @@ export default class UserService {
await this.updateAuthInfo(authInfo, { blockedIps }); await this.updateAuthInfo(authInfo, { blockedIps });
return blockedIps; return blockedIps;
} }
private async insertDb(payload: SystemInfo): Promise<SystemInfo> { private async insertDb(payload: SystemInfo): Promise<SystemInfo> {
const doc = await SystemModel.create({ ...payload }, { returning: true }); const doc = await SystemModel.insert({ ...payload });
return doc; return doc;
} }
@serializeAuthMutation @serializeAuthMutation
public async initializeUser({ public async initializeUser({
username, username,
@@ -366,7 +343,6 @@ export default class UserService {
}); });
return { code: 200, message: t('更新成功') }; return { code: 200, message: t('更新成功') };
} }
@serializeAuthMutation @serializeAuthMutation
public async updateUsernameAndPassword({ public async updateUsernameAndPassword({
username, username,
@@ -389,14 +365,12 @@ export default class UserService {
}); });
return { code: 200, message: t('更新成功') }; return { code: 200, message: t('更新成功') };
} }
@serializeAuthMutation @serializeAuthMutation
public async updateAvatar(avatar: string) { public async updateAvatar(avatar: string) {
const authInfo = await this.getAuthInfo(); const authInfo = await this.getAuthInfo();
await this.updateAuthInfo(authInfo, { avatar }); await this.updateAuthInfo(authInfo, { avatar });
return { code: 200, data: avatar, message: t('更新成功') }; return { code: 200, data: avatar, message: t('更新成功') };
} }
@serializeAuthMutation @serializeAuthMutation
public async initTwoFactor() { public async initTwoFactor() {
const secret = authenticator.generateSecret(); const secret = authenticator.generateSecret();
@@ -408,7 +382,6 @@ export default class UserService {
await this.updateAuthInfo(authInfo, { twoFactorSecret: secret }); await this.updateAuthInfo(authInfo, { twoFactorSecret: secret });
return { secret, url: otpauth }; return { secret, url: otpauth };
} }
@serializeAuthMutation @serializeAuthMutation
public async activeTwoFactor(code: string) { public async activeTwoFactor(code: string) {
const authInfo = await this.getAuthInfo(); const authInfo = await this.getAuthInfo();
@@ -427,14 +400,17 @@ export default class UserService {
} }
return isValid; return isValid;
} }
@serializeAuthMutation @serializeAuthMutation
public async twoFactorLogin( public async twoFactorLogin(
{ {
username, username,
password, password,
code, code,
}: { username: string; password: string; code: string }, }: {
username: string;
password: string;
code: string;
},
req: any, req: any,
) { ) {
const authInfo = await this.getAuthInfo(); const authInfo = await this.getAuthInfo();
@@ -482,7 +458,6 @@ export default class UserService {
return { code: 430, message: t('验证失败') }; return { code: 430, message: t('验证失败') };
} }
} }
@serializeAuthMutation @serializeAuthMutation
public async deactivateTwoFactor() { public async deactivateTwoFactor() {
const authInfo = await this.getAuthInfo(); const authInfo = await this.getAuthInfo();
@@ -496,7 +471,6 @@ export default class UserService {
}); });
return true; return true;
} }
public async getAuthInfo() { public async getAuthInfo() {
const authInfo = await shareStore.getAuthInfo(); const authInfo = await shareStore.getAuthInfo();
if (authInfo) { if (authInfo) {
@@ -505,7 +479,6 @@ export default class UserService {
const doc = await this.getDb({ type: AuthDataType.authConfig }); const doc = await this.getDb({ type: AuthDataType.authConfig });
return (doc.info || {}) as AuthInfo; return (doc.info || {}) as AuthInfo;
} }
private async updateAuthInfo(authInfo: AuthInfo, info: Partial<AuthInfo>) { private async updateAuthInfo(authInfo: AuthInfo, info: Partial<AuthInfo>) {
const result = { ...authInfo, ...info }; const result = { ...authInfo, ...info };
await shareStore.updateAuthInfo(result); await shareStore.updateAuthInfo(result);
@@ -517,34 +490,27 @@ export default class UserService {
this.sockService.getClients().forEach((conn) => conn.close('401')); this.sockService.getClients().forEach((conn) => conn.close('401'));
} }
} }
public async getNotificationMode(): Promise<NotificationInfo> { public async getNotificationMode(): Promise<NotificationInfo> {
const doc = await this.getDb({ type: AuthDataType.notification }); const doc = await this.getDb({ type: AuthDataType.notification });
return (doc.info || {}) as NotificationInfo; return (doc.info || {}) as NotificationInfo;
} }
private async updateAuthDb(payload: SystemInfo): Promise<any> { private async updateAuthDb(payload: SystemInfo): Promise<any> {
let doc = await SystemModel.findOne({ where: { type: payload.type } }); let doc = await SystemModel.findOne({ type: payload.type }, {});
if (doc) { if (doc) {
const updateResult = await SystemModel.update(payload, { await SystemModel.nativeUpdate({ id: doc.id }, payload);
where: { id: doc.id }, doc = await SystemModel.findOne({ id: doc.id });
returning: true,
});
doc = updateResult[1][0];
} else { } else {
doc = await SystemModel.create(payload, { returning: true }); doc = await SystemModel.insert(payload);
} }
return doc; return doc;
} }
public async getDb(query: any): Promise<SystemInfo> { public async getDb(query: any): Promise<SystemInfo> {
const doc = await SystemModel.findOne({ where: { ...query } }); const doc = await SystemModel.findOne({ ...query }, {});
if (!doc) { if (!doc) {
throw new Error(`${JSON.stringify(query)} not found`); throw new Error(`${JSON.stringify(query)} not found`);
} }
return doc.get({ plain: true }); return doc;
} }
public async updateNotificationMode(notificationInfo: NotificationInfo) { public async updateNotificationMode(notificationInfo: NotificationInfo) {
const code = Math.random().toString().slice(-6); const code = Math.random().toString().slice(-6);
const isSuccess = await this.notificationService.testNotify( const isSuccess = await this.notificationService.testNotify(
@@ -562,12 +528,10 @@ export default class UserService {
return { code: 400, message: t('通知发送失败,请检查参数') }; return { code: 400, message: t('通知发送失败,请检查参数') };
} }
} }
private normalizeTokens( private normalizeTokens(
tokens: Record<string, string | TokenInfo[]>, tokens: Record<string, string | TokenInfo[]>,
): Record<string, TokenInfo[]> { ): Record<string, TokenInfo[]> {
const normalized: Record<string, TokenInfo[]> = {}; const normalized: Record<string, TokenInfo[]> = {};
for (const [platform, value] of Object.entries(tokens)) { for (const [platform, value] of Object.entries(tokens)) {
if (typeof value === 'string') { if (typeof value === 'string') {
// Legacy format: convert string token to TokenInfo array // Legacy format: convert string token to TokenInfo array
@@ -589,10 +553,8 @@ export default class UserService {
normalized[platform] = value || []; normalized[platform] = value || [];
} }
} }
return normalized; return normalized;
} }
private addTokenToList( private addTokenToList(
tokens: Record<string, string | TokenInfo[]>, tokens: Record<string, string | TokenInfo[]>,
platform: string, platform: string,
@@ -606,16 +568,12 @@ export default class UserService {
); );
maxTokensPerPlatform = config.maxTokensPerPlatform; maxTokensPerPlatform = config.maxTokensPerPlatform;
} }
const normalized = this.normalizeTokens(tokens); const normalized = this.normalizeTokens(tokens);
if (!normalized[platform]) { if (!normalized[platform]) {
normalized[platform] = []; normalized[platform] = [];
} }
// Add new token // Add new token
normalized[platform].unshift(tokenInfo); normalized[platform].unshift(tokenInfo);
// Limit the number of active tokens per platform // Limit the number of active tokens per platform
if (normalized[platform].length > maxTokensPerPlatform) { if (normalized[platform].length > maxTokensPerPlatform) {
normalized[platform] = normalized[platform].slice( normalized[platform] = normalized[platform].slice(
@@ -623,40 +581,32 @@ export default class UserService {
maxTokensPerPlatform, maxTokensPerPlatform,
); );
} }
return normalized; return normalized;
} }
private removeTokenFromList( private removeTokenFromList(
tokens: Record<string, string | TokenInfo[]>, tokens: Record<string, string | TokenInfo[]>,
platform: string, platform: string,
tokenValue: string, tokenValue: string,
): Record<string, TokenInfo[]> { ): Record<string, TokenInfo[]> {
const normalized = this.normalizeTokens(tokens); const normalized = this.normalizeTokens(tokens);
if (normalized[platform]) { if (normalized[platform]) {
normalized[platform] = normalized[platform].filter( normalized[platform] = normalized[platform].filter(
(t) => t.value !== tokenValue, (t) => t.value !== tokenValue,
); );
} }
return normalized; return normalized;
} }
private findTokenInList( private findTokenInList(
tokens: Record<string, string | TokenInfo[]>, tokens: Record<string, string | TokenInfo[]>,
platform: string, platform: string,
tokenValue: string, tokenValue: string,
): TokenInfo | undefined { ): TokenInfo | undefined {
const normalized = this.normalizeTokens(tokens); const normalized = this.normalizeTokens(tokens);
if (normalized[platform]) { if (normalized[platform]) {
return normalized[platform].find((t) => t.value === tokenValue); return normalized[platform].find((t) => t.value === tokenValue);
} }
return undefined; return undefined;
} }
@serializeAuthMutation @serializeAuthMutation
public async resetAuthInfo(info: Partial<AuthInfo>) { public async resetAuthInfo(info: Partial<AuthInfo>) {
const { retries, twoFactorActivated, password, username } = info; const { retries, twoFactorActivated, password, username } = info;
@@ -673,7 +623,6 @@ export default class UserService {
}, },
(x) => !isNil(x), (x) => !isNil(x),
); );
if (password !== undefined) { if (password !== undefined) {
payload.password = await hashPassword(password); payload.password = await hashPassword(password);
} }
+4 -2
View File
@@ -135,8 +135,10 @@ export function parseCronSchedule(schedule: unknown): CronSchedule {
}; };
Object.freeze(patterns); Object.freeze(patterns);
Object.freeze(parsed); Object.freeze(parsed);
if (scheduleCache.size >= 512) if (scheduleCache.size >= 512) {
scheduleCache.delete(scheduleCache.keys().next().value); const oldest = scheduleCache.keys().next().value;
if (oldest !== undefined) scheduleCache.delete(oldest);
}
scheduleCache.set(cacheKey, parsed); scheduleCache.set(cacheKey, parsed);
return parsed; return parsed;
} }
+2
View File
@@ -1,6 +1,8 @@
const messages: Record<string, Record<string, string>> = { const messages: Record<string, Record<string, string>> = {
zh: {}, zh: {},
en: { en: {
'认证服务暂时不可用,请稍后重试':
'Authentication service is temporarily unavailable; try again shortly.',
'升级任务启动失败,请检查日志后重试': '升级任务启动失败,请检查日志后重试':
'Failed to start the update. Check the logs and try again.', 'Failed to start the update. Check the logs and try again.',
'[schedule][停止已运行任务] 任务ID: %s, PID: %s': '[schedule][停止已运行任务] 任务ID: %s, PID: %s':
+14 -15
View File
@@ -1,17 +1,16 @@
import { Op } from 'sequelize'; import { Query, QuerySpec, orderBy } from '../data/queries';
import { import {
SystemModel, SystemModel,
AuthDataType, AuthDataType,
NotifyStatus, NotifyStatus,
NotifyLogInfo, NotifyLogInfo,
} from '../data/system'; } from '../data/system';
// Both backend sends and script reports use the same bounded, best-effort sink. // Both backend sends and script reports use the same bounded, best-effort sink.
export async function persistNotificationAudit( export async function persistNotificationAudit(
info: NotifyLogInfo, info: NotifyLogInfo,
): Promise<void> { ): Promise<void> {
try { try {
await SystemModel.create({ await SystemModel.insert({
type: AuthDataType.notifyLog, type: AuthDataType.notifyLog,
info: { info: {
timestamp: Date.now(), timestamp: Date.now(),
@@ -27,19 +26,19 @@ export async function persistNotificationAudit(
taskId: /^\d+$/.test(info.taskId || '') ? info.taskId : undefined, taskId: /^\d+$/.test(info.taskId || '') ? info.taskId : undefined,
}, },
}); });
const oldest = await SystemModel.findAll({ const oldest = await SystemModel.find(
where: { type: AuthDataType.notifyLog }, { type: AuthDataType.notifyLog },
attributes: ['id'], {
order: [['id', 'DESC']], fields: ['id'],
offset: 200, orderBy: orderBy([['id', 'DESC']]),
limit: 1, offset: 200,
}); limit: 1,
},
);
if (oldest[0]?.id) { if (oldest[0]?.id) {
await SystemModel.destroy({ await SystemModel.nativeDelete({
where: { type: AuthDataType.notifyLog,
type: AuthDataType.notifyLog, id: { ['$lte']: oldest[0].id },
id: { [Op.lte]: oldest[0].id },
},
}); });
} }
} catch (error) { } catch (error) {
+14 -7
View File
@@ -137,13 +137,20 @@ class TaskLimit {
this.manualCronoLimit.concurrency = limit; this.manualCronoLimit.concurrency = limit;
return; return;
} }
await SystemModel.sync();
const doc = await SystemModel.findOne({ try {
where: { type: AuthDataType.systemConfig }, const doc = await SystemModel.findOne({
}); type: AuthDataType.systemConfig,
if (doc?.info?.cronConcurrency) { });
this.cronLimit.concurrency = doc.info.cronConcurrency; if (doc?.info?.cronConcurrency) {
this.manualCronoLimit.concurrency = doc.info.cronConcurrency; this.cronLimit.concurrency = doc.info.cronConcurrency;
this.manualCronoLimit.concurrency = doc.info.cronConcurrency;
}
} catch (error) {
// Imports can precede the isolated initializer on a fresh installation.
// Use defaults until that initializer creates Auths; never create schema here.
if ((error as { code?: string }).code !== 'DATABASE_SCHEMA_MISSING')
throw error;
} }
} }
+7 -9
View File
@@ -20,7 +20,7 @@ export function runCron(
// Check if the cron is already running and stop it (only if multiple instances are not allowed) // Check if the cron is already running and stop it (only if multiple instances are not allowed)
try { try {
const existingCron = await CrontabModel.findOne({ const existingCron = await CrontabModel.findOne({
where: { id: Number(cron.id) }, id: Number(cron.id),
}); });
// A mutation may commit while this queued run is reading the database. // A mutation may commit while this queued run is reading the database.
@@ -45,19 +45,17 @@ export function runCron(
await killTask(existingCron.pid, true); await killTask(existingCron.pid, true);
// Mark old running instances as stopped // Mark old running instances as stopped
const stoppedAt = dayjs().unix(); const stoppedAt = dayjs().unix();
await RunningInstanceModel.update( await RunningInstanceModel.nativeUpdate(
{ status: InstanceStatus.stopped, finished_at: stoppedAt },
{ {
where: { cron_id: Number(cron.id),
cron_id: Number(cron.id), status: InstanceStatus.running,
status: InstanceStatus.running,
},
}, },
{ status: InstanceStatus.stopped, finished_at: stoppedAt },
); );
// Update the status to idle after killing // Update the status to idle after killing
await CrontabModel.update( await CrontabModel.nativeUpdate(
{ id: Number(cron.id) },
{ status: CrontabStatus.idle, pid: undefined }, { status: CrontabStatus.idle, pid: undefined },
{ where: { id: Number(cron.id) } },
); );
} }
} catch (error) { } catch (error) {
+5 -15
View File
@@ -1,4 +1,4 @@
import { QueryTypes, Sequelize } from 'sequelize'; import { DatabaseClient } from '../data/databaseClient';
// Append new entries; IDs are persisted and must not be renumbered or reused. // Append new entries; IDs are persisted and must not be renumbered or reused.
const columns = [ const columns = [
@@ -27,25 +27,19 @@ const columns = [
{ table: 'Crontabs', column: 'queued_token', type: 'VARCHAR(255)' }, { table: 'Crontabs', column: 'queued_token', type: 'VARCHAR(255)' },
]; ];
export async function migrateSchema(database: Sequelize): Promise<void> { export async function migrateSchema(database: DatabaseClient): Promise<void> {
await database.transaction(async (transaction) => { await database.transaction(async () => {
await database.query( await database.query(
'CREATE TABLE IF NOT EXISTS "SchemaMigrations" ("id" TEXT PRIMARY KEY, "applied_at" TEXT NOT NULL)', 'CREATE TABLE IF NOT EXISTS "SchemaMigrations" ("id" TEXT PRIMARY KEY, "applied_at" TEXT NOT NULL)',
{ transaction },
); );
const applied = await database.query<{ id: string }>( const applied = await database.query<{ id: string }>(
'SELECT "id" FROM "SchemaMigrations"', 'SELECT "id" FROM "SchemaMigrations"',
{ type: QueryTypes.SELECT, transaction },
); );
const appliedIds = new Set(applied.map(({ id }) => id)); const appliedIds = new Set(applied.map(({ id }) => id));
for (const { table, column, type } of columns) { for (const { table, column, type } of columns) {
const id = `add-${table}-${column}`; const id = `add-${table}-${column}`;
const fields = await database.query<{ name: string }>( const fields = await database.query<{ name: string }>(
`PRAGMA table_info("${table}")`, `PRAGMA table_info("${table}")`,
{
type: QueryTypes.SELECT,
transaction,
},
); );
if (fields.length === 0) if (fields.length === 0)
throw new Error(`Migration table is missing: ${table}`); throw new Error(`Migration table is missing: ${table}`);
@@ -53,16 +47,12 @@ export async function migrateSchema(database: Sequelize): Promise<void> {
// table/column/type come only from the static migration manifest above. // table/column/type come only from the static migration manifest above.
await database.query( await database.query(
`ALTER TABLE "${table}" ADD COLUMN "${column}" ${type}`, `ALTER TABLE "${table}" ADD COLUMN "${column}" ${type}`,
{ transaction },
); );
} }
if (!appliedIds.has(id)) { if (!appliedIds.has(id)) {
await database.query( await database.query(
'INSERT INTO "SchemaMigrations" ("id", "applied_at") VALUES (:id, :appliedAt)', 'INSERT INTO "SchemaMigrations" ("id", "applied_at") VALUES (?, ?)',
{ [id, new Date().toISOString()],
replacements: { id, appliedAt: new Date().toISOString() },
transaction,
},
); );
} }
} }
+7 -5
View File
@@ -1,9 +1,7 @@
import { AuthInfo } from '../data/system'; import { AuthInfo } from '../data/system';
import { App } from '../data/open'; import { App } from '../data/open';
import Keyv from 'keyv'; import Keyv from 'keyv';
import KeyvSqlite from '@keyv/sqlite'; import { CacheStore } from '../data/cacheStore';
import config from '../config';
import path from 'path';
export enum EKeyv { export enum EKeyv {
'apps' = 'apps', 'apps' = 'apps',
@@ -17,8 +15,12 @@ export interface IKeyvStore {
lang: string; lang: string;
} }
const keyvSqlite = new KeyvSqlite(path.join(config.dbPath, 'keyv.sqlite')); const keyvSqlite = new CacheStore();
export const keyvStore = new Keyv<IKeyvStore>({ store: keyvSqlite }); // A stopped database is unavailable, not a cache miss or an expired session.
export const keyvStore = new Keyv<IKeyvStore>({
store: keyvSqlite,
throwOnErrors: true,
});
export const shareStore = { export const shareStore = {
getAuthInfo() { getAuthInfo() {
+12 -31
View File
@@ -2,21 +2,15 @@ import express, { Application, Request } from 'express';
import { AuthDataType, SystemModel } from '../data/system'; import { AuthDataType, SystemModel } from '../data/system';
import { normalizeClientIp } from './clientIp'; import { normalizeClientIp } from './clientIp';
import { t } from './i18n'; import { t } from './i18n';
const DEFAULT_TRUST_PROXY = 'loopback'; const DEFAULT_TRUST_PROXY = 'loopback';
type TrustProxyValue = boolean | number | string; type TrustProxyValue = boolean | number | string;
let activeApp: Application | undefined; let activeApp: Application | undefined;
function getEnvironmentSetting(): string { function getEnvironmentSetting(): string {
return process.env.QL_TRUST_PROXY?.trim() || ''; return process.env.QL_TRUST_PROXY?.trim() || '';
} }
function normalizeSetting(value?: string): string { function normalizeSetting(value?: string): string {
return value?.trim() || DEFAULT_TRUST_PROXY; return value?.trim() || DEFAULT_TRUST_PROXY;
} }
export function resolveTrustProxy(value?: string): TrustProxyValue { export function resolveTrustProxy(value?: string): TrustProxyValue {
const setting = normalizeSetting(value); const setting = normalizeSetting(value);
if (setting === 'true' || setting === 'false') { if (setting === 'true' || setting === 'false') {
@@ -27,7 +21,6 @@ export function resolveTrustProxy(value?: string): TrustProxyValue {
} }
return setting; return setting;
} }
function validateTrustProxy(value: string): string { function validateTrustProxy(value: string): string {
const setting = normalizeSetting(value); const setting = normalizeSetting(value);
if (setting.length > 500 || /[\r\n]/.test(setting)) { if (setting.length > 500 || /[\r\n]/.test(setting)) {
@@ -36,25 +29,22 @@ function validateTrustProxy(value: string): string {
if (/^\d+$/.test(setting) && Number(setting) > 20) { if (/^\d+$/.test(setting) && Number(setting) > 20) {
throw new Error(t('代理层数不能超过 20')); throw new Error(t('代理层数不能超过 20'));
} }
const probe = express(); const probe = express();
probe.set('trust proxy', resolveTrustProxy(setting)); probe.set('trust proxy', resolveTrustProxy(setting));
return setting; return setting;
} }
async function getStoredSetting(): Promise<string> { async function getStoredSetting(): Promise<string> {
const doc = await SystemModel.findOne({ const doc = await SystemModel.findOne(
where: { type: AuthDataType.systemConfig }, { type: AuthDataType.systemConfig },
}); {},
const info = (doc?.get('info') || {}) as Record<string, unknown>; );
const info = (doc?.info || {}) as Record<string, unknown>;
return typeof info.trustProxy === 'string' ? info.trustProxy : ''; return typeof info.trustProxy === 'string' ? info.trustProxy : '';
} }
export async function getTrustProxyConfig() { export async function getTrustProxyConfig() {
const environmentSetting = getEnvironmentSetting(); const environmentSetting = getEnvironmentSetting();
const storedSetting = await getStoredSetting(); const storedSetting = await getStoredSetting();
const trustProxy = normalizeSetting(environmentSetting || storedSetting); const trustProxy = normalizeSetting(environmentSetting || storedSetting);
return { return {
trustProxy, trustProxy,
source: environmentSetting source: environmentSetting
@@ -65,36 +55,31 @@ export async function getTrustProxyConfig() {
editable: !environmentSetting, editable: !environmentSetting,
}; };
} }
export async function initializeTrustProxy(app: Application) { export async function initializeTrustProxy(app: Application) {
activeApp = app; activeApp = app;
const { trustProxy } = await getTrustProxyConfig(); const { trustProxy } = await getTrustProxyConfig();
app.set('trust proxy', resolveTrustProxy(trustProxy)); app.set('trust proxy', resolveTrustProxy(trustProxy));
} }
export async function updateTrustProxy(value: string) { export async function updateTrustProxy(value: string) {
if (getEnvironmentSetting()) { if (getEnvironmentSetting()) {
throw new Error(t('环境变量 QL_TRUST_PROXY 已生效,系统设置不可覆盖')); throw new Error(t('环境变量 QL_TRUST_PROXY 已生效,系统设置不可覆盖'));
} }
const trustProxy = validateTrustProxy(value); const trustProxy = validateTrustProxy(value);
const doc = await SystemModel.findOne({ const doc = await SystemModel.findOne(
where: { type: AuthDataType.systemConfig }, { type: AuthDataType.systemConfig },
}); {},
);
if (!doc) { if (!doc) {
throw new Error(t('系统配置不存在')); throw new Error(t('系统配置不存在'));
} }
const plain = doc;
const plain = doc.get({ plain: true }); await SystemModel.nativeUpdate(
await SystemModel.update( { id: plain.id },
{ info: { ...(plain.info || {}), trustProxy } as any }, { info: { ...(plain.info || {}), trustProxy } as any },
{ where: { id: plain.id } },
); );
activeApp?.set('trust proxy', resolveTrustProxy(trustProxy)); activeApp?.set('trust proxy', resolveTrustProxy(trustProxy));
return getTrustProxyConfig(); return getTrustProxyConfig();
} }
function parseForwardedFor(value: string | string[] | undefined): string[] { function parseForwardedFor(value: string | string[] | undefined): string[] {
const values = Array.isArray(value) ? value : value ? [value] : []; const values = Array.isArray(value) ? value : value ? [value] : [];
return values return values
@@ -102,7 +87,6 @@ function parseForwardedFor(value: string | string[] | undefined): string[] {
.map((item) => normalizeClientIp(item)) .map((item) => normalizeClientIp(item))
.filter(Boolean); .filter(Boolean);
} }
export async function diagnoseClientIp(req: Request) { export async function diagnoseClientIp(req: Request) {
const remoteAddress = normalizeClientIp(req.socket.remoteAddress); const remoteAddress = normalizeClientIp(req.socket.remoteAddress);
const forwardedFor = parseForwardedFor(req.headers['x-forwarded-for']); const forwardedFor = parseForwardedFor(req.headers['x-forwarded-for']);
@@ -112,7 +96,6 @@ export async function diagnoseClientIp(req: Request) {
const trust = req.app.get('trust proxy fn') as const trust = req.app.get('trust proxy fn') as
| ((ip: string, hop: number) => boolean) | ((ip: string, hop: number) => boolean)
| undefined; | undefined;
let selectedIndex = Math.max(hopsFromApp.length - 1, 0); let selectedIndex = Math.max(hopsFromApp.length - 1, 0);
for (let index = 0; index < hopsFromApp.length - 1; index += 1) { for (let index = 0; index < hopsFromApp.length - 1; index += 1) {
if (!trust?.(hopsFromApp[index], index)) { if (!trust?.(hopsFromApp[index], index)) {
@@ -120,7 +103,6 @@ export async function diagnoseClientIp(req: Request) {
break; break;
} }
} }
const hops = hopsFromApp.map((ip, index) => ({ const hops = hopsFromApp.map((ip, index) => ({
ip, ip,
hop: index, hop: index,
@@ -131,7 +113,6 @@ export async function diagnoseClientIp(req: Request) {
? 'client' ? 'client'
: 'not_checked', : 'not_checked',
})); }));
return { return {
...(await getTrustProxyConfig()), ...(await getTrustProxyConfig()),
remoteAddress, remoteAddress,
+1 -1
View File
@@ -23,5 +23,5 @@
"esModuleInterop": true "esModuleInterop": true
}, },
"include": ["./**/*"], "include": ["./**/*"],
"exclude": ["node_modules"] "exclude": ["node_modules", "./data/databaseWorker.ts"]
} }
+15 -2
View File
@@ -164,7 +164,10 @@ async function readState(tmp) {
} }
} }
async function getStatus(tmp) { async function getStatus(
tmp,
{ root = path.resolve(__dirname, '../../../..') } = {},
) {
const state = await readState(tmp); const state = await readState(tmp);
if (active.has(state.status)) { if (active.has(state.status)) {
let alive = false; let alive = false;
@@ -198,7 +201,17 @@ async function getStatus(tmp) {
await handle.close(); await handle.close();
} }
} }
return state; const pkg = JSON.parse(
await fs
.readFile(path.join(root, 'package.json'), 'utf8')
.catch((error) => {
if (error.code !== 'ENOENT') throw error;
return '{}';
}),
);
// Read the installed package on every poll: the worker's completion record
// survives replacement, while this marker disappears only with the new runtime.
return { ...state, migrationRequired: Boolean(pkg.qinglongMigrationBridge) };
} }
// Manual restarts accept the currently prepared package, including one retained // Manual restarts accept the currently prepared package, including one retained
+179 -30
View File
@@ -1,5 +1,6 @@
// One installed verifier handles release downloads and local offline archives. // One installed verifier handles release downloads and local offline archives.
const fs = require('node:fs/promises'); const fs = require('node:fs/promises');
const { constants, createReadStream } = require('node:fs');
const path = require('node:path'); const path = require('node:path');
const crypto = require('node:crypto'); const crypto = require('node:crypto');
const os = require('node:os'); const os = require('node:os');
@@ -176,23 +177,39 @@ async function verifyPayload(selected, manifest, target) {
}); });
const names = Object.keys(actual).sort(), const names = Object.keys(actual).sort(),
expected = Object.keys(manifest.files).sort(); expected = Object.keys(manifest.files).sort();
if ( const differences = [...new Set([...expected, ...names])].filter(
JSON.stringify(names) !== JSON.stringify(expected) || (name) =>
names.some( !Object.hasOwn(actual, name) ||
(name) => !Object.hasOwn(manifest.files, name) ||
JSON.stringify(actual[name]) !== JSON.stringify(manifest.files[name]), JSON.stringify(actual[name]) !== JSON.stringify(manifest.files[name]),
) );
) if (differences.length)
throw new Error( throw new Error(
'Platform bundle files or dependency checksums do not match', 'Platform bundle files or dependency checksums do not match: ' +
JSON.stringify({
count: differences.length,
files: differences.slice(0, 5).map((name) => ({
name,
expected: manifest.files[name] || null,
actual: actual[name] || null,
})),
}),
); );
if ( if (
sha256(await fs.readFile(path.join(selected.source, 'pnpm-lock.yaml'))) !== sha256(await fs.readFile(path.join(selected.source, 'pnpm-lock.yaml'))) !==
manifest.lockfileSha256 manifest.lockfileSha256
) )
throw new Error('Platform bundle lockfile mismatch'); throw new Error('Platform bundle lockfile mismatch');
const pkg = JSON.parse(
await fs.readFile(path.join(selected.source, 'package.json'), 'utf8'),
);
await fs.access( await fs.access(
path.join(selected.source, 'node_modules/sqlite3/package.json'), path.join(
selected.source,
pkg.dependencies?.['@mikro-orm/sql']
? 'node_modules/@mikro-orm/sql/package.json'
: 'node_modules/sqlite3/package.json',
),
); );
} }
@@ -202,34 +219,58 @@ async function smokeDependencies(source, signal) {
const path = require('node:path'),fs = require('node:fs'); const path = require('node:path'),fs = require('node:fs');
const root = fs.realpathSync(process.cwd()); const root = fs.realpathSync(process.cwd());
const r = require('node:module').createRequire(path.join(root,'package.json')); const r = require('node:module').createRequire(path.join(root,'package.json'));
for (const name of ['sqlite3','sequelize','cron-parser','@grpc/grpc-js','undici']) { const pkg=r('./package.json'), modern=Boolean(pkg.dependencies?.['@mikro-orm/sql']);
if (modern) {
const [major,minor]=process.versions.node.split('.').map(Number);
if(major<22 || (major===22 && minor<17)) throw new Error('Bundle requires Node >=22.17; the existing Node is retained');
}
for (const name of [...(modern?['@mikro-orm/core','@mikro-orm/sql']:['sqlite3','sequelize']),'cron-parser','@grpc/grpc-js','undici']) {
if (!fs.realpathSync(r.resolve(name)).startsWith(root + path.sep)) throw new Error('Dependency outside staged payload: '+name); if (!fs.realpathSync(r.resolve(name)).startsWith(root + path.sep)) throw new Error('Dependency outside staged payload: '+name);
r(name); r(name);
} }
if (modern) {
const {DatabaseSync}=require('node:sqlite'), db=new DatabaseSync(':memory:');
try {
db.exec('CREATE TABLE verify_bundle(value INTEGER);INSERT INTO verify_bundle VALUES(42)');
if(db.prepare('SELECT value FROM verify_bundle').get().value!==42) throw new Error('SQLite readback failed');
} finally {db.close();}
} else {
const sqlite=r('sqlite3'), db=new sqlite.Database(':memory:'); const sqlite=r('sqlite3'), db=new sqlite.Database(':memory:');
db.serialize(()=>{db.run('CREATE TABLE verify_bundle(value INTEGER)');db.run('INSERT INTO verify_bundle VALUES(42)'); db.serialize(()=>{db.run('CREATE TABLE verify_bundle(value INTEGER)');db.run('INSERT INTO verify_bundle VALUES(42)');
db.get('SELECT value FROM verify_bundle',(err,row)=>{if(err||row.value!==42){console.error(err||'SQLite readback failed');process.exitCode=1;}db.close();});}); db.get('SELECT value FROM verify_bundle',(err,row)=>{if(err||row.value!==42){console.error(err||'SQLite readback failed');process.exitCode=1;}db.close();});});
}
`; `;
// Match the server's normal runtime. Node 20.18's built-in HTTP parser needs // Match the server's normal runtime. Node 20.18's built-in HTTP parser needs
// WebAssembly, which --jitless disables; only archive workers need that flag. // WebAssembly, which --jitless disables; only archive workers need that flag.
await execute(process.execPath, [...serverRuntimeArgs(process), '-e', probe], { await execute(
cwd: source, process.execPath,
env: { ...process.env, NODE_PATH: '', NODE_OPTIONS: '' }, [...serverRuntimeArgs(process), '-e', probe],
// ARM1176 devices need the same startup allowance as the backend probe. {
// Keep this bounded and cancellable without forcing an unnecessary repair. cwd: source,
timeout: nodeRuntimeArgs().length ? 300000 : 30000, env: { ...process.env, NODE_PATH: '', NODE_OPTIONS: '' },
signal, // ARM1176 devices need the same startup allowance as the backend probe.
}); // Keep this bounded and cancellable without forcing an unnecessary repair.
timeout: nodeRuntimeArgs().length ? 300000 : 30000,
signal,
},
);
} }
async function extractArchive(archive, destination) { async function extractArchive(archive, destination) {
const entries = new Map(), const entries = new Map(),
links = new Set(); directories = new Map([['qinglong', 0o755]]);
const header = (entry) => ({
type: entry.type,
size: entry.size,
mode: entry.mode,
linkpath: entry.linkpath,
});
let unpacked = 0, let unpacked = 0,
invalid; invalid;
await tar.t({ await tar.t({
file: archive, file: archive,
strict: true, strict: true,
maxReadSize: 256 * 1024,
onReadEntry(entry) { onReadEntry(entry) {
try { try {
const name = entry.path.replace(/\/$/, ''); const name = entry.path.replace(/\/$/, '');
@@ -237,16 +278,19 @@ async function extractArchive(archive, destination) {
!name || !name ||
name.includes('\\') || name.includes('\\') ||
/[\x00-\x1f\x7f]/.test(name) || /[\x00-\x1f\x7f]/.test(name) ||
name.split('/').length > 1024 ||
name.split('/').some((p) => !p || p === '.' || p === '..') || name.split('/').some((p) => !p || p === '.' || p === '..') ||
!(name === 'qinglong' || name.startsWith('qinglong/')) || !(name === 'qinglong' || name.startsWith('qinglong/')) ||
entries.has(name) || entries.has(name) ||
!['File', 'OldFile', 'Directory', 'SymbolicLink'].includes(entry.type) !['File', 'OldFile', 'Directory', 'SymbolicLink'].includes(entry.type)
) )
throw new Error(`Unsafe bundle archive entry: ${name}`); throw new Error(`Unsafe bundle archive entry: ${name}`);
if (!Number.isSafeInteger(entry.size) || entry.size < 0)
throw new Error('Invalid bundle archive entry size');
unpacked += entry.size; unpacked += entry.size;
if (unpacked > 2 * 1024 ** 3 || entries.size > 200000) if (unpacked > 2 * 1024 ** 3 || entries.size > 200000)
throw new Error('Bundle archive exceeds extraction limits'); throw new Error('Bundle archive exceeds extraction limits');
entries.set(name, entry.type); entries.set(name, header(entry));
if (entry.type === 'SymbolicLink') { if (entry.type === 'SymbolicLink') {
if ( if (
path.posix.isAbsolute(entry.linkpath) || path.posix.isAbsolute(entry.linkpath) ||
@@ -257,7 +301,6 @@ async function extractArchive(archive, destination) {
) )
) )
throw new Error('Unsafe bundle symlink'); throw new Error('Unsafe bundle symlink');
links.add(name);
} }
} catch (error) { } catch (error) {
invalid ||= error; invalid ||= error;
@@ -265,22 +308,101 @@ async function extractArchive(archive, destination) {
}, },
}); });
if (invalid) throw invalid; if (invalid) throw invalid;
for (const name of entries.keys()) { for (const [name, entry] of entries) {
if (entry.type === 'Directory')
directories.set(name, ((entry.mode ?? 0o755) & 0o777) | 0o700);
let parent = path.posix.dirname(name); let parent = path.posix.dirname(name);
while (parent !== '.') { while (parent !== '.') {
if (links.has(parent)) if (entries.has(parent) && entries.get(parent).type !== 'Directory')
throw new Error('Archive writes through a symlink'); throw new Error('Archive writes through a symlink or non-directory');
if (!directories.has(parent)) directories.set(parent, 0o755);
parent = path.posix.dirname(parent); parent = path.posix.dirname(parent);
} }
} }
await tar.x({ // The extraction root must be new and private. Create every parent before
file: archive, // links, and keep directories private/searchable until all writes finish.
const directoryNames = [...directories.keys()].sort(
(a, b) => a.split('/').length - b.split('/').length,
);
for (const name of directoryNames) {
const filename = path.join(destination, name);
await fs.mkdir(filename, { mode: 0o700 });
await fs.chmod(filename, 0o700);
}
let writes = Promise.resolve(),
failure;
const active = new Set(),
seen = new Set();
const parser = tar.t({
cwd: destination, cwd: destination,
strict: true, strict: true,
preservePaths: false, noResume: true,
noChown: true, onReadEntry(entry) {
umask: 0, active.add(entry);
// A parser failure can destroy queued entries before their turn starts.
entry.on('error', () => {});
writes = writes
.then(async () => {
if (failure) return entry.resume();
const name = entry.path.replace(/\/$/, '');
if (
seen.has(name) ||
JSON.stringify(entries.get(name)) !== JSON.stringify(header(entry))
)
throw new Error('Archive changed during extraction');
seen.add(name);
const filename = path.join(destination, name);
if (entry.type === 'Directory') return entry.resume();
if (entry.type === 'SymbolicLink') {
await fs.symlink(entry.linkpath, filename);
return entry.resume();
}
const mode = (entry.mode ?? 0o644) & 0o777;
const handle = await fs.open(
filename,
constants.O_WRONLY |
constants.O_CREAT |
constants.O_EXCL |
constants.O_NOFOLLOW,
0o600,
);
try {
// tar's fs-minipass writer discards successful partial write/writev
// results. Node's writeFile loops until each chunk is fully written.
// One entry/chunk at a time bounds memory and open FDs on ARMv6.
for await (const chunk of entry) await handle.writeFile(chunk);
await handle.chmod(mode);
} finally {
await handle.close();
}
})
.catch((error) => {
failure ||= error;
entry.resume();
})
.finally(() => active.delete(entry));
},
}); });
const input = createReadStream(archive, { highWaterMark: 256 * 1024 });
try {
await new Promise((resolve, reject) => {
input.once('error', reject);
parser.once('error', reject);
parser.once('end', resolve);
input.pipe(parser);
});
} catch (error) {
failure ||= error;
for (const entry of active) entry.destroy(error);
} finally {
input.destroy();
}
await writes;
if (failure) throw failure;
if (seen.size !== entries.size)
throw new Error('Archive changed during extraction');
for (const name of directoryNames.reverse())
await fs.chmod(path.join(destination, name), directories.get(name));
} }
async function stageBundle({ async function stageBundle({
@@ -297,6 +419,8 @@ async function stageBundle({
repair = repairDependencies, repair = repairDependencies,
}) { }) {
let local = env.QL_UPGRADE_FILE; let local = env.QL_UPGRADE_FILE;
let pendingMigration = false,
requiresModern = false;
if (!local) { if (!local) {
const current = JSON.parse( const current = JSON.parse(
await fs await fs
@@ -307,6 +431,9 @@ async function stageBundle({
}), }),
); );
if (current.qinglongUpgradeFormat !== 2) return null; // One-time compatibility path for older releases. if (current.qinglongUpgradeFormat !== 2) return null; // One-time compatibility path for older releases.
pendingMigration = Boolean(current.qinglongMigrationBridge);
requiresModern =
pendingMigration || Boolean(current.dependencies?.['@mikro-orm/sql']);
} }
const target = await runtimeTarget(); const target = await runtimeTarget();
const branch = ['develop', 'debian-dev'].includes(env.QL_BRANCH) const branch = ['develop', 'debian-dev'].includes(env.QL_BRANCH)
@@ -361,7 +488,16 @@ async function stageBundle({
indexUrl, indexUrl,
]); ]);
const index = JSON.parse(await fs.readFile(indexFile, 'utf8')); const index = JSON.parse(await fs.readFile(indexFile, 'utf8'));
const entry = index.packages?.[targetKey(target)]; if (pendingMigration && !Object.hasOwn(index, 'runtimePackages'))
throw new Error('更新索引尚未提供第二步运行包,请稍后重试');
requiresModern ||= Object.hasOwn(index, 'runtimePackages');
// Old format-2 clients read packages and receive an updater bridge. Once
// installed, this selector uses only the actual runtime packages. A broken
// runtime entry must never fall back to the bridge or a different platform.
const packages = Object.hasOwn(index, 'runtimePackages')
? index.runtimePackages
: index.packages;
const entry = packages?.[targetKey(target)];
if ( if (
index.format !== 2 || index.format !== 2 ||
!/^[a-f0-9]{40}$/.test(index.sourceCommit) || !/^[a-f0-9]{40}$/.test(index.sourceCommit) ||
@@ -403,6 +539,10 @@ async function stageBundle({
const manifest = JSON.parse( const manifest = JSON.parse(
await fs.readFile(path.join(incoming, 'bundle.json'), 'utf8'), await fs.readFile(path.join(incoming, 'bundle.json'), 'utf8'),
); );
if (manifest.migrationBridge)
throw new Error(
'This is a legacy upgrader bridge; download the runtime package instead',
);
if (expectedCommit && manifest.sourceCommit !== expectedCommit) if (expectedCommit && manifest.sourceCommit !== expectedCommit)
throw new Error('Update index and bundle commit differ'); throw new Error('Update index and bundle commit differ');
const selected = { const selected = {
@@ -413,6 +553,15 @@ async function stageBundle({
await fs.rename(path.join(incoming, 'static'), selected.static); await fs.rename(path.join(incoming, 'static'), selected.static);
await report({ phase: 'verifying', percent: null }); await report({ phase: 'verifying', percent: null });
await verifyPayload(selected, manifest, target); await verifyPayload(selected, manifest, target);
if (
requiresModern &&
!JSON.parse(
await fs.readFile(path.join(selected.source, 'package.json'), 'utf8'),
).dependencies?.['@mikro-orm/sql']
)
throw new Error(
'第二步更新包仍是旧数据库后端,已保留当前服务,请稍后重试',
);
const build = await verifyStatic(selected.source, selected.static, env); const build = await verifyStatic(selected.source, selected.static, env);
if (build.sourceCommit !== manifest.sourceCommit) if (build.sourceCommit !== manifest.sourceCommit)
throw new Error('Bundle and build commit differ'); throw new Error('Bundle and build commit differ');
@@ -210,6 +210,28 @@ async function prestartBackend(
} }
} }
async function cleanSqliteBuildArtifacts(directory) {
const entries = await fs.readdir(directory, { withFileTypes: true }).catch((error) => {
if (error.code !== 'ENOENT') throw error;
return [];
});
if (!entries.length) return;
const pkg = JSON.parse(await fs.readFile(path.join(directory, 'package.json'), 'utf8'));
const runtimePath = path.posix.normalize(pkg.binary?.module_path || '.');
const copiedRuntime = runtimePath.startsWith('lib/binding/');
// Our legacy fork copies its runtime to lib/binding. Other sqlite3 versions
// load build/Release directly: retain that addon and remove only the compiler
// helper directory. Never traverse a build-directory symlink while cleaning.
for (const entry of entries) {
if (entry.name !== 'build' && !/^build-tmp-napi-v\d+$/.test(entry.name)) continue;
const build = path.join(directory, entry.name);
await fs.rm(
copiedRuntime || entry.isSymbolicLink() ? build : path.join(build, 'node_gyp_bins'),
{ recursive: true, force: true },
);
}
}
async function repairDependencies(source, { target, signal, env }) { async function repairDependencies(source, { target, signal, env }) {
const workspace = await fs.mkdtemp( const workspace = await fs.mkdtemp(
path.join(path.dirname(source), 'repair-'), path.join(path.dirname(source), 'repair-'),
@@ -299,7 +321,12 @@ async function repairDependencies(source, { target, signal, env }) {
'--child-concurrency=1', '--child-concurrency=1',
'--network-concurrency=2', '--network-concurrency=2',
]); ]);
await run(['rebuild', 'sqlite3', '@whyour/sqlite3']); if (pkg.dependencies?.sqlite3) {
await run(['rebuild', 'sqlite3', '@whyour/sqlite3']);
await cleanSqliteBuildArtifacts(
path.join(workspace, 'node_modules/sqlite3'),
);
}
if ( if (
!(await fs.readFile(path.join(source, 'pnpm-lock.yaml'))).equals( !(await fs.readFile(path.join(source, 'pnpm-lock.yaml'))).equals(
await fs.readFile(path.join(workspace, 'pnpm-lock.yaml')), await fs.readFile(path.join(workspace, 'pnpm-lock.yaml')),
@@ -390,5 +417,6 @@ module.exports = {
prestartBackend, prestartBackend,
dependencyFailure, dependencyFailure,
repairDependencies, repairDependencies,
cleanSqliteBuildArtifacts,
probeLimits, probeLimits,
}; };
+18 -2
View File
@@ -1,6 +1,6 @@
# Run Node package installation natively on the builder. Node/npm can spin at # Run Node package installation natively on the builder. Node/npm can spin at
# 100% CPU when Alpine s390x is emulated through QEMU. # 100% CPU when Alpine s390x is emulated through QEMU.
FROM --platform=$BUILDPLATFORM node:18-alpine3.18 AS builder FROM --platform=$BUILDPLATFORM node:24-alpine3.24 AS builder
ARG TARGETARCH ARG TARGETARCH
ENV NPM_CONFIG_PREFIX=/opt/node-global ENV NPM_CONFIG_PREFIX=/opt/node-global
@@ -20,10 +20,18 @@ RUN set -x \
&& npm_config_target_platform=linux \ && npm_config_target_platform=linux \
npm_config_target_arch="${NODE_ARCH}" \ npm_config_target_arch="${NODE_ARCH}" \
npm_config_target_libc=musl \ npm_config_target_libc=musl \
pnpm install --prod --frozen-lockfile QL_BUILD_ONLY=1 pnpm install --prod --frozen-lockfile
# The amd64 APK runtime has crashed in native URL parsing on dev. Keep its
# URL implementation bundled with the pinned upstream musl Node binary.
FROM --platform=linux/amd64 node:24.18.1-alpine3.24@sha256:f70403e87646dc51b45295f4b8b70cdad0b63d2297c4c9899119b03f7af7a6b3 AS node-runtime-amd64
FROM python:3.11-alpine FROM python:3.11-alpine
ARG TARGETARCH
ARG TARGETVARIANT
COPY --from=node-runtime-amd64 /usr/local/bin/node /tmp/node-runtime-amd64
ARG QL_MAINTAINER="whyour" ARG QL_MAINTAINER="whyour"
LABEL maintainer="${QL_MAINTAINER}" LABEL maintainer="${QL_MAINTAINER}"
ARG QL_URL=https://github.com/${QL_MAINTAINER}/qinglong.git ARG QL_URL=https://github.com/${QL_MAINTAINER}/qinglong.git
@@ -62,6 +70,10 @@ RUN set -x \
netcat-openbsd \ netcat-openbsd \
unzip \ unzip \
npm \ npm \
&& if [ "$TARGETARCH" = amd64 ]; then \
mv /tmp/node-runtime-amd64 /usr/local/bin/node; \
ln -sf /usr/local/bin/node /usr/bin/node; \
else rm /tmp/node-runtime-amd64; fi \
&& rm -rf /var/cache/apk/* \ && rm -rf /var/cache/apk/* \
&& apk update \ && apk update \
&& ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \ && ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
@@ -86,6 +98,10 @@ COPY static/ /ql/static/
COPY docker/verify-build.cjs docker/build-manifest.cjs /tmp/ COPY docker/verify-build.cjs docker/build-manifest.cjs /tmp/
COPY --from=builder /tmp/build/pnpm-lock.yaml /tmp/dependency-lock.yaml COPY --from=builder /tmp/build/pnpm-lock.yaml /tmp/dependency-lock.yaml
RUN cd ${QL_DIR} && node /tmp/verify-build.cjs /tmp/dependency-lock.yaml RUN cd ${QL_DIR} && node /tmp/verify-build.cjs /tmp/dependency-lock.yaml
RUN if [ "$TARGETARCH" = arm ] && [ "${TARGETVARIANT:-}" = v6 ]; then \
QEMU_CPU=arm1176 node --jitless ${QL_DIR}/shell/node-runtime-probe.cjs; \
else node ${QL_DIR}/shell/node-runtime-probe.cjs; fi
ENV PNPM_HOME=${QL_DIR}/data/dep_cache/node \ ENV PNPM_HOME=${QL_DIR}/data/dep_cache/node \
PYTHON_HOME=${QL_DIR}/data/dep_cache/python3 \ PYTHON_HOME=${QL_DIR}/data/dep_cache/python3 \
+18 -2
View File
@@ -1,6 +1,6 @@
# Run Node package installation natively on the builder. Node/npm can spin at # Run Node package installation natively on the builder. Node/npm can spin at
# 100% CPU when Alpine s390x is emulated through QEMU. # 100% CPU when Alpine s390x is emulated through QEMU.
FROM --platform=$BUILDPLATFORM node:18-alpine3.18 AS builder FROM --platform=$BUILDPLATFORM node:24-alpine3.24 AS builder
ARG TARGETARCH ARG TARGETARCH
ENV NPM_CONFIG_PREFIX=/opt/node-global ENV NPM_CONFIG_PREFIX=/opt/node-global
@@ -20,10 +20,18 @@ RUN set -x \
&& npm_config_target_platform=linux \ && npm_config_target_platform=linux \
npm_config_target_arch="${NODE_ARCH}" \ npm_config_target_arch="${NODE_ARCH}" \
npm_config_target_libc=musl \ npm_config_target_libc=musl \
pnpm install --prod --frozen-lockfile QL_BUILD_ONLY=1 pnpm install --prod --frozen-lockfile
# The amd64 APK runtime has crashed in native URL parsing on dev. Keep its
# URL implementation bundled with the pinned upstream musl Node binary.
FROM --platform=linux/amd64 node:24.18.1-alpine3.24@sha256:f70403e87646dc51b45295f4b8b70cdad0b63d2297c4c9899119b03f7af7a6b3 AS node-runtime-amd64
FROM python:3.10-alpine FROM python:3.10-alpine
ARG TARGETARCH
ARG TARGETVARIANT
COPY --from=node-runtime-amd64 /usr/local/bin/node /tmp/node-runtime-amd64
ARG QL_MAINTAINER="whyour" ARG QL_MAINTAINER="whyour"
LABEL maintainer="${QL_MAINTAINER}" LABEL maintainer="${QL_MAINTAINER}"
ARG QL_URL=https://github.com/${QL_MAINTAINER}/qinglong.git ARG QL_URL=https://github.com/${QL_MAINTAINER}/qinglong.git
@@ -62,6 +70,10 @@ RUN set -x \
netcat-openbsd \ netcat-openbsd \
unzip \ unzip \
npm \ npm \
&& if [ "$TARGETARCH" = amd64 ]; then \
mv /tmp/node-runtime-amd64 /usr/local/bin/node; \
ln -sf /usr/local/bin/node /usr/bin/node; \
else rm /tmp/node-runtime-amd64; fi \
&& rm -rf /var/cache/apk/* \ && rm -rf /var/cache/apk/* \
&& apk update \ && apk update \
&& ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \ && ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
@@ -86,6 +98,10 @@ COPY static/ /ql/static/
COPY docker/verify-build.cjs docker/build-manifest.cjs /tmp/ COPY docker/verify-build.cjs docker/build-manifest.cjs /tmp/
COPY --from=builder /tmp/build/pnpm-lock.yaml /tmp/dependency-lock.yaml COPY --from=builder /tmp/build/pnpm-lock.yaml /tmp/dependency-lock.yaml
RUN cd ${QL_DIR} && node /tmp/verify-build.cjs /tmp/dependency-lock.yaml RUN cd ${QL_DIR} && node /tmp/verify-build.cjs /tmp/dependency-lock.yaml
RUN if [ "$TARGETARCH" = arm ] && [ "${TARGETVARIANT:-}" = v6 ]; then \
QEMU_CPU=arm1176 node --jitless ${QL_DIR}/shell/node-runtime-probe.cjs; \
else node ${QL_DIR}/shell/node-runtime-probe.cjs; fi
ENV PNPM_HOME=${QL_DIR}/data/dep_cache/node \ ENV PNPM_HOME=${QL_DIR}/data/dep_cache/node \
PYTHON_HOME=${QL_DIR}/data/dep_cache/python3 \ PYTHON_HOME=${QL_DIR}/data/dep_cache/python3 \
+7 -4
View File
@@ -1,6 +1,9 @@
# Node 20 Bookworm is the latest official Node image variant that covers the FROM python:3.11.14-slim-bookworm AS nodebuilder
# full Debian build matrix, including arm/v7, ppc64le, and s390x. ARG TARGETARCH
FROM node:20-bookworm-slim AS nodebuilder ARG TARGETVARIANT
COPY docker/install-node-runtime.sh /tmp/install-node-runtime.sh
RUN apt-get update && apt-get install --no-install-recommends -y ca-certificates curl xz-utils libatomic1 libstdc++6 && \
sh /tmp/install-node-runtime.sh /usr/local
FROM python:3.11.14-slim-bookworm AS builder FROM python:3.11.14-slim-bookworm AS builder
COPY package.json .npmrc pnpm-lock.yaml /tmp/build/ COPY package.json .npmrc pnpm-lock.yaml /tmp/build/
@@ -13,7 +16,7 @@ RUN set -x && \
npm i -g pnpm@8.3.1 && \ npm i -g pnpm@8.3.1 && \
test "$(pnpm --version)" = "8.3.1" && \ test "$(pnpm --version)" = "8.3.1" && \
cd /tmp/build && \ cd /tmp/build && \
pnpm install --prod --frozen-lockfile QL_BUILD_ONLY=1 pnpm install --prod --frozen-lockfile
FROM python:3.11.14-slim-bookworm FROM python:3.11.14-slim-bookworm
+7 -4
View File
@@ -1,6 +1,9 @@
# Node 20 Bookworm is the latest official Node image variant that covers the FROM python:3.10.19-slim-bookworm@sha256:23f63358922e79a794f71be8f3723c84e5ccca9638af3f74456dc73d6184499e AS nodebuilder
# full Debian build matrix, including arm/v7, ppc64le, and s390x. ARG TARGETARCH
FROM node:20-bookworm-slim AS nodebuilder ARG TARGETVARIANT
COPY docker/install-node-runtime.sh /tmp/install-node-runtime.sh
RUN apt-get update && apt-get install --no-install-recommends -y ca-certificates curl xz-utils libatomic1 libstdc++6 && \
sh /tmp/install-node-runtime.sh /usr/local
# Keep the full release matrix, including s390x, on the same immutable base. # Keep the full release matrix, including s390x, on the same immutable base.
# The floating 3.10-slim-bookworm tag no longer includes s390x. # The floating 3.10-slim-bookworm tag no longer includes s390x.
@@ -14,7 +17,7 @@ RUN set -x && \
apt-get install --no-install-recommends -y libatomic1 && \ apt-get install --no-install-recommends -y libatomic1 && \
npm i -g pnpm@8.3.1 && \ npm i -g pnpm@8.3.1 && \
cd /tmp/build && \ cd /tmp/build && \
pnpm install --prod --frozen-lockfile QL_BUILD_ONLY=1 pnpm install --prod --frozen-lockfile
FROM python:3.10.19-slim-bookworm@sha256:23f63358922e79a794f71be8f3723c84e5ccca9638af3f74456dc73d6184499e FROM python:3.10.19-slim-bookworm@sha256:23f63358922e79a794f71be8f3723c84e5ccca9638af3f74456dc73d6184499e
+22
View File
@@ -0,0 +1,22 @@
ARG LEGACY_IMAGE
ARG BASE_IMAGE
FROM ${LEGACY_IMAGE} AS legacy
FROM ${BASE_IMAGE} AS bridge
ARG BASE_IMAGE
ENV QL_BUNDLE_BASE_IMAGE=${BASE_IMAGE}
ARG TARGETARCH
ARG TARGETVARIANT
USER root
COPY --from=legacy /ql/ /legacy/
RUN ulimit -c 0; if [ "$TARGETARCH/$TARGETVARIANT" = arm/v6 ]; then \
export QEMU_CPU=arm1176; \
apk add --no-cache build-base nodejs-dev && \
node --jitless /ql/scripts/build-migration-bridge.cjs /legacy /ql /out && \
for archive in /out/qinglong-*-linux-armv6-musl.tar.gz; do \
case "$archive" in */qinglong-migration-*) continue ;; esac; \
node /ql/shell/install-bundle.cjs --root /tmp/ql-bundle-offline-smoke --file "$archive" --install && \
rm -rf /tmp/ql-bundle-offline-smoke || exit 1; \
done; \
else node /ql/scripts/build-migration-bridge.cjs /legacy /ql /out; fi
FROM scratch
COPY --from=bridge /out/ /
+1 -12
View File
@@ -6,18 +6,7 @@ ENV QL_BUNDLE_BASE_IMAGE=${BASE_IMAGE}
ARG TARGETARCH ARG TARGETARCH
ARG TARGETVARIANT ARG TARGETVARIANT
USER root USER root
# The generic "arm" sqlite release is ARMv7. ARMv6 must compile its own addon. # SQLite comes from the retained Node runtime; no architecture-specific addon.
RUN ulimit -c 0; if [ "$TARGETARCH/$TARGETVARIANT" = arm/v6 ]; then \
apk add --no-cache --virtual .ql-bundle-build build-base binutils python3 nodejs-dev && \
cd "$(readlink -f /ql/node_modules/sqlite3)" && \
CFLAGS='-march=armv6 -mfpu=vfp -mfloat-abi=hard' \
CXXFLAGS='-march=armv6 -mfpu=vfp -mfloat-abi=hard' \
npm_config_nodedir=/usr npm_config_build_from_source=true QEMU_CPU=arm1176 \
node -e 'require("node:child_process").execFileSync(process.execPath,[require.resolve("@mapbox/node-pre-gyp/bin/node-pre-gyp"),"rebuild","--napi_build_version=6"],{stdio:"inherit"})' && \
readelf -A lib/binding/napi-v6-linux-arm/node_sqlite3.node | grep -E 'Tag_CPU_arch: v6($|K)' && \
rm -rf build && \
apk del .ql-bundle-build; \
fi
RUN ulimit -c 0; if [ "$TARGETARCH/$TARGETVARIANT" = arm/v6 ]; then \ RUN ulimit -c 0; if [ "$TARGETARCH/$TARGETVARIANT" = arm/v6 ]; then \
export QEMU_CPU=arm1176; \ export QEMU_CPU=arm1176; \
node --jitless /ql/scripts/build-upgrade-bundle.cjs /ql /out && \ node --jitless /ql/scripts/build-upgrade-bundle.cjs /ql /out && \
+30
View File
@@ -0,0 +1,30 @@
#!/bin/sh
set -eu
# Official binaries, pinned to their release checksums. Node 24 no longer
# publishes ARMv7 binaries, so that existing Debian target uses Node 22 LTS.
node_version=24.18.1
case "${TARGETARCH:?}/${TARGETVARIANT:-}" in
amd64/*) node_arch=x64; node_sha=d6c664df3f3f61458e8c277585571328522d705166723a7c7823a9253a4d15a0 ;;
arm64/*) node_arch=arm64; node_sha=7201e3a09dc825bac57867c81913e2b8f0ef87d04cb9082af4cda82f6ff3d88c ;;
ppc64le/*) node_arch=ppc64le; node_sha=c47812c13862be7f6e71194c434c4d78a99096e72efb9c31a2627437e056d669 ;;
s390x/*) node_arch=s390x; node_sha=acf57bd726f35afc2ba8475194e5a8bd532c00e1a3b2436295b2fbdde199d0e9 ;;
arm/v7) node_version=22.23.2; node_arch=armv7l; node_sha=ef8f26a3de19acd8c23548e6c3cfc2052610b0e67abb5fd64dbd92c8b1c1245b ;;
*) echo "Unsupported Debian Node architecture: ${TARGETARCH}/${TARGETVARIANT:-}" >&2; exit 1 ;;
esac
if [ "${1:-}" = --print ]; then
printf '%s %s %s\n' "$node_version" "$node_arch" "$node_sha"
exit 0
fi
node_prefix=${1:-/usr/local}
node_work_dir=$(mktemp -d)
trap 'rm -rf "$node_work_dir"' EXIT
node_archive="node-v${node_version}-linux-${node_arch}.tar.xz"
curl --fail --location --retry 3 --connect-timeout 20 --max-time 300 \
"https://nodejs.org/dist/v${node_version}/${node_archive}" -o "$node_work_dir/$node_archive"
printf '%s %s\n' "$node_sha" "$node_work_dir/$node_archive" | sha256sum -c -
mkdir -p "$node_prefix"
tar -xJf "$node_work_dir/$node_archive" -C "$node_prefix" --strip-components=1 --no-same-owner
"$node_prefix/bin/node" -e 'const {DatabaseSync}=require("node:sqlite"); const db=new DatabaseSync(":memory:"); try {db.exec("CREATE TABLE node_probe(value INTEGER);INSERT INTO node_probe VALUES(42)");if(db.prepare("SELECT value FROM node_probe").get().value!==42)throw Error("Node SQLite probe failed");}finally{db.close();}'
+5
View File
@@ -0,0 +1,5 @@
{
"sourceCommit": "99a4d0eff8431898d7638fefff305591fe326ed5",
"musl": "ghcr.io/whyour/qinglong@sha256:0847b0035083cd439c00ff3087392f67035b26842209c110ee976f51ec13fbcd",
"glibc": "ghcr.io/whyour/qinglong@sha256:d2775474223793a5dfe0060910acdcd9bab4c2b70ff6b2ef770ec536afb206f6"
}
+25 -11
View File
@@ -1,12 +1,18 @@
# 平台完整升级包 # 平台完整升级包
CI 从同次构建的不可变镜像摘要导出完整 tar.gz,包含源码中的运行脚本和模板、前后端编译产物、锁文件和已经安装的生产 `node_modules`。不包含 Node 可执行文件、用户数据、`.env`、Git 历史或 npm 缓存。保留现有 Node 和用户脚本依赖。 CI 从同次构建的不可变镜像摘要导出完整 tar.gz,包含源码中的运行脚本和模板、前后端编译产物、锁文件和已经安装的生产 `node_modules`。不包含 Node 可执行文件、用户数据、`.env`、Git 历史或 npm 缓存。保留现有 Node 和用户脚本依赖。面板最低需要 Node 22.17,推荐 Node 24;Node 20 安装先升级运行时,应用更新包不会替换 Node。
## 在线升级 ## 在线升级
`package.json` 中 `qinglongUpgradeFormat: 2` 表示使用完整包。旧版升级器仍可通过原归档路径完成首次迁移;完整包版本不会在平台包缺失、损坏或不兼容时退回设备端 npm 安装。仅当完整包已通过校验、依赖加载或后端预启动明确报告依赖错误时,在线升级会按同一冻结锁文件修复一次生产依赖,然后再次预启动;第二次失败即停止,保留正在运行的旧服务。 `package.json` 中 `qinglongUpgradeFormat: 2` 表示使用完整包。满足 Node 22.17 的旧格式 2 面板可在面板内分两步迁移,不需要手动运行安装器或更换镜像:第一次下载独立的兼容升级器包,预启动通过后手动重启;此时仍运行旧数据库后端,界面提示“升级器已更新,请继续升级新版”。点击“继续升级新版”完成第二次下载和新版后端预启动,再手动重启,健康与版本检查通过后才显示“升级成功”并隐藏侧栏状态。关闭页面、刷新或换浏览器不会丢失第二步提示;Node 不满足最低要求会在第一步准备阶段被旧校验器拒绝,旧服务继续运行。
面板下载任务与本地离线入口共用 `platformBundle.cjs`:平台匹配 → 归档安全检查 → 全文件/链接/权限和锁文件校验 → SQLite/Sequelize/cron-parser/gRPC/undici 加载及 SQLite 内存库读写 → 隔离后端预启动 → 发布就绪指针。预启动使用现有 Node、临时数据目录和 loopback 随机端口,运行真实数据库建表/迁移和事务读写、gRPC 服务,以及与正常启动共用的 HTTP 中间件、路由和 WebSocket 初始化,再通过 `/api/health` 确认双服务与目标版本、提交和后端摘要;不运行任务、订阅、通知、用户启动钩子和系统依赖安装。临时库不是用户数据库副本,因此实际应用后的健康检查和回退仍不可省略。点击重启时,接口只检查就绪记录并提交后台任务;完整文件与依赖复验由独立工作进程执行,复验通过后才停止旧服务,避免小设备的预检耗时超过 HTTP 请求期限。离线升级同步面板任务状态,并与在线任务互斥;中断准备时保留之前的可用就绪包。原有服务停止、文件备份、启动健康检查及失败恢复仍由共享升级引擎处理。 兼容包单独命名为 `qinglong-migration-<commit>-<platform>.tar.gz`,来自固定摘要、固定源码提交的旧镜像,保留真实旧 SQLite/Sequelize 依赖和旧数据库后端,只替换升级器、前端和构建身份;它不伪造 SQLite 模块,也不执行用户数据迁移。正常运行包仍为 `qinglong-<commit>-<platform>.tar.gz`,只使用 node:sqlite。索引的 `packages` 为旧客户端保留兼容包,`runtimePackages` 指向正式运行包;新版升级器优先且只使用完整的 `runtimePackages`,不因损坏或缺失而退回兼容包。新版在线/离线入口会拒绝误用兼容包。生产索引发布前,每个平台都必须使用未经修改的旧升级器真实下载、校验、预启动第一步,再使用包内新版升级器下载、校验、预启动第二步。两步失败均沿用原有回滚流程。
固定旧镜像中的 generic ARM SQLite addon 实际使用 ARMv7 指令。ARMv6 musl 兼容包会在 CI 构建时从同一旧依赖源码重新编译该 addon,使用匹配构建 Node 的本地头文件、单并发及 ARMv6/VFP 硬浮点参数,并检查 ELF 指令集,拒绝 ARMv7、Thumb-2 或 VFPv3 产物。旧锁文件及后端保持不变,其他架构不执行此步骤;正常运行包仍不包含旧 SQLite,目标设备无需为第一步迁移联网编译。
旧源码归档升级器继续保留原兼容路径;完整包版本不会在平台包缺失、损坏或不兼容时退回设备端 npm 安装。仅当完整包已通过校验、依赖加载或后端预启动明确报告依赖错误时,在线升级会按同一冻结锁文件修复一次生产依赖,然后再次预启动;第二次失败即停止,保留正在运行的旧服务。第一步旧后端的检查仍使用真实旧依赖,第二步才切换到 MikroORM 和 Node 内置 SQLite。
面板下载任务与本地离线入口共用 `platformBundle.cjs`:平台匹配 → 归档安全检查 → 全文件/链接/权限和锁文件校验 → MikroORM/cron-parser/gRPC/undici 加载及 node:sqlite 内存库读写(旧包继续检查 SQLite/Sequelize) → 隔离后端预启动 → 发布就绪指针。预启动使用现有 Node、临时数据目录和 loopback 随机端口,运行真实数据库建表/迁移和事务读写、gRPC 服务,以及与正常启动共用的 HTTP 中间件、路由和 WebSocket 初始化,再通过 `/api/health` 确认双服务与目标版本、提交和后端摘要;不运行任务、订阅、通知、用户启动钩子和系统依赖安装。临时库不是用户数据库副本,因此实际应用后的健康检查和回退仍不可省略。点击重启时,接口只检查就绪记录并提交后台任务;完整文件与依赖复验由独立工作进程执行,复验通过后才停止旧服务,避免小设备的预检耗时超过 HTTP 请求期限。离线升级同步面板任务状态,并与在线任务互斥;中断准备时保留之前的可用就绪包。原有服务停止、文件备份、启动健康检查及失败恢复仍由共享升级引擎处理。
正式升级包跟随 master 同一次版本镜像构建;develop 的普通提交也自动构建开发包。两者都直接使用该次 Alpine/Debian 镜像的不可变 digest。Git tag 工作流和 `static_sync_only` 不生成完整包。需要重建当前开发包时,可手动运行同一工作流: 正式升级包跟随 master 同一次版本镜像构建;develop 的普通提交也自动构建开发包。两者都直接使用该次 Alpine/Debian 镜像的不可变 digest。Git tag 工作流和 `static_sync_only` 不生成完整包。需要重建当前开发包时,可手动运行同一工作流:
@@ -24,7 +30,7 @@ gh workflow run build-docker-image.yml --ref develop
## 手动离线安装和升级 ## 手动离线安装和升级
从 `upgrade-v<version>` 的说明或 `latest.json` 中取得当前批次的下载链接,下载匹配设备的 `.tar.gz` 和独立 `install-bundle.cjs`;两者都来自同一个 `upgrade-build-v<version>-<run-id>-<run-attempt>` release。独立安装器已内置解析依赖,在目标设备正常情况无需 npm、pnpm 或重新安装 SQLite。离线准备和预启动默认不联网;依赖不符合要求时直接报错。可对照同批次 release 的 `latest.json` 核对归档 SHA256。 从 `upgrade-v<version>` 的说明或 `latest.json` 的 `runtimePackages` 中取得当前批次的下载链接,下载匹配设备的正常运行 `.tar.gz` 和独立 `install-bundle.cjs`;两者都来自同一个 `upgrade-build-v<version>-<run-id>-<run-attempt>` release。不要下载 `qinglong-migration-` 包作为离线安装包。独立安装器已内置解析依赖,在目标设备正常情况无需 npm、pnpm 或重新安装 SQLite。离线准备和预启动默认不联网;依赖不符合要求时直接报错。可对照同批次 release 的 `latest.json` 核对归档 SHA256。
已有安装(默认应用并重启): 已有安装(默认应用并重启):
@@ -32,7 +38,7 @@ gh workflow run build-docker-image.yml --ref develop
node install-bundle.cjs --root /ql --file /absolute/path/qinglong-<commit>-linux-arm64-musl.tar.gz node install-bundle.cjs --root /ql --file /absolute/path/qinglong-<commit>-linux-arm64-musl.tar.gz
``` ```
只准备,稍后手动重启: 已迁移的安装只准备,稍后手动重启(首次从旧格式 2 迁移应直接执行上一条新版独立安装器命令,不能调用旧版 `ql reload system`):
```sh ```sh
node install-bundle.cjs --root /ql --file /absolute/path/package.tar.gz --download-only node install-bundle.cjs --root /ql --file /absolute/path/package.tar.gz --download-only
@@ -45,7 +51,7 @@ ql reload system
node install-bundle.cjs --root /ql --file /absolute/path/package.tar.gz --download-only --repair-deps node install-bundle.cjs --root /ql --file /absolute/path/package.tar.gz --download-only --repair-deps
``` ```
修复需要已有 pnpm 8.3.1 及必要编译工具。依赖在私有目录按 `--prod --frozen-lockfile` 重装,仅重建 SQLite;ARMv6 强制源码编译和 ARMv6 硬浮点指令,禁止下载 generic ARM 的 ARMv7 二进制。修复仅执行一次;不安装或替换 Node、不改锁文件、不改已校验源码和静态产物。应用前以原包校验清单和独立记录的修复依赖摘要复验。 修复需要已有 pnpm 8.3.1。新包在私有目录按 `--prod --frozen-lockfile` 重装生产 JavaScript 依赖,不编译 SQLite。只有旧包显式依赖 sqlite3 时才需要编译工具并重建 SQLite;旧 ARMv6 包强制源码编译和 ARMv6 硬浮点指令,禁止下载 generic ARM 的 ARMv7 二进制。修复仅执行一次;不安装或替换 Node、不改锁文件、不改已校验源码和静态产物。应用前以原包校验清单和独立记录的修复依赖摘要复验。
修复继承调用环境中的标准 HTTP/HTTPS/ALL/NO_PROXY(含小写形式),其余 Node 预加载和编译参数仍隔离。修复安装或 SQLite 重建失败时,清理私有目录,保留当前服务、当前依赖及上一次就绪包;不会标记本次更新就绪。预启动或修复主进程退出时立即终止其进程组,防止持有输出管道的子进程继续写入已清理目录。 修复继承调用环境中的标准 HTTP/HTTPS/ALL/NO_PROXY(含小写形式),其余 Node 预加载和编译参数仍隔离。修复安装或 SQLite 重建失败时,清理私有目录,保留当前服务、当前依赖及上一次就绪包;不会标记本次更新就绪。预启动或修复主进程退出时立即终止其进程组,防止持有输出管道的子进程继续写入已清理目录。
@@ -74,9 +80,9 @@ npm 会排除根目录的 `pnpm-lock.yaml`。主包的原生 `npm pack/publish`
`package.json#files` 是 npm 包内容的唯一项目规则,不再维护根 `.npmignore`:npm 在根包使用 `files` 时会停用该目录的 `.npmignore` 和 `.gitignore` 规则。打包测试在没有 `.npmignore` 的独立源码目录中注入私有配置、密钥占位文件、缓存及生成的预加载配置,执行真实 `npm pack` 和安装,检查运行文件完整、私有文件及其内容不进入归档。此规则只用于 npm 包,完整平台 tar.gz 仍由平台包构建器处理。 `package.json#files` 是 npm 包内容的唯一项目规则,不再维护根 `.npmignore`:npm 在根包使用 `files` 时会停用该目录的 `.npmignore` 和 `.gitignore` 规则。打包测试在没有 `.npmignore` 的独立源码目录中注入私有配置、密钥占位文件、缓存及生成的预加载配置,执行真实 `npm pack` 和安装,检查运行文件完整、私有文件及其内容不进入归档。此规则只用于 npm 包,完整平台 tar.gz 仍由平台包构建器处理。
干净源码完成 `pnpm build:front`、`pnpm build:back`、`pnpm build:info` 后,运行 `node scripts/verify-panel-package.cjs pack /tmp/qinglong-npm` 生成并检查真实 npm 归档。CI 在一次性 Linux 容器中,以 Node 20.18.1 和 Node 24 安装同一个归档,使用真实生产依赖和已安装的 `qinglong`、`ql` 入口,验证启动、损坏下载保留旧服务、在线准备与重启、离线准备与 Bash 重启及账号、配置、环境变量、任务和脚本保留。测试还安装完整性摘要固定的已发布 `@whyour/qinglong@2.23.0`,在旧服务中建立真实数据,再用新 `.tgz` 执行 npm 覆盖安装,验证账号、配置、环境变量、任务和脚本保留。之后升级到同次 Debian 镜像不可变 digest 通过正式 `Dockerfile.upgrade-bundle` 导出的原始平台包,确认运行目录变为 pnpm 布局。该单平台验收包仅上传短期 CI artifact,不发布 Release,也不推进开发或正式更新索引。 干净源码完成 `pnpm build:front`、`pnpm build:back`、`pnpm build:info` 后,运行 `node scripts/verify-panel-package.cjs pack /tmp/qinglong-npm` 生成并检查真实 npm 归档。CI 在一次性 Linux 容器中,以 Node 22.17.0 和 Node 24 安装同一个归档,使用真实生产依赖和已安装的 `qinglong`、`ql` 入口,验证启动、损坏下载保留旧服务、在线准备与重启、离线准备与 Bash 重启及账号、配置、环境变量、任务和脚本保留。测试还安装完整性摘要固定的已发布 `@whyour/qinglong@2.23.0`,在旧服务中建立真实数据,再用新 `.tgz` 执行 npm 覆盖安装,验证账号、配置、环境变量、任务和脚本保留。之后升级到同次 Debian 镜像不可变 digest 通过正式 `Dockerfile.upgrade-bundle` 导出的原始平台包,确认运行目录变为 pnpm 布局。该单平台验收包仅上传短期 CI artifact,不发布 Release,也不推进开发或正式更新索引。
Umi 4.4.4 的构建依赖仍使用 Node 24 已移除的 `http_parser`。前端源码构建使用 `.nvmrc` 中的 Node 22,CI 从该文件读取版本,Vercel 前后端项目均配置为 Node 22.x。`package.json` 不声明 `engines.node`,避免 Vercel 用开放的最低运行版本范围覆盖项目的构建版本;`pnpm build:front` 直接执行构建,不下载或切换 Node。面板运行时仍支持 Node 20.18.1 及以上,平台包的清单与安装器独立校验最低 Node 要求,npm 实装验收继续覆盖 Node 20/24。 Umi 4.4.4 的构建依赖仍使用 Node 24 已移除的 `http_parser`。前端源码构建使用 `.nvmrc` 中的 Node 22,CI 从该文件读取版本,Vercel 前后端项目均配置为 Node 22.x。`package.json` 不声明 `engines.node`,避免 Vercel 用开放的最低运行版本范围覆盖项目的构建版本;`pnpm build:front` 直接执行构建,不下载或切换 Node。面板运行时最低为 Node 22.17,平台包清单、npm preinstall 和数据库进程均检查最低 Node 要求。CI 的 Node 20.18.1 项使用实际新 npm 归档验证拒绝安装,以及本地/全局旧包、`.env`、外置数据和旧服务保留;它不启动新后端。构建专用环境可跳过安装阶段的运行时探测,后端启动不跳过版本检查。
服务集成测试只能在专用容器中运行:设置 `QL_PANEL_PACKAGE_TEST_ISOLATED=1`、指向隔离 PM2 的 `QL_PANEL_TEST_PM2` 和指向正式构建脚本输出目录的 `QL_PANEL_TEST_BUNDLE_DIR`,然后执行 `node scripts/verify-panel-package.cjs verify /tmp/qinglong-npm /tmp/qinglong-npm-results`。容器需已有 Bash、curl、OpenSSL、Python、jq、nginx 和 procps。旧正式包会重置 PATH,因此验收容器将 PM2 6.0.14 安装到 `/usr/local/bin/pm2`,并以此设置 `QL_PANEL_TEST_PM2`;隔离通过专属容器和独立 `PM2_HOME` 保证。测试创建独立的 npm prefix、HOME、PM2_HOME 和数据目录,结束后清理。 服务集成测试只能在专用容器中运行:设置 `QL_PANEL_PACKAGE_TEST_ISOLATED=1`、指向隔离 PM2 的 `QL_PANEL_TEST_PM2` 和指向正式构建脚本输出目录的 `QL_PANEL_TEST_BUNDLE_DIR`,然后执行 `node scripts/verify-panel-package.cjs verify /tmp/qinglong-npm /tmp/qinglong-npm-results`。容器需已有 Bash、curl、OpenSSL、Python、jq、nginx 和 procps。旧正式包会重置 PATH,因此验收容器将 PM2 6.0.14 安装到 `/usr/local/bin/pm2`,并以此设置 `QL_PANEL_TEST_PM2`;隔离通过专属容器和独立 `PM2_HOME` 保证。测试创建独立的 npm prefix、HOME、PM2_HOME 和数据目录,结束后清理。
@@ -84,13 +90,21 @@ Umi 4.4.4 的构建依赖仍使用 Node 24 已移除的 `http_parser`。前端
master 发布任务等待上述测试通过,直接发布已验证的 `.tgz`,不重新构建或打包。develop 只生成测试归档与验收记录,不发布 npm。 master 发布任务等待上述测试通过,直接发布已验证的 `.tgz`,不重新构建或打包。develop 只生成测试归档与验收记录,不发布 npm。
## 数据库兼容与回滚
正常运行包使用 MikroORM 7 的官方 NodeSqliteDialect。Sequelize 和 sqlite3 只作为开发测试依赖,用于验证旧库读写,不进入正常运行包的生产依赖;独立的第一步兼容包仍携带固定旧镜像的真实旧依赖和后端,以满足已安装旧升级器的检查。九张业务表及既有迁移记录保留原名和结构,不运行 ORM 自动改表;日期保持旧版 UTC 文本格式,JSON、ID、排序精度及 Keyv 缓存格式保持兼容,旧版可读新版写入的数据。
每个后端进程延迟启动一个有界数据库子进程,避免同步 node:sqlite 阻塞 HTTP 和调度事件循环。事务独占请求队列,嵌套事务使用 SAVEPOINT;事务异常回滚。数据库进程退出或超时后拒绝后续请求、健康检查失败,不自动重放写入;恢复由服务重启完成。后端正常退出会终止其数据库子进程,预启动监督器还会终止整个临时进程组。数据库进程在打开 SQLite 前启动一个仅监视父进程的轻量线程;即使主 SQL 线程阻塞,父进程被强杀后也会终止整个数据库进程并释放事务锁,避免阻止旧服务重启。
## 平台兼容 ## 平台兼容
- 最低 Node 为 20.18.1,Node-API 至少 6;安装器只检查,不替换 Node。 - 最低 Node 为 22.17,Node-API 至少 6;安装器只检查,不替换 Node。Debian amd64、arm64、ppc64le、s390x 使用 Node 24;官方 Node 24 不提供 Linux ARMv7 二进制,Debian ARMv7 使用 Node 22 LTS。
- 以 Node 实际运行架构、ARM 指令集和 libc 类型/版本选择包;不依据 Docker 架构别名猜测。 - 以 Node 实际运行架构、ARM 指令集和 libc 类型/版本选择包;不依据 Docker 架构别名猜测。
- glibc、musl 分包,并记录构建系统 libc 下限。系统库仍以目标设备加载测试作为最后检查。 - glibc、musl 分包,并记录构建系统 libc 下限。系统库仍以目标设备加载测试作为最后检查。
- ARMv6 与 ARMv7 使用不同包名。ARMv6 在目标镜像中从源码编译 SQLite,禁止复用 generic `arm` 的 ARMv7 预编译件;构建需通过加载和内存库读写,失败则不发布渠道。 - ARMv6 与 ARMv7 使用不同包名。当前生产依赖没有 SQLite 原生 addon,使用设备 Node 自带的 node:sqlite。ARMv6 Node 本身仍须兼容 ARM1176 和硬浮点;构建需通过内置 SQLite 的加载和内存库读写,失败则不发布渠道。
- ARMv6 的归档构建、在线更新工作进程及独立离线安装器使用 `--jitless`,避开 ARM1176 上 V8 的优化编译崩溃;保留现有 Node 二进制,Node 24 及以上的 ARMv6 面板进程通过共享 `nodeRuntime` 使用 `--no-opt --no-sparkplug --no-maglev`,其他架构和较低 Node 版本保持原启动参数。ARMv6 的 gRPC 工作进程首次启动及重建最多等待 2 分钟;升级后的整体健康检查最多等待 5 分钟,覆盖慢设备按顺序启动数据库、调度与 HTTP 工作进程的耗时;其他平台仍为 60 秒,超时或版本校验失败仍执行回退。依赖加载与后端预启动使用现有 Node 的正常模式,使 Node 20.18.1 的内置 HTTP/WebAssembly 组件保持可用。预启动只有一个后端进程,按顺序初始化数据库、gRPC、HTTP,不启动 PM2/cluster;ARMv6 V8 堆上限 128 MiB、期限 5 分钟,其他平台为 192 MiB/60 秒。堆上限不是 RSS 总内存上限,系统仍需为现有服务和 OpenSSL 留出内存。依赖修复的包管理器及编译使用单并发、低网络并发和有界时长;ARMv6 包管理器采用 `--jitless`、256 MiB 堆上限,并优先复用版本匹配的现有 Node 头文件与实际运行时编译配置;超时、中断会终止整个进程组并清理临时目录。 - ARMv6 的归档构建、在线更新工作进程及独立离线安装器使用 `--jitless`,避开 ARM1176 上 V8 的优化编译崩溃;保留现有 Node 二进制,Node 24 及以上的 ARMv6 面板进程通过共享 `nodeRuntime` 使用 `--no-opt --no-sparkplug --no-maglev`,其他架构和较低 Node 版本保持原启动参数。ARMv6 的 gRPC 工作进程首次启动及重建最多等待 2 分钟;升级后的整体健康检查最多等待 5 分钟,覆盖慢设备按顺序启动数据库、调度与 HTTP 工作进程的耗时;其他平台仍为 60 秒,超时或版本校验失败仍执行回退。依赖加载与后端预启动保留 HTTP/WebAssembly 所需能力。预启动使用一个后端和一个数据库子进程,按顺序初始化数据库、gRPC、HTTP,不启动 PM2/cluster;后端的 ARMv6 V8 堆上限 128 MiB、期限 5 分钟,其他平台为 192 MiB/60 秒。数据库执行主线程的 V8 堆上限 128 MiB,ARMv6 使用 `--jitless`;监视线程不加载 ORM、HTTP 或 ts-node,并保留进程的 V8 模式。主线程和监视线程的 V8 限制不是 RSS 总内存上限,系统仍需为现有服务和 OpenSSL 留出内存。依赖修复的包管理器及编译使用单并发、低网络并发和有界时长;ARMv6 包管理器采用 `--jitless`、256 MiB 堆上限,并优先复用版本匹配的现有 Node 头文件与实际运行时编译配置;超时、中断会终止整个进程组并清理临时目录。
- tar 中允许 pnpm 的包内符号链接;拒绝绝对路径、越界路径、包外链接、硬链接、设备条目和经符号链接写入的条目。重启前再次校验依赖,暂存失败保留上一次可用的就绪包。 - tar 中允许 pnpm 的包内符号链接;拒绝绝对路径、越界路径、包外链接、硬链接、设备条目和经符号链接写入的条目。重启前再次校验依赖,暂存失败保留上一次可用的就绪包。
归档先完整预检路径、类型、大小与链接,再向新建的私有目录逐个流式写入文件。文件写入使用 Node 的完整写入接口,处理短写后才关闭文件;目录权限在写入结束后恢复,避免受调用进程的 umask 影响。读取、解析或写入失败时,等候待写文件关闭后再清理,不发布就绪状态。校验失败会列出最多五个文件的预期和实际权限、哈希或链接信息,不输出文件内容。
自托管镜像可设置 `QL_UPGRADE_INDEX_URL` 指向兼容索引;要求 HTTPS,测试时仅放行 loopback HTTP。索引和所有平台归档应部署到同一可访问源,避免依赖额外的 raw 域名。 自托管镜像可设置 `QL_UPGRADE_INDEX_URL` 指向兼容索引;要求 HTTPS,测试时仅放行 loopback HTTP。索引和所有平台归档应部署到同一可访问源,避免依赖额外的 raw 域名。
+14 -11
View File
@@ -49,6 +49,7 @@
"version.yaml" "version.yaml"
], ],
"scripts": { "scripts": {
"preinstall": "node -e \"if (process.env.QL_BUILD_ONLY !== '1' && process.env.VERCEL !== '1') require('./shell/runtime-guard.cjs').assertNodeRuntime()\"",
"build:cli": "node cli/scripts/build.cjs", "build:cli": "node cli/scripts/build.cjs",
"check:cli": "npm --prefix cli run check", "check:cli": "npm --prefix cli run check",
"test:cli": "npm run build:cli && node cli/scripts/test.cjs", "test:cli": "npm run build:cli && node cli/scripts/test.cjs",
@@ -57,7 +58,7 @@
"start:back": "nodemon ./back/app.ts", "start:back": "nodemon ./back/app.ts",
"start:front": "max dev", "start:front": "max dev",
"build:front": "max build", "build:front": "max build",
"build:back": "tsc -p back/tsconfig.json", "build:back": "tsc -p back/tsconfig.json && tsc -p back/data/tsconfig.worker.json",
"test": "TS_NODE_PROJECT=back/tsconfig.json node -r ts-node/register/transpile-only --test test/back/*.test.cjs test/front/*.test.cjs", "test": "TS_NODE_PROJECT=back/tsconfig.json node -r ts-node/register/transpile-only --test test/back/*.test.cjs test/front/*.test.cjs",
"benchmark:execution": "TS_NODE_PROJECT=back/tsconfig.json node -r ts-node/register/transpile-only scripts/benchmark-execution.cjs", "benchmark:execution": "TS_NODE_PROJECT=back/tsconfig.json node -r ts-node/register/transpile-only scripts/benchmark-execution.cjs",
"build:info": "node scripts/write-build-info.cjs", "build:info": "node scripts/write-build-info.cjs",
@@ -121,11 +122,11 @@
} }
}, },
"dependencies": { "dependencies": {
"@ant-design/plots": "^2.6.8",
"@bufbuild/protobuf": "^2.10.0", "@bufbuild/protobuf": "^2.10.0",
"@grpc/grpc-js": "1.14.5", "@grpc/grpc-js": "1.14.5",
"@grpc/proto-loader": "^0.8.0", "@grpc/proto-loader": "^0.8.0",
"@keyv/sqlite": "^4.0.1", "@mikro-orm/core": "7.2.5",
"@mikro-orm/sql": "7.2.5",
"@otplib/preset-default": "^12.0.1", "@otplib/preset-default": "^12.0.1",
"body-parser": "^1.20.3", "body-parser": "^1.20.3",
"celebrate": "^15.0.3", "celebrate": "^15.0.3",
@@ -147,7 +148,7 @@
"ip2region": "2.3.0", "ip2region": "2.3.0",
"js-yaml": "^4.1.0", "js-yaml": "^4.1.0",
"jsonwebtoken": "^9.0.2", "jsonwebtoken": "^9.0.2",
"keyv": "^5.2.3", "keyv": "5.6.0",
"lodash": "^4.17.21", "lodash": "^4.17.21",
"multer": "2.4.0", "multer": "2.4.0",
"node-cron": "4.6.0", "node-cron": "4.6.0",
@@ -157,19 +158,18 @@
"ps-tree": "^1.2.0", "ps-tree": "^1.2.0",
"reflect-metadata": "^0.2.2", "reflect-metadata": "^0.2.2",
"request-ip": "3.3.0", "request-ip": "3.3.0",
"sequelize": "6.37.8",
"sockjs": "^0.3.24", "sockjs": "^0.3.24",
"sqlite3": "npm:@whyour/sqlite3@1.1.2", "tar": "7.5.20",
"toad-scheduler": "^3.0.1", "toad-scheduler": "^3.0.1",
"typedi": "^0.10.0", "typedi": "^0.10.0",
"undici": "^7.9.0", "undici": "^7.9.0",
"uuid": "^11.0.3", "uuid": "^11.0.3",
"winston": "^3.17.0", "winston": "^3.17.0",
"winston-daily-rotate-file": "^5.0.0", "winston-daily-rotate-file": "^5.0.0"
"tar": "7.5.20"
}, },
"devDependencies": { "devDependencies": {
"@ant-design/icons": "^5.0.1", "@ant-design/icons": "^5.0.1",
"@ant-design/plots": "^2.6.8",
"@ant-design/pro-layout": "6.38.22", "@ant-design/pro-layout": "6.38.22",
"@codemirror/state": "6.5.4", "@codemirror/state": "6.5.4",
"@codemirror/view": "6.39.16", "@codemirror/view": "6.39.16",
@@ -187,7 +187,7 @@
"@types/jsonwebtoken": "^8.5.8", "@types/jsonwebtoken": "^8.5.8",
"@types/lodash": "^4.14.185", "@types/lodash": "^4.14.185",
"@types/multer": "^1.4.7", "@types/multer": "^1.4.7",
"@types/node": "^17.0.21", "@types/node": "^22.20.5",
"@types/nodemailer": "^6.4.4", "@types/nodemailer": "^6.4.4",
"@types/proper-lockfile": "^4.1.4", "@types/proper-lockfile": "^4.1.4",
"@types/ps-tree": "^1.1.6", "@types/ps-tree": "^1.1.6",
@@ -233,13 +233,16 @@
"react-intl-universal": "^2.12.0", "react-intl-universal": "^2.12.0",
"react-router-dom": "6.26.1", "react-router-dom": "6.26.1",
"react-split-pane": "^0.1.92", "react-split-pane": "^0.1.92",
"sequelize": "6.37.8",
"sockjs-client": "^1.6.0", "sockjs-client": "^1.6.0",
"sqlite3": "npm:@whyour/sqlite3@1.1.2",
"ts-node": "^10.9.2", "ts-node": "^10.9.2",
"ts-proto": "^2.6.1", "ts-proto": "^2.6.1",
"tslib": "^2.4.0", "tslib": "^2.4.0",
"typescript": "5.2.2", "typescript": "5.9.3",
"vh-check": "^2.0.5", "vh-check": "^2.0.5",
"virtualizedtableforantd4": "1.3.0" "virtualizedtableforantd4": "1.3.0"
}, },
"qinglongUpgradeFormat": 2 "qinglongUpgradeFormat": 2,
"qinglongRuntimeMinimum": "22.17.0"
} }
+539 -646
View File
File diff suppressed because it is too large Load Diff
+272
View File
@@ -0,0 +1,272 @@
// A separate first-hop package for installed format-2 verifiers. Its database
// backend and dependencies remain the pinned legacy implementation. Only the
// updater and frontend are replaced; the next download uses runtimePackages.
const fs = require('node:fs/promises');
const { createReadStream } = require('node:fs');
const path = require('node:path');
const http = require('node:http');
const assert = require('node:assert/strict');
const { pipeline } = require('node:stream/promises');
const { once } = require('node:events');
const { collectBuildFiles } = require('../docker/build-manifest.cjs');
const { buildBundle } = require('./build-upgrade-bundle.cjs');
const artifacts = require('../cli/src/internal/maintenance/upgradeArtifacts.cjs');
const bundle = require('../cli/src/internal/maintenance/platformBundle.cjs');
const baseline = require('../docker/migration-bridge-images.json');
const { rebuildMigrationSqlite } = require('./rebuild-migration-sqlite.cjs');
const read = async (file) => JSON.parse(await fs.readFile(file, 'utf8'));
async function prepareBridge(legacy, current, destination) {
const oldBuild = await artifacts.verifyUpgrade(
legacy,
path.join(legacy, 'static'),
);
const build = await artifacts.verifyUpgrade(
current,
path.join(current, 'static'),
);
const pkg = await read(path.join(legacy, 'package.json'));
const next = await read(path.join(current, 'package.json'));
if (
oldBuild.sourceCommit !== baseline.sourceCommit ||
pkg.qinglongUpgradeFormat !== 2 ||
!pkg.dependencies?.sqlite3 ||
!pkg.dependencies?.sequelize ||
!next.dependencies?.['@mikro-orm/sql']
)
throw new Error(
'Migration bridge requires the pinned legacy backend and a modern runtime',
);
// Never copy user state, even if this builder is used outside Docker.
const excluded = new Set(['.git', '.tmp', '.env', 'data', 'output']);
await fs.cp(legacy, destination, {
recursive: true,
verbatimSymlinks: true,
filter: (file) =>
!excluded.has(path.relative(legacy, file).split(path.sep)[0]),
});
const maintenance = 'cli/src/internal/maintenance';
await fs.rm(path.join(destination, maintenance), {
recursive: true,
force: true,
});
await fs.cp(
path.join(current, maintenance),
path.join(destination, maintenance),
{ recursive: true },
);
await fs.rm(path.join(destination, 'static/dist'), {
recursive: true,
force: true,
});
await fs.cp(
path.join(current, 'static/dist'),
path.join(destination, 'static/dist'),
{ recursive: true },
);
const migrationBridge = {
format: 1,
legacySourceCommit: oldBuild.sourceCommit,
sourceCommit: build.sourceCommit,
targetVersion: next.version,
};
await fs.writeFile(
path.join(destination, 'package.json'),
JSON.stringify(
{
...pkg,
qinglongRuntimeMinimum: '22.17.0',
qinglongMigrationBridge: migrationBridge,
},
null,
2,
),
);
await fs.writeFile(
path.join(destination, 'static/build-info.json'),
JSON.stringify({
...oldBuild,
sourceCommit: build.sourceCommit,
files: collectBuildFiles(path.join(destination, 'static')),
}),
);
// No backend bytes or dependency lock may change in the updater hop.
assert.equal(
bundle.sha256(
await fs.readFile(path.join(destination, 'static/build/app.js')),
),
oldBuild.files['build/app.js'],
);
assert.equal(
bundle.sha256(await fs.readFile(path.join(destination, 'pnpm-lock.yaml'))),
oldBuild.lockfileSha256,
);
return migrationBridge;
}
async function verifyBothDownloads(legacy, current, directory, records) {
let index, workspace;
const files = new Map(
records.map((r) => ['/' + r.filename, path.join(directory, r.filename)]),
);
const server = http.createServer((req, res) => {
if (!index) {
res.writeHead(503);
res.end();
return;
}
if (req.url === '/latest.json') {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(index));
} else if (files.has(req.url)) {
const record = records.find((r) => '/' + r.filename === req.url);
res.setHeader('Content-Length', record.size);
pipeline(createReadStream(files.get(req.url)), res).catch(() =>
res.destroy(),
);
} else {
res.writeHead(404);
res.end();
}
});
try {
server.listen(0, '127.0.0.1');
await once(server, 'listening');
const base = `http://127.0.0.1:${server.address().port}`;
const entry = (r) => ({
size: r.size,
sha256: r.sha256,
url: base + '/' + r.filename,
});
const [bridge, runtime] = records;
index = {
format: 2,
sourceCommit: runtime.sourceCommit,
packages: { [bridge.target]: entry(bridge) },
runtimePackages: { [runtime.target]: entry(runtime) },
};
workspace = await fs.mkdtemp(path.join(directory, '.legacy-client-'));
const env = {
...process.env,
QL_BRANCH: 'develop',
QL_UPGRADE_INDEX_URL: base + '/latest.json',
QL_UPGRADE_FILE: '',
NO_PROXY: '127.0.0.1',
no_proxy: '127.0.0.1',
};
const old = require(path.join(
legacy,
'cli/src/internal/maintenance/upgradeArtifacts.cjs',
));
const firstTmp = path.join(workspace, 'first');
await fs.mkdir(firstTmp);
// These are the unchanged, genuinely installed old download, dependency
// smoke, prestart and readiness verifiers. No injected probe or verifier.
const selected = await old.stageUpgrade({
root: legacy,
tmp: firstTmp,
env,
run: (program, args, capture) =>
old.runProcess(env, program, args, capture),
});
await old.selectedUpgrade(firstTmp, 'develop', env);
assert(
(await read(path.join(selected.source, 'package.json')))
.qinglongMigrationBridge,
);
const upgraded = require(path.join(
selected.source,
'cli/src/internal/maintenance/upgradeArtifacts.cjs',
));
const secondTmp = path.join(workspace, 'second');
await fs.mkdir(secondTmp);
const next = await upgraded.stageUpgrade({
root: selected.source,
tmp: secondTmp,
env,
run: (program, args, capture) =>
upgraded.runProcess(env, program, args, capture),
});
await upgraded.selectedUpgrade(secondTmp, 'develop', env);
const pkg = await read(path.join(next.source, 'package.json'));
assert(pkg.dependencies['@mikro-orm/sql']);
assert.equal(pkg.qinglongMigrationBridge, undefined);
await assert.rejects(
fs.access(path.join(next.source, 'node_modules/sqlite3')),
{ code: 'ENOENT' },
);
await assert.rejects(
fs.access(path.join(next.source, 'node_modules/sequelize')),
{ code: 'ENOENT' },
);
const oldBundle = require(path.join(
legacy,
'cli/src/internal/maintenance/platformBundle.cjs',
));
const manifest = await read(
path.join(path.dirname(selected.source), 'bundle.json'),
);
assert.throws(
() =>
oldBundle.assertCompatible(
manifest,
{ ...manifest.target, libcVersion: manifest.target.libcMinimum },
'20.18.1',
),
/requires Node/,
);
return {
legacyOnlineVerified: true,
modernOnlineVerified: true,
legacyNode20Rejected: true,
};
} finally {
server.closeAllConnections();
if (server.listening) await new Promise((resolve) => server.close(resolve));
if (workspace) await fs.rm(workspace, { recursive: true, force: true });
}
}
async function buildMigrationBridge(legacy, current, output) {
legacy = path.resolve(legacy);
current = path.resolve(current);
output = path.resolve(output);
await fs.mkdir(output, { recursive: true });
const workspace = await fs.mkdtemp(path.join(output, '.migration-'));
let bridge;
try {
const root = path.join(workspace, 'root');
await prepareBridge(legacy, current, root);
await rebuildMigrationSqlite(root);
const runtime = await buildBundle(current, output);
bridge = await buildBundle(root, output);
const verification = await verifyBothDownloads(legacy, current, output, [
bridge,
runtime,
]);
bridge = { ...bridge, migrationVerified: true, ...verification };
await fs.writeFile(
path.join(output, bridge.filename + '.json'),
JSON.stringify(bridge) + '\n',
);
console.log(JSON.stringify(bridge));
return bridge;
} catch (error) {
if (bridge)
await fs.rm(path.join(output, bridge.filename + '.json'), {
force: true,
});
throw error;
} finally {
await fs.rm(workspace, { recursive: true, force: true });
}
}
module.exports = { prepareBridge, verifyBothDownloads, buildMigrationBridge };
if (require.main === module)
buildMigrationBridge(process.argv[2], process.argv[3], process.argv[4]).catch(
(error) => {
console.error(error);
process.exitCode = 1;
},
);
+31 -18
View File
@@ -1,10 +1,15 @@
const fs = require('node:fs/promises'); const fs = require('node:fs/promises');
const { createWriteStream } = require('node:fs');
const path = require('node:path'); const path = require('node:path');
const { pipeline } = require('node:stream/promises');
const tar = require('tar'); const tar = require('tar');
const bundle = require('../cli/src/internal/maintenance/platformBundle.cjs'); const bundle = require('../cli/src/internal/maintenance/platformBundle.cjs');
const { const {
verifyUpgrade, verifyUpgrade,
} = require('../cli/src/internal/maintenance/upgradeArtifacts.cjs'); } = require('../cli/src/internal/maintenance/upgradeArtifacts.cjs');
const {
prestartBackend,
} = require('../cli/src/internal/maintenance/upgradePrestart.cjs');
async function normalizeModes(directory) { async function normalizeModes(directory) {
for (const entry of await fs.readdir(directory, { withFileTypes: true })) { for (const entry of await fs.readdir(directory, { withFileTypes: true })) {
@@ -19,6 +24,17 @@ async function normalizeModes(directory) {
} }
} }
async function writeBundleArchive(workspace, filename) {
// Core streams complete partial writes; tar's file option uses fs-minipass,
// whose writer can silently discard a successful short write/writev result.
await pipeline(
tar.c({ gzip: true, portable: true, noMtime: true, cwd: workspace }, [
'qinglong',
]),
createWriteStream(filename, { highWaterMark: 256 * 1024 }),
);
}
async function buildBundle(root, output) { async function buildBundle(root, output) {
root = path.resolve(root); root = path.resolve(root);
output = path.resolve(output); output = path.resolve(output);
@@ -26,6 +42,10 @@ async function buildBundle(root, output) {
if (process.platform !== 'linux') if (process.platform !== 'linux')
throw new Error('Release bundles must be built and probed on Linux'); throw new Error('Release bundles must be built and probed on Linux');
const build = await verifyUpgrade(root, path.join(root, 'static')); const build = await verifyUpgrade(root, path.join(root, 'static'));
const pkg = JSON.parse(
await fs.readFile(path.join(root, 'package.json'), 'utf8'),
);
const migrationBridge = pkg.qinglongMigrationBridge;
await bundle.smokeDependencies(root); await bundle.smokeDependencies(root);
await fs.mkdir(output, { recursive: true }); await fs.mkdir(output, { recursive: true });
const workspace = await fs.mkdtemp(path.join(output, '.bundle-')); const workspace = await fs.mkdtemp(path.join(output, '.bundle-'));
@@ -64,9 +84,8 @@ async function buildBundle(root, output) {
const manifest = { const manifest = {
format: 2, format: 2,
sourceCommit: build.sourceCommit, sourceCommit: build.sourceCommit,
version: JSON.parse( version: pkg.version,
await fs.readFile(path.join(root, 'package.json'), 'utf8'), ...(migrationBridge ? { migrationBridge } : {}),
).version,
lockfileSha256: build.lockfileSha256, lockfileSha256: build.lockfileSha256,
target: { target: {
platform: target.platform, platform: target.platform,
@@ -76,7 +95,7 @@ async function buildBundle(root, output) {
endianness: target.endianness, endianness: target.endianness,
armFloatAbi: target.armFloatAbi, armFloatAbi: target.armFloatAbi,
}, },
nodeMinimum: '20.18.1', nodeMinimum: '22.17.0',
nodeApi: 6, nodeApi: 6,
files: await bundle.inventory({ source, static: staticRoot }), files: await bundle.inventory({ source, static: staticRoot }),
}; };
@@ -85,19 +104,10 @@ async function buildBundle(root, output) {
JSON.stringify(manifest), JSON.stringify(manifest),
); );
console.log('Bundle inventory verified; writing archive'); console.log('Bundle inventory verified; writing archive');
const filename = `qinglong-${build.sourceCommit}-${bundle.targetKey( const filename = `qinglong-${migrationBridge ? 'migration-' : ''}${
target, build.sourceCommit
)}.tar.gz`; }-${bundle.targetKey(target)}.tar.gz`;
await tar.c( await writeBundleArchive(workspace, path.join(output, filename));
{
gzip: true,
portable: true,
noMtime: true,
cwd: workspace,
file: path.join(output, filename),
},
['qinglong'],
);
// Verify the bytes users receive, not just the pre-archive build directory. // Verify the bytes users receive, not just the pre-archive build directory.
const roundtrip = path.join(workspace, 'roundtrip'); const roundtrip = path.join(workspace, 'roundtrip');
await fs.mkdir(roundtrip); await fs.mkdir(roundtrip);
@@ -111,6 +121,8 @@ async function buildBundle(root, output) {
await verifyUpgrade(extracted.source, extracted.static); await verifyUpgrade(extracted.source, extracted.static);
await bundle.smokeDependencies(extracted.source); await bundle.smokeDependencies(extracted.source);
console.log('Archived production dependencies and SQLite verified'); console.log('Archived production dependencies and SQLite verified');
await prestartBackend(extracted, { target });
console.log('Archived backend database, gRPC and HTTP startup verified');
const bytes = await fs.readFile(path.join(output, filename)); const bytes = await fs.readFile(path.join(output, filename));
const record = { const record = {
format: 2, format: 2,
@@ -119,6 +131,7 @@ async function buildBundle(root, output) {
target: bundle.targetKey(target), target: bundle.targetKey(target),
filename, filename,
baseImage: process.env.QL_BUNDLE_BASE_IMAGE, baseImage: process.env.QL_BUNDLE_BASE_IMAGE,
...(migrationBridge ? { migrationBridge } : {}),
size: bytes.length, size: bytes.length,
sha256: bundle.sha256(bytes), sha256: bundle.sha256(bytes),
}; };
@@ -132,7 +145,7 @@ async function buildBundle(root, output) {
await fs.rm(workspace, { recursive: true, force: true }); await fs.rm(workspace, { recursive: true, force: true });
} }
} }
module.exports = { buildBundle, normalizeModes }; module.exports = { buildBundle, normalizeModes, writeBundleArchive };
if (require.main === module) if (require.main === module)
buildBundle(process.argv[2] || '/ql', process.argv[3] || '/out').catch( buildBundle(process.argv[2] || '/ql', process.argv[3] || '/out').catch(
(e) => { (e) => {
+65 -36
View File
@@ -19,39 +19,63 @@ const platforms = [
'linux-s390x-glibc', 'linux-s390x-glibc',
]; ];
function releaseIndex(directory, commit, repository, options = {}) { function releaseIndex(directory, commit, repository, options = {}) {
const { version, releaseTag = `upgrade-${commit}` } = options; const {
version,
releaseTag = `upgrade-${commit}`,
migration = false,
} = options;
if ( if (
!/^[a-f0-9]{40}$/.test(commit) || !/^[a-f0-9]{40}$/.test(commit) ||
!/^[\w.-]+\/[\w.-]+$/.test(repository) || !/^[\w.-]+\/[\w.-]+$/.test(repository) ||
!/^[\w.+-]+$/.test(releaseTag) !/^[\w.+-]+$/.test(releaseTag)
) )
throw new Error('Invalid release source'); throw new Error('Invalid release source');
const packages = {}; const readPackages = (bridge) => {
for (const target of platforms) { const packages = {};
const filename = `qinglong-${commit}-${target}.tar.gz`; for (const target of platforms) {
const record = JSON.parse( const filename = `qinglong-${
fs.readFileSync(path.join(directory, filename + '.json')), bridge ? 'migration-' : ''
); }${commit}-${target}.tar.gz`;
const bytes = fs.readFileSync(path.join(directory, filename)); const record = JSON.parse(
if ( fs.readFileSync(path.join(directory, filename + '.json')),
record.format !== 2 || );
record.sourceCommit !== commit || const bytes = fs.readFileSync(path.join(directory, filename));
(version !== undefined && record.version !== version) || if (
record.target !== target || record.format !== 2 ||
record.filename !== filename || record.sourceCommit !== commit ||
record.size !== bytes.length || (version !== undefined &&
record.sha256 !== sha256(bytes) (bridge ? record.migrationBridge?.targetVersion : record.version) !==
) version) ||
throw new Error(`Unverified platform bundle: ${target}`); (bridge &&
packages[target] = { (record.migrationVerified !== true ||
size: record.size, record.migrationBridge?.sourceCommit !== commit ||
sha256: record.sha256, record.migrationBridge?.legacySourceCommit !==
url: `https://github.com/${repository}/releases/download/${encodeURIComponent( require('../docker/migration-bridge-images.json')
releaseTag, .sourceCommit)) ||
)}/${filename}`, (!bridge && record.migrationBridge) ||
}; record.target !== target ||
} record.filename !== filename ||
return { format: 2, sourceCommit: commit, packages }; record.size !== bytes.length ||
record.sha256 !== sha256(bytes)
)
throw new Error(`Unverified platform bundle: ${target}`);
packages[target] = {
size: record.size,
sha256: record.sha256,
url: `https://github.com/${repository}/releases/download/${encodeURIComponent(
releaseTag,
)}/${filename}`,
};
}
return packages;
};
const runtimePackages = readPackages(false);
return {
format: 2,
sourceCommit: commit,
packages: migration ? readPackages(true) : runtimePackages,
...(migration ? { runtimePackages } : {}),
};
} }
function publishBundles({ function publishBundles({
directory, directory,
@@ -83,6 +107,7 @@ function publishBundles({
...releaseIndex(directory, commit, repository, { ...releaseIndex(directory, commit, repository, {
version, version,
releaseTag: tag, releaseTag: tag,
migration: true,
}), }),
version, version,
buildId, buildId,
@@ -115,10 +140,12 @@ function publishBundles({
}; };
const notes = const notes =
`Verified platform bundles for v${version}, source ${commit}, build ${buildId}. Existing Node runtime is retained.\n\n` + `Verified platform bundles for v${version}, source ${commit}, build ${buildId}. Existing Node runtime is retained.\n\n` +
Object.entries(index.packages) Object.entries(index.runtimePackages)
.map(([target, entry]) => `- [${target}](${entry.url})`) .map(([target, entry]) => `- [${target}](${entry.url})`)
.join('\n') + .join('\n') +
`\n- [Standalone offline installer](${index.installer.url})`; `\n- [Standalone offline installer](${index.installer.url})\n\n` +
'Legacy format-2 panels update the upgrader first, then download the runtime. ' +
'The packages index contains the separate legacy bridge; runtimePackages contains the normal online/offline packages.';
const ensureRelease = (ref, title) => { const ensureRelease = (ref, title) => {
if (!lookup(ref)) if (!lookup(ref))
gh( gh(
@@ -150,13 +177,15 @@ function publishBundles({
} }
ensureRelease(tag, `Platform upgrade v${version} (${buildId})`); ensureRelease(tag, `Platform upgrade v${version} (${buildId})`);
const assets = platforms const assets = platforms
.flatMap((target) => { .flatMap((target) =>
const archive = path.join( [false, true].flatMap((bridge) => {
directory, const archive = path.join(
`qinglong-${commit}-${target}.tar.gz`, directory,
); `qinglong-${bridge ? 'migration-' : ''}${commit}-${target}.tar.gz`,
return [archive, archive + '.json']; );
}) return [archive, archive + '.json'];
}),
)
.concat( .concat(
path.join(directory, 'install-bundle.cjs'), path.join(directory, 'install-bundle.cjs'),
path.join(directory, 'latest.json'), path.join(directory, 'latest.json'),
+64
View File
@@ -0,0 +1,64 @@
// The legacy image's generic ARM prebuild contains ARMv7 instructions. Rebuild
// only the first-hop bridge on the builder; never add sqlite3 to normal bundles
// or ask an ARMv6 device to compile it during the migration.
const fs = require('node:fs/promises');
const path = require('node:path');
const { createRequire } = require('node:module');
const { execFile } = require('node:child_process');
const { promisify } = require('node:util');
const bundle = require('../cli/src/internal/maintenance/platformBundle.cjs');
const { repairDependencies, cleanSqliteBuildArtifacts } = require('../cli/src/internal/maintenance/upgradePrestart.cjs');
function assertArmv6Addon(attributes) {
// Alpine's ARM CRT objects are ARMv6KZ (the ARM1176 architecture). Linking
// them raises a base-v6 object's ELF tag without introducing ARMv7/Thumb-2.
if (
!/Tag_CPU_arch:\s+v6(?:K|KZ|ZK)?\s*$/m.test(attributes) ||
/Tag_THUMB_ISA_use:\s+Thumb-2|Tag_FP_arch:\s+VFPv[3-9]|Tag_Advanced_SIMD_arch:\s+\S/.test(attributes) ||
!/Tag_ABI_VFP_args:\s+VFP registers/.test(attributes)
)
throw new Error('Legacy migration SQLite must use ARMv6 hard-float instructions: ' +
String(attributes).split('\n').filter((line) =>
/Tag_(CPU_arch|THUMB_ISA_use|FP_arch|ABI_VFP_args|Advanced_SIMD_arch):/.test(line),
).slice(0, 5).map((line) => line.trim()).join('; '));
}
async function rebuildMigrationSqlite(root, {
target,
run = promisify(execFile),
runtime = process,
repair = repairDependencies,
} = {}) {
target ||= await bundle.runtimeTarget();
if (target.arch !== 'armv6' || target.libc !== 'musl') return false;
const pkg = JSON.parse(await fs.readFile(path.join(root, 'package.json')));
if (!pkg.qinglongMigrationBridge || !pkg.dependencies?.sqlite3)
throw new Error('Only a legacy migration bridge may rebuild SQLite');
const prefix = path.resolve(path.dirname(runtime.execPath), '..');
const header = await fs.readFile(path.join(prefix, 'include/node/node_version.h'), 'utf8');
const version = ['MAJOR', 'MINOR', 'PATCH'].map((part) =>
header.match(new RegExp(`#define\\s+NODE_${part}_VERSION\\s+(\\d+)`))?.[1],
).join('.');
if (version !== runtime.versions.node)
throw new Error('Migration SQLite requires headers matching the existing Node');
const lock = await fs.readFile(path.join(root, 'pnpm-lock.yaml'));
// The Node 18 legacy builder skipped node-gyp 12's optional dependency.
// The existing frozen repair installs that locked graph using the current
// Node, then forces an ARMv6 source build with local headers and one job.
await repair(root, { target, env: runtime.env });
// Repair replaces node_modules, so resolve its new layout after completion.
const rebuilt = path.dirname(createRequire(path.join(root, 'package.json')).resolve('sqlite3/package.json'));
const addon = path.join(rebuilt, 'lib/binding/napi-v6-linux-arm/node_sqlite3.node');
const { stdout } = await run('readelf', ['-A', addon], { timeout: 30000 });
assertArmv6Addon(stdout);
if (!lock.equals(await fs.readFile(path.join(root, 'pnpm-lock.yaml'))))
throw new Error('Migration SQLite rebuild changed the frozen lockfile');
// node-pre-gyp copies the addon to lib/binding and then renames build to
// build-tmp-napi-v<N>. Both layouts contain compiler objects and an absolute
// node_gyp_bins/python3 link; none belongs in a portable runtime archive.
await cleanSqliteBuildArtifacts(rebuilt);
console.log('Legacy bridge SQLite rebuilt and verified for ARMv6 hard-float');
return true;
}
module.exports = { rebuildMigrationSqlite, assertArmv6Addon };
+525 -112
View File
@@ -16,61 +16,134 @@ const execute = promisify(execFile);
const repository = path.resolve(__dirname, '..'); const repository = path.resolve(__dirname, '..');
const hash = (bytes) => createHash('sha256').update(bytes).digest('hex'); const hash = (bytes) => createHash('sha256').update(bytes).digest('hex');
const read = async (file) => JSON.parse(await fs.readFile(file, 'utf8')); const read = async (file) => JSON.parse(await fs.readFile(file, 'utf8'));
const run = (program, args, options = {}) => execute(program, args, { const run = (program, args, options = {}) =>
encoding: 'utf8', maxBuffer: 8 * 1024 * 1024, timeout: 120000, execute(program, args, {
...options, encoding: 'utf8',
}); maxBuffer: 8 * 1024 * 1024,
timeout: 120000,
...options,
});
const requiredFiles = [ const requiredFiles = [
'.env.example', 'back/protos/api.proto', 'ecosystem.config.js', 'version.yaml', '.env.example',
'shell/upgrade.cjs', 'shell/install-bundle.cjs', 'shell/install-package.cjs', 'back/protos/api.proto',
'shell/upgrade-lock.yaml', 'shell/start.sh', 'shell/start-simplify.sh', 'ecosystem.config.js',
'shell/task.sh', 'shell/preload/client.js', 'shell/preload/client.py', 'version.yaml',
'shell/preload/sitecustomize.js', 'shell/preload/sitecustomize.py', 'shell/upgrade.cjs',
'shell/preload/esm-loader.mjs', 'shell/preload/notificationAudit.js', 'shell/install-bundle.cjs',
'sample/config.sample.sh', 'sample/auth.sample.json', 'shell/install-package.cjs',
'static/build-info.json', 'static/build/app.js', 'static/build/token.js', 'shell/upgrade-lock.yaml',
'shell/runtime-guard.cjs',
'shell/node-runtime-probe.cjs',
'shell/start.sh',
'shell/start-simplify.sh',
'shell/task.sh',
'shell/preload/client.js',
'shell/preload/client.py',
'shell/preload/sitecustomize.js',
'shell/preload/sitecustomize.py',
'shell/preload/esm-loader.mjs',
'shell/preload/notificationAudit.js',
'sample/config.sample.sh',
'sample/auth.sample.json',
'static/build-info.json',
'static/build/app.js',
'static/build/token.js',
'static/build/bootstrap/upgradeProbe.js', 'static/build/bootstrap/upgradeProbe.js',
...[
'databaseClient',
'databaseErrors',
'databaseProcess',
'databaseWorker',
'repository',
'schema',
'queries',
'cacheStore',
'statistics',
].map((name) => `static/build/data/${name}.js`),
'static/dist/index.html', 'static/dist/index.html',
...['upgradeArtifacts', 'platformBundle', 'upgradePrestart', 'panelUpgrade', 'nodeRuntime'] ...[
.map((name) => `cli/src/internal/maintenance/${name}.cjs`), 'upgradeArtifacts',
'platformBundle',
'upgradePrestart',
'panelUpgrade',
'nodeRuntime',
].map((name) => `cli/src/internal/maintenance/${name}.cjs`),
]; ];
function checkPackedFiles(files) { function checkPackedFiles(files) {
const names = new Set(files.map((file) => file.path)); const names = new Set(files.map((file) => file.path));
for (const name of requiredFiles) assert.ok(names.has(name), `Missing npm runtime: ${name}`); for (const name of requiredFiles)
assert.ok(names.has(name), `Missing npm runtime: ${name}`);
for (const name of names) { for (const name of names) {
assert.ok(/^(?:package\.json$|LICENSE$|README[^/]*\.md$|\.env\.example$|ecosystem\.config\.js$|version\.yaml$|back\/protos\/api\.proto$|cli\/(?:README[^/]*\.md$|src\/internal\/maintenance\/[^/]+\.cjs$)|sample\/|shell\/|static\/)/.test(name), `Unexpected npm file: ${name}`); assert.ok(
assert.ok(!/^shell\/(?:pub\.sh$|preload\/(?:env\.|notify\.|__ql_notify__\.|lang_env\.sh|.*-notify\.json))/.test(name), `Generated or release-only runtime file: ${name}`); /^(?:package\.json$|LICENSE$|README[^/]*\.md$|\.env\.example$|ecosystem\.config\.js$|version\.yaml$|back\/protos\/api\.proto$|cli\/(?:README[^/]*\.md$|src\/internal\/maintenance\/[^/]+\.cjs$)|sample\/|shell\/|static\/)/.test(
assert.ok(!/(?:^|\/)__pycache__\/|\.pyc$|(?:^|\/)\.DS_Store$/.test(name), name); name,
),
`Unexpected npm file: ${name}`,
);
assert.ok(
!/^shell\/(?:pub\.sh$|preload\/(?:env\.|notify\.|__ql_notify__\.|lang_env\.sh|.*-notify\.json))/.test(
name,
),
`Generated or release-only runtime file: ${name}`,
);
assert.ok(
!/(?:^|\/)__pycache__\/|\.pyc$|(?:^|\/)\.DS_Store$/.test(name),
name,
);
assert.notEqual(name, 'sample/tool.ts'); assert.notEqual(name, 'sample/tool.ts');
} }
} }
async function packPanel(output) { async function packPanel(output) {
const git = (...args) => run('git', args, { cwd: repository }); const git = (...args) => run('git', args, { cwd: repository });
assert.equal((await git('status', '--porcelain', '--untracked-files=all')).stdout.trim(), '', 'Pack only from a clean source checkout'); assert.equal(
await run(process.execPath, [path.join(repository, 'docker/verify-build.cjs')], { cwd: repository }); (await git('status', '--porcelain', '--untracked-files=all')).stdout.trim(),
'',
'Pack only from a clean source checkout',
);
await run(
process.execPath,
[path.join(repository, 'docker/verify-build.cjs')],
{ cwd: repository },
);
await fs.mkdir(output, { recursive: true }); await fs.mkdir(output, { recursive: true });
const cache = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-npm-pack-cache-')); const cache = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-npm-pack-cache-'));
try { try {
const result = await run('npm', ['pack', '--json', '--cache', cache, '--pack-destination', output], { cwd: repository }); const result = await run(
'npm',
['pack', '--json', '--cache', cache, '--pack-destination', output],
{ cwd: repository },
);
const [manifest] = JSON.parse(result.stdout); const [manifest] = JSON.parse(result.stdout);
checkPackedFiles(manifest.files); checkPackedFiles(manifest.files);
assert.equal(manifest.name, '@whyour/qinglong'); assert.equal(manifest.name, '@whyour/qinglong');
const build = await read(path.join(repository, 'static/build-info.json')); const build = await read(path.join(repository, 'static/build-info.json'));
const names = new Set(manifest.files.map((file) => file.path)); const names = new Set(manifest.files.map((file) => file.path));
for (const name of Object.keys(build.files)) assert.ok(names.has(`static/${name}`), `npm filtered a verified build artifact: ${name}`); for (const name of Object.keys(build.files))
assert.ok(
names.has(`static/${name}`),
`npm filtered a verified build artifact: ${name}`,
);
const receipt = { const receipt = {
filename: manifest.filename, sha256: hash(await fs.readFile(path.join(output, manifest.filename))), filename: manifest.filename,
sourceCommit: build.sourceCommit, size: manifest.size, fileCount: manifest.files.length, sha256: hash(await fs.readFile(path.join(output, manifest.filename))),
sourceCommit: build.sourceCommit,
size: manifest.size,
fileCount: manifest.files.length,
}; };
await fs.writeFile(path.join(output, 'package-receipt.json'), JSON.stringify(receipt, null, 2)); await fs.writeFile(
path.join(output, 'package-receipt.json'),
JSON.stringify(receipt, null, 2),
);
console.log(JSON.stringify(receipt)); console.log(JSON.stringify(receipt));
return receipt; return receipt;
} finally { } finally {
await fs.rm(cache, { recursive: true, force: true }); await fs.rm(cache, { recursive: true, force: true });
// Also recover an interrupted npm pack whose postpack hook did not execute. // Also recover an interrupted npm pack whose postpack hook did not execute.
await fs.rm(path.join(repository, 'shell/upgrade-lock.yaml'), { force: true }); await fs.rm(path.join(repository, 'shell/upgrade-lock.yaml'), {
force: true,
});
} }
} }
@@ -84,8 +157,13 @@ async function unusedPort() {
// Consume the unmodified output of Dockerfile.upgrade-bundle / buildBundle. // Consume the unmodified output of Dockerfile.upgrade-bundle / buildBundle.
async function platformFixture(root, directory, expectedCommit) { async function platformFixture(root, directory, expectedCommit) {
assert.ok(directory, 'Set QL_PANEL_TEST_BUNDLE_DIR to the official platform bundle output'); assert.ok(
const names = (await fs.readdir(directory)).filter((name) => name.endsWith('.tar.gz')); directory,
'Set QL_PANEL_TEST_BUNDLE_DIR to the official platform bundle output',
);
const names = (await fs.readdir(directory)).filter((name) =>
name.endsWith('.tar.gz'),
);
assert.equal(names.length, 1, 'Expected one official platform package'); assert.equal(names.length, 1, 'Expected one official platform package');
const archive = path.resolve(directory, names[0]); const archive = path.resolve(directory, names[0]);
const record = await read(archive + '.json'); const record = await read(archive + '.json');
@@ -105,64 +183,140 @@ async function platformFixture(root, directory, expectedCommit) {
async function legacyArchive(temp, env) { async function legacyArchive(temp, env) {
const directory = path.join(temp, 'legacy'); const directory = path.join(temp, 'legacy');
await fs.mkdir(directory); await fs.mkdir(directory);
const result = await run('npm', ['pack', '@whyour/qinglong@2.23.0', '--ignore-scripts', '--json', '--registry=https://registry.npmjs.org', '--pack-destination', directory, '--cache', path.join(temp, 'cache')], { cwd: temp, env, timeout: 300000 }); const result = await run(
'npm',
[
'pack',
'@whyour/qinglong@2.23.0',
'--ignore-scripts',
'--json',
'--registry=https://registry.npmjs.org',
'--pack-destination',
directory,
'--cache',
path.join(temp, 'cache'),
],
{ cwd: temp, env, timeout: 300000 },
);
const [record] = JSON.parse(result.stdout); const [record] = JSON.parse(result.stdout);
assert.equal(record.name, '@whyour/qinglong'); assert.equal(record.name, '@whyour/qinglong');
assert.equal(record.version, '2.23.0'); assert.equal(record.version, '2.23.0');
assert.equal(record.filename, 'whyour-qinglong-2.23.0.tgz'); assert.equal(record.filename, 'whyour-qinglong-2.23.0.tgz');
const archive = path.join(directory, record.filename); const archive = path.join(directory, record.filename);
const integrity = 'sha512-' + createHash('sha512').update(await fs.readFile(archive)).digest('base64'); const integrity =
assert.equal(integrity, 'sha512-x6B9s1/SRKkHe7jRQtb3LzMluJ3O3PAAssrr/5jaFMMl7DZvABSEiqkkZMu0r1BNtiCF72uj8OqXN9pCnF2WVw==', 'Published legacy package integrity changed'); 'sha512-' +
createHash('sha512')
.update(await fs.readFile(archive))
.digest('base64');
assert.equal(
integrity,
'sha512-x6B9s1/SRKkHe7jRQtb3LzMluJ3O3PAAssrr/5jaFMMl7DZvABSEiqkkZMu0r1BNtiCF72uj8OqXN9pCnF2WVw==',
'Published legacy package integrity changed',
);
return { archive, integrity, version: record.version }; return { archive, integrity, version: record.version };
} }
async function verifyPanel(input, output) { async function verifyPanel(input, output) {
assert.equal(process.platform, 'linux', 'Service tests require Linux'); assert.equal(process.platform, 'linux', 'Service tests require Linux');
assert.equal(process.env.QL_PANEL_PACKAGE_TEST_ISOLATED, '1', 'Run only inside a disposable Linux container: QL_PANEL_PACKAGE_TEST_ISOLATED=1'); assert.equal(
process.env.QL_PANEL_PACKAGE_TEST_ISOLATED,
'1',
'Run only inside a disposable Linux container: QL_PANEL_PACKAGE_TEST_ISOLATED=1',
);
const stat = await fs.stat(input); const stat = await fs.stat(input);
if (stat.isDirectory()) { if (stat.isDirectory()) {
const names = (await fs.readdir(input)).filter((name) => name.endsWith('.tgz')); const names = (await fs.readdir(input)).filter((name) =>
name.endsWith('.tgz'),
);
assert.equal(names.length, 1, 'Expected one exact npm archive'); assert.equal(names.length, 1, 'Expected one exact npm archive');
input = path.join(input, names[0]); input = path.join(input, names[0]);
} }
const expected = await read(path.join(path.dirname(input), 'package-receipt.json')); const expected = await read(
path.join(path.dirname(input), 'package-receipt.json'),
);
assert.equal(path.basename(input), expected.filename); assert.equal(path.basename(input), expected.filename);
assert.equal(hash(await fs.readFile(input)), expected.sha256, 'npm archive differs from its build receipt'); assert.equal(
hash(await fs.readFile(input)),
expected.sha256,
'npm archive differs from its build receipt',
);
assert.equal((await fs.stat(input)).size, expected.size); assert.equal((await fs.stat(input)).size, expected.size);
assert.match(expected.sourceCommit, /^[a-f0-9]{40}$/); assert.match(expected.sourceCommit, /^[a-f0-9]{40}$/);
if (process.env.GITHUB_SHA) assert.equal(expected.sourceCommit, process.env.GITHUB_SHA); if (process.env.GITHUB_SHA)
assert.equal(expected.sourceCommit, process.env.GITHUB_SHA);
const nodeSha = hash(await fs.readFile(process.execPath)); const nodeSha = hash(await fs.readFile(process.execPath));
const pm2 = process.env.QL_PANEL_TEST_PM2; const pm2 = process.env.QL_PANEL_TEST_PM2;
assert.ok(pm2 && path.isAbsolute(pm2), 'Set QL_PANEL_TEST_PM2 to an isolated PM2 executable'); assert.ok(
const temp = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'ql-npm-install-'))); pm2 && path.isAbsolute(pm2),
const home = path.join(temp, 'home'), prefix = path.join(temp, 'prefix'), data = path.join(temp, 'data'); 'Set QL_PANEL_TEST_PM2 to an isolated PM2 executable',
);
const temp = await fs.realpath(
await fs.mkdtemp(path.join(os.tmpdir(), 'ql-npm-install-')),
);
const home = path.join(temp, 'home'),
prefix = path.join(temp, 'prefix'),
data = path.join(temp, 'data');
await fs.mkdir(home); await fs.mkdir(home);
await fs.mkdir(data); await fs.mkdir(data);
const port = await unusedPort(); const port = await unusedPort();
let grpcPort = await unusedPort(); let grpcPort = await unusedPort();
while (grpcPort === port) grpcPort = await unusedPort(); while (grpcPort === port) grpcPort = await unusedPort();
const env = { ...process.env, HOME: home, PM2_HOME: path.join(temp, '.pm2'), const env = {
QL_DATA_DIR: data, QL_BRANCH: 'develop', NODE_ENV: 'production', QL_CONTAINER: 'false', ...process.env,
BACK_PORT: String(port), GRPC_PORT: String(grpcPort), QlPort: String(port), QlGrpcPort: String(grpcPort), HOME: home,
BIND_HOST: '127.0.0.1', BIND_HOST_GRPC: '127.0.0.1', PM2_HOME: path.join(temp, '.pm2'),
NODE_PATH: '', NODE_OPTIONS: '', QL_DATA_DIR: data,
PATH: [path.join(prefix, 'bin'), path.dirname(pm2), process.env.PATH].join(path.delimiter), QL_BRANCH: 'develop',
NODE_ENV: 'production',
QL_CONTAINER: 'false',
BACK_PORT: String(port),
GRPC_PORT: String(grpcPort),
QlPort: String(port),
QlGrpcPort: String(grpcPort),
BIND_HOST: '127.0.0.1',
BIND_HOST_GRPC: '127.0.0.1',
NODE_PATH: '',
NODE_OPTIONS: '',
PATH: [path.join(prefix, 'bin'), path.dirname(pm2), process.env.PATH].join(
path.delimiter,
),
}; };
for (const key of ['QL_CLI_ROOT', 'QL_DIR', 'QL_UPGRADE_FILE', 'QL_UPGRADE_REPAIR_DEPS', 'QL_UPGRADE_LAUNCHER_PID', 'QL_UPGRADE_INDEX_URL']) delete env[key]; for (const key of [
'QL_CLI_ROOT',
'QL_DIR',
'QL_UPGRADE_FILE',
'QL_UPGRADE_REPAIR_DEPS',
'QL_UPGRADE_LAUNCHER_PID',
'QL_UPGRADE_INDEX_URL',
])
delete env[key];
let root, fixtureServer, token; let root, fixtureServer, token;
const api = async (route, method = 'GET', body, anonymous = false) => { const api = async (route, method = 'GET', body, anonymous = false) => {
const response = await fetch(`http://127.0.0.1:${port}/api/${route}`, { const response = await fetch(`http://127.0.0.1:${port}/api/${route}`, {
method, headers: { 'Content-Type': 'application/json', ...(!anonymous && token ? { Authorization: `Bearer ${token}` } : {}) }, method,
body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(5000), headers: {
'Content-Type': 'application/json',
...(!anonymous && token ? { Authorization: `Bearer ${token}` } : {}),
},
body: body === undefined ? undefined : JSON.stringify(body),
signal: AbortSignal.timeout(5000),
}); });
const result = await response.json(); const result = await response.json();
assert.equal(result.code, 200, `${method} ${route}: ${result.message}`); try {
assert.equal(result.code, 200, `${method} ${route}: ${result.message}`);
} catch (error) {
error.status = response.status;
throw error;
}
return result.data; return result.data;
}; };
const healthy = async () => { const healthy = async () => {
const deadline = Date.now() + 120000; const deadline = Date.now() + 120000;
while (Date.now() < deadline) { while (Date.now() < deadline) {
try { const value = await api('health', 'GET', undefined, true); if (value.status === 'ok') return value; } catch {} try {
const value = await api('health', 'GET', undefined, true);
if (value.status === 'ok') return value;
} catch {}
await delay(300); await delay(300);
} }
throw new Error('Installed npm backend did not become healthy'); throw new Error('Installed npm backend did not become healthy');
@@ -174,26 +328,81 @@ async function verifyPanel(input, output) {
const value = await api('system/update-status'); const value = await api('system/update-status');
assert.equal(value.id, id); assert.equal(value.id, id);
if (value.status === status) return value; if (value.status === status) return value;
if (value.status === 'failed' && status !== 'failed') throw new Error(value.error); if (value.status === 'failed' && status !== 'failed')
throw new Error(value.error);
} catch (error) { } catch (error) {
if (!/fetch failed|ECONNRESET|ECONNREFUSED|aborted|timeout/i.test(error.message)) throw error; if (
error.status !== 503 &&
!/fetch failed|ECONNRESET|ECONNREFUSED|aborted|timeout/i.test(
error.message,
)
)
throw error;
} }
await delay(250); await delay(250);
} }
throw new Error(`Installed npm update never reached ${status}`); throw new Error(`Installed npm update never reached ${status}`);
}; };
try { try {
const install = (archive) => run('npm', ['install', '--global', '--prefix', prefix, '--force', '--omit=dev', '--no-audit', '--no-fund', '--cache', path.join(temp, 'cache'), archive], { cwd: temp, env, timeout: 600000 }); const install = (archive) =>
const start = () => run(path.join(prefix, 'bin/qinglong'), ['reload'], { cwd: temp, env, timeout: 180000 }); run(
const stop = () => run(pm2, ['delete', 'ecosystem.config.js'], { cwd: root, env }); 'npm',
console.log(`Installing actual npm archive with production lifecycle scripts on ${process.version}`); [
'install',
'--global',
'--prefix',
prefix,
'--force',
'--omit=dev',
'--no-audit',
'--no-fund',
'--cache',
path.join(temp, 'cache'),
archive,
],
{ cwd: temp, env, timeout: 600000 },
);
const start = () =>
run(path.join(prefix, 'bin/qinglong'), ['reload'], {
cwd: temp,
env,
timeout: 180000,
});
const stop = () =>
run(pm2, ['delete', 'ecosystem.config.js'], { cwd: root, env });
console.log(
`Installing actual npm archive with production lifecycle scripts on ${process.version}`,
);
await install(input); await install(input);
const globalRoot = (await run('npm', ['root', '--global', '--prefix', prefix], { cwd: temp, env })).stdout.trim(); const globalRoot = (
await run('npm', ['root', '--global', '--prefix', prefix], {
cwd: temp,
env,
})
).stdout.trim();
root = path.join(globalRoot, '@whyour/qinglong'); root = path.join(globalRoot, '@whyour/qinglong');
env.QL_DIR = root; env.QL_DIR = root;
const runtimeProbes = [];
const checkRuntime = async () => {
const result = await run(
process.execPath,
[path.join(root, 'shell/node-runtime-probe.cjs')],
{ cwd: temp, env },
);
runtimeProbes.push(JSON.parse(result.stdout.trim()));
};
await checkRuntime();
// Use a subprocess: cached candidate modules must not mask missing files // Use a subprocess: cached candidate modules must not mask missing files
// after old-package installation and the real npm overwrite below. // after old-package installation and the real npm overwrite below.
await run(process.execPath, ['-e', `const root=process.argv[1],p=require('node:path');const m=require(root+'/package.json');for(const target of Object.values(m.bin))require('node:fs').accessSync(p.join(root,target));for(const name of ['platformBundle','upgradePrestart','panelUpgrade'])require(root+'/cli/src/internal/maintenance/'+name+'.cjs');require(root+'/shell/upgrade.cjs').verifyUpgrade(root,p.join(root,'static')).catch(e=>{console.error(e);process.exitCode=1});`, root], { env, cwd: temp }); await run(
process.execPath,
[
'-e',
`const root=process.argv[1],p=require('node:path');const m=require(root+'/package.json');for(const target of Object.values(m.bin))require('node:fs').accessSync(p.join(root,target));for(const name of ['platformBundle','upgradePrestart','panelUpgrade'])require(root+'/cli/src/internal/maintenance/'+name+'.cjs');require(root+'/shell/upgrade.cjs').verifyUpgrade(root,p.join(root,'static')).catch(e=>{console.error(e);process.exitCode=1});`,
root,
],
{ env, cwd: temp },
);
env.QL_DATA_DIR = path.join(temp, 'fresh/data'); env.QL_DATA_DIR = path.join(temp, 'fresh/data');
await fs.mkdir(env.QL_DATA_DIR, { recursive: true }); await fs.mkdir(env.QL_DATA_DIR, { recursive: true });
console.log('Starting the freshly installed npm package'); console.log('Starting the freshly installed npm package');
@@ -201,40 +410,75 @@ async function verifyPanel(input, output) {
const fresh = await healthy(); const fresh = await healthy();
assert.equal(fresh.build.sourceCommit, expected.sourceCommit); assert.equal(fresh.build.sourceCommit, expected.sourceCommit);
assert.equal(fresh.services.grpc, true); assert.equal(fresh.services.grpc, true);
assert.match(await fetch(`http://127.0.0.1:${port}/`).then((r) => r.text()), /<html/i); assert.match(
await fetch(`http://127.0.0.1:${port}/`).then((r) => r.text()),
/<html/i,
);
await stop(); await stop();
await fs.rm(path.join(temp, 'fresh'), { recursive: true, force: true }); await fs.rm(path.join(temp, 'fresh'), { recursive: true, force: true });
env.QL_DATA_DIR = data; env.QL_DATA_DIR = data;
console.log('Installing the integrity-pinned published legacy npm package'); console.log('Installing the integrity-pinned published legacy npm package');
const legacy = await legacyArchive(temp, env); const legacy = await legacyArchive(temp, env);
await install(legacy.archive); await install(legacy.archive);
assert.equal((await read(path.join(root, 'package.json'))).version, legacy.version); assert.equal(
await assert.rejects(fs.access(path.join(root, 'cli/src/internal/maintenance/platformBundle.cjs')), { code: 'ENOENT' }); (await read(path.join(root, 'package.json'))).version,
const legacyBackendSha = hash(await fs.readFile(path.join(root, 'static/build/app.js'))); legacy.version,
);
await assert.rejects(
fs.access(
path.join(root, 'cli/src/internal/maintenance/platformBundle.cjs'),
),
{ code: 'ENOENT' },
);
const legacyBackendSha = hash(
await fs.readFile(path.join(root, 'static/build/app.js')),
);
await start(); await start();
await healthy(); await healthy();
const credentials = { username: 'npm_package_verification', password: `Test_${randomUUID()}!` }; const credentials = {
username: 'npm_package_verification',
password: `Test_${randomUUID()}!`,
};
await api('user/init', 'PUT', credentials, true); await api('user/init', 'PUT', credentials, true);
token = (await api('user/login', 'POST', credentials, true)).token; token = (await api('user/login', 'POST', credentials, true)).token;
const nonce = randomUUID(); const nonce = randomUUID();
await api('envs', 'POST', [{ name: 'NPM_PACKAGE_PRESERVE', value: nonce }]); await api('envs', 'POST', [{ name: 'NPM_PACKAGE_PRESERVE', value: nonce }]);
const script = path.join(data, 'scripts/npm-package-check.js'); const script = path.join(data, 'scripts/npm-package-check.js');
await fs.writeFile(script, `console.log(${JSON.stringify(nonce)});\n`); await fs.writeFile(script, `console.log(${JSON.stringify(nonce)});\n`);
const cron = await api('crons', 'POST', { name: 'npm package retained task', command: `node ${script}`, schedule: '0 0 1 1 *' }); const cron = await api('crons', 'POST', {
name: 'npm package retained task',
command: `node ${script}`,
schedule: '0 0 1 1 *',
});
const userConfig = path.join(data, 'config/config.sh'); const userConfig = path.join(data, 'config/config.sh');
await fs.appendFile(userConfig, `\n# retained npm package setting ${nonce}\n`); await fs.appendFile(
userConfig,
`\n# retained npm package setting ${nonce}\n`,
);
const userConfigSha = hash(await fs.readFile(userConfig)); const userConfigSha = hash(await fs.readFile(userConfig));
const preservedData = async () => { const preservedData = async () => {
assert.equal((await api('user')).username, credentials.username); assert.equal((await api('user')).username, credentials.username);
assert.ok((await api('envs')).some((x) => x.name === 'NPM_PACKAGE_PRESERVE' && x.value === nonce)); assert.ok(
(await api('envs')).some(
(x) => x.name === 'NPM_PACKAGE_PRESERVE' && x.value === nonce,
),
);
assert.equal((await api(`crons/${cron.id}`)).command, cron.command); assert.equal((await api(`crons/${cron.id}`)).command, cron.command);
assert.equal(hash(await fs.readFile(userConfig)), userConfigSha); assert.equal(hash(await fs.readFile(userConfig)), userConfigSha);
assert.equal(await fs.readFile(script, 'utf8'), `console.log(${JSON.stringify(nonce)});\n`); assert.equal(
await fs.readFile(script, 'utf8'),
`console.log(${JSON.stringify(nonce)});\n`,
);
}; };
await preservedData(); await preservedData();
console.log('Stopping legacy service and verifying npm automatically preserves its .env during overwrite'); console.log(
'Stopping legacy service and verifying npm automatically preserves its .env during overwrite',
);
await stop(); await stop();
await fs.appendFile(path.join(root, '.env'), `\n# retained custom npm environment ${nonce}\n`); await fs.appendFile(
path.join(root, '.env'),
`\n# retained custom npm environment ${nonce}\n`,
);
const savedEnv = await fs.readFile(path.join(root, '.env')); const savedEnv = await fs.readFile(path.join(root, '.env'));
await install(input); await install(input);
assert.deepEqual(await fs.readFile(path.join(root, '.env')), savedEnv); assert.deepEqual(await fs.readFile(path.join(root, '.env')), savedEnv);
@@ -242,25 +486,62 @@ async function verifyPanel(input, output) {
const configSha = hash(savedEnv); const configSha = hash(savedEnv);
const local = createRequire(path.join(root, 'package.json')); const local = createRequire(path.join(root, 'package.json'));
const artifacts = local('./shell/upgrade.cjs'); const artifacts = local('./shell/upgrade.cjs');
const build = await artifacts.verifyUpgrade(root, path.join(root, 'static')); const build = await artifacts.verifyUpgrade(
root,
path.join(root, 'static'),
);
assert.equal(build.sourceCommit, expected.sourceCommit); assert.equal(build.sourceCommit, expected.sourceCommit);
assert.notEqual(hash(await fs.readFile(path.join(root, 'static/build/app.js'))), legacyBackendSha); assert.notEqual(
assert.equal(hash(await fs.readFile(path.join(root, 'pnpm-lock.yaml'))), build.lockfileSha256); hash(await fs.readFile(path.join(root, 'static/build/app.js'))),
const fixture = await platformFixture(root, process.env.QL_PANEL_TEST_BUNDLE_DIR, expected.sourceCommit); legacyBackendSha,
let badChecksum = true, requests = 0; );
assert.equal(
hash(await fs.readFile(path.join(root, 'pnpm-lock.yaml'))),
build.lockfileSha256,
);
const fixture = await platformFixture(
root,
process.env.QL_PANEL_TEST_BUNDLE_DIR,
expected.sourceCommit,
);
let badChecksum = true,
requests = 0;
fixtureServer = http.createServer((request, response) => { fixtureServer = http.createServer((request, response) => {
requests++; requests++;
if (request.url === '/latest.json') { if (request.url === '/latest.json') {
response.setHeader('Content-Type', 'application/json'); response.setHeader('Content-Type', 'application/json');
response.end(JSON.stringify({ format: 2, sourceCommit: fixture.sourceCommit, packages: { [fixture.target]: { size: fixture.size, sha256: badChecksum ? '0'.repeat(64) : fixture.sha256, url: `http://127.0.0.1:${fixtureServer.address().port}/bundle.tar.gz` } } })); response.end(
JSON.stringify({
format: 2,
sourceCommit: fixture.sourceCommit,
packages: {
[fixture.target]: {
size: fixture.size,
sha256: badChecksum ? '0'.repeat(64) : fixture.sha256,
url: `http://127.0.0.1:${
fixtureServer.address().port
}/bundle.tar.gz`,
},
},
}),
);
} else if (request.url === '/bundle.tar.gz') { } else if (request.url === '/bundle.tar.gz') {
response.setHeader('Content-Length', fixture.size); response.setHeader('Content-Length', fixture.size);
sync.createReadStream(fixture.archive).pipe(response); sync.createReadStream(fixture.archive).pipe(response);
} else { response.statusCode = 404; response.end(); } } else {
response.statusCode = 404;
response.end();
}
}); });
await new Promise((resolve) => fixtureServer.listen(0, '127.0.0.1', resolve)); await new Promise((resolve) =>
env.QL_UPGRADE_INDEX_URL = `http://127.0.0.1:${fixtureServer.address().port}/latest.json`; fixtureServer.listen(0, '127.0.0.1', resolve),
console.log('Starting the overwritten npm package against legacy user data'); );
env.QL_UPGRADE_INDEX_URL = `http://127.0.0.1:${
fixtureServer.address().port
}/latest.json`;
console.log(
'Starting the overwritten npm package against legacy user data',
);
await start(); await start();
const original = await healthy(); const original = await healthy();
assert.equal(original.build.sourceCommit, build.sourceCommit); assert.equal(original.build.sourceCommit, build.sourceCommit);
@@ -277,32 +558,82 @@ async function verifyPanel(input, output) {
assert.equal((await healthy()).build.instanceId, original.build.instanceId); assert.equal((await healthy()).build.instanceId, original.build.instanceId);
await preserved(); await preserved();
badChecksum = false; badChecksum = false;
console.log('Testing installed panel download, prestart and explicit application'); console.log(
'Testing installed panel download, prestart and explicit application',
);
const accepted = await api('system/update', 'PUT'); const accepted = await api('system/update', 'PUT');
const ready = await job(accepted.id, 'ready'); const ready = await job(accepted.id, 'ready');
assert.equal((await healthy()).build.instanceId, original.build.instanceId); assert.equal((await healthy()).build.instanceId, original.build.instanceId);
const pointer = await read(path.join(root, '.tmp/upgrade-ready-develop.json')); const pointer = await read(
const receipt = await read(path.join(root, '.tmp', pointer.directory, 'prestart.json')); path.join(root, '.tmp/upgrade-ready-develop.json'),
assert.ok(receipt.completedAt <= ready.updatedAt, 'Readiness was recorded before backend prestart'); );
const receipt = await read(
path.join(root, '.tmp', pointer.directory, 'prestart.json'),
);
assert.ok(
receipt.completedAt <= ready.updatedAt,
'Readiness was recorded before backend prestart',
);
const applying = await api('update/system', 'PUT'); const applying = await api('update/system', 'PUT');
await job(applying.id, 'completed'); await job(applying.id, 'completed');
const afterOnline = await healthy(); const afterOnline = await healthy();
await checkRuntime();
assert.notEqual(afterOnline.build.instanceId, original.build.instanceId); assert.notEqual(afterOnline.build.instanceId, original.build.instanceId);
assert.equal(afterOnline.build.sourceCommit, fixture.sourceCommit); assert.equal(afterOnline.build.sourceCommit, fixture.sourceCommit);
await fs.access(path.join(root, 'node_modules/.pnpm')); await fs.access(path.join(root, 'node_modules/.pnpm'));
assert.match(await fs.realpath(path.join(root, 'node_modules/sqlite3')), /[/\\]node_modules[/\\]\.pnpm[/\\]/); assert.match(
await fs.realpath(path.join(root, 'node_modules/@mikro-orm/sql')),
/[/\\]node_modules[/\\]\.pnpm[/\\]/,
);
await assert.rejects(fs.access(path.join(root, 'node_modules/sqlite3')), {
code: 'ENOENT',
});
await assert.rejects(fs.access(path.join(root, 'node_modules/sequelize')), {
code: 'ENOENT',
});
await fs.access(path.join(root, 'scripts/build-upgrade-bundle.cjs')); await fs.access(path.join(root, 'scripts/build-upgrade-bundle.cjs'));
await artifacts.verifyUpgrade(root, path.join(root, 'static')); await artifacts.verifyUpgrade(root, path.join(root, 'static'));
assert.match(await fetch(`http://127.0.0.1:${port}/`).then((r) => r.text()), /<html/i); assert.match(
await fetch(`http://127.0.0.1:${port}/`).then((r) => r.text()),
/<html/i,
);
await preserved(); await preserved();
console.log('Testing offline installer and Bash reload from the installed npm package'); console.log(
'Testing offline installer and Bash reload from the installed npm package',
);
const requestsBeforeOffline = requests; const requestsBeforeOffline = requests;
await run(process.execPath, [path.join(root, 'shell/install-bundle.cjs'), '--root', root, '--file', fixture.archive, '--download-only'], { cwd: temp, env, timeout: 240000 }); await run(
assert.equal(requests, requestsBeforeOffline, 'Offline preparation contacted the index'); process.execPath,
assert.equal((await healthy()).build.instanceId, afterOnline.build.instanceId); [
await run(path.join(prefix, 'bin/ql'), ['reload', 'system'], { cwd: temp, env, timeout: 240000 }); path.join(root, 'shell/install-bundle.cjs'),
'--root',
root,
'--file',
fixture.archive,
'--download-only',
],
{ cwd: temp, env, timeout: 240000 },
);
assert.equal(
requests,
requestsBeforeOffline,
'Offline preparation contacted the index',
);
assert.equal(
(await healthy()).build.instanceId,
afterOnline.build.instanceId,
);
await run(path.join(prefix, 'bin/ql'), ['reload', 'system'], {
cwd: temp,
env,
timeout: 240000,
});
const afterOffline = await healthy(); const afterOffline = await healthy();
assert.notEqual(afterOffline.build.instanceId, afterOnline.build.instanceId); await checkRuntime();
assert.notEqual(
afterOffline.build.instanceId,
afterOnline.build.instanceId,
);
assert.equal(afterOffline.build.sourceCommit, fixture.sourceCommit); assert.equal(afterOffline.build.sourceCommit, fixture.sourceCommit);
await fs.access(path.join(root, 'node_modules/.pnpm')); await fs.access(path.join(root, 'node_modules/.pnpm'));
assert.equal(hash(await fs.readFile(process.execPath)), nodeSha); assert.equal(hash(await fs.readFile(process.execPath)), nodeSha);
@@ -312,36 +643,66 @@ async function verifyPanel(input, output) {
let taskExecuted = false; let taskExecuted = false;
for (let i = 0; i < 100; i++) { for (let i = 0; i < 100; i++) {
const log = String(await api(`crons/${cron.id}/log`)); const log = String(await api(`crons/${cron.id}/log`));
if (log.includes(nonce) && /完成|执行结束|finished|completed/i.test(log)) { taskExecuted = true; break; } if (
log.includes(nonce) &&
/完成|执行结束|finished|completed/i.test(log)
) {
taskExecuted = true;
break;
}
await delay(300); await delay(300);
} }
assert.ok(taskExecuted, 'Retained task did not execute after npm package upgrades'); assert.ok(
taskExecuted,
'Retained task did not execute after npm package upgrades',
);
const result = { const result = {
node: process.version, platform: fixture.target, node: process.version,
archiveSha256: hash(await fs.readFile(input)), sourceCommit: build.sourceCommit, platform: fixture.target,
installedProductionDependencies: true, restoredLockVerified: true, archiveSha256: hash(await fs.readFile(input)),
legacyNpmVersion: legacy.version, legacyNpmIntegrity: legacy.integrity, sourceCommit: build.sourceCommit,
legacyNpmOverwriteInstalled: true, legacyExternalDataPreserved: true, installedProductionDependencies: true,
restoredLockVerified: true,
legacyNpmVersion: legacy.version,
legacyNpmIntegrity: legacy.integrity,
legacyNpmOverwriteInstalled: true,
legacyExternalDataPreserved: true,
customRootEnvironmentPreservedAutomatically: true, customRootEnvironmentPreservedAutomatically: true,
officialPlatformBundle: { baseImage: fixture.baseImage, sourceCommit: fixture.sourceCommit, sha256: fixture.sha256 }, officialPlatformBundle: {
baseImage: fixture.baseImage,
sourceCommit: fixture.sourceCommit,
sha256: fixture.sha256,
},
migratedFromNpmToOfficialPnpmLayout: true, migratedFromNpmToOfficialPnpmLayout: true,
existingNodePreserved: true, existingNodePreserved: true,
frontendAndBackendStarted: true, checksumFailureKeptOldService: true, nativeUrlRuntimeProbes: runtimeProbes,
prestartBeforeReady: true, onlineActualRestart: true, frontendAndBackendStarted: true,
offlinePreparationDidNotContactIndex: true, offlineBashActualRestart: true, checksumFailureKeptOldService: true,
accountEnvironmentConfigurationAndScriptsPreserved: true, retainedTaskExecuted: true, prestartBeforeReady: true,
initialBuild: original.build, afterOnline: afterOnline.build, afterOffline: afterOffline.build, onlineActualRestart: true,
offlinePreparationDidNotContactIndex: true,
offlineBashActualRestart: true,
accountEnvironmentConfigurationAndScriptsPreserved: true,
retainedTaskExecuted: true,
initialBuild: original.build,
afterOnline: afterOnline.build,
afterOffline: afterOffline.build,
}; };
if (output) { if (output) {
await fs.mkdir(output, { recursive: true }); await fs.mkdir(output, { recursive: true });
await fs.writeFile(path.join(output, `verification-${process.version}.json`), JSON.stringify(result, null, 2)); await fs.writeFile(
path.join(output, `verification-${process.version}.json`),
JSON.stringify(result, null, 2),
);
} }
console.log(JSON.stringify(result, null, 2)); console.log(JSON.stringify(result, null, 2));
return result; return result;
} catch (error) { } catch (error) {
if (root) { if (root) {
for (const file of ['qinglong-error.log', 'qinglong-out.log']) { for (const file of ['qinglong-error.log', 'qinglong-out.log']) {
const text = await fs.readFile(path.join(env.PM2_HOME, 'logs', file), 'utf8').catch(() => ''); const text = await fs
.readFile(path.join(env.PM2_HOME, 'logs', file), 'utf8')
.catch(() => '');
if (text) console.error(text.slice(-6000)); if (text) console.error(text.slice(-6000));
} }
} }
@@ -351,19 +712,71 @@ async function verifyPanel(input, output) {
fixtureServer.closeAllConnections(); fixtureServer.closeAllConnections();
await new Promise((resolve) => fixtureServer.close(resolve)); await new Promise((resolve) => fixtureServer.close(resolve));
} }
// A failed assertion must not remove a prefix that its detached updater is
// still extracting into. Identify only this private installation's job.
if (root) {
const state = await read(
path.join(root, '.tmp/panel-upgrade.json'),
).catch(() => undefined);
if (
Number.isSafeInteger(state?.pid) &&
state.pid > 1 &&
typeof state.id === 'string'
) {
const command = (
await fs
.readFile(`/proc/${state.pid}/cmdline`, 'utf8')
.catch(() => '')
).split('\0');
if (
command.includes(
path.join(root, 'cli/src/internal/maintenance/panelUpgrade.cjs'),
) &&
command.includes('worker') &&
command.includes(state.id)
) {
const signal = (name) => {
try {
process.kill(-state.pid, name);
return true;
} catch (error) {
if (error.code !== 'ESRCH') throw error;
return false;
}
};
signal('SIGTERM');
const deadline = Date.now() + 30000;
while (signal(0) && Date.now() < deadline) await delay(100);
if (signal(0)) signal('SIGKILL');
}
}
}
await run(pm2, ['kill'], { env, cwd: temp }).catch(() => {}); await run(pm2, ['kill'], { env, cwd: temp }).catch(() => {});
await run('nginx', ['-s', 'quit'], { env, cwd: temp }).catch(() => {}); await run('nginx', ['-s', 'quit'], { env, cwd: temp }).catch(() => {});
await fs.rm(temp, { recursive: true, force: true }); await fs.rm(temp, {
recursive: true,
force: true,
maxRetries: 3,
retryDelay: 100,
});
} }
} }
module.exports = { checkPackedFiles, packPanel, verifyPanel }; module.exports = { checkPackedFiles, packPanel, verifyPanel };
if (require.main === module) { if (require.main === module) {
const [action, input, output] = process.argv.slice(2); const [action, input, output] = process.argv.slice(2);
const operation = action === 'pack' && input const operation =
? packPanel(path.resolve(input)) action === 'pack' && input
: action === 'verify' && input ? packPanel(path.resolve(input))
: action === 'verify' && input
? verifyPanel(path.resolve(input), output && path.resolve(output)) ? verifyPanel(path.resolve(input), output && path.resolve(output))
: Promise.reject(new Error('Usage: node scripts/verify-panel-package.cjs pack OUTPUT | verify ARCHIVE_OR_DIRECTORY [RESULT_DIRECTORY]')); : Promise.reject(
operation.catch((error) => { console.error(error); process.exitCode = 1; }); new Error(
'Usage: node scripts/verify-panel-package.cjs pack OUTPUT | verify ARCHIVE_OR_DIRECTORY [RESULT_DIRECTORY]',
),
);
operation.catch((error) => {
console.error(error);
process.exitCode = 1;
});
} }
+180
View File
@@ -0,0 +1,180 @@
// Verify the actual archive under an unsupported Node without touching any
// existing installation. This is also the Node 20 negative CI matrix entry.
const fs = require('node:fs/promises');
const path = require('node:path');
const os = require('node:os');
const assert = require('node:assert/strict');
const { execFile, spawn } = require('node:child_process');
const { promisify } = require('node:util');
const execute = promisify(execFile);
async function verifyRuntimeFloor(input, output) {
assert.equal(
Number(process.versions.node.split('.')[0]),
20,
'Run this rejection test with Node 20',
);
const names = (await fs.readdir(input)).filter((name) =>
name.endsWith('.tgz'),
);
assert.equal(names.length, 1, 'Expected one actual npm archive');
const archive = path.resolve(input, names[0]);
const temp = await fs.realpath(
await fs.mkdtemp(path.join(os.tmpdir(), 'ql-node-floor-')),
);
const env = {
PATH: process.env.PATH,
HOME: temp,
TMPDIR: temp,
npm_config_cache: path.join(temp, 'cache'),
npm_config_userconfig: path.join(temp, 'npmrc'),
npm_config_globalconfig: path.join(temp, 'global-npmrc'),
};
const npm = (args) =>
execute('npm', args, {
cwd: temp,
env,
encoding: 'utf8',
timeout: 600000,
maxBuffer: 4 * 1024 * 1024,
});
try {
await fs.writeFile(env.npm_config_userconfig, '');
await fs.writeFile(env.npm_config_globalconfig, '');
const legacy = path.join(temp, 'legacy');
await fs.mkdir(legacy);
await fs.writeFile(
path.join(legacy, 'package.json'),
JSON.stringify({ name: '@whyour/qinglong', version: '2.22.0' }),
);
await fs.writeFile(path.join(legacy, 'legacy-marker'), 'old application');
const [old] = JSON.parse(
(
await execute('npm', ['pack', '--json', '--ignore-scripts'], {
cwd: legacy,
env,
})
).stdout,
);
for (const global of [false, true]) {
const prefix = path.join(temp, global ? 'global' : 'local');
const installed = path.join(
prefix,
global
? 'lib/node_modules/@whyour/qinglong'
: 'node_modules/@whyour/qinglong',
);
const args = [
'install',
...(global ? ['--global'] : []),
'--prefix',
prefix,
'--omit=dev',
'--no-audit',
'--no-fund',
];
await npm([...args, path.join(legacy, old.filename)]);
const environment = Buffer.from(
'QL_DATA_DIR=/external/data\r\nSYNTHETIC_SECRET=keep\n',
);
const data = path.join(temp, global ? 'global-data' : 'local-data');
await fs.writeFile(path.join(installed, '.env'), environment);
await fs.writeFile(data, 'synthetic old data');
const child = spawn(
process.execPath,
[
'-e',
`
const fs=require('node:fs'),http=require('node:http');
const value=fs.readFileSync(process.argv[1],'utf8')+fs.readFileSync(process.argv[2],'utf8');
const server=http.createServer((q,r)=>r.end(value));
server.listen(0,'127.0.0.1',()=>process.send(server.address().port));
`,
path.join(installed, '.env'),
data,
],
{ env, stdio: ['ignore', 'ignore', 'pipe', 'ipc'] },
);
const stopped = new Promise((resolve) => child.once('exit', resolve));
try {
const port = await new Promise((resolve, reject) => {
const timer = setTimeout(
() => reject(new Error('Old service startup timed out')),
10000,
);
child.once('message', (value) => {
clearTimeout(timer);
resolve(value);
});
child.once('error', (error) => {
clearTimeout(timer);
reject(error);
});
child.once('exit', () => {
clearTimeout(timer);
reject(new Error('Old service exited'));
});
});
const url = `http://127.0.0.1:${port}`;
const before = await (await fetch(url)).text();
await assert.rejects(npm([...args, archive]), (error) => {
assert.equal(
error.killed,
false,
`npm did not reach the runtime guard: ${error.message}`,
);
assert.match(error.stderr + error.stdout, /requires Node >=22\.17/);
return true;
});
assert.equal(
JSON.parse(await fs.readFile(path.join(installed, 'package.json')))
.version,
'2.22.0',
);
assert.deepEqual(
await fs.readFile(path.join(installed, '.env')),
environment,
);
assert.equal(await fs.readFile(data, 'utf8'), 'synthetic old data');
assert.equal(child.exitCode, null);
assert.equal(await (await fetch(url)).text(), before);
} finally {
child.kill('SIGKILL');
await stopped;
}
}
const result = {
node: process.version,
unsupportedRuntimeRejected: true,
localAndGlobalOldPackagePreserved: true,
environmentAndDataPreserved: true,
oldServiceStayedHealthy: true,
};
if (output) {
await fs.mkdir(output, { recursive: true });
await fs.writeFile(
path.join(output, `verification-${process.version}.json`),
JSON.stringify(result, null, 2),
);
}
console.log(JSON.stringify(result, null, 2));
return result;
} finally {
await fs.rm(temp, { recursive: true, force: true });
}
}
module.exports = { verifyRuntimeFloor };
if (require.main === module) {
const [input, output] = process.argv.slice(2);
if (!input)
throw new Error(
'Usage: node scripts/verify-panel-runtime-floor.cjs ARCHIVE_DIRECTORY [RESULT_DIRECTORY]',
);
verifyRuntimeFloor(path.resolve(input), output && path.resolve(output)).catch(
(error) => {
console.error(error);
process.exitCode = 1;
},
);
}
+62
View File
@@ -0,0 +1,62 @@
const assert = require('node:assert/strict');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { createHash } = require('node:crypto');
const { spawnSync } = require('node:child_process');
// Run in new processes: a healthy parent can outlive a broken native runtime.
function probeNodeRuntime(executable = process.execPath) {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-node-url-'));
const source = `
const assert = require('node:assert/strict');
const { pathToFileURL, fileURLToPath } = require('node:url');
for (let i = 0; i < 1000; i++) {
assert.equal(new URL('/任务?q=' + i, 'https://example.com').hostname, 'example.com');
assert.equal(new URL('https://例子.测试').hostname, 'xn--fsqu00a.xn--0zwm56d');
assert.equal(new URL('http://[::1]:5700').port, '5700');
const filename = '/tmp/中文 path/' + i + '.cjs';
assert.equal(fileURLToPath(pathToFileURL(filename)), filename);
}
console.log('QINGLONG_NODE_URL_OK');
`;
const file = path.join(directory, 'probe.cjs');
fs.writeFileSync(file, source);
try {
for (let i = 0; i < 4; i++) {
const args = process.execArgv.includes('--jitless') ? ['--jitless'] : [];
args.push(...(i % 2 ? [file] : ['-e', source]));
const child = spawnSync(executable, args, {
encoding: 'utf8',
// Multi-architecture CI runs this same probe through QEMU.
timeout: 120000,
maxBuffer: 65536,
env: { ...process.env, NODE_OPTIONS: '' },
});
assert.equal(
child.status,
0,
`Node URL runtime probe failed (${i % 2 ? 'file' : 'eval'}): signal=${
child.signal
}; status=${child.status}; ${child.error?.message || child.stderr}`,
);
assert.match(child.stdout, /QINGLONG_NODE_URL_OK/);
}
return {
ok: true,
node: process.version,
architecture: process.arch,
ada: process.versions.ada,
processes: 4,
urlIterationsPerProcess: 1000,
nodeSha256: createHash('sha256')
.update(fs.readFileSync(executable))
.digest('hex'),
};
} finally {
fs.rmSync(directory, { recursive: true, force: true });
}
}
module.exports = { probeNodeRuntime };
if (require.main === module) console.log(JSON.stringify(probeNodeRuntime()));
+17
View File
@@ -0,0 +1,17 @@
const minimum = '22.17.0';
function assertNodeRuntime(version = process.versions.node) {
const [major, minor] = version.split('.').map(Number);
if (!Number.isInteger(major) || !Number.isInteger(minor) || major < 22 || (major === 22 && minor < 17)) {
throw Object.assign(new Error(`Qinglong requires Node >=${minimum}; upgrade Node before installing. Application updates retain the existing Node.`), { code: 'UNSUPPORTED_NODE' });
}
if (version === process.versions.node) {
try {
if (typeof require('node:sqlite').DatabaseSync !== 'function') throw new Error();
} catch {
throw Object.assign(new Error('This Node build does not provide node:sqlite; use a Node build with SQLite support.'), { code: 'UNSUPPORTED_NODE_SQLITE' });
}
}
}
module.exports = { minimum, assertNodeRuntime };
+4 -2
View File
@@ -20,14 +20,16 @@ export default function UpgradeNotice({
const showNotice = const showNotice =
activeStatuses.includes(job.status) || activeStatuses.includes(job.status) ||
job.status === 'ready' || job.status === 'ready' ||
job.status === 'failed'; job.status === 'failed' ||
Boolean(job.migrationRequired);
if (!showNotice && !open) return null; if (!showNotice && !open) return null;
const text = !connected const text = !connected
? intl.get('连接已断开,正在重新连接') ? intl.get('连接已断开,正在重新连接')
: intl.get(upgradeStatusLabel(job)); : intl.get(upgradeStatusLabel(job));
const label = const label =
activeStatuses.includes(job.status) && activeStatuses.includes(job.status) &&
job.phase === 'downloading' && job.percent != null job.phase === 'downloading' &&
job.percent != null
? `${text} ${job.percent}%` ? `${text} ${job.percent}%`
: text; : text;
return ( return (
+11 -2
View File
@@ -15,6 +15,7 @@ export interface UpgradeJob {
version?: string; version?: string;
error?: string; error?: string;
log?: string; log?: string;
migrationRequired?: boolean;
} }
export const activeStatuses = ['starting', 'running', 'restarting']; export const activeStatuses = ['starting', 'running', 'restarting'];
@@ -34,6 +35,8 @@ export const phaseLabels: Record<string, string> = {
// Status is authoritative; old progress fields must not claim completion. // Status is authoritative; old progress fields must not claim completion.
export function upgradeStatusLabel(job: UpgradeJob): string { export function upgradeStatusLabel(job: UpgradeJob): string {
if (job.migrationRequired && ['idle', 'completed'].includes(job.status))
return '升级器已更新,请继续升级新版';
if (job.status === 'ready') return '下载完成,等待重启'; if (job.status === 'ready') return '下载完成,等待重启';
if (job.status === 'completed') return '升级成功'; if (job.status === 'completed') return '升级成功';
if (job.status === 'failed') return '更新失败'; if (job.status === 'failed') return '更新失败';
@@ -94,8 +97,14 @@ export function createUpgradeStore() {
) { ) {
sessionStorage.removeItem(completionKey); sessionStorage.removeItem(completionKey);
notification.success({ notification.success({
message: intl.get('升级成功'), message: intl.get(
description: intl.get('已重新启动并运行新版本,无需再次重启。'), result.data.migrationRequired ? '升级器已更新' : '升级成功',
),
description: intl.get(
result.data.migrationRequired
? '当前仍运行旧版,请继续下载并重启以完成新版升级。'
: '已重新启动并运行新版本,无需再次重启。',
),
duration: 8, duration: 8,
}); });
} }
+8
View File
@@ -758,6 +758,14 @@
"暂存更新已变化,请重新下载更新。": "The staged update changed. Download the update again.", "暂存更新已变化,请重新下载更新。": "The staged update changed. Download the update again.",
"启动版本与更新包不一致": "The running build does not match the update package.", "启动版本与更新包不一致": "The running build does not match the update package.",
"下载完成,等待重启": "Download complete, restart to update", "下载完成,等待重启": "Download complete, restart to update",
"升级器已更新,请继续升级新版": "Upgrader updated; continue to the new version",
"升级器已更新": "Upgrader updated",
"当前仍运行旧版,请继续下载并重启以完成新版升级。": "The previous version is still running. Download the new version and restart to finish upgrading.",
"继续升级新版": "Continue upgrading",
"升级器已更新。第二步将下载并预启动新版,准备成功后再重启应用。": "The upgrader is updated. Step 2 downloads and tests the new backend before you restart to apply it.",
"下载新版": "Download new version",
"更新索引尚未提供第二步运行包,请稍后重试": "The update index does not yet provide the second-step runtime. Please try again later.",
"第二步更新包仍是旧数据库后端,已保留当前服务,请稍后重试": "The second-step package still uses the legacy database backend. Your current service is unchanged. Please try again later.",
"升级成功": "Update successful", "升级成功": "Update successful",
"已重新启动并运行新版本,无需再次重启。": "Restarted and running the new version. No further restart is needed." "已重新启动并运行新版本,无需再次重启。": "Restarted and running the new version. No further restart is needed."
} }
+8
View File
@@ -758,6 +758,14 @@
"暂存更新已变化,请重新下载更新。": "暂存更新已变化,请重新下载更新。", "暂存更新已变化,请重新下载更新。": "暂存更新已变化,请重新下载更新。",
"启动版本与更新包不一致": "启动版本与更新包不一致", "启动版本与更新包不一致": "启动版本与更新包不一致",
"下载完成,等待重启": "下载完成,等待重启", "下载完成,等待重启": "下载完成,等待重启",
"升级器已更新,请继续升级新版": "升级器已更新,请继续升级新版",
"升级器已更新": "升级器已更新",
"当前仍运行旧版,请继续下载并重启以完成新版升级。": "当前仍运行旧版,请继续下载并重启以完成新版升级。",
"继续升级新版": "继续升级新版",
"升级器已更新。第二步将下载并预启动新版,准备成功后再重启应用。": "升级器已更新。第二步将下载并预启动新版,准备成功后再重启应用。",
"下载新版": "下载新版",
"更新索引尚未提供第二步运行包,请稍后重试": "更新索引尚未提供第二步运行包,请稍后重试",
"第二步更新包仍是旧数据库后端,已保留当前服务,请稍后重试": "第二步更新包仍是旧数据库后端,已保留当前服务,请稍后重试",
"升级成功": "升级成功", "升级成功": "升级成功",
"已重新启动并运行新版本,无需再次重启。": "已重新启动并运行新版本,无需再次重启。" "已重新启动并运行新版本,无需再次重启。": "已重新启动并运行新版本,无需再次重启。"
} }
+26 -4
View File
@@ -35,6 +35,18 @@ const CheckUpdate = ({ systemInfo }: any) => {
}; };
const checkUpgrade = async () => { const checkUpgrade = async () => {
if (job.migrationRequired) {
Modal.confirm({
title: intl.get('继续升级新版'),
content: intl.get(
'升级器已更新。第二步将下载并预启动新版,准备成功后再重启应用。',
),
okText: intl.get('下载新版'),
cancelText: intl.get('以后再说'),
onOk: startUpdate,
});
return;
}
if (!upgradeStore.beginMutation()) return; if (!upgradeStore.beginMutation()) return;
try { try {
const { code, data } = await request.put( const { code, data } = await request.put(
@@ -101,7 +113,9 @@ const CheckUpdate = ({ systemInfo }: any) => {
}); });
const label = intl.get(upgradeStatusLabel(job)); const label = intl.get(upgradeStatusLabel(job));
const hasJob = job.status !== 'idle' && job.status !== 'loading'; const hasJob =
Boolean(job.migrationRequired) ||
(job.status !== 'idle' && job.status !== 'loading');
const actions = !active && ( const actions = !active && (
<Space wrap> <Space wrap>
<Button <Button
@@ -110,9 +124,15 @@ const CheckUpdate = ({ systemInfo }: any) => {
disabled={!connected || job.status === 'loading'} disabled={!connected || job.status === 'loading'}
onClick={ready ? showReloadModal : checkUpgrade} onClick={ready ? showReloadModal : checkUpgrade}
> >
{intl.get(ready ? '重启并更新' : '检查更新')} {intl.get(
ready
? '重启并更新'
: job.migrationRequired
? '继续升级新版'
: '检查更新',
)}
</Button> </Button>
{!ready && !completed && ( {!ready && !completed && !job.migrationRequired && (
<Button <Button
disabled={busy || !connected || job.status === 'loading'} disabled={busy || !connected || job.status === 'loading'}
onClick={() => reloadSystem(false)} onClick={() => reloadSystem(false)}
@@ -148,7 +168,9 @@ const CheckUpdate = ({ systemInfo }: any) => {
? intl.get(job.error || '更新失败,请检查网络及日志或稍后再试') || ? intl.get(job.error || '更新失败,请检查网络及日志或稍后再试') ||
job.error job.error
: intl.get( : intl.get(
completed job.migrationRequired && !ready && !active
? '当前仍运行旧版,请继续下载并重启以完成新版升级。'
: completed
? '已重新启动并运行新版本,无需再次重启。' ? '已重新启动并运行新版本,无需再次重启。'
: ready : ready
? '更新已就绪,重启后应用新版本' ? '更新已就绪,重启后应用新版本'
+16 -15
View File
@@ -14,7 +14,6 @@ function temporary(t) {
t.after(() => fs.rmSync(dir, { recursive: true, force: true })); t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
return dir; return dir;
} }
test('environment values remain literal when the generated JavaScript loads', async () => { test('environment values remain literal when the generated JavaScript loads', async () => {
const output = {}; const output = {};
const Env = load(path.resolve('back/services/env.ts'), { const Env = load(path.resolve('back/services/env.ts'), {
@@ -42,7 +41,6 @@ test('environment values remain literal when the generated JavaScript loads', as
assert.equal(context.process.env[`VALUE_${i}`], value), assert.equal(context.process.env[`VALUE_${i}`], value),
); );
}); });
test('subscription shell parameters round trip without expansion or command substitution', () => { test('subscription shell parameters round trip without expansion or command substitution', () => {
const { formatCommand } = load(path.resolve('back/config/subscription.ts')); const { formatCommand } = load(path.resolve('back/config/subscription.ts'));
const payload = const payload =
@@ -83,7 +81,6 @@ test('subscription shell parameters round trip without expansion or command subs
} }
} }
}); });
test('SSH alias traversal and symlinks cannot write or remove outside files', async (t) => { test('SSH alias traversal and symlinks cannot write or remove outside files', async (t) => {
const base = temporary(t); const base = temporary(t);
const root = path.join(base, 'ssh'); const root = path.join(base, 'ssh');
@@ -141,14 +138,17 @@ test('SSH alias traversal and symlinks cannot write or remove outside files', as
service.generateSingleSshConfig('safe', 'host', '$(printf BAD)'), service.generateSingleSshConfig('safe', 'host', '$(printf BAD)'),
); );
}); });
function cronService(config, updates = []) { function cronService(config, updates = []) {
const Cron = load(path.resolve('back/services/cron.ts'), { const Cron = load(path.resolve('back/services/cron.ts'), {
typedi: decorators, typedi: decorators,
'../config': config, '../config': config,
'../data/cron': { '../data/cron': {
CrontabStatus: { running: 0, idle: 1, queued: 3 }, CrontabStatus: { running: 0, idle: 1, queued: 3 },
CrontabModel: { update: async (v) => updates.push(v) }, CrontabModel: {
nativeUpdate: async (where, v) => {
return updates.push(v);
},
},
}, },
'../data/runningInstance': { RunningInstanceModel: {}, InstanceStatus: {} }, '../data/runningInstance': { RunningInstanceModel: {}, InstanceStatus: {} },
'../config/util': { fileExist: async (p) => fs.existsSync(p) }, '../config/util': { fileExist: async (p) => fs.existsSync(p) },
@@ -183,7 +183,6 @@ function subscriptionService(config, model = {}) {
}).default; }).default;
return new Subscription(logger, {}, {}, {}, {}); return new Subscription(logger, {}, {}, {}, {});
} }
test('stored cron/subscription paths cannot read or enumerate outside the log root', async (t) => { test('stored cron/subscription paths cannot read or enumerate outside the log root', async (t) => {
const base = temporary(t); const base = temporary(t);
const root = path.join(base, 'log'); const root = path.join(base, 'log');
@@ -224,7 +223,6 @@ test('stored cron/subscription paths cannot read or enumerate outside the log ro
(await subscription.logs(1)).some((x) => x.filename === 'normal.log'), (await subscription.logs(1)).some((x) => x.filename === 'normal.log'),
); );
}); });
test('subscription aliases are checked before persistence and forced deletion', async (t) => { test('subscription aliases are checked before persistence and forced deletion', async (t) => {
const base = temporary(t); const base = temporary(t);
for (const dir of ['scripts', 'repo']) fs.mkdirSync(path.join(base, dir)); for (const dir of ['scripts', 'repo']) fs.mkdirSync(path.join(base, dir));
@@ -235,8 +233,12 @@ test('subscription aliases are checked before persistence and forced deletion',
repoPath: path.join(base, 'repo'), repoPath: path.join(base, 'repo'),
}, },
{ {
findAll: async () => [{ alias: '../victim' }], find: async (where, read = {}) => {
destroy: async () => writes++, return [{ alias: '../victim' }];
},
nativeDelete: async (where, read = {}) => {
return writes++;
},
}, },
); );
service.insert = service.updateDb = async () => writes++; service.insert = service.updateDb = async () => writes++;
@@ -254,14 +256,17 @@ test('subscription aliases are checked before persistence and forced deletion',
); );
assert.equal(writes, 0); assert.equal(writes, 0);
}); });
function systemService(config, overrides = {}) { function systemService(config, overrides = {}) {
const System = load(path.resolve('back/services/system.ts'), { const System = load(path.resolve('back/services/system.ts'), {
typedi: decorators, typedi: decorators,
'../config': config, '../config': config,
'../config/const': {}, '../config/const': {},
'../data/dependence': { '../data/dependence': {
DependenceModel: { findAll: async () => [] }, DependenceModel: {
find: async (where, read = {}) => {
return [];
},
},
DependenceTypes: {}, DependenceTypes: {},
DependenceStatus: {}, DependenceStatus: {},
}, },
@@ -277,7 +282,6 @@ function systemService(config, overrides = {}) {
}).default; }).default;
return new System(logger, {}, {}); return new System(logger, {}, {});
} }
test('backup rejects unrecognized paths before spawning; valid export uses argv without a shell', async (t) => { test('backup rejects unrecognized paths before spawning; valid export uses argv without a shell', async (t) => {
const base = temporary(t); const base = temporary(t);
const calls = []; const calls = [];
@@ -333,7 +337,6 @@ test('backup rejects unrecognized paths before spawning; valid export uses argv
assert.equal(calls[0][2].shell, undefined); assert.equal(calls[0][2].shell, undefined);
assert.equal(downloaded, true); assert.equal(downloaded, true);
}); });
test('proxy, language and mirror settings stay literal at their shell boundaries', async (t) => { test('proxy, language and mirror settings stay literal at their shell boundaries', async (t) => {
const base = temporary(t); const base = temporary(t);
const commands = []; const commands = [];
@@ -373,7 +376,6 @@ test('proxy, language and mirror settings stay literal at their shell boundaries
} }
assert.equal((await service.reloadSystem('$(printf BAD)')).code, 400); assert.equal((await service.reloadSystem('$(printf BAD)')).code, 400);
}); });
test('the ignored-log sentinel remains supported without exposing device files', async (t) => { test('the ignored-log sentinel remains supported without exposing device files', async (t) => {
const root = temporary(t); const root = temporary(t);
const cron = cronService({ logPath: root }); const cron = cronService({ logPath: root });
@@ -385,7 +387,6 @@ test('the ignored-log sentinel remains supported without exposing device files',
cron.status({ ids: [], status: 0, log_path: '/dev/zero' }), cron.status({ ids: [], status: 0, log_path: '/dev/zero' }),
); );
}); });
test('normal backup creates an archive containing the selected data directories', async (t) => { test('normal backup creates an archive containing the selected data directories', async (t) => {
const base = temporary(t); const base = temporary(t);
const dataPath = path.join(base, 'data'); const dataPath = path.join(base, 'data');
+129
View File
@@ -0,0 +1,129 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const path = require('node:path');
const os = require('node:os');
const fs = require('node:fs/promises');
const express = require('express');
const jwt = require('jsonwebtoken');
const load = require('../helpers/load-security-module.cjs');
test('authentication cache propagates database read and write failures', async () => {
const failure = new Error('synthetic unavailable database');
const CacheStore = load(path.resolve('back/data/cacheStore.ts'), {
'.': {
database: {
request: async () => {
throw failure;
},
},
},
}).CacheStore;
const { shareStore } = load(path.resolve('back/shared/store.ts'), {
'../data/cacheStore': { CacheStore },
});
await assert.rejects(shareStore.getAuthInfo(), (error) => error === failure);
await assert.rejects(
shareStore.updateAuthInfo({ username: 'qa' }),
(error) => error === failure,
);
});
test('HTTP authentication distinguishes unavailable storage from an invalid session', async (t) => {
const root = await fs.mkdtemp(
path.join(os.tmpdir(), 'ql-auth-availability-'),
);
const secret = 'synthetic-auth-availability';
const token = jwt.sign({}, secret, { algorithm: 'HS384', expiresIn: '1h' });
const auth = { username: 'qa', password: 'configured', token };
let unavailable = false;
let initialized = 0;
const read = async (value) => {
if (unavailable) throw new Error(`SQL failure must not expose ${token}`);
return value;
};
const router = express.Router();
router.get('/system/update-status', (_req, res) => res.json({ code: 200 }));
router.get('/envs', (_req, res) => res.json({ code: 200 }));
router.get('/health', (_req, res) => res.json({ code: 200 }));
router.put('/user/init', (_req, res) => {
initialized++;
res.json({ code: 200 });
});
const app = express();
load(path.resolve('back/loaders/express.ts'), {
'../config': {
rootPath: root,
uploadPath: path.join(root, 'upload'),
configPath: path.join(root, 'config'),
api: { prefix: '/api' },
apiWhiteList: ['/api/health', '/api/user/init'],
jwt: { secret },
},
'../api': () => router,
'../config/util': {
getToken: (req) =>
(req.headers.authorization || '').replace(/^Bearer /, ''),
getPlatform: () => 'desktop',
},
'../shared/store': {
shareStore: {
getAuthInfo: () => read(auth),
getApps: () =>
read([
{
scopes: ['envs'],
tokens: [
{ value: 'app-token', expiration: Date.now() / 1000 + 3600 },
],
},
]),
},
},
'../shared/i18n': { t: (value) => value },
'../config/serverEnv': { serveEnv: (_req, res) => res.end() },
'../data/open': {},
'../data/system': {},
}).default({ app });
const server = app.listen(0, '127.0.0.1');
t.after(async () => {
await new Promise((resolve) => server.close(resolve));
await fs.rm(root, { recursive: true, force: true });
});
await new Promise((resolve) => server.once('listening', resolve));
const request = async (route, value, method = 'GET') => {
const response = await fetch(
`http://127.0.0.1:${server.address().port}${route}`,
{
method,
headers: value ? { Authorization: `Bearer ${value}` } : {},
signal: AbortSignal.timeout(3000),
},
);
return { status: response.status, body: await response.text() };
};
assert.equal((await request('/api/system/update-status', token)).status, 200);
unavailable = true;
for (const [route, value, method] of [
['/api/system/update-status', token],
['/open/envs', 'app-token'],
['/api/user/init', undefined, 'PUT'],
]) {
const response = await request(route, value, method);
assert.equal(response.status, 503);
assert.equal(JSON.parse(response.body).code, 503);
assert.ok(!response.body.includes(token));
assert.ok(!response.body.includes('SQL failure'));
}
assert.equal(initialized, 0);
assert.equal((await request('/api/health')).status, 200);
unavailable = false;
assert.equal((await request('/api/system/update-status', token)).status, 200);
const invalid = jwt.sign({ unrecognized: true }, secret, {
algorithm: 'HS384',
expiresIn: '1h',
});
assert.equal(
(await request('/api/system/update-status', invalid)).status,
401,
);
});
+69
View File
@@ -0,0 +1,69 @@
const assert = require('node:assert/strict');
const test = require('node:test');
const express = require('express');
const { DatabaseClient } = require('../../back/data/databaseClient');
const { Repository } = require('../../back/data/repository');
const { initialize } = require('../helpers/repository-fixture.cjs');
function mockModule(modulePath, exports) {
const filename = require.resolve(modulePath);
require.cache[filename] = {
id: filename,
filename,
loaded: true,
exports,
children: [],
paths: [],
};
}
test('instance API retains newest-first ordering independently of insertion order', async (t) => {
const db = new DatabaseClient(':memory:');
t.after(() => db.close());
await initialize(db);
const instances = new Repository('RunningInstance', () => db);
for (const [cron_id, started_at] of [
[7, 200],
[7, 300],
[8, 900],
[7, 100],
]) {
await instances.insert({ cron_id, started_at, status: 2 });
}
for (const service of ['cron', 'cronView']) {
mockModule(`../../back/services/${service}`, {
__esModule: true,
default: class {},
});
}
mockModule('../../back/schedule/client', { __esModule: true, default: {} });
mockModule('../../back/shared/i18n', { t: (message) => message });
mockModule('../../back/data/runningInstance', {
RunningInstanceModel: instances,
InstanceStatus: { running: 0 },
});
const app = express.Router();
require('../../back/api/cron').default(app);
const router = app.stack.find((layer) => layer.name === 'router').handle;
const handler = router.stack
.find((layer) => layer.route?.path === '/:id/instances')
.route.stack.at(-1).handle;
let response;
await handler(
{ params: { id: 7 } },
{
send: (body) => {
response = body;
},
},
(error) => {
throw error;
},
);
assert.equal(response.code, 200);
assert.deepEqual(
response.data.map((row) => row.started_at),
[300, 200, 100],
);
assert(response.data.every((row) => row.cron_id === 7));
});
+69 -12
View File
@@ -10,15 +10,40 @@ test('system and node modes route extended cron once and keep portable rules in
const end = source.indexOf(' private async getLogName', begin); const end = source.indexOf(' private async getLogName', begin);
const saveBegin = source.indexOf(' private async setCrontab('); const saveBegin = source.indexOf(' private async setCrontab(');
const saveEnd = source.indexOf(' public importCrontab(', saveBegin); const saveEnd = source.indexOf(' public importCrontab(', saveBegin);
const js = ts.transpileModule(`class Fixture {${source.slice(begin, end)}${source.slice(saveBegin, saveEnd)}}; module.exports = Fixture;`, { const js = ts.transpileModule(
compilerOptions: { target: ts.ScriptTarget.ES2020 }, `class Fixture {${source.slice(begin, end)}${source.slice(
}).outputText; saveBegin,
saveEnd,
)}}; module.exports = Fixture;`,
{
compilerOptions: { target: ts.ScriptTarget.ES2020 },
},
).outputText;
for (const mode of ['system', 'node']) { for (const mode of ['system', 'node']) {
const module = { exports: {} }; const module = { exports: {} };
let file = '', installs = 0; let file = '',
new Function('module', 'process', 'execSync', 'ScheduleType', 'config', 'writeFileWithLock', 'CrontabModel', js)( installs = 0;
module, { env: { QL_SCHEDULER: mode } }, () => { installs++; }, ScheduleType, new Function(
{ crontabFile: '/unused' }, async (_, data) => { file = data; }, { update: async () => {} }, 'module',
'process',
'execSync',
'ScheduleType',
'config',
'writeFileWithLock',
'CrontabModel',
js,
)(
module,
{ env: { QL_SCHEDULER: mode } },
() => {
installs++;
},
ScheduleType,
{ crontabFile: '/unused' },
async (_, data) => {
file = data;
},
{ nativeUpdate: async () => {} },
); );
const service = new module.exports(); const service = new module.exports();
service.makeCommand = (row) => `task ${row.id}.js`; service.makeCommand = (row) => `task ${row.id}.js`;
@@ -40,20 +65,52 @@ test('system and node modes route extended cron once and keep portable rules in
{ id: 'named', schedule: '0 0 * JAN MON' }, { id: 'named', schedule: '0 0 * JAN MON' },
{ id: 'macro', schedule: '@daily' }, { id: 'macro', schedule: '@daily' },
{ id: 'short', schedule: '*' }, { id: 'short', schedule: '*' },
{ id: 'extra', schedule: '* * * * *', extra_schedules: [{ schedule: '0 * * * *' }] }, {
id: 'extra',
schedule: '* * * * *',
extra_schedules: [{ schedule: '0 * * * *' }],
},
{ id: 'once', schedule: '@once' }, { id: 'once', schedule: '@once' },
{ id: 'boot', schedule: '@boot' }, { id: 'boot', schedule: '@boot' },
]; ];
for (const row of rows) { for (const row of rows) {
const special = ['once', 'boot'].includes(row.id); const special = ['once', 'boot'].includes(row.id);
const portable = ['plain', 'spaces'].includes(row.id); const portable = ['plain', 'spaces'].includes(row.id);
assert.equal(service.shouldUseCronClient(row), !special && (mode === 'node' || !portable), `${mode}: ${row.id}`); assert.equal(
service.shouldUseCronClient(row),
!special && (mode === 'node' || !portable),
`${mode}: ${row.id}`,
);
} }
await service.setCrontab({ data: rows, total: rows.length }); await service.setCrontab({ data: rows, total: rows.length });
assert.equal(installs, mode === 'system' ? 1 : 0); assert.equal(installs, mode === 'system' ? 1 : 0);
for (const id of ['numeric-step', 'sunday-seven', 'day-step', 'restricted-days', 'restricted-week', 'annually', 'midnight', 'minutely', 'seconds', 'question', 'last', 'nth', 'named', 'macro', 'short', 'extra', 'once', 'boot']) { for (const id of [
const line = file.split('\n').find((line) => line.endsWith(`task ${id}.js`)); 'numeric-step',
assert.ok(line.startsWith('# '), `${mode}: ${id} cannot also run from system crontab`); 'sunday-seven',
'day-step',
'restricted-days',
'restricted-week',
'annually',
'midnight',
'minutely',
'seconds',
'question',
'last',
'nth',
'named',
'macro',
'short',
'extra',
'once',
'boot',
]) {
const line = file
.split('\n')
.find((line) => line.endsWith(`task ${id}.js`));
assert.ok(
line.startsWith('# '),
`${mode}: ${id} cannot also run from system crontab`,
);
} }
} }
}); });
+12 -21
View File
@@ -1,28 +1,19 @@
const { DatabaseClient } = require('../../back/data/databaseClient');
const { repository, initialize } = require('../helpers/repository-fixture.cjs');
const { orderBy } = require('../../back/data/queries');
require('ts-node/register/transpile-only'); require('ts-node/register/transpile-only');
const assert = require('node:assert/strict'); const assert = require('node:assert/strict');
const { test } = require('node:test'); const { test } = require('node:test');
const { Sequelize, DataTypes } = require('sequelize'); const { Sequelize, DataTypes } = require('sequelize');
const dayjs = require('dayjs'); const dayjs = require('dayjs');
const express = require('express'); const express = require('express');
test('today result lists include mixed and deleted tasks and exclude other dates', async (t) => { test('today result lists include mixed and deleted tasks and exclude other dates', async (t) => {
const db = new Sequelize({ const db = new DatabaseClient(':memory:');
dialect: 'sqlite',
storage: ':memory:',
logging: false,
});
t.after(() => db.close()); t.after(() => db.close());
const crons = db.define('Cron', { const crons = repository(db, 'Crontab');
name: DataTypes.STRING, const stats = repository(db, 'CrontabStat');
command: DataTypes.STRING,
});
const stats = db.define('Stat', {
ref_id: DataTypes.INTEGER,
date: DataTypes.STRING,
fail_count: DataTypes.INTEGER,
success_count: DataTypes.INTEGER,
});
const replacements = { const replacements = {
'../../back/data': { database: db },
'../../back/data/cron': { CrontabModel: crons }, '../../back/data/cron': { CrontabModel: crons },
'../../back/data/cronStats': { CrontabStatModel: stats }, '../../back/data/cronStats': { CrontabStatModel: stats },
'../../back/data/runningInstance': {}, '../../back/data/runningInstance': {},
@@ -42,14 +33,14 @@ test('today result lists include mixed and deleted tasks and exclude other dates
else delete require.cache[resolved]; else delete require.cache[resolved];
}); });
} }
await db.sync(); await initialize(db);
await crons.bulkCreate([ await crons.insertMany([
{ id: 1, name: 'Recovered task', command: 'task recovered.js' }, { id: 1, name: 'Recovered task', command: 'task recovered.js' },
{ id: 2, name: '', command: 'task unnamed.js' }, { id: 2, name: '', command: 'task unnamed.js' },
{ id: 3, name: 'Successful task', command: 'task success.js' }, { id: 3, name: 'Successful task', command: 'task success.js' },
]); ]);
const today = dayjs().format('YYYY-MM-DD'); const today = dayjs().format('YYYY-MM-DD');
await stats.bulkCreate([ await stats.insertMany([
{ ref_id: 1, date: today, fail_count: 2, success_count: 1 }, { ref_id: 1, date: today, fail_count: 2, success_count: 1 },
{ ref_id: 2, date: today, fail_count: 1, success_count: 0 }, { ref_id: 2, date: today, fail_count: 1, success_count: 0 },
{ ref_id: 3, date: today, fail_count: 0, success_count: 4 }, { ref_id: 3, date: today, fail_count: 0, success_count: 4 },
@@ -135,7 +126,7 @@ test('today result lists include mixed and deleted tasks and exclude other dates
}, },
], ],
}); });
await stats.destroy({ where: {} }); await stats.nativeDelete({});
await successHandler( await successHandler(
{}, {},
{ {
@@ -160,7 +151,7 @@ test('today result lists include mixed and deleted tasks and exclude other dates
}, },
); );
assert.deepEqual(response, { code: 200, data: [] }); assert.deepEqual(response, { code: 200, data: [] });
await stats.drop(); await db.query('DROP TABLE CrontabStats');
let forwarded; let forwarded;
await handler( await handler(
{}, {},
+222
View File
@@ -0,0 +1,222 @@
const test = require('node:test'),
assert = require('node:assert/strict');
const fs = require('node:fs/promises'),
path = require('node:path'),
os = require('node:os');
const {
Sequelize,
DataTypes,
Op,
QueryTypes,
where,
col,
} = require('sequelize');
const { DatabaseClient } = require('../../back/data/databaseClient');
const { Repository } = require('../../back/data/repository');
const { initialize } = require('../helpers/repository-fixture.cjs');
const names = {
App: 'Apps',
Auth: 'Auths',
Crontab: 'Crontabs',
CrontabStat: 'CrontabStats',
CrontabView: 'CrontabViews',
Dependence: 'Dependences',
Env: 'Envs',
RunningInstance: 'RunningInstances',
Subscription: 'Subscriptions',
};
async function legacy(file) {
const database = new Sequelize({
dialect: 'sqlite',
storage: file,
logging: false,
});
const models = {};
for (const [name, tableName] of Object.entries(names)) {
const columns = await database.query(`PRAGMA table_info("${tableName}")`, {
type: QueryTypes.SELECT,
});
const attrs = Object.fromEntries(
columns.map((c) => [
c.name,
{
type:
c.type === 'JSON'
? DataTypes.JSON
: c.type === 'DATETIME'
? DataTypes.DATE
: c.type === 'TINYINT(1)'
? DataTypes.BOOLEAN
: /INT|NUMBER/.test(c.type)
? DataTypes.NUMBER
: DataTypes.STRING,
...(c.pk ? { primaryKey: true, autoIncrement: true } : {}),
},
]),
);
models[name] = database.define(name, attrs, {
tableName,
freezeTableName: true,
});
}
return { database, models };
}
test('all nine legacy tables remain readable in both versions without changing schema, IDs, dates or .env', async (t) => {
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-orm-legacy-'));
t.after(() => fs.rm(dir, { recursive: true, force: true }));
const file = path.join(dir, 'database.sqlite'),
envFile = path.join(dir, '.env');
const environment = 'QL_DATA_DIR=/synthetic/old-data\nSYNTHETIC_ENV=keep\n';
await fs.writeFile(envFile, environment);
const seed = new Sequelize({
dialect: 'sqlite',
storage: file,
logging: false,
});
try {
const sql = await fs.readFile(
path.resolve('test/fixtures/sequelize-2.23-schema.sql'),
'utf8',
);
for (const statement of sql
.replace(/^--.*$/gm, '')
.split(';')
.map((x) => x.trim())
.filter(Boolean))
await seed.query(statement);
} finally {
await seed.close();
}
let old = await legacy(file);
const payloads = {
App: { name: 'legacy', scopes: ['envs'], client_id: 'synthetic-client' },
Auth: {
type: 'authConfig',
info: {
username: 'synthetic-user',
password: 'synthetic-hash',
tokens: { desktop: ['keep'] },
},
},
Crontab: {
name: 'legacy',
command: 'echo synthetic',
schedule: '0 0 * * *',
saved: true,
labels: ['中文'],
},
CrontabStat: { ref_id: 7, date: '2026-10-21', run_count: 2 },
CrontabView: {
name: 'legacy',
filters: [{ property: 'status', operation: 'In', value: [1] }],
},
Dependence: { name: 'legacy', type: 0, log: ['旧日志'] },
Env: {
name: 'legacy',
value: '旧值',
labels: ['中文'],
position: 4500000000000000,
},
RunningInstance: { cron_id: 7, started_at: 100, status: 2, exit_code: 143 },
Subscription: {
name: 'legacy',
alias: 'legacy',
pull_option: { username: 'synthetic', password: 'synthetic-only' },
},
};
const date = new Date('2020-01-02T03:04:05.006Z');
for (const [name, payload] of Object.entries(payloads))
await old.models[name].create({
id: 7,
...payload,
createdAt: date,
updatedAt: date,
});
const schema = await old.database.query(
"SELECT type,name,sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' ORDER BY type,name",
{ type: QueryTypes.SELECT },
);
await old.database.close();
const client = new DatabaseClient(file);
t.after(() => client.close());
await initialize(client);
for (const [name, payload] of Object.entries(payloads)) {
const repository = new Repository(name, () => client),
row = await repository.findOne({ id: 7 });
for (const [field, value] of Object.entries(payload))
assert.deepEqual(row[field], value, `${name}.${field}`);
assert.equal(row.createdAt.toISOString(), date.toISOString());
const added = await repository.insert({
...payload,
id: undefined,
...(payload.name ? { name: 'new' } : {}),
...(payload.alias ? { alias: 'new' } : {}),
...(payload.date ? { date: '2026-10-22' } : {}),
});
assert(added.id > 7, name);
assert(added.createdAt instanceof Date);
}
assert.deepEqual(
await client.query(
"SELECT type,name,sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' ORDER BY type,name",
),
schema,
);
await client.close();
old = await legacy(file);
try {
for (const name of Object.keys(payloads)) {
const rows = await old.models[name].findAll({ order: [['id', 'ASC']] });
assert.equal(rows.length, 2);
assert.equal(rows[0].createdAt.toISOString(), date.toISOString());
assert(rows[1].createdAt instanceof Date);
assert(!Number.isNaN(+rows[1].createdAt));
}
assert.equal(await fs.readFile(envFile, 'utf8'), environment);
} finally {
await old.database.close();
}
});
test('native filters retain legacy NULL, IN, nested groups and JSON text matching', async (t) => {
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-orm-filters-'));
t.after(() => fs.rm(dir, { recursive: true, force: true }));
const file = path.join(dir, 'database.sqlite');
const client = new DatabaseClient(file);
t.after(() => client.close());
await initialize(client);
const repository = new Repository('Env', () => client);
for (const row of [
{ name: 'one', value: 'a', status: 0, labels: ['中文', 'keep'] },
{ name: 'two', value: 'b', status: 1, labels: [] },
{ name: null, value: 'c', status: 0, labels: null },
])
await repository.insert(row);
const old = await legacy(file);
t.after(() => old.database.close());
const cases = [
[{ status: [0, 1] }, { status: [0, 1] }],
[{ name: { [Op.not]: null } }, { name: { $ne: null } }],
[
{ [Op.or]: [{ name: 'missing' }, {}] },
{ $or: [{ name: 'missing' }, {}] },
],
[{ [Op.and]: [{ status: 0 }, {}] }, { $and: [{ status: 0 }, {}] }],
[{ [Op.or]: [] }, { $or: [] }],
[
where(col('labels'), { [Op.like]: '%keep%' }),
{ labels: { $like: '%keep%' } },
],
[{ name: { [Op.notIn]: ['one'] } }, { name: { $nin: ['one'] } }],
];
for (const [before, after] of cases) {
const expected = (
await old.models.Env.findAll({ where: before, order: [['id', 'ASC']] })
).map((x) => x.id);
const actual = (
await repository.find(after, { orderBy: [{ id: 'asc' }] })
).map((x) => x.id);
assert.deepEqual(actual, expected, JSON.stringify(after));
}
});
@@ -0,0 +1,77 @@
const test = require('node:test'),
assert = require('node:assert/strict');
const fs = require('node:fs/promises'),
path = require('node:path'),
os = require('node:os'),
{ spawnSync } = require('node:child_process');
const { minimum, assertNodeRuntime } = require('../../shell/runtime-guard.cjs');
const { databaseStartupFailure } = require('../../back/data/databaseErrors');
const {
dependencyFailure,
} = require('../../cli/src/internal/maintenance/upgradePrestart.cjs');
test('runtime policy matches package metadata and accepts the exact Node minimum', () => {
assert.equal(minimum, require('../../package.json').qinglongRuntimeMinimum);
for (const version of ['22.17.0', '22.23.2', '24.18.1', '26.0.0'])
assert.doesNotThrow(() => assertNodeRuntime(version));
for (const version of ['20.20.2', '22.16.9', 'invalid'])
assert.throws(() => assertNodeRuntime(version), {
code: 'UNSUPPORTED_NODE',
});
});
test('database startup repairs only third-party dependency import failures', () => {
for (const message of [
"Cannot find module '@mikro-orm/sql'",
"Cannot find module '/candidate/node_modules/@mikro-orm/core/missing.js'",
"Cannot find module './missing.js'\nRequire stack:\n- /candidate/node_modules/@mikro-orm/sql/index.js",
]) {
const result = databaseStartupFailure(
Object.assign(new Error(message), { code: 'MODULE_NOT_FOUND' }),
);
assert.equal(dependencyFailure(result), true);
}
for (const value of [
new Error('heap out of memory'),
undefined,
Object.assign(
new Error(
"Cannot find module './schema'\nRequire stack:\n- /candidate/static/build/data/databaseWorker.js",
),
{ code: 'MODULE_NOT_FOUND' },
),
Object.assign(new Error("Cannot find module 'node:sqlite'"), {
code: 'MODULE_NOT_FOUND',
}),
])
assert.equal(dependencyFailure(databaseStartupFailure(value)), false);
});
test('Node 20 rejects backend initialization before changing its old database or .env', async (t) => {
const node20 = process.env.QL_TEST_NODE20;
if (!node20) {
t.skip('Set QL_TEST_NODE20 to verify an existing Node 20 runtime');
return;
}
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-runtime-floor-'));
t.after(() => fs.rm(dir, { recursive: true, force: true }));
const data = path.join(dir, 'data'),
db = path.join(data, 'db/database.sqlite'),
env = path.join(dir, '.env');
await fs.mkdir(path.dirname(db), { recursive: true });
await fs.writeFile(db, 'old synthetic database marker');
await fs.writeFile(env, 'SYNTHETIC_ENV=keep\n');
const result = spawnSync(
node20,
[path.resolve('static/build/bootstrap/database.js')],
{
env: { ...process.env, QL_DIR: dir, QL_DATA_DIR: data },
encoding: 'utf8',
timeout: 10000,
},
);
assert.equal(result.status, 1, result.stderr);
assert.match(result.stderr, /Node >=22\.17/);
assert.equal(await fs.readFile(db, 'utf8'), 'old synthetic database marker');
assert.equal(await fs.readFile(env, 'utf8'), 'SYNTHETIC_ENV=keep\n');
});
+515
View File
@@ -0,0 +1,515 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs/promises');
const path = require('node:path');
const os = require('node:os');
const { DatabaseClient } = require('../../back/data/databaseClient');
const { Repository } = require('../../back/data/repository');
const { initialSchema } = require('../../back/data/schema');
const { migrateSchema } = require('../../back/shared/schemaMigrations');
test('requests buffered before bootstrap and during owner-monitor startup retain their order', async (t) => {
const { fork } = require('node:child_process');
const directory = await fs.mkdtemp(
path.join(os.tmpdir(), 'ql-db-bootstrap-queue-'),
);
const script = path.join(directory, 'delayed.ts');
await fs.writeFile(
script,
`setTimeout(()=>require(${JSON.stringify(
require.resolve('../../back/data/databaseProcess'),
)}),100)`,
);
const child = fork(script, [], {
execArgv: [
'--require',
require.resolve('ts-node/register/transpile-only'),
'--max-old-space-size=128',
],
env: {
...process.env,
TS_NODE_PROJECT: path.resolve('back/data/tsconfig.worker.json'),
},
stdio: ['ignore', 'ignore', 'pipe', 'ipc'],
});
const stopped = new Promise((resolve) => child.once('exit', resolve));
let stderr = '';
child.stderr.on('data', (chunk) => {
stderr += chunk;
});
t.after(async () => {
child.kill('SIGKILL');
await stopped;
await fs.rm(directory, { recursive: true, force: true });
});
const responses = new Promise((resolve, reject) => {
const replies = [];
const timer = setTimeout(
() => reject(new Error('Buffered startup requests were lost')),
30000,
);
child.on('message', (reply) => {
if (reply.error) {
clearTimeout(timer);
reject(new Error(reply.error.message));
return;
}
replies.push(reply);
if (replies.length === 2) {
clearTimeout(timer);
resolve(replies);
}
});
child.once('exit', () => {
clearTimeout(timer);
reject(new Error('Database exited during bootstrap: ' + stderr));
});
child.once('error', reject);
});
child.send({ storage: ':memory:', ownerPid: process.pid });
child.send({ id: 1, operation: 'query', args: ['SELECT 42 AS value'] });
child.send({ id: 2, operation: 'query', args: ['SELECT 43 AS value'] });
assert.deepEqual(await responses, [
{ id: 1, value: [{ value: 42 }] },
{ id: 2, value: [{ value: 43 }] },
]);
});
test('killing the owner during synchronous SQL releases the write lock and rolls back before restart', async (t) => {
const { fork } = require('node:child_process');
const { DatabaseSync } = require('node:sqlite');
const directory = await fs.mkdtemp(
path.join(os.tmpdir(), 'ql-db-owner-exit-'),
);
const file = path.join(directory, 'data.sqlite');
const script = path.join(directory, 'owner.cjs');
await fs.writeFile(
script,
`
const {DatabaseClient}=require(${JSON.stringify(
require.resolve('../../back/data/databaseClient'),
)});
const db=new DatabaseClient(process.argv[2]);
(async()=>{
await db.query('CREATE TABLE probe(value INTEGER)');
await db.query('INSERT INTO probe VALUES(1)');
await db.transaction(async()=>{
await db.query('UPDATE probe SET value=2');
const running=db.query('WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<200000000000) SELECT SUM(x) FROM n');
setTimeout(()=>process.send({databasePid:db.worker.pid}),300);
await running;
});
})().catch(error=>{console.error(error);process.exitCode=1});
`,
);
const owner = fork(script, [file], {
execArgv: ['--require', require.resolve('ts-node/register/transpile-only')],
env: {
...process.env,
TS_NODE_PROJECT: path.resolve('back/tsconfig.json'),
},
stdio: ['ignore', 'ignore', 'pipe', 'ipc'],
});
const exited = new Promise((resolve) => owner.once('exit', resolve));
let databasePid,
probe,
stderr = '',
unlocked = false;
owner.stderr.on('data', (chunk) => {
stderr += chunk;
});
t.after(async () => {
probe?.close();
owner.kill('SIGKILL');
await exited;
if (databasePid && !unlocked) {
try {
process.kill(databasePid, 'SIGKILL');
} catch (error) {
if (error.code !== 'ESRCH') throw error;
}
}
await fs.rm(directory, { recursive: true, force: true });
});
databasePid = await new Promise((resolve, reject) => {
const timer = setTimeout(
() => reject(new Error('Owner startup timed out: ' + stderr)),
30000,
);
owner.once('message', (message) => {
clearTimeout(timer);
resolve(message.databasePid);
});
owner.once('exit', () => {
clearTimeout(timer);
reject(new Error('Owner exited before SQL: ' + stderr));
});
owner.once('error', reject);
});
probe = new DatabaseSync(file);
probe.exec('PRAGMA busy_timeout=50');
assert.throws(() => probe.exec('BEGIN IMMEDIATE'), /locked/);
owner.kill('SIGKILL');
await exited;
const deadline = Date.now() + 5000;
do {
try {
probe.exec('BEGIN IMMEDIATE;ROLLBACK');
unlocked = true;
break;
} catch (error) {
if (!/locked/.test(error.message)) throw error;
}
await new Promise((resolve) => setTimeout(resolve, 50));
} while (Date.now() < deadline);
assert.equal(
unlocked,
true,
'The orphan database process still blocks the old service',
);
assert.equal(probe.prepare('SELECT value FROM probe').get().value, 1);
});
async function fixture(t, file = ':memory:') {
const db = new DatabaseClient(file);
t.after(() => db.close());
await db.transaction(async () => {
for (const sql of initialSchema) await db.query(sql);
await migrateSchema(db);
});
return {
db,
env: new Repository('Env', () => db),
auth: new Repository('Auth', () => db),
};
}
test('direct ts-node CLI startup registers the isolated database worker compiler', async () => {
const execute = require('node:util').promisify(
require('node:child_process').execFile,
);
const result = await execute(
process.execPath,
[
path.resolve('node_modules/ts-node/dist/bin.js'),
'--transpile-only',
'--project',
'back/tsconfig.json',
'-e',
`const {DatabaseClient}=require('./back/data/databaseClient');const db=new DatabaseClient(':memory:');db.query('SELECT 42 AS value').then(async rows=>{console.log(JSON.stringify(rows));await db.close()}).catch(async error=>{console.error(error);process.exitCode=1;await db.close().catch(()=>{})})`,
],
{ cwd: path.resolve('.'), timeout: 30000 },
);
assert.deepEqual(JSON.parse(result.stdout), [{ value: 42 }]);
});
test('worker round-trips legacy field names, JSON, dates, null and precise positions', async (t) => {
const { db, env } = await fixture(t);
const row = await env.insert({
name: '中文',
value: 'keep',
position: 4500000000000000,
labels: ['old', '中文'],
remarks: null,
});
assert.deepEqual((await env.findOne({ id: row.id })).labels, row.labels);
assert.equal(row.position, 4500000000000000);
assert.equal(row.remarks, null);
assert(row.createdAt instanceof Date);
const stored = (
await db.query('SELECT typeof(createdAt) AS type,createdAt FROM Envs')
)[0];
assert.equal(stored.type, 'text');
assert.match(
stored.createdAt,
/^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\.\d{3} \+00:00$/,
);
assert.equal(
(await db.query('SELECT count(*) AS n FROM SchemaMigrations'))[0].n,
15,
);
});
test('JSON LIKE uses serialized text without quoting the pattern', async (t) => {
const { env } = await fixture(t);
await env.insert({ name: 'a', value: 'b', labels: ['中文', 'old'] });
assert.equal((await env.find({ labels: { $like: '%old%' } })).length, 1);
assert.equal(
(await env.find({ $not: { labels: { $like: '%old%' } } })).length,
0,
);
});
test('transaction commit, failure and nested savepoints preserve ownership', async (t) => {
const { db, env } = await fixture(t);
await env.insert({ name: 'a', value: 'old' });
await db.transaction(async () => {
await env.nativeUpdate({ name: 'a' }, { value: 'committed' });
await assert.rejects(
db.transaction(async () => {
await env.nativeUpdate({ name: 'a' }, { value: 'rollback' });
throw Error('injected');
}),
/injected/,
);
assert.equal((await env.findOne({ name: 'a' })).value, 'committed');
});
await assert.rejects(
db.transaction(async () => {
await env.nativeUpdate({ name: 'a' }, { value: 'rollback all' });
throw Error('outer failure');
}),
/outer failure/,
);
assert.equal((await env.findOne({ name: 'a' })).value, 'committed');
});
test('unrelated reads wait for rollback and cannot see uncommitted data', async (t) => {
const { db, env } = await fixture(t);
await env.insert({ name: 'a', value: 'old' });
let release, writing;
const started = new Promise((r) => {
writing = r;
});
const gate = new Promise((r) => {
release = r;
});
const transaction = db.transaction(async () => {
await env.nativeUpdate({ name: 'a' }, { value: 'uncommitted' });
writing();
await gate;
throw Error('rollback');
});
const rejected = assert.rejects(transaction, /rollback/);
await started;
const read = env.findOne({ name: 'a' });
release();
await rejected;
assert.equal((await read).value, 'old');
});
test('expired async transaction scopes cannot write after commit', async (t) => {
const { db, env } = await fixture(t);
// Capture an asynchronous resource inside the scope, then release it afterwards.
let release, result;
const gate = new Promise((r) => {
release = r;
});
await db.transaction(async () => {
result = gate.then(() => env.insert({ name: 'late' }));
});
const rejected = assert.rejects(result, { code: 'TRANSACTION_SCOPE' });
release();
await rejected;
assert.equal(await env.count(), 0);
});
test('conditional updates keep compare-and-swap counts and do not clear undefined data', async (t) => {
const { env } = await fixture(t);
const row = await env.insert({ name: 'a', value: 'old' });
assert.equal(
await env.nativeUpdate({ id: row.id, value: 'newer' }, { value: 'bad' }),
0,
);
assert.equal(
await env.nativeUpdate(
{ id: row.id, value: 'old' },
{ value: 'new', remarks: undefined },
),
1,
);
assert.equal((await env.findOne({ id: row.id })).value, 'new');
await assert.rejects(env.nativeUpdate({ id: undefined }, { value: 'bad' }));
});
test('missing predicates cannot become unrestricted writes or disappear inside an IN filter', async (t) => {
const { env } = await fixture(t);
const row = await env.insert({ name: 'retained', value: 'old' });
for (const where of [undefined, null, { id: { $in: [row.id, undefined] } }]) {
await assert.rejects(env.nativeUpdate(where, { value: 'must not write' }));
await assert.rejects(env.nativeDelete(where));
}
assert.equal(await env.count(), 1);
assert.equal((await env.findOne({ id: row.id })).value, 'old');
});
test('worker failures reject writes, never replay them, and roll back SQLite', async (t) => {
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-db-worker-'));
t.after(() => fs.rm(dir, { recursive: true, force: true }));
const file = path.join(dir, 'database.sqlite');
const { db, env } = await fixture(t, file);
await env.insert({ name: 'a', value: 'old' });
await assert.rejects(
db.transaction(async () => {
await env.nativeUpdate({ name: 'a' }, { value: 'uncommitted' });
db.worker.kill('SIGKILL');
await db.stopped;
await env.count();
}),
);
assert.equal(db.healthy, false);
await assert.rejects(env.insert({ name: 'replayed' }));
const reopened = new DatabaseClient(file);
t.after(() => reopened.close());
const records = new Repository('Env', () => reopened);
assert.equal((await records.findOne({ name: 'a' })).value, 'old');
assert.equal(await records.count(), 1);
});
test('a long database query leaves main-thread timers responsive', async (t) => {
const { db } = await fixture(t);
const started = performance.now();
const expensive = db.query(
'WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<2000000) SELECT SUM(x) AS n FROM n',
);
const timer = await new Promise((r) =>
setTimeout(() => r(performance.now() - started), 10),
);
assert(timer < 150, `main-thread timer delayed ${timer}ms`);
assert.equal((await expensive)[0].n, 2000001000000);
});
test('existing Keyv authentication cache stays byte-compatible with legacy readers', async (t) => {
const { DatabaseSync } = require('node:sqlite');
const Keyv = require('keyv').default;
const { CacheStore } = require('../../back/data/cacheStore');
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-cache-compat-'));
t.after(() => fs.rm(dir, { recursive: true, force: true }));
const cacheFile = path.join(dir, 'keyv.sqlite');
const old = new DatabaseSync(cacheFile);
old.exec('CREATE TABLE keyv(key VARCHAR(255) PRIMARY KEY,value TEXT)');
const value = {
username: 'synthetic-user',
password: 'synthetic-hash',
tokens: { desktop: ['keep'] },
};
old
.prepare('INSERT INTO keyv VALUES(?,?)')
.run('keyv:authInfo', JSON.stringify({ value, expires: null }));
old.close();
const db = new DatabaseClient(':memory:', cacheFile);
t.after(() => db.close());
const cache = new Keyv({ store: new CacheStore(() => db) });
assert.deepEqual(await cache.get('authInfo'), value);
const next = { ...value, username: 'new-synthetic-user' };
await cache.set('authInfo', next);
const legacy = new DatabaseSync(cacheFile, { readOnly: true });
try {
assert.deepEqual(
JSON.parse(
legacy
.prepare('SELECT value FROM keyv WHERE key=?')
.get('keyv:authInfo').value,
).value,
next,
);
} finally {
legacy.close();
}
});
test('optional DTO fields and nested JSON values may be undefined on insert', async (t) => {
const { env, auth } = await fixture(t);
const created = await env.insert({
name: 'optional',
value: 'keep',
id: undefined,
remarks: undefined,
});
assert.equal(created.value, 'keep');
const row = await auth.insert({
type: 'notifyLog',
info: { title: 'keep', taskId: undefined },
});
assert.deepEqual(row.info, { title: 'keep' });
});
test('legacy null pin flags sort like zero flags before comparing positions', async (t) => {
const { env } = await fixture(t);
await env.insert({ name: 'null', position: 20, isPinned: null });
await env.insert({ name: 'zero', position: 10, isPinned: 0 });
await env.insert({ name: 'pinned', position: 1, isPinned: 1 });
assert.deepEqual(
(
await env.find(
{},
{ orderBy: [{ isPinned: 'desc' }, { position: 'desc' }] },
)
).map((x) => x.name),
['pinned', 'null', 'zero'],
);
});
test('a stalled transaction callback times out and rolls back before unrelated reads', async (t) => {
const { db, env } = await fixture(t);
db.timeoutMs = 100;
await env.insert({ name: 'a', value: 'old' });
let release;
const gate = new Promise((r) => {
release = r;
});
await assert.rejects(
db.transaction(async () => {
await env.nativeUpdate({ name: 'a' }, { value: 'uncommitted' });
await gate;
await env.insert({ name: 'too-late' });
}),
{ code: 'DATABASE_TRANSACTION_TIMEOUT' },
);
release();
assert.equal((await env.findOne({ name: 'a' })).value, 'old');
assert.equal(await env.count(), 1);
assert.equal(db.healthy, true);
});
test('a stuck database worker is terminated, rejects its queue and never replays writes', async (t) => {
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-db-timeout-'));
t.after(() => fs.rm(dir, { recursive: true, force: true }));
const file = path.join(dir, 'database.sqlite');
const { db, env } = await fixture(t, file);
await env.insert({ name: 'a', value: 'old' });
db.timeoutMs = 100;
await assert.rejects(
db.transaction(async () => {
await env.nativeUpdate({ name: 'a' }, { value: 'uncommitted' });
await db.query(
'WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<200000000) SELECT SUM(x) FROM n',
);
}),
(e) =>
['DATABASE_TRANSACTION_TIMEOUT', 'DATABASE_WORKER_TIMEOUT'].includes(
e.code,
),
);
assert.equal(db.healthy, false);
await assert.rejects(env.insert({ name: 'replayed' }));
await db.close();
const next = new DatabaseClient(file);
t.after(() => next.close());
const records = new Repository('Env', () => next);
assert.equal((await records.findOne({ name: 'a' })).value, 'old');
assert.equal(await records.count(), 1);
});
test('the database queue is bounded while a transaction owns its connection', async (t) => {
const { db, env } = await fixture(t);
db.queueLimit = 2;
db.timeoutMs = 3000;
let entered, release;
const started = new Promise((r) => {
entered = r;
});
const gate = new Promise((r) => {
release = r;
});
const transaction = db.transaction(async () => {
entered();
await gate;
});
await started;
const waiting = env.count();
await assert.rejects(env.count(), { code: 'DATABASE_BUSY' });
release();
await transaction;
assert.equal(await waiting, 0);
});
+2 -9
View File
@@ -1,6 +1,5 @@
const assert = require('node:assert/strict'); const assert = require('node:assert/strict');
const test = require('node:test'); const test = require('node:test');
test('dependency listing marks cache entries missing from disk for reinstall', async (t) => { test('dependency listing marks cache entries missing from disk for reinstall', async (t) => {
const moduleStubs = new Map(); const moduleStubs = new Map();
const stubModule = (modulePath, exports) => { const stubModule = (modulePath, exports) => {
@@ -15,7 +14,6 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
paths: [], paths: [],
}; };
}; };
const DependenceStatus = { const DependenceStatus = {
installing: 0, installing: 0,
installed: 1, installed: 1,
@@ -28,7 +26,7 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
]; ];
const updates = []; const updates = [];
const DependenceModel = { const DependenceModel = {
findAll: async ({ where }) => { find: async (where, read = {}) => {
if (where.status === DependenceStatus.installed) { if (where.status === DependenceStatus.installed) {
return docs; return docs;
} }
@@ -39,11 +37,10 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
: doc.status, : doc.status,
})); }));
}, },
update: async ({ status }, { where }) => { nativeUpdate: async (where, { status }) => {
updates.push({ status, ids: where.id }); updates.push({ status, ids: where.id });
}, },
}; };
stubModule('../../back/config', { stubModule('../../back/config', {
__esModule: true, __esModule: true,
default: {}, default: {},
@@ -83,7 +80,6 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
__esModule: true, __esModule: true,
default: class SockService {}, default: class SockService {},
}); });
const servicePath = require.resolve('../../back/services/dependence'); const servicePath = require.resolve('../../back/services/dependence');
const originalService = require.cache[servicePath]; const originalService = require.cache[servicePath];
delete require.cache[servicePath]; delete require.cache[servicePath];
@@ -101,7 +97,6 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
} }
} }
}); });
const DependenceService = require('../../back/services/dependence').default; const DependenceService = require('../../back/services/dependence').default;
const service = new DependenceService({}, {}); const service = new DependenceService({}, {});
const result = await service.dependencies({ const result = await service.dependencies({
@@ -109,7 +104,6 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
type: 'nodejs', type: 'nodejs',
status: '', status: '',
}); });
assert.deepEqual(updates, [ assert.deepEqual(updates, [
{ status: DependenceStatus.installFailed, ids: [1] }, { status: DependenceStatus.installFailed, ids: [1] },
]); ]);
@@ -120,7 +114,6 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
{ id: 2, status: DependenceStatus.installed }, { id: 2, status: DependenceStatus.installed },
], ],
); );
await service.dependencies({ searchValue: '', type: 'linux', status: '' }); await service.dependencies({ searchValue: '', type: 'linux', status: '' });
assert.equal(updates.length, 1); assert.equal(updates.length, 1);
}); });
+5 -7
View File
@@ -1,7 +1,6 @@
const assert = require('node:assert/strict'); const assert = require('node:assert/strict');
const test = require('node:test'); const test = require('node:test');
const express = require('express'); const express = require('express');
function mockModule(modulePath, exports) { function mockModule(modulePath, exports) {
const filename = require.resolve(modulePath); const filename = require.resolve(modulePath);
require.cache[filename] = { require.cache[filename] = {
@@ -13,7 +12,6 @@ function mockModule(modulePath, exports) {
paths: [], paths: [],
}; };
} }
mockModule('../../back/config', { mockModule('../../back/config', {
__esModule: true, __esModule: true,
default: { default: {
@@ -49,15 +47,17 @@ for (const service of ['config', 'script', 'log']) {
} }
mockModule('../../back/data/runningInstance', { mockModule('../../back/data/runningInstance', {
InstanceStatus: { running: 'running' }, InstanceStatus: { running: 'running' },
RunningInstanceModel: { findOne: async () => null }, RunningInstanceModel: {
findOne: async (where, read = {}) => {
return null;
},
},
}); });
const deprecatedRoutes = [ const deprecatedRoutes = [
['config', '/configs/detail'], ['config', '/configs/detail'],
['script', '/scripts/detail'], ['script', '/scripts/detail'],
['log', '/logs/detail'], ['log', '/logs/detail'],
]; ];
for (const [moduleName, replacement] of deprecatedRoutes) { for (const [moduleName, replacement] of deprecatedRoutes) {
test(`${moduleName} filename route points callers to its detail API`, () => { test(`${moduleName} filename route points callers to its detail API`, () => {
const app = express.Router(); const app = express.Router();
@@ -68,12 +68,10 @@ for (const [moduleName, replacement] of deprecatedRoutes) {
); );
const handler = deprecatedRoute.route.stack.at(-1).handle; const handler = deprecatedRoute.route.stack.at(-1).handle;
let responseBody; let responseBody;
handler( handler(
{ params: { file: 'Example.js' }, query: {} }, { params: { file: 'Example.js' }, query: {} },
{ send: (body) => (responseBody = body) }, { send: (body) => (responseBody = body) },
); );
assert.deepEqual(responseBody, { assert.deepEqual(responseBody, {
code: 410, code: 410,
message: `接口已下线,请使用 ${replacement} 接口`, message: `接口已下线,请使用 ${replacement} 接口`,
+20 -16
View File
@@ -5,7 +5,6 @@ const os = require('node:os');
const path = require('node:path'); const path = require('node:path');
const { execFileSync } = require('node:child_process'); const { execFileSync } = require('node:child_process');
const load = require('../helpers/load-security-module.cjs'); const load = require('../helpers/load-security-module.cjs');
const types = { nodejs: 0, python3: 1, linux: 2 }; const types = { nodejs: 0, python3: 1, linux: 2 };
const statuses = { installed: 1, installFailed: 2, queued: 6, cancelled: 7 }; const statuses = { installed: 1, installFailed: 2, queued: 6, cancelled: 7 };
const util = load('back/config/util.ts', { const util = load('back/config/util.ts', {
@@ -15,7 +14,6 @@ const util = load('back/config/util.ts', {
'../shared/utils': {}, '../shared/utils': {},
'../data/dependence': { DependenceTypes: types }, '../data/dependence': { DependenceTypes: types },
}); });
function serviceFor(docs) { function serviceFor(docs) {
const probes = [], const probes = [],
removed = [], removed = [],
@@ -28,18 +26,21 @@ function serviceFor(docs) {
DependenceStatus: statuses, DependenceStatus: statuses,
versionDependenceCommandTypes: { 0: '@', 1: '==', 2: '=' }, versionDependenceCommandTypes: { 0: '@', 1: '==', 2: '=' },
DependenceModel: { DependenceModel: {
findAll: async ({ where }) => find: async (where, read = {}) => {
docs.filter( return docs.filter(
(d) => (d) =>
(where.status === undefined || d.status === where.status) && (where.status === undefined || d.status === where.status) &&
(!where.id || where.id.includes(d.id)), (!where.id || where.id.includes(d.id)),
), );
update: async (values, { where }) => { },
nativeUpdate: async (where, values) => {
docs docs
.filter((d) => where.id.includes(d.id)) .filter((d) => where.id.includes(d.id))
.forEach((d) => Object.assign(d, values)); .forEach((d) => Object.assign(d, values));
}, },
destroy: async ({ where }) => removed.push(...where.id), nativeDelete: async (where, read = {}) => {
return removed.push(...where.id);
},
}, },
}, },
'../config/util': { '../config/util': {
@@ -62,7 +63,6 @@ function serviceFor(docs) {
}; };
return { service, probes, removed, uninstalls }; return { service, probes, removed, uninstalls };
} }
test('listing preserves Python source results and still detects missing named packages', async () => { test('listing preserves Python source results and still detects missing named packages', async () => {
const docs = [ const docs = [
' requirements with spaces/requirements.txt ', ' requirements with spaces/requirements.txt ',
@@ -88,7 +88,6 @@ test('listing preserves Python source results and still detects missing named pa
// Source installs must continue to execute pip rather than treating the source as a package name. // Source installs must continue to execute pip rather than treating the source as a package name.
assert.equal(util.getGetCommand(1, docs[0].name), 'echo ""'); assert.equal(util.getGetCommand(1, docs[0].name), 'echo ""');
}); });
test('deletion cleans previously misclassified Python sources and only queues intended records', async () => { test('deletion cleans previously misclassified Python sources and only queues intended records', async () => {
const docs = [ const docs = [
{ id: 1, name: 'requirements.txt', type: 1, status: 2 }, { id: 1, name: 'requirements.txt', type: 1, status: 2 },
@@ -106,7 +105,6 @@ test('deletion cleans previously misclassified Python sources and only queues in
]); ]);
assert.equal(docs[2].status, 2); assert.equal(docs[2].status, 2);
}); });
test('Alpine container mirror command runs without sudo and writes valid repositories', async (t) => { test('Alpine container mirror command runs without sudo and writes valid repositories', async (t) => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-alpine-mirror-')); const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-alpine-mirror-'));
t.after(() => fs.rmSync(root, { recursive: true, force: true })); t.after(() => fs.rmSync(root, { recursive: true, force: true }));
@@ -151,7 +149,6 @@ test('Alpine container mirror command runs without sudo and writes valid reposit
), ),
); );
}); });
test('force deletion removes a source record even when its requirements file is gone', async () => { test('force deletion removes a source record even when its requirements file is gone', async () => {
const removed = [], const removed = [],
updates = []; updates = [];
@@ -167,8 +164,12 @@ test('force deletion removes a source record even when its requirements file is
DependenceTypes: types, DependenceTypes: types,
DependenceStatus: { ...statuses, removeFailed: 5 }, DependenceStatus: { ...statuses, removeFailed: 5 },
DependenceModel: { DependenceModel: {
destroy: async ({ where }) => removed.push(...where.id), nativeDelete: async (where, read = {}) => {
update: async (values) => updates.push(values), return removed.push(...where.id);
},
nativeUpdate: async (where, values) => {
return updates.push(values);
},
}, },
}, },
'../shared/pLimit': { '../shared/pLimit': {
@@ -186,7 +187,6 @@ test('force deletion removes a source record even when its requirements file is
await service.installOrUninstallDependency(dep, false, true); await service.installOrUninstallDependency(dep, false, true);
assert.deepEqual(removed, [1]); assert.deepEqual(removed, [1]);
}); });
test('Python extras run pip even when the base package version is already installed', async (t) => { test('Python extras run pip even when the base package version is already installed', async (t) => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-extra-install-')); const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-extra-install-'));
t.after(() => fs.rmSync(root, { recursive: true, force: true })); t.after(() => fs.rmSync(root, { recursive: true, force: true }));
@@ -217,8 +217,12 @@ test('Python extras run pip even when the base package version is already instal
DependenceStatus: { ...statuses, installing: 0, cancelled: 7 }, DependenceStatus: { ...statuses, installing: 0, cancelled: 7 },
versionDependenceCommandTypes: { 0: '@', 1: '==', 2: '=' }, versionDependenceCommandTypes: { 0: '@', 1: '==', 2: '=' },
DependenceModel: { DependenceModel: {
findAll: async () => [dep], find: async (where, read = {}) => {
update: async (values) => Object.assign(dep, values), return [dep];
},
nativeUpdate: async (where, values) => {
return Object.assign(dep, values);
},
}, },
}, },
'../shared/pLimit': { '../shared/pLimit': {
+5 -10
View File
@@ -9,10 +9,8 @@ const path = require('node:path');
const load = require('../helpers/load-security-module.cjs'); const load = require('../helpers/load-security-module.cjs');
const { observeChildProcess } = require('../../back/shared/childProcess'); const { observeChildProcess } = require('../../back/shared/childProcess');
const { LogStreamManager } = require('../../back/shared/logStreamManager'); const { LogStreamManager } = require('../../back/shared/logStreamManager');
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
const logger = { info() {}, error() {} }; const logger = { info() {}, error() {} };
test( test(
'spawn failure settles without exit, and the next scheduled run can proceed', 'spawn failure settles without exit, and the next scheduled run can proceed',
{ timeout: 3000 }, { timeout: 3000 },
@@ -36,7 +34,11 @@ test(
}, },
'../loaders/logger': logger, '../loaders/logger': logger,
'../data/cron': { '../data/cron': {
CrontabModel: { findOne: async () => ({ isDisabled: 0 }) }, CrontabModel: {
findOne: async (where, read = {}) => {
return { isDisabled: 0 };
},
},
CrontabStatus: {}, CrontabStatus: {},
}, },
'../data/runningInstance': { '../data/runningInstance': {
@@ -51,7 +53,6 @@ test(
assert.equal(releases, 2); assert.equal(releases, 2);
}, },
); );
test( test(
'completion waits for slow log consumers and preserves the final output', 'completion waits for slow log consumers and preserves the final output',
{ timeout: 5000 }, { timeout: 5000 },
@@ -79,7 +80,6 @@ test(
assert.match(chunks.join(''), /错误/); assert.match(chunks.join(''), /错误/);
}, },
); );
test('exit is not completion, and UTF-8 split across writes remains intact', async () => { test('exit is not completion, and UTF-8 split across writes remains intact', async () => {
const child = new EventEmitter(); const child = new EventEmitter();
child.stdout = new PassThrough(); child.stdout = new PassThrough();
@@ -105,7 +105,6 @@ test('exit is not completion, and UTF-8 split across writes remains intact', asy
await observed.completed; await observed.completed;
assert.equal(chunks.join(''), '末尾中文'); assert.equal(chunks.join(''), '末尾中文');
}); });
test( test(
'failed log sink drains large output instead of blocking the child', 'failed log sink drains large output instead of blocking the child',
{ timeout: 5000 }, { timeout: 5000 },
@@ -124,7 +123,6 @@ test(
assert.equal(result.error.message, 'ENOSPC'); assert.equal(result.error.message, 'ENOSPC');
}, },
); );
function scheduleFixture() { function scheduleFixture() {
let active = 0; let active = 0;
const limit = async (_params, fn) => { const limit = async (_params, fn) => {
@@ -150,7 +148,6 @@ function scheduleFixture() {
}).default; }).default;
return { service: new Schedule(logger), active: () => active }; return { service: new Schedule(logger), active: () => active };
} }
test( test(
'PID response keeps queue capacity occupied until cleanup completes', 'PID response keeps queue capacity occupied until cleanup completes',
{ timeout: 4000 }, { timeout: 4000 },
@@ -178,7 +175,6 @@ test(
assert.equal(fixture.active(), 0); assert.equal(fixture.active(), 0);
}, },
); );
test( test(
'before/error/end callback failures settle and release capacity', 'before/error/end callback failures settle and release capacity',
{ timeout: 3000 }, { timeout: 3000 },
@@ -207,7 +203,6 @@ test(
assert.equal(fixture.active(), 0); assert.equal(fixture.active(), 0);
}, },
); );
test( test(
'concurrent writes and closes preserve all bytes, and failed files can be closed', 'concurrent writes and closes preserve all bytes, and failed files can be closed',
{ timeout: 5000 }, { timeout: 5000 },
+5 -5
View File
@@ -5,7 +5,6 @@ const os = require('node:os');
const path = require('node:path'); const path = require('node:path');
const express = require('express'); const express = require('express');
const load = require('../helpers/load-security-module.cjs'); const load = require('../helpers/load-security-module.cjs');
function fixture(t, customData = false) { function fixture(t, customData = false) {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-access-scope-')); const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-access-scope-'));
t.after(() => fs.rmSync(root, { recursive: true, force: true })); t.after(() => fs.rmSync(root, { recursive: true, force: true }));
@@ -46,12 +45,15 @@ function fixture(t, customData = false) {
'../config/const': {}, '../config/const': {},
'../data/runningInstance': { '../data/runningInstance': {
InstanceStatus: { running: 'running' }, InstanceStatus: { running: 'running' },
RunningInstanceModel: { findOne: async () => null }, RunningInstanceModel: {
findOne: async (where, read = {}) => {
return null;
},
},
}, },
}; };
return { config, mocks, secret }; return { config, mocks, secret };
} }
function api(name, service, mocks) { function api(name, service, mocks) {
mocks.typedi = { Container: { get: () => service } }; mocks.typedi = { Container: { get: () => service } };
const app = express.Router(); const app = express.Router();
@@ -79,7 +81,6 @@ function api(name, service, mocks) {
return result; return result;
}; };
} }
for (const customData of [false, true]) { for (const customData of [false, true]) {
test(`config APIs reject script compatibility paths (custom=${customData})`, async (t) => { test(`config APIs reject script compatibility paths (custom=${customData})`, async (t) => {
const { config, mocks, secret } = fixture(t, customData); const { config, mocks, secret } = fixture(t, customData);
@@ -144,7 +145,6 @@ for (const customData of [false, true]) {
assert.equal(fs.readFileSync(secret, 'utf8'), 'PANEL-SECRET'); assert.equal(fs.readFileSync(secret, 'utf8'), 'PANEL-SECRET');
}); });
} }
test('log APIs allow configuration-like names while retaining directory boundaries', async (t) => { test('log APIs allow configuration-like names while retaining directory boundaries', async (t) => {
const { config, mocks, secret } = fixture(t); const { config, mocks, secret } = fixture(t);
const Log = load(path.resolve('back/services/log.ts'), mocks).default; const Log = load(path.resolve('back/services/log.ts'), mocks).default;
+5 -20
View File
@@ -2,7 +2,7 @@ const test = require('node:test');
const assert = require('node:assert/strict'); const assert = require('node:assert/strict');
const fs = require('node:fs'); const fs = require('node:fs');
const ts = require('typescript'); const ts = require('typescript');
const { Sequelize, DataTypes, Model } = require('sequelize');
const load = require('../helpers/load-security-module.cjs'); const load = require('../helpers/load-security-module.cjs');
const { CronResponse } = load('back/protos/api.ts'); const { CronResponse } = load('back/protos/api.ts');
const source = fs.readFileSync('back/schedule/api.ts', 'utf8'); const source = fs.readFileSync('back/schedule/api.ts', 'utf8');
@@ -11,31 +11,16 @@ const end = source.indexOf('export const getCronDetail', start);
const code = ts.transpileModule(source.slice(start, end), { const code = ts.transpileModule(source.slice(start, end), {
compilerOptions: { target: ts.ScriptTarget.ES2020 }, compilerOptions: { target: ts.ScriptTarget.ES2020 },
}).outputText; }).outputText;
const normalize = new Function('Model', `${code}\nreturn normalizeCronData;`)( const normalize = new Function(`${code}\nreturn normalizeCronData;`)();
Model,
);
test('gRPC cron response preserves Sequelize model fields through protobuf serialization', async (t) => { test('gRPC cron response preserves plain repository fields through protobuf serialization', async (t) => {
const db = new Sequelize({ const model = {
dialect: 'sqlite',
storage: ':memory:',
logging: false,
});
t.after(() => db.close());
const Cron = db.define('Cron', {
name: DataTypes.STRING,
command: DataTypes.STRING,
schedule: DataTypes.STRING,
labels: DataTypes.JSON,
extra_schedules: DataTypes.JSON,
});
const model = Cron.build({
id: 7, id: 7,
name: 'smoke', name: 'smoke',
command: 'true', command: 'true',
schedule: '* * * * *', schedule: '* * * * *',
labels: ['test'], labels: ['test'],
}); };
const decoded = CronResponse.decode( const decoded = CronResponse.decode(
CronResponse.encode({ code: 200, data: normalize(model) }).finish(), CronResponse.encode({ code: 200, data: normalize(model) }).finish(),
); );
+1 -5
View File
@@ -5,7 +5,6 @@ const os = require('node:os');
const path = require('node:path'); const path = require('node:path');
const load = require('../helpers/load-security-module.cjs'); const load = require('../helpers/load-security-module.cjs');
const { LogStreamManager } = require('../../back/shared/logStreamManager'); const { LogStreamManager } = require('../../back/shared/logStreamManager');
async function fixture(t) { async function fixture(t) {
const base = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-log-boundary-')); const base = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-log-boundary-'));
t.after(() => fs.rm(base, { recursive: true, force: true })); t.after(() => fs.rm(base, { recursive: true, force: true }));
@@ -31,7 +30,6 @@ async function fixture(t) {
]; ];
return { root, outside, victim, invalid }; return { root, outside, victim, invalid };
} }
test('log streams reject traversal, sibling prefixes and escaping symlinks before writing', async (t) => { test('log streams reject traversal, sibling prefixes and escaping symlinks before writing', async (t) => {
const { root, outside, victim, invalid } = await fixture(t); const { root, outside, victim, invalid } = await fixture(t);
const manager = new LogStreamManager(root); const manager = new LogStreamManager(root);
@@ -58,7 +56,6 @@ test('log streams reject traversal, sibling prefixes and escaping symlinks befor
await manager.closeAll(); await manager.closeAll();
assert.equal(await fs.readFile(log, 'utf8'), '开始\n结束\n'); assert.equal(await fs.readFile(log, 'utf8'), '开始\n结束\n');
}); });
test('log initialization rejects unsafe paths before mkdir or file writes', async (t) => { test('log initialization rejects unsafe paths before mkdir or file writes', async (t) => {
const { root, outside, victim, invalid } = await fixture(t); const { root, outside, victim, invalid } = await fixture(t);
const { handleLogPath } = load(path.resolve('back/config/util.ts'), { const { handleLogPath } = load(path.resolve('back/config/util.ts'), {
@@ -87,7 +84,6 @@ test('log initialization rejects unsafe paths before mkdir or file writes', asyn
assert.equal(await handleLogPath(log, 'ignored'), log); assert.equal(await handleLogPath(log, 'ignored'), log);
assert.equal(await fs.readFile(log, 'utf8'), 'initial'); assert.equal(await fs.readFile(log, 'utf8'), 'initial');
}); });
test('manual execution rejects escaping log names before creating directories or spawning', async (t) => { test('manual execution rejects escaping log names before creating directories or spawning', async (t) => {
const { root, outside, victim } = await fixture(t); const { root, outside, victim } = await fixture(t);
let spawned = 0; let spawned = 0;
@@ -97,7 +93,7 @@ test('manual execution rejects escaping log names before creating directories or
'../config': { logPath: root }, '../config': { logPath: root },
'../data/cron': { '../data/cron': {
CrontabStatus: { queued: 3, idle: 1 }, CrontabStatus: { queued: 3, idle: 1 },
CrontabModel: { update: async () => {} }, CrontabModel: { nativeUpdate: async (where, data) => {} },
}, },
'../data/runningInstance': { RunningInstanceModel: {}, InstanceStatus: {} }, '../data/runningInstance': { RunningInstanceModel: {}, InstanceStatus: {} },
'../config/util': {}, '../config/util': {},
+5 -3
View File
@@ -6,7 +6,6 @@ const os = require('node:os');
const { spawn } = require('node:child_process'); const { spawn } = require('node:child_process');
const load = require('../helpers/load-security-module.cjs'); const load = require('../helpers/load-security-module.cjs');
const { LogStreamManager } = require('../../back/shared/logStreamManager'); const { LogStreamManager } = require('../../back/shared/logStreamManager');
test( test(
'manual execution captures a short child before slow status storage and flushes before release', 'manual execution captures a short child before slow status storage and flushes before release',
{ timeout: 5000 }, { timeout: 5000 },
@@ -22,10 +21,13 @@ test(
'../data/cron': { '../data/cron': {
CrontabStatus: { queued: 3, running: 0, idle: 1 }, CrontabStatus: { queued: 3, running: 0, idle: 1 },
CrontabModel: { CrontabModel: {
update: async (data, options) => { nativeUpdate: async (where, data) => {
const options = {
where: where,
};
await new Promise((resolve) => setTimeout(resolve, 40)); await new Promise((resolve) => setTimeout(resolve, 40));
updates.push({ data, options }); updates.push({ data, options });
return [1]; return 1;
}, },
}, },
}, },
+38 -43
View File
@@ -1,3 +1,6 @@
const { DatabaseClient } = require('../../back/data/databaseClient');
const { repository, initialize } = require('../helpers/repository-fixture.cjs');
const { orderBy } = require('../../back/data/queries');
const test = require('node:test'); const test = require('node:test');
const assert = require('node:assert/strict'); const assert = require('node:assert/strict');
const fs = require('node:fs/promises'); const fs = require('node:fs/promises');
@@ -9,36 +12,21 @@ const { Sequelize, DataTypes } = require('sequelize');
const load = require('../helpers/load-security-module.cjs'); const load = require('../helpers/load-security-module.cjs');
const { killTask } = require('../../back/config/util'); const { killTask } = require('../../back/config/util');
const { LogStreamManager } = require('../../back/shared/logStreamManager'); const { LogStreamManager } = require('../../back/shared/logStreamManager');
for (const conflict of ['stop', 'newer-queue', 'stop-requeue']) { for (const conflict of ['stop', 'newer-queue', 'stop-requeue']) {
test( test(
`manual startup loses its conditional claim after ${conflict} and terminates the late child`, `manual startup loses its conditional claim after ${conflict} and terminates the late child`,
{ timeout: 10000 }, { timeout: 10000 },
async (t) => { async (t) => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-manual-claim-')); const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-manual-claim-'));
const db = new Sequelize({ const db = new DatabaseClient(':memory:');
dialect: 'sqlite',
storage: ':memory:',
logging: false,
});
t.after(async () => { t.after(async () => {
await db.close(); await db.close();
await fs.rm(root, { recursive: true, force: true }); await fs.rm(root, { recursive: true, force: true });
}); });
const crons = db.define('Cron', { const crons = repository(db, 'Crontab');
status: DataTypes.INTEGER, const instances = repository(db, 'RunningInstance');
pid: DataTypes.INTEGER, await initialize(db);
log_path: DataTypes.STRING, await crons.insert({
queued_token: DataTypes.STRING,
command: DataTypes.STRING,
});
const instances = db.define('Instance', {
cron_id: DataTypes.INTEGER,
status: DataTypes.INTEGER,
pid: DataTypes.INTEGER,
});
await db.sync();
await crons.create({
id: 1, id: 1,
status: 3, status: 3,
pid: null, pid: null,
@@ -102,7 +90,9 @@ for (const conflict of ['stop', 'newer-queue', 'stop-requeue']) {
}).default; }).default;
const service = new CronService({ info() {}, error() {} }); const service = new CronService({ info() {}, error() {} });
service.getDb = async () => service.getDb = async () =>
(await crons.findByPk(1)).get({ plain: true }); await crons.findOne({
id: 1,
});
service.makeCommand = () => 'ignored'; service.makeCommand = () => 'ignored';
const running = service.runSingle(1); const running = service.runSingle(1);
await pending; await pending;
@@ -115,16 +105,22 @@ for (const conflict of ['stop', 'newer-queue', 'stop-requeue']) {
assert.equal(pendingRuns.length, 1); assert.equal(pendingRuns.length, 1);
assert.equal( assert.equal(
pendingRuns[0].token, pendingRuns[0].token,
(await crons.findByPk(1)).queued_token, (
await crons.findOne({
id: 1,
})
).queued_token,
); );
} else if (conflict === 'stop') await service.stop([1]); } else if (conflict === 'stop') await service.stop([1]);
else await crons.update({ log_path: 'newer.log' }, { where: { id: 1 } }); else await crons.nativeUpdate({ id: 1 }, { log_path: 'newer.log' });
release(); release();
const result = await running; const result = await running;
assert.match(result.error.message, /stopped or superseded/); assert.match(result.error.message, /stopped or superseded/);
assert.ok(child.signalCode || child.exitCode !== null); assert.ok(child.signalCode || child.exitCode !== null);
assert.throws(() => process.kill(child.pid, 0), { code: 'ESRCH' }); assert.throws(() => process.kill(child.pid, 0), { code: 'ESRCH' });
const row = await crons.findByPk(1); const row = await crons.findOne({
id: 1,
});
assert.equal(row.status, conflict === 'stop' ? 1 : 3); assert.equal(row.status, conflict === 'stop' ? 1 : 3);
assert.equal( assert.equal(
row.log_path, row.log_path,
@@ -135,7 +131,6 @@ for (const conflict of ['stop', 'newer-queue', 'stop-requeue']) {
}, },
); );
} }
test( test(
'verified termination waits for a SIGTERM-resistant target to exit', 'verified termination waits for a SIGTERM-resistant target to exit',
{ timeout: 5000 }, { timeout: 5000 },
@@ -158,22 +153,12 @@ test(
assert.equal(child.signalCode, 'SIGKILL'); assert.equal(child.signalCode, 'SIGKILL');
}, },
); );
test('a runner waiting for a concurrency slot cannot adopt a newer queued generation', async (t) => { test('a runner waiting for a concurrency slot cannot adopt a newer queued generation', async (t) => {
const db = new Sequelize({ const db = new DatabaseClient(':memory:');
dialect: 'sqlite',
storage: ':memory:',
logging: false,
});
t.after(() => db.close()); t.after(() => db.close());
const crons = db.define('Cron', { const crons = repository(db, 'Crontab');
status: DataTypes.INTEGER, await initialize(db);
pid: DataTypes.INTEGER, await crons.insert({ id: 1, status: 1, log_path: 'same.log' });
log_path: DataTypes.STRING,
queued_token: DataTypes.STRING,
});
await db.sync();
await crons.create({ id: 1, status: 1, log_path: 'same.log' });
const waiting = []; const waiting = [];
const CronService = load('back/services/cron.ts', { const CronService = load('back/services/cron.ts', {
'../config': {}, '../config': {},
@@ -182,7 +167,7 @@ test('a runner waiting for a concurrency slot cannot adopt a newer queued genera
CrontabStatus: { queued: 3, idle: 1, running: 0 }, CrontabStatus: { queued: 3, idle: 1, running: 0 },
}, },
'../data/runningInstance': { '../data/runningInstance': {
RunningInstanceModel: { findAll: async () => [] }, RunningInstanceModel: { find: async () => [] },
InstanceStatus: { running: 0 }, InstanceStatus: { running: 0 },
}, },
'../config/util': { '../config/util': {
@@ -205,14 +190,24 @@ test('a runner waiting for a concurrency slot cannot adopt a newer queued genera
const errors = []; const errors = [];
const service = new CronService({ error: (...args) => errors.push(args) }); const service = new CronService({ error: (...args) => errors.push(args) });
await service.run([1]); await service.run([1]);
const first = (await crons.findByPk(1)).queued_token; const first = (
await crons.findOne({
id: 1,
})
).queued_token;
await service.stop([1]); await service.stop([1]);
await service.run([1]); await service.run([1]);
const second = (await crons.findByPk(1)).queued_token; const second = (
await crons.findOne({
id: 1,
})
).queued_token;
assert.notEqual(first, second); assert.notEqual(first, second);
assert.equal(waiting.length, 2); assert.equal(waiting.length, 2);
await waiting[0](); await waiting[0]();
const row = await crons.findByPk(1); const row = await crons.findOne({
id: 1,
});
assert.equal(row.status, 3); assert.equal(row.status, 3);
assert.equal(row.queued_token, second); assert.equal(row.queued_token, second);
assert.equal(row.log_path, 'same.log'); assert.equal(row.log_path, 'same.log');

Some files were not shown because too many files have changed in this diff Show More