Compare commits

..
Author SHA1 Message Date
whyour c9f1b2d4bb fix: preserve legacy SQLite addons loaded from build directories 2026-10-23 14:33:13 +08:00
whyour 5fdfd9077d fix: clean SQLite compiler artifacts before publishing repaired dependencies 2026-10-23 14:15:08 +08:00
whyour ce2e396b3e fix: remove legacy SQLite compiler artifacts from migration bundles 2026-10-23 14:04:42 +08:00
whyour 49cec13fcd test: classify subprocess crashes without exposing credentials 2026-10-23 13:50:37 +08:00
whyour 81de8f7094 fix: accept ARM1176-compatible ELF architecture attributes 2026-10-23 13:10:42 +08:00
whyour dce4f60c47 fix: restore locked build tools for the ARMv6 migration bridge 2026-10-23 12:16:28 +08:00
whyour 1623e89a71 fix: rebuild legacy migration SQLite for ARMv6 on the builder 2026-10-23 11:54:52 +08:00
whyour 1a49e81ffd fix: prevent second-hop migration from returning to a legacy backend 2026-10-23 11:08:20 +08:00
whyour 656b9419d6 test: budget startup for minimum-Node concurrent cache workers 2026-10-23 10:51:16 +08:00
whyour 8d2f67e263 Merge QEMU Node probe timeout fix into migration review 2026-10-23 10:40:27 +08:00
whyour 2a32f528f1 fix: initialize sample environment in migration candidate tests 2026-10-23 10:37:30 +08:00
whyour 384ce170c7 feat: bridge legacy panel upgrades in two verified steps 2026-10-23 10:33:00 +08:00
whyour cc001ecebd fix: allow emulated architectures time for Node URL probes 2026-10-23 01:53:28 +08:00
whyour 04c8a63518 Merge Alpine Node runtime fix into migration review branch 2026-10-23 01:49:18 +08:00
whyour 8976447b1e fix: pin Alpine amd64 Node runtime and verify URL recovery 2026-10-23 01:43:56 +08:00
whyour 0a586fbd1d fix: preserve newest-first execution instance ordering 2026-10-23 01:42:58 +08:00
whyour 15d54866a8 feat: migrate SQLite runtime and verify complete upgrade bundles
Use MikroORM's official Node SQLite dialect with bounded database isolation, preserve legacy schema and data for rollback, and require Node 22.17 or newer. Update Debian runtimes and retain ARMv6 musl compatibility without a SQLite addon.

Complete short archive writes, stop orphaned SQL workers after owner death, preserve buffered startup requests, and prestart the archived backend on every platform. Cover real package installation, migration, failure recovery, and legacy database compatibility.

Keep authentication storage failures distinct from expired sessions with a pinned Keyv runtime, propagated cache errors, and retryable HTTP 503 responses. Preserve genuine 401 failures and verify legacy cache bytes, initialization safety, and session recovery.
2026-10-22 13:47:26 +08:00
134 changed files with 8422 additions and 3569 deletions
+26 -12
View File
@@ -550,10 +550,14 @@ jobs:
cache-from: type=registry,ref=whyour/qinglong:cache-debian-python3.10
cache-to: type=registry,ref=whyour/qinglong:cache-debian-python3.10,mode=max
# One unmodified amd64 package for migration acceptance only. This does not
# Unmodified amd64 packages for glibc and musl migration acceptance. This does not
# publish bundles or advance development/formal channel indexes.
build-npm-migration-bundle:
needs: build-debian
needs: [build-debian, build-alpine]
strategy:
fail-fast: false
matrix:
libc: [glibc, musl]
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
@@ -561,7 +565,7 @@ jobs:
- uses: docker/setup-buildx-action@v4
- name: Export the official platform package from this run's image digest
env:
IMAGE_DIGEST: ${{ needs.build-debian.outputs.image-digest }}
IMAGE_DIGEST: ${{ matrix.libc == 'musl' && needs.build-alpine.outputs.image-digest || needs.build-debian.outputs.image-digest }}
run: |
set -euo pipefail
[[ "$IMAGE_DIGEST" =~ ^sha256:[a-f0-9]{64}$ ]]
@@ -569,20 +573,21 @@ jobs:
docker buildx build --platform linux/amd64 --build-arg "BASE_IMAGE=$image" --file docker/Dockerfile.upgrade-bundle --output "type=local,dest=$RUNNER_TEMP/npm-migration-bundle" .
- uses: actions/upload-artifact@v6
with:
name: npm-migration-bundle-${{ github.sha }}
name: npm-migration-bundle-${{ matrix.libc }}-${{ github.sha }}
path: ${{ runner.temp }}/npm-migration-bundle/*
if-no-files-found: error
retention-days: 3
verify-npm-package:
needs: [build-static, build-npm-migration-bundle]
needs: [build-static, build-npm-migration-bundle, build-alpine]
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
node: ['20.18.1', '24']
container: node:${{ matrix.node }}-bookworm
node: ['20.18.1', '22.17.0', '24', 'alpine']
container:
image: ${{ matrix.node == 'alpine' && format('ghcr.io/{0}@{1}', github.repository, needs.build-alpine.outputs.image-digest) || format('node:{0}-bookworm', matrix.node) }}
steps:
- uses: actions/checkout@v6
- uses: actions/download-artifact@v6
@@ -591,22 +596,30 @@ jobs:
path: ${{ runner.temp }}/npm-panel/
- uses: actions/download-artifact@v6
with:
name: npm-migration-bundle-${{ github.sha }}
name: npm-migration-bundle-${{ matrix.node == 'alpine' && 'musl' || 'glibc' }}-${{ github.sha }}
path: ${{ runner.temp }}/npm-migration-bundle/
- name: Install isolated service test tools
run: |
apt-get update
apt-get install -y --no-install-recommends python3 openssl curl jq nginx procps
if [ -f /etc/alpine-release ]; then
apk add --no-cache python3 openssl curl jq nginx procps make g++
else
apt-get update
apt-get install -y --no-install-recommends python3 openssl curl jq nginx procps
fi
# The published legacy launcher resets PATH to standard directories.
# /usr/local is confined to this disposable job container; PM2_HOME
# remains private to each verification run.
npm install --global --prefix /usr/local --no-audit --no-fund pm2@6.0.14
- name: Install and upgrade the exact npm archive
if: matrix.node != '20.18.1'
env:
QL_PANEL_PACKAGE_TEST_ISOLATED: '1'
QL_PANEL_TEST_PM2: /usr/local/bin/pm2
QL_PANEL_TEST_BUNDLE_DIR: ${{ runner.temp }}/npm-migration-bundle
run: node scripts/verify-panel-package.cjs verify "$RUNNER_TEMP/npm-panel" "$RUNNER_TEMP/npm-panel-results"
- name: Reject an unsupported runtime while preserving the old installation
if: matrix.node == '20.18.1'
run: node scripts/verify-panel-runtime-floor.cjs "$RUNNER_TEMP/npm-panel" "$RUNNER_TEMP/npm-panel-results"
- uses: actions/upload-artifact@v6
with:
name: qinglong-npm-verification-${{ matrix.node }}-${{ github.sha }}
@@ -701,7 +714,8 @@ jobs:
if [ "$BUNDLE_LIBC" = glibc ]; then digest="$DEBIAN_DIGEST"; fi
test -n "$digest"
image="ghcr.io/${GITHUB_REPOSITORY}@${digest}"
docker buildx build --platform "$BUNDLE_PLATFORM" --build-arg "BASE_IMAGE=$image" --file docker/Dockerfile.upgrade-bundle --output type=local,dest=output/upgrade .
legacy=$(node -p "require('./docker/migration-bridge-images.json')['$BUNDLE_LIBC']")
docker buildx build --platform "$BUNDLE_PLATFORM" --build-arg "BASE_IMAGE=$image" --build-arg "LEGACY_IMAGE=$legacy" --file docker/Dockerfile.upgrade-bridge --output type=local,dest=output/upgrade .
- uses: actions/upload-artifact@v6
with:
name: upgrade-${{ matrix.name }}-${{ github.sha }}
@@ -721,7 +735,7 @@ jobs:
- uses: actions/checkout@v6
- uses: actions/setup-node@v6
with:
node-version: '20'
node-version: '24'
- uses: pnpm/action-setup@v6
with:
version: '8.3.1'
+10 -44
View File
@@ -6,11 +6,6 @@ on:
- '.github/workflows/release-readiness.yml'
- 'package.json'
- 'pnpm-lock.yaml'
- 'docker/Dockerfile*'
- 'shell/node-runtime-probe.cjs'
- 'scripts/verify-panel-package.cjs'
- 'scripts/verify-image-node-layers.py'
- 'test/back/node-runtime-probe.test.cjs'
workflow_dispatch:
permissions:
@@ -36,7 +31,7 @@ jobs:
version: '8.3.1'
- uses: actions/setup-node@v6
with:
node-version: '20'
node-version: '22'
cache: pnpm
- run: pnpm install --frozen-lockfile
- run: pnpm build:front && pnpm build:back && pnpm build:info
@@ -46,13 +41,6 @@ jobs:
path: static/
include-hidden-files: true
if-no-files-found: error
- name: Pack the verified npm archive for isolated runtime validation
run: node scripts/verify-panel-package.cjs pack "$RUNNER_TEMP/npm-panel"
- uses: actions/upload-artifact@v6
with:
name: release-readiness-npm-panel
path: ${{ runner.temp }}/npm-panel/
if-no-files-found: error
images:
needs: [regression, static]
@@ -132,39 +120,17 @@ jobs:
run: |
docker run --rm --network none --entrypoint node \
qinglong-readiness:${{ matrix.image }} \
-e 'const assert=require("node:assert/strict"); const {createRequire}=require("node:module"); for(const [name,want] of Object.entries({"@grpc/grpc-js":"1.14.5",compression:"1.8.2",multer:"2.4.0",sequelize:"6.37.8"}))assert.equal(require(name+"/package.json").version,want); const expressRequire=createRequire(require.resolve("express")); assert.equal(expressRequire("proxy-addr/package.json").version,"2.0.8"); const sockRequire=createRequire(require.resolve("sockjs")); assert.equal(sockRequire("websocket-driver/package.json").version,"0.7.5"); require("sqlite3"); require("nodemailer"); require("undici"); console.log("Minimal production patches and runtime imports verified")'
- name: Export ARMv6 image layers for Node regression checks
if: startsWith(matrix.image, 'alpine')
uses: docker/build-push-action@v7
with:
context: .
file: ${{ matrix.file }}
platforms: linux/arm/v6
build-args: |
QL_URL=${{ steps.source.outputs.url }}
QL_BRANCH=${{ steps.source.outputs.branch }}
SOURCE_COMMIT=${{ steps.source.outputs.sha }}
outputs: type=docker,dest=${{ runner.temp }}/node-layers-arm.tar
push: false
provenance: false
sbom: false
- name: Check Node binaries in every serialized image layer
if: startsWith(matrix.image, 'alpine')
-e 'const assert=require("node:assert/strict"); const {createRequire}=require("node:module"); for(const [name,want] of Object.entries({"@grpc/grpc-js":"1.14.5",compression:"1.8.2",multer:"2.4.0"}))assert.equal(require(name+"/package.json").version,want); const expressRequire=createRequire(require.resolve("express")); assert.equal(expressRequire("proxy-addr/package.json").version,"2.0.8"); const sockRequire=createRequire(require.resolve("sockjs")); assert.equal(sockRequire("websocket-driver/package.json").version,"0.7.5"); assert.equal(typeof require("@mikro-orm/sql").NodeSqliteDialect,"function"); const db=new (require("node:sqlite").DatabaseSync)(":memory:"); assert.equal(db.prepare("SELECT 42 AS value").get().value,42); db.close(); require("nodemailer"); require("undici"); console.log("Minimal production patches and runtime imports verified")'
- name: Export native Debian image for isolated dev validation
if: matrix.image == 'debian'
run: |
docker save qinglong-readiness:${{ matrix.image }} -o "$RUNNER_TEMP/node-layers-amd64.tar"
python3 scripts/verify-image-node-layers.py "$RUNNER_TEMP/node-layers-amd64.tar" amd64
python3 scripts/verify-image-node-layers.py "$RUNNER_TEMP/node-layers-arm.tar" arm
rm "$RUNNER_TEMP/node-layers-amd64.tar" "$RUNNER_TEMP/node-layers-arm.tar"
- name: Export native image for isolated dev validation
if: matrix.image == 'debian' || matrix.image == 'alpine'
run: |
docker save qinglong-readiness:${{ matrix.image }} | gzip -1 > release-readiness-${{ matrix.image }}-image.tar.gz
sha256sum release-readiness-${{ matrix.image }}-image.tar.gz > release-readiness-${{ matrix.image }}-image.sha256
docker image inspect qinglong-readiness:${{ matrix.image }} > release-readiness-${{ matrix.image }}-image.json
docker save qinglong-readiness:debian | gzip -1 > release-readiness-debian-image.tar.gz
sha256sum release-readiness-debian-image.tar.gz > release-readiness-debian-image.sha256
docker image inspect qinglong-readiness:debian > release-readiness-debian-image.json
- uses: actions/upload-artifact@v6
if: matrix.image == 'debian' || matrix.image == 'alpine'
if: matrix.image == 'debian'
with:
name: release-readiness-${{ matrix.image }}-image
path: release-readiness-${{ matrix.image }}-image.*
name: release-readiness-debian-image
path: release-readiness-debian-image.*
compression-level: 0
if-no-files-found: error
@@ -0,0 +1,201 @@
name: SQLite migration candidate
on:
pull_request:
branches: [develop]
workflow_dispatch:
permissions:
contents: read
concurrency:
group: sqlite-migration-candidate-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
SOURCE_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
jobs:
build-static:
if: ${{ (github.event_name == 'pull_request' && github.head_ref == 'codex/sqlite-runtime-migration' && github.event.pull_request.head.repo.full_name == github.repository) || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/codex/sqlite-runtime-migration') }}
runs-on: ubuntu-latest
timeout-minutes: 25
steps:
- uses: actions/checkout@v6
with:
ref: ${{ env.SOURCE_COMMIT }}
- uses: pnpm/action-setup@v6
with:
version: '8.3.1'
- uses: actions/setup-node@v6
with:
node-version: '22.17.0'
cache: pnpm
- run: pnpm install --frozen-lockfile
- run: cp .env.example .env
- run: npm test
- run: npm run build:front && npm run build:back && npm run build:info
- uses: actions/upload-artifact@v6
with:
name: migration-static-${{ env.SOURCE_COMMIT }}
path: static/
include-hidden-files: true
if-no-files-found: error
retention-days: 7
test-node24:
needs: build-static
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v6
with:
ref: ${{ env.SOURCE_COMMIT }}
- uses: pnpm/action-setup@v6
with:
version: '8.3.1'
- uses: actions/setup-node@v6
with:
node-version: '24'
cache: pnpm
- run: pnpm install --frozen-lockfile
- run: cp .env.example .env
- run: npm test
build-images:
needs: build-static
runs-on: ubuntu-22.04
timeout-minutes: 90
permissions:
contents: read
packages: write
strategy:
fail-fast: false
matrix:
include:
- libc: musl
file: docker/Dockerfile
platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x,linux/386
- libc: glibc
file: docker/Dockerfile.debian
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
steps:
- uses: actions/checkout@v6
with:
ref: ${{ env.SOURCE_COMMIT }}
- uses: actions/download-artifact@v6
with:
name: migration-static-${{ env.SOURCE_COMMIT }}
path: static/
- uses: docker/setup-qemu-action@v4
with:
cache-image: false
- uses: docker/setup-buildx-action@v4
- uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ github.token }}
- uses: docker/build-push-action@v7
id: image
with:
context: .
file: ${{ matrix.file }}
platforms: ${{ matrix.platforms }}
build-args: |
QL_BRANCH=codex/sqlite-runtime-migration
SOURCE_COMMIT=${{ env.SOURCE_COMMIT }}
push: true
tags: ghcr.io/${{ github.repository }}:migration-candidate-${{ env.SOURCE_COMMIT }}-${{ matrix.libc }}
cache-from: type=registry,ref=whyour/qinglong:cache-${{ matrix.libc == 'musl' && 'alpine' || 'debian' }}
- name: Record the immutable candidate image
env:
IMAGE_DIGEST: ${{ steps.image.outputs.digest }}
BUNDLE_LIBC: ${{ matrix.libc }}
run: |
[[ "$IMAGE_DIGEST" =~ ^sha256:[a-f0-9]{64}$ ]]
printf '%s\n' "ghcr.io/$GITHUB_REPOSITORY@$IMAGE_DIGEST" > "$BUNDLE_LIBC-image.txt"
- uses: actions/upload-artifact@v6
with:
name: migration-${{ matrix.libc }}-image-${{ env.SOURCE_COMMIT }}
path: ${{ matrix.libc }}-image.txt
retention-days: 7
build-bundles:
needs: build-images
runs-on: ubuntu-22.04
timeout-minutes: 90
strategy:
fail-fast: false
matrix:
include:
- { platform: linux/amd64, libc: musl, name: x64-musl }
- { platform: linux/arm/v6, libc: musl, name: armv6-musl }
- { platform: linux/arm/v7, libc: musl, name: armv7-musl }
- { platform: linux/arm64, libc: musl, name: arm64-musl }
- { platform: linux/ppc64le, libc: musl, name: ppc64-musl }
- { platform: linux/s390x, libc: musl, name: s390x-musl }
- { platform: linux/386, libc: musl, name: ia32-musl }
- { platform: linux/amd64, libc: glibc, name: x64-glibc }
- { platform: linux/arm/v7, libc: glibc, name: armv7-glibc }
- { platform: linux/arm64, libc: glibc, name: arm64-glibc }
- { platform: linux/ppc64le, libc: glibc, name: ppc64-glibc }
- { platform: linux/s390x, libc: glibc, name: s390x-glibc }
steps:
- uses: actions/checkout@v6
with:
ref: ${{ env.SOURCE_COMMIT }}
- uses: actions/download-artifact@v6
with:
name: migration-${{ matrix.libc }}-image-${{ env.SOURCE_COMMIT }}
- uses: docker/setup-qemu-action@v4
with:
cache-image: false
- uses: docker/setup-buildx-action@v4
- name: Build and verify both online hops using the unchanged legacy client
env:
BUNDLE_PLATFORM: ${{ matrix.platform }}
BUNDLE_LIBC: ${{ matrix.libc }}
run: |
set -euo pipefail
image=$(cat "$BUNDLE_LIBC-image.txt")
legacy=$(node -p "require('./docker/migration-bridge-images.json')['$BUNDLE_LIBC']")
docker buildx build --platform "$BUNDLE_PLATFORM" --build-arg "BASE_IMAGE=$image" --build-arg "LEGACY_IMAGE=$legacy" --file docker/Dockerfile.upgrade-bridge --output type=local,dest=output/upgrade .
- uses: actions/upload-artifact@v6
with:
name: migration-bundles-${{ matrix.name }}-${{ env.SOURCE_COMMIT }}
path: output/upgrade/*
if-no-files-found: error
retention-days: 7
verify-index:
needs: [build-bundles, test-node24]
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v6
with:
ref: ${{ env.SOURCE_COMMIT }}
- uses: actions/setup-node@v6
with:
node-version: '24'
- uses: pnpm/action-setup@v6
with:
version: '8.3.1'
- run: pnpm install --prod --frozen-lockfile && npm --prefix cli ci
- uses: actions/download-artifact@v6
with:
pattern: migration-bundles-*-${{ env.SOURCE_COMMIT }}
path: output/upgrade/
merge-multiple: true
- run: node scripts/build-bundle-installer.cjs output/upgrade/install-bundle.cjs
- name: Validate every platform without publishing a channel
run: |
node -e "const fs=require('fs'),{releaseIndex}=require('./scripts/publish-upgrade-bundles.cjs'); const index=releaseIndex('output/upgrade',process.env.SOURCE_COMMIT,process.env.GITHUB_REPOSITORY,{migration:true}); fs.writeFileSync('output/upgrade/candidate-index.json',JSON.stringify(index));"
- uses: actions/upload-artifact@v6
with:
name: migration-candidate-index-${{ env.SOURCE_COMMIT }}
path: |
output/upgrade/candidate-index.json
output/upgrade/install-bundle.cjs
retention-days: 7
+1 -1
View File
@@ -18,7 +18,7 @@ jobs:
version: '8.3.1'
- uses: actions/setup-node@v6
with:
node-version: '20'
node-version: '24'
cache: pnpm
- run: pnpm install --frozen-lockfile
- run: cp .env.example .env
+2
View File
@@ -1,5 +1,7 @@
# Node.js 原生部署与 Python venv
面板要求 Node.js >=22.17,推荐 Node.js 24,并要求该 Node 构建提供 `node:sqlite`。应用升级保留现有 Node;旧 Node 20 环境需先升级运行时。远程管理 CLI 的最低版本要求独立于面板。
设置 `QL_DIR` 为安装目录,`QL_DATA_DIR` 为以 `/data` 结尾的绝对持久化数据路径。
如需隔离 Python,请在启动进程的环境中显式设置 `PYTHON_VENV_DIR`(绝对路径或相对 `QL_DIR` 的路径),例如 `export PYTHON_VENV_DIR=/srv/qinglong/.venv`。未设置时保留现有 Python prefix 行为。
+21 -14
View File
@@ -5,10 +5,7 @@ import CronService from '../services/cron';
import CronViewService from '../services/cronView';
import { celebrate, Joi } from 'celebrate';
import { commonCronSchema } from '../validation/schedule';
import {
RunningInstanceModel,
InstanceStatus,
} from '../data/runningInstance';
import { RunningInstanceModel, InstanceStatus } from '../data/runningInstance';
import { t } from '../shared/i18n';
import cronClient from '../schedule/client';
@@ -19,8 +16,17 @@ export default (app: Router) => {
route.use(async (req, res, next) => {
// Keep stop/status callbacks available even when the scheduler is down.
if (['POST', 'PUT', 'DELETE'].includes(req.method) &&
['/', '/run', '/enable', '/disable', '/views/enable', '/views/disable'].includes(req.path)) {
if (
['POST', 'PUT', 'DELETE'].includes(req.method) &&
[
'/',
'/run',
'/enable',
'/disable',
'/views/enable',
'/views/disable',
].includes(req.path)
) {
try {
await cronClient.readiness.ensureReady();
} catch (error) {
@@ -497,13 +503,10 @@ export default (app: Router) => {
}),
async (req: Request<{ id: number }>, res: Response, next: NextFunction) => {
try {
const instances = await RunningInstanceModel.findAll({
where: {
cron_id: req.params.id,
},
order: [['started_at', 'DESC']],
raw: true,
});
const instances = await RunningInstanceModel.find(
{ cron_id: req.params.id },
{ orderBy: [{ started_at: 'desc' }] },
);
return res.send({ code: 200, data: instances });
} catch (e) {
return next(e);
@@ -519,7 +522,11 @@ export default (app: Router) => {
instanceId: Joi.number().required(),
}),
}),
async (req: Request<{ id: number; instanceId: number }>, res: Response, next: NextFunction) => {
async (
req: Request<{ id: number; instanceId: number }>,
res: Response,
next: NextFunction,
) => {
try {
const cronService = Container.get(CronService);
const data = await cronService.stopInstance(req.params.instanceId);
+93 -157
View File
@@ -1,38 +1,32 @@
import { statistics } from '../data/statistics';
import { Query, QuerySpec, orderBy } from '../data/queries';
import { Router, Request, Response, NextFunction } from 'express';
import { Container } from 'typedi';
import { fn, col, where, Op } from 'sequelize';
import { CrontabModel } from '../data/cron';
import { CrontabStatModel } from '../data/cronStats';
import {
RunningInstanceModel,
InstanceStatus,
} from '../data/runningInstance';
import { RunningInstanceModel, InstanceStatus } from '../data/runningInstance';
import dayjs from 'dayjs';
import os from 'os';
import { isEmpty } from 'lodash';
import { t, tf } from '../shared/i18n';
const route = Router();
export default (app: Router) => {
app.use('/dashboard', route);
route.post('/record', async (req: Request, res: Response) => {
try {
const { ref_id, code, elapsed } = req.body;
if (!ref_id) return res.send({ code: 400, message: 'ref_id required' });
const today = dayjs().format('YYYY-MM-DD');
const isSuccess = code === 0 ? 1 : 0;
const isFail = code !== 0 ? 1 : 0;
const elapsedMs = (Number(elapsed) || 0) * 1000;
const existing = await CrontabStatModel.findOne({
where: { ref_id: Number(ref_id), date: today },
});
const existing = await CrontabStatModel.findOne(
{ ref_id: Number(ref_id), date: today },
{},
);
if (existing) {
await CrontabStatModel.update(
await CrontabStatModel.nativeUpdate(
{ id: existing.id },
{
run_count: (existing.run_count || 0) + 1,
success_count: (existing.success_count || 0) + isSuccess,
@@ -40,10 +34,9 @@ export default (app: Router) => {
total_time: (existing.total_time || 0) + elapsedMs,
max_time: Math.max(existing.max_time || 0, elapsedMs),
},
{ where: { id: existing.id } },
);
} else {
await CrontabStatModel.create({
await CrontabStatModel.insert({
ref_id: Number(ref_id),
date: today,
run_count: 1,
@@ -53,41 +46,27 @@ export default (app: Router) => {
max_time: elapsedMs,
});
}
res.send({ code: 200 });
} catch (e) {
res.send({ code: 500 });
}
});
route.get(
'/overview',
async (req: Request, res: Response, next: NextFunction) => {
try {
const today = dayjs().format('YYYY-MM-DD');
const [total, enabled, disabled, stats] = await Promise.all([
CrontabModel.count(),
CrontabModel.count({ where: { isDisabled: 0 } }),
CrontabModel.count({ where: { isDisabled: 1 } }),
CrontabStatModel.findOne({
attributes: [
[fn('SUM', col('run_count')), 'total_runs'],
[fn('SUM', col('success_count')), 'total_success'],
[fn('SUM', col('fail_count')), 'total_fail'],
[fn('SUM', col('total_time')), 'total_time'],
],
where: { date: today },
raw: true,
}),
CrontabModel.count({}),
CrontabModel.count({ isDisabled: 0 }),
CrontabModel.count({ isDisabled: 1 }),
statistics.totals(today),
]);
const row = stats as any;
const totalRuns = Number(row?.total_runs) || 0;
const totalSuccess = Number(row?.total_success) || 0;
const totalFail = Number(row?.total_fail) || 0;
const totalTime = Number(row?.total_time) || 0;
res.send({
code: 200,
data: {
@@ -97,7 +76,10 @@ export default (app: Router) => {
todayRuns: totalRuns,
todaySuccess: totalSuccess,
todayFail: totalFail,
successRate: totalRuns > 0 ? ((totalSuccess / totalRuns) * 100).toFixed(1) : '0',
successRate:
totalRuns > 0
? ((totalSuccess / totalRuns) * 100).toFixed(1)
: '0',
avgTime: totalRuns > 0 ? Math.round(totalTime / totalRuns) : 0,
},
});
@@ -106,25 +88,25 @@ export default (app: Router) => {
}
},
);
for (const isSuccess of [false, true]) {
route.get(
isSuccess ? '/successes' : '/failures',
async (req: Request, res: Response, next: NextFunction) => {
try {
const countColumn = isSuccess ? 'success_count' : 'fail_count';
const rows = (await CrontabStatModel.findAll({
attributes: ['ref_id', [fn('SUM', col(countColumn)), 'result_count']],
where: { date: dayjs().format('YYYY-MM-DD'), [countColumn]: { [Op.gt]: 0 } },
group: ['ref_id'],
order: [[fn('SUM', col(countColumn)), 'DESC'], ['ref_id', 'ASC']],
raw: true,
})) as any[];
const crons = rows.length > 0 ? await CrontabModel.findAll({
attributes: ['id', 'name', 'command'],
where: { id: { [Op.in]: rows.map((row) => Number(row.ref_id)) } },
raw: true,
}) : [];
const rows = (await statistics.results(
dayjs().format('YYYY-MM-DD'),
countColumn,
)) as any[];
const crons =
rows.length > 0
? await CrontabModel.find(
{ id: { ['$in']: rows.map((row) => Number(row.ref_id)) } },
{
fields: ['id', 'name', 'command'],
},
)
: [];
const cronMap = new Map(crons.map((cron) => [cron.id, cron]));
res.send({
code: 200,
@@ -135,7 +117,9 @@ export default (app: Router) => {
id,
name: cron?.name || cron?.command || tf('任务#%s', id),
command: cron?.command || '',
[isSuccess ? 'successCount' : 'failCount']: Number(row.result_count),
[isSuccess ? 'successCount' : 'failCount']: Number(
row.result_count,
),
deleted: !cron,
};
}),
@@ -146,7 +130,6 @@ export default (app: Router) => {
},
);
}
route.get(
'/trend',
async (req: Request, res: Response, next: NextFunction) => {
@@ -156,22 +139,7 @@ export default (app: Router) => {
for (let i = days - 1; i >= 0; i--) {
dates.push(dayjs().subtract(i, 'day').format('YYYY-MM-DD'));
}
const rows = (await CrontabStatModel.findAll({
attributes: [
'date',
[fn('SUM', col('run_count')), 'total_runs'],
[fn('SUM', col('success_count')), 'total_success'],
[fn('SUM', col('fail_count')), 'total_fail'],
],
where: {
date: { [Op.in]: dates },
},
group: ['date'],
order: [['date', 'ASC']],
raw: true,
})) as any[];
const rows = (await statistics.trend(dates)) as any[];
const dataMap: Record<string, any> = {};
rows.forEach((r: any) => {
dataMap[r.date] = {
@@ -180,89 +148,52 @@ export default (app: Router) => {
fail: Number(r.total_fail) || 0,
};
});
const data = dates.map((d) => ({
date: dayjs(d).format('MM-DD'),
...(dataMap[d] || { total: 0, success: 0, fail: 0 }),
}));
res.send({ code: 200, data });
} catch (e) {
next(e);
}
},
);
route.get(
'/top-time',
async (req: Request, res: Response, next: NextFunction) => {
try {
const today = dayjs().format('YYYY-MM-DD');
const rows = (await CrontabStatModel.findAll({
attributes: [
'ref_id',
[fn('SUM', col('total_time')), 'total_time'],
[fn('SUM', col('run_count')), 'run_count'],
[fn('MAX', col('max_time')), 'max_time'],
],
where: { date: today, run_count: { [Op.gt]: 0 } },
group: ['ref_id'],
order: [[fn('SUM', col('total_time')), 'DESC']],
limit: 5,
raw: true,
})) as any[];
const rows = (await statistics.ranking(today, 'total_time')) as any[];
const ids = rows.map((r) => Number(r.ref_id));
const crons = await CrontabModel.findAll({
where: { id: { [Op.in]: ids } },
raw: true,
});
const crons = await CrontabModel.find({ id: { ['$in']: ids } }, {});
const nameMap: Record<number, string> = {};
crons.forEach((c: any) => { nameMap[c.id] = c.name || c.command; });
crons.forEach((c: any) => {
nameMap[c.id] = c.name || c.command;
});
const data = rows.map((r: any, i) => ({
rank: i + 1,
name: nameMap[Number(r.ref_id)] || tf('任务#%s', r.ref_id),
avgTime: Math.round(Number(r.total_time) / Number(r.run_count)),
maxTime: Number(r.max_time),
}));
res.send({ code: 200, data });
} catch (e) {
next(e);
}
},
);
route.get(
'/top-count',
async (req: Request, res: Response, next: NextFunction) => {
try {
const today = dayjs().format('YYYY-MM-DD');
const rows = (await CrontabStatModel.findAll({
attributes: [
'ref_id',
[fn('SUM', col('run_count')), 'run_count'],
[fn('SUM', col('total_time')), 'total_time'],
[fn('SUM', col('success_count')), 'success_count'],
],
where: { date: today, run_count: { [Op.gt]: 0 } },
group: ['ref_id'],
order: [[fn('SUM', col('run_count')), 'DESC']],
limit: 5,
raw: true,
})) as any[];
const rows = (await statistics.ranking(today, 'run_count')) as any[];
const ids = rows.map((r) => Number(r.ref_id));
const crons = await CrontabModel.findAll({
where: { id: { [Op.in]: ids } },
raw: true,
});
const crons = await CrontabModel.find({ id: { ['$in']: ids } }, {});
const nameMap: Record<number, string> = {};
crons.forEach((c: any) => { nameMap[c.id] = c.name || c.command; });
crons.forEach((c: any) => {
nameMap[c.id] = c.name || c.command;
});
const data = rows.map((r: any, i) => ({
rank: i + 1,
name: nameMap[Number(r.ref_id)] || tf('任务#%s', r.ref_id),
@@ -270,48 +201,42 @@ export default (app: Router) => {
avgTime: Math.round(Number(r.total_time) / Number(r.run_count)),
successRate:
Number(r.run_count) > 0
? ((Number(r.success_count) / Number(r.run_count)) * 100).toFixed(1)
? ((Number(r.success_count) / Number(r.run_count)) * 100).toFixed(
1,
)
: '0',
}));
res.send({ code: 200, data });
} catch (e) {
next(e);
}
},
);
route.get(
'/runtime',
async (req: Request, res: Response, next: NextFunction) => {
try {
const runningInstances = await RunningInstanceModel.findAll({
where: {
const runningInstances = await RunningInstanceModel.find(
{
status: InstanceStatus.running,
},
raw: true,
});
const queuedCrons = await CrontabModel.findAll({
where: {
{},
);
const queuedCrons = await CrontabModel.find(
{
status: 3, // queued
},
raw: true,
});
{},
);
// Fetch cron names for running instances
const cronIds = [
...new Set(runningInstances.map((i: any) => i.cron_id)),
];
const crons =
cronIds.length > 0
? await CrontabModel.findAll({
where: { id: cronIds },
raw: true,
})
? await CrontabModel.find({ id: cronIds }, {})
: [];
const cronMap = new Map(crons.map((c: any) => [c.id, c]));
const now = dayjs().unix();
const running = runningInstances.map((inst: any) => {
const cron = cronMap.get(inst.cron_id);
@@ -324,19 +249,18 @@ export default (app: Router) => {
logPath: inst.log_path,
};
});
const dayAgo = dayjs().subtract(24, 'hour').unix();
const idleTasks = await CrontabModel.findAll({
where: {
const idleTasks = await CrontabModel.find(
{
isDisabled: 0,
status: 1,
last_execution_time: { [Op.lt]: dayAgo },
last_execution_time: { ['$lt']: dayAgo },
},
order: [['last_execution_time', 'ASC']],
limit: 5,
raw: true,
});
{
orderBy: orderBy([['last_execution_time', 'ASC']]),
limit: 5,
},
);
res.send({
code: 200,
data: {
@@ -357,33 +281,45 @@ export default (app: Router) => {
}
},
);
route.get(
'/labels',
async (req: Request, res: Response, next: NextFunction) => {
try {
const today = dayjs().format('YYYY-MM-DD');
const [crons, stats] = (await Promise.all([
CrontabModel.findAll({ where: { isDisabled: 0 }, raw: true }),
CrontabStatModel.findAll({ where: { date: today }, raw: true }),
]));
const [crons, stats] = await Promise.all([
CrontabModel.find({ isDisabled: 0 }, {}),
CrontabStatModel.find({ date: today }, {}),
]);
const statMap: Record<number, any> = {};
stats.forEach((s: any) => { statMap[s.ref_id] = s; });
const labelMap: Record<string, { count: number; runs: number; success: number; totalTime: number }> = {};
stats.forEach((s: any) => {
statMap[s.ref_id] = s;
});
const labelMap: Record<
string,
{
count: number;
runs: number;
success: number;
totalTime: number;
}
> = {};
crons.forEach((c) => {
let rawLabels = c.labels;
if (typeof rawLabels === 'string') rawLabels = JSON.parse(rawLabels);
const labels: string[] = Array.isArray(rawLabels)
? [...new Set((rawLabels as string[]).filter((l: string) => !isEmpty(l)))]
? [
...new Set(
(rawLabels as string[]).filter((l: string) => !isEmpty(l)),
),
]
: [];
if (labels.length === 0) {
labels.push(t('未分类'));
}
const st = statMap[c.id!];
labels.forEach((label: string) => {
if (!labelMap[label]) labelMap[label] = { count: 0, runs: 0, success: 0, totalTime: 0 };
if (!labelMap[label])
labelMap[label] = { count: 0, runs: 0, success: 0, totalTime: 0 };
labelMap[label].count += 1;
if (st) {
labelMap[label].runs += Number(st.run_count) || 0;
@@ -392,24 +328,22 @@ export default (app: Router) => {
}
});
});
const data = Object.entries(labelMap)
.map(([label, v]) => ({
label,
count: v.count,
todayRuns: v.runs,
successRate: v.runs > 0 ? ((v.success / v.runs) * 100).toFixed(1) : '0',
successRate:
v.runs > 0 ? ((v.success / v.runs) * 100).toFixed(1) : '0',
avgTime: v.runs > 0 ? Math.round(v.totalTime / v.runs) : 0,
}))
.sort((a, b) => b.todayRuns - a.todayRuns);
res.send({ code: 200, data });
} catch (e) {
next(e);
}
},
);
route.get(
'/system',
async (req: Request, res: Response, next: NextFunction) => {
@@ -422,7 +356,9 @@ export default (app: Router) => {
uptime: Math.floor(process.uptime()),
memTotal: os.totalmem(),
memFree: os.freemem(),
memUsagePercent: ((1 - os.freemem() / os.totalmem()) * 100).toFixed(1),
memUsagePercent: ((1 - os.freemem() / os.totalmem()) * 100).toFixed(
1,
),
heapUsed: Math.round(memUsage.heapUsed / 1024 / 1024),
heapTotal: Math.round(memUsage.heapTotal / 1024 / 1024),
loadAvg: os.loadavg().map((v) => Number(v.toFixed(2))),
+2 -5
View File
@@ -1,19 +1,16 @@
import { errStack } from '../shared/errors';
// Also terminate if the primary disappears without forwarding a signal.
const onParentDisconnect = () => process.exit(1);
process.once('disconnect', onParentDisconnect);
async function initializeDatabase() {
const { sequelize } = await import('../data');
const { database } = await import('../data');
try {
const { default: loadDatabase } = await import('../loaders/db');
await loadDatabase();
} finally {
await sequelize.close();
await database.close();
}
}
initializeDatabase()
.catch((error) => {
console.error(`[boot] Database initialization failed:\n${errStack(error)}`);
+21 -19
View File
@@ -3,11 +3,12 @@
import 'reflect-metadata';
import dns from 'dns';
import http from 'http';
// The first scheduler connection can outlive its short RPC deadline. Let the
// normal single-flight recovery finish within the supervised startup budget.
async function waitForUpgradeScheduler(
readiness: { recover(): Promise<boolean> },
readiness: {
recover(): Promise<boolean>;
},
timeoutMs: number,
retryMs = 250,
) {
@@ -22,13 +23,12 @@ async function waitForUpgradeScheduler(
} while (Date.now() < deadline);
throw new Error('Upgrade scheduler readiness failed');
}
async function probeUpgrade() {
if (!process.send || process.env.QL_UPGRADE_PROBE !== '1')
throw new Error('Upgrade probe requires its isolated supervisor');
dns.setDefaultResultOrder('ipv4first');
const { default: config } = await import('../config');
const { sequelize } = await import('../data');
const { database } = await import('../data');
const { default: loadDatabase } = await import('../loaders/db');
const { Container } = await import('typedi');
const { GrpcServerService } = await import('../services/grpc');
@@ -43,17 +43,17 @@ async function probeUpgrade() {
try {
await loadDatabase();
// Exercise an actual on-disk transaction as well as schema initialization.
await sequelize.transaction(async (transaction) => {
await sequelize.query('CREATE TABLE upgrade_probe (value INTEGER)', {
transaction,
});
await sequelize.query('INSERT INTO upgrade_probe VALUES (42)', {
transaction,
});
const [rows] = await sequelize.query('SELECT value FROM upgrade_probe', {
transaction,
});
if ((rows[0] as { value: number })?.value !== 42)
await database.transaction(async () => {
await database.query('CREATE TABLE upgrade_probe (value INTEGER)');
await database.query('INSERT INTO upgrade_probe VALUES (42)');
const rows = await database.query('SELECT value FROM upgrade_probe');
if (
(
rows[0] as {
value: number;
}
)?.value !== 42
)
throw new Error('Upgrade database readback failed');
});
inject();
@@ -64,8 +64,11 @@ async function probeUpgrade() {
cronClient.readiness,
process.arch === 'arm' &&
Number(
(process.config.variables as { arm_version?: string | number })
.arm_version,
(
process.config.variables as {
arm_version?: string | number;
}
).arm_version,
) === 6
? 60000
: 10000,
@@ -119,10 +122,9 @@ async function probeUpgrade() {
} finally {
await server.shutdown();
grpc.getServer().forceShutdown();
await sequelize.close();
await database.close();
}
}
process.once('disconnect', () => process.exit(1));
probeUpgrade()
.then(() => process.exit(0))
+28
View File
@@ -0,0 +1,28 @@
import { EventEmitter } from 'events';
import { database } from '.';
import { DatabaseClient } from './databaseClient';
// Keep the existing Keyv serialization and namespace. Only its SQLite transport
// changes; old cached authentication records remain readable in either version.
export class CacheStore extends EventEmitter {
readonly opts = { dialect: 'sqlite' };
namespace?: string;
constructor(private client: () => DatabaseClient = () => database) {
super();
}
get(key: string): Promise<string | undefined> {
return this.client().request('cache', 'get', key);
}
async set(key: string, value: string) {
return this.client().request('cache', 'set', key, value);
}
async delete(key: string): Promise<boolean> {
return this.client().request('cache', 'delete', key);
}
async clear(): Promise<void> {
throw new Error('Clearing all authentication namespaces is not supported');
}
async disconnect(): Promise<void> {
/* The shared database client owns the worker. */
}
}
+3 -36
View File
@@ -1,5 +1,4 @@
import { sequelize } from '.';
import { DataTypes, Model, ModelDefined } from 'sequelize';
import { Repository, RecordRow } from './repository';
export class Crontab {
name?: string;
@@ -62,37 +61,5 @@ export enum CrontabStatus {
'queued' = 3,
}
export interface CronInstance extends Model<Crontab, Crontab>, Crontab {}
export const CrontabModel = sequelize.define<CronInstance>('Crontab', {
name: {
unique: 'compositeIndex',
type: DataTypes.STRING,
},
command: {
unique: 'compositeIndex',
type: DataTypes.STRING,
},
schedule: {
unique: 'compositeIndex',
type: DataTypes.STRING,
},
timestamp: DataTypes.STRING,
saved: DataTypes.BOOLEAN,
status: DataTypes.NUMBER,
isSystem: DataTypes.NUMBER,
pid: DataTypes.NUMBER,
isDisabled: DataTypes.NUMBER,
isPinned: DataTypes.NUMBER,
log_path: DataTypes.STRING,
queued_token: DataTypes.STRING,
labels: DataTypes.JSON,
last_running_time: DataTypes.NUMBER,
last_execution_time: DataTypes.NUMBER,
sub_id: { type: DataTypes.NUMBER, allowNull: true },
extra_schedules: DataTypes.JSON,
task_before: DataTypes.STRING,
task_after: DataTypes.STRING,
log_name: DataTypes.STRING,
allow_multiple_instances: DataTypes.NUMBER,
work_dir: DataTypes.STRING,
});
export type CronInstance = RecordRow<Crontab>;
export const CrontabModel = new Repository<Crontab>('Crontab');
+3 -48
View File
@@ -1,5 +1,4 @@
import { DataTypes, Model } from 'sequelize';
import { sequelize } from '.';
import { Repository, RecordRow } from './repository';
export class CrontabStat {
id?: number;
@@ -23,49 +22,5 @@ export class CrontabStat {
}
}
export interface CrontabStatInstance extends Model<CrontabStat, CrontabStat>, CrontabStat {}
export const CrontabStatModel = sequelize.define<CrontabStatInstance>(
'CrontabStat',
{
id: {
type: DataTypes.INTEGER,
primaryKey: true,
autoIncrement: true,
},
ref_id: {
type: DataTypes.NUMBER,
allowNull: false,
},
date: {
type: DataTypes.STRING,
allowNull: false,
},
run_count: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
success_count: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
fail_count: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
total_time: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
max_time: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
},
{
indexes: [
{ unique: true, fields: ['ref_id', 'date'] },
{ fields: ['date'] },
],
},
);
export type CrontabStatInstance = RecordRow<CrontabStat>;
export const CrontabStatModel = new Repository<CrontabStat>('CrontabStat');
+3 -23
View File
@@ -1,5 +1,4 @@
import { sequelize } from '.';
import { DataTypes, Model } from 'sequelize';
import { Repository, RecordRow } from './repository';
export enum CronViewType {
'系统' = 1,
@@ -39,24 +38,5 @@ export class CrontabView {
}
}
export interface CronViewInstance
extends Model<CrontabView, CrontabView>,
CrontabView {}
export const CrontabViewModel = sequelize.define<CronViewInstance>(
'CrontabView',
{
name: {
unique: 'name',
type: DataTypes.STRING,
},
position: DataTypes.NUMBER,
isDisabled: DataTypes.NUMBER,
filters: DataTypes.JSON,
sorts: DataTypes.JSON,
filterRelation: {
type: DataTypes.STRING,
allowNull: true,
},
type: DataTypes.NUMBER,
},
);
export type CronViewInstance = RecordRow<CrontabView>;
export const CrontabViewModel = new Repository<CrontabView>('CrontabView');
+325
View File
@@ -0,0 +1,325 @@
import { AsyncLocalStorage } from 'async_hooks';
import { ChildProcess, fork } from 'child_process';
import { join } from 'path';
import { setTimeout, clearTimeout } from 'timers';
import { DatabaseError } from './databaseErrors';
type Scope = { active: boolean; child: boolean; token: number };
type Reply = {
id: number;
value?: any;
error?: { code: string; message: string };
};
type Limits = { timeoutMs?: number; queueLimit?: number };
// One worker per backend process; SQLite itself coordinates independent processes.
// A transaction owns this client's queue until its callback and commit finish.
export class DatabaseClient {
private worker?: ChildProcess;
private stopped: Promise<void> = Promise.resolve();
private failed?: Error;
private closed = false;
private nextId = 0;
private queued = 0;
private tail: Promise<void> = Promise.resolve();
private pending = new Map<
number,
{
resolve(value: any): void;
reject(error: Error): void;
timer: NodeJS.Timeout;
}
>();
private scope = new AsyncLocalStorage<Scope>();
private readonly timeoutMs: number;
private readonly queueLimit: number;
constructor(
public readonly storage: string,
private cacheStorage?: string,
limits: Limits = {},
) {
this.timeoutMs =
limits.timeoutMs ?? (process.arch === 'arm' ? 300000 : 60000);
this.queueLimit = limits.queueLimit ?? 256;
if (
!Number.isSafeInteger(this.timeoutMs) ||
this.timeoutMs < 1 ||
!Number.isSafeInteger(this.queueLimit) ||
this.queueLimit < 1
)
throw new Error('Invalid database worker limits');
}
private abort(error: Error) {
this.failed ||= error;
for (const request of this.pending.values()) {
clearTimeout(request.timer);
request.reject(this.failed);
}
this.pending.clear();
this.unref();
this.worker?.kill('SIGKILL');
}
private unref() {
this.worker?.unref();
this.worker?.channel?.unref();
}
private start() {
if (this.worker) return this.worker;
const [major, minor] = process.versions.node.split('.').map(Number);
if (major < 22 || (major === 22 && minor < 17))
throw new DatabaseError(
'Qinglong requires Node >=22.17; keep the previous installation until Node is upgraded',
'UNSUPPORTED_NODE',
);
// ts-node's CLI places its executable in execArgv, rather than using -r.
// Source workers register their own compiler; compiled workers need none.
const preloads = __filename.endsWith('.ts')
? ['--require', require.resolve('ts-node/register/transpile-only')]
: [];
const armv6 =
process.arch === 'arm' &&
Number(
(process.config.variables as { arm_version?: string | number })
.arm_version,
) === 6;
const worker = fork(require.resolve('./databaseProcess'), [], {
serialization: 'advanced',
stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
execArgv: [
...preloads,
...(armv6 ? ['--jitless'] : []),
'--max-old-space-size=128',
],
env: {
...process.env,
NODE_OPTIONS: '',
...(__filename.endsWith('.ts')
? { TS_NODE_PROJECT: join(__dirname, 'tsconfig.worker.json') }
: {}),
},
});
this.stopped = new Promise((resolve) =>
worker.once('close', () => resolve()),
);
const onOwnerExit = () => worker.kill('SIGKILL');
process.once('exit', onOwnerExit);
worker.once('close', () => process.removeListener('exit', onOwnerExit));
this.worker = worker;
worker.on('message', (reply: Reply & { type?: string }) => {
if (reply.type === 'fatal' && reply.error) {
this.abort(new DatabaseError(reply.error.message, reply.error.code));
return;
}
const request = this.pending.get(reply.id);
if (!request) return;
this.pending.delete(reply.id);
clearTimeout(request.timer);
if (!this.pending.size) this.unref();
if (reply.error)
request.reject(
new DatabaseError(reply.error.message, reply.error.code),
);
else request.resolve(reply.value);
});
const fail = () =>
this.abort(
new DatabaseError(
'Database worker stopped; restart the service before retrying a write',
'DATABASE_WORKER_EXIT',
),
);
worker.on('error', fail);
worker.on('exit', () => {
if (!this.closed) fail();
});
worker.send({
storage: this.storage,
cacheStorage: this.cacheStorage,
ownerPid: process.pid,
});
this.unref();
return worker;
}
private send(
operation: string,
args: any[] = [],
token?: number,
): Promise<any> {
if (this.failed) return Promise.reject(this.failed);
if (this.closed)
return Promise.reject(
new DatabaseError('Database is closed', 'DATABASE_CLOSED'),
);
return new Promise((resolve, reject) => {
try {
const worker = this.start();
const id = ++this.nextId;
const timer = setTimeout(
() =>
this.abort(
new DatabaseError(
'Database worker timed out; writes are never automatically replayed',
'DATABASE_WORKER_TIMEOUT',
),
),
this.timeoutMs,
);
timer.unref();
this.pending.set(id, { resolve, reject, timer });
worker.ref();
worker.channel?.ref();
try {
worker.send({ id, operation, args, token }, (error) => {
if (error)
this.abort(
new DatabaseError(
'Database IPC disconnected',
'DATABASE_WORKER_EXIT',
),
);
});
} catch (error) {
this.pending.delete(id);
clearTimeout(timer);
if (!this.pending.size) this.unref();
reject(error);
}
} catch (error) {
reject(error);
}
});
}
private exclusive<T>(
operation: () => Promise<T>,
shutdown = false,
): Promise<T> {
if (!shutdown && this.queued >= this.queueLimit)
return Promise.reject(
new DatabaseError('Database request queue is full', 'DATABASE_BUSY'),
);
this.queued++;
const previous = this.tail;
let release!: () => void;
this.tail = new Promise<void>((resolve) => {
release = resolve;
});
return previous.then(operation).finally(() => {
this.queued--;
release();
});
}
request<T = any>(operation: string, ...args: any[]): Promise<T> {
const scope = this.scope.getStore();
if (scope) {
if (!scope.active || scope.child)
return Promise.reject(
new DatabaseError(
'Transaction scope is no longer available',
'TRANSACTION_SCOPE',
),
);
if (this.pending.size >= this.queueLimit)
return Promise.reject(
new DatabaseError('Database request queue is full', 'DATABASE_BUSY'),
);
return this.send(operation, args, scope.token);
}
return this.exclusive(() => this.send(operation, args));
}
async transaction<T>(operation: () => Promise<T>): Promise<T> {
const parent = this.scope.getStore();
if (parent && (!parent.active || parent.child))
throw new DatabaseError(
'Concurrent or expired nested transaction',
'TRANSACTION_SCOPE',
);
const run = async () => {
const token = ++this.nextId;
if (parent) parent.child = true;
const scope: Scope = { active: true, child: false, token };
let began = false;
let timer: NodeJS.Timeout | undefined;
try {
await this.send('begin', [], token);
began = true;
const deadline = new Promise<never>((_, reject) => {
timer = setTimeout(
() =>
reject(
new DatabaseError(
'Database transaction timed out',
'DATABASE_TRANSACTION_TIMEOUT',
),
),
this.timeoutMs,
);
});
const value = await Promise.race([
this.scope.run(scope, operation),
deadline,
]);
scope.active = false;
await this.send('commit', [], token);
return value;
} catch (error) {
scope.active = false;
if (began) {
try {
await this.send('rollback', [], token);
} catch {
this.abort(
new DatabaseError(
'Transaction rollback failed; restart the service',
'DATABASE_ROLLBACK_FAILED',
),
);
}
}
throw error;
} finally {
clearTimeout(timer);
if (parent) parent.child = false;
}
};
return parent ? run() : this.exclusive(run);
}
query<T = Record<string, unknown>>(
sql: string,
params: unknown[] = [],
): Promise<T[]> {
return this.request('query', sql, params);
}
get healthy() {
return !this.failed && !this.closed;
}
async close() {
if (this.scope.getStore())
throw new DatabaseError(
'Cannot close within a transaction',
'TRANSACTION_SCOPE',
);
await this.exclusive(async () => {
if (this.closed) return;
try {
if (this.worker && !this.failed) await this.send('close');
} finally {
this.closed = true;
this.worker?.ref();
this.worker?.kill('SIGKILL');
await this.stopped;
}
}, true);
}
}
+36
View File
@@ -0,0 +1,36 @@
// Error details (SQL, bindings, credentials) stay inside the database worker.
export class DatabaseError extends Error {
constructor(message: string, public readonly code: string) {
super(message);
this.name = 'DatabaseError';
}
}
export function databaseStartupFailure(value: unknown) {
const error = (value && typeof value === 'object' ? value : {}) as Error & { code?: string };
const missing = /Cannot find (?:module|package) ['"]([^'"]+)['"]/.exec(
error.message || '',
)?.[1];
const importer = /Require stack:\r?\n-\s+([^\r\n]+)/.exec(
error.message || '',
)?.[1];
const dependency =
/MODULE_NOT_FOUND/.test(error.code || '') &&
Boolean(
missing &&
((!missing.startsWith('.') &&
!missing.startsWith('/') &&
!missing.startsWith('node:')) ||
/node_modules[/\\]/.test(missing) ||
/node_modules[/\\]/.test(importer || '')),
);
return dependency
? {
code: 'MODULE_NOT_FOUND',
message: "Cannot find module '@mikro-orm/sql'",
}
: {
code: 'DATABASE_WORKER_EXIT',
message: 'Database process failed to start',
};
}
+61
View File
@@ -0,0 +1,61 @@
import { databaseStartupFailure } from './databaseErrors';
import { Worker } from 'worker_threads';
// IPC disconnect cannot run while DatabaseSync is executing SQL. Start this
// independent, allocation-light monitor before opening SQLite so a SIGKILLed
// backend cannot leave an orphan holding the old service's database write lock.
async function monitorOwner(ownerPid: number) {
if (!Number.isSafeInteger(ownerPid) || ownerPid < 1)
throw new Error('Invalid database owner');
const monitor = new Worker(
`
const {parentPort,workerData}=require('node:worker_threads');
process.on('uncaughtException',()=>process.kill(process.pid,'SIGKILL'));
const check=()=>{
if(process.ppid!==workerData.ownerPid)process.kill(process.pid,'SIGKILL');
};
check();
setInterval(check,500);
parentPort.postMessage('ready');
`,
{
eval: true,
workerData: { ownerPid },
// V8's process-wide --jitless remains active on ARMv6. Do not load the
// source worker's ts-node compiler into this tiny, plain-JavaScript monitor.
execArgv: [],
resourceLimits: {
maxOldGenerationSizeMb: 8,
maxYoungGenerationSizeMb: 2,
stackSizeMb: 1,
},
},
);
monitor.once('exit', () => process.exit(1));
await new Promise<void>((resolve, reject) => {
monitor.once('message', () => resolve());
monitor.once('error', reject);
});
monitor.unref();
}
// The bootstrap catches dependency import errors before the ORM module loads.
// The advanced IPC serializer preserves Date, Buffer and undefined values.
process.once('disconnect', () => process.exit(1));
process.once('message', async (settings) => {
const pending: unknown[] = [];
const buffer = (request: unknown) => pending.push(request);
// Keep a listener throughout asynchronous monitor startup. Node may flush
// several buffered IPC messages while delivering the initial settings.
process.on('message', buffer);
try {
await monitorOwner((settings as { ownerPid: number }).ownerPid);
require('./databaseWorker').startDatabaseWorker(settings);
process.removeListener('message', buffer);
for (const request of pending) process.emit('message', request, undefined);
} catch (error) {
process.send?.({ type: 'fatal', error: databaseStartupFailure(error) });
} finally {
process.removeListener('message', buffer);
}
});
+280
View File
@@ -0,0 +1,280 @@
import { mkdirSync } from 'fs';
import { dirname } from 'path';
import { EntitySchema, Type } from '@mikro-orm/core';
import { MikroORM, SqliteDriver, NodeSqliteDialect, raw } from '@mikro-orm/sql';
import { entityDefinitions } from './schema';
// Sequelize DATE readers expect TEXT. Numeric epoch dates would break rollback.
class LegacyDateType extends Type<Date, string> {
convertToDatabaseValue(value: Date) {
return value == null
? value
: value.toISOString().replace('T', ' ').replace('Z', ' +00:00');
}
convertToJSValue(value: string | Date) {
return value == null
? value
: value instanceof Date
? value
: new Date(value);
}
getColumnType() {
return 'DATETIME';
}
compareAsType() {
return 'string' as const;
}
get runtimeType() {
return 'Date';
}
}
const entities = Object.values(entityDefinitions).map(
(definition) =>
new EntitySchema({
...definition,
properties: Object.fromEntries(
Object.entries(definition.properties).map(([name, field]) => [
name,
{
...field,
type:
field.type === 'legacyDate' ? new LegacyDateType() : field.type,
},
]),
),
} as any),
);
let orm: Awaited<ReturnType<typeof MikroORM.init<SqliteDriver>>>;
let cache: any;
const transactions: number[] = [];
let ready: Promise<void>;
let workerData: { storage: string; cacheStorage?: string; ownerPid: number };
async function initialize() {
if (workerData.storage !== ':memory:')
mkdirSync(dirname(workerData.storage), { recursive: true });
orm = await MikroORM.init<SqliteDriver>({
driver: SqliteDriver,
driverOptions: new NodeSqliteDialect(workerData.storage),
dbName: workerData.storage,
entities,
debug: false,
allowGlobalContext: false,
discovery: { warnWhenNoEntities: false },
});
await orm.em.getConnection().execute('PRAGMA busy_timeout=10000');
}
function filter(name: keyof typeof entityDefinitions, value: any): any {
if (value === undefined) throw new Error('Undefined database predicate');
if (value == null || typeof value !== 'object' || value instanceof Date)
return value;
if (Array.isArray(value)) return value.map((part) => filter(name, part));
const properties = entityDefinitions[name].properties;
return Object.fromEntries(
Object.entries(value).map(([key, item]: [string, any]) => {
if (item === undefined) throw new Error('Undefined database predicate');
// LIKE searches JSON's serialized text, matching the pre-migration behavior.
if (
key in properties &&
(properties as any)[key].type === 'json' &&
item &&
typeof item === 'object' &&
'$like' in item
)
return [raw(`cast("${key}" as text)`), item];
return [key, filter(name, item)];
}),
);
}
function values(
name: keyof typeof entityDefinitions,
payload: Record<string, any>,
insert = false,
) {
const properties = entityDefinitions[name].properties;
const data = Object.fromEntries(
Object.entries(payload).filter(
([key, value]) => key in properties && value !== undefined,
),
);
for (const [key, value] of Object.entries(data)) {
if ((properties as any)[key].type === 'json' && value !== null)
data[key] = JSON.parse(JSON.stringify(value));
}
const now = new Date();
if (insert && !data.createdAt) data.createdAt = now;
data.updatedAt = now;
return data;
}
function plain(name: keyof typeof entityDefinitions, row: any) {
if (!row) return null;
return Object.fromEntries(
Object.keys(entityDefinitions[name].properties)
.filter((key) => row[key] !== undefined)
.map((key) => [key, row[key]]),
);
}
async function execute(
operation: string,
args: any[],
token?: number,
): Promise<any> {
await ready;
// Reject queued work after owner death; the monitor interrupts running SQL.
if (process.ppid !== workerData.ownerPid) process.exit(1);
const connection = orm.em.getConnection();
if (operation === 'begin') {
await connection.execute(
transactions.length ? `SAVEPOINT "ql_${token}"` : 'BEGIN IMMEDIATE',
);
transactions.push(token!);
return;
}
if (token !== transactions[transactions.length - 1])
throw new Error('Database transaction ownership mismatch');
if (operation === 'commit' || operation === 'rollback') {
if (!transactions.length) throw new Error('No active database transaction');
const nested = transactions.length > 1;
if (operation === 'rollback' && nested)
await connection.execute(`ROLLBACK TO SAVEPOINT "ql_${token}"`);
await connection.execute(
nested ? `RELEASE SAVEPOINT "ql_${token}"` : operation.toUpperCase(),
);
transactions.pop();
return;
}
if (operation === 'close') {
if (transactions.length)
throw new Error('Cannot close an active transaction');
cache?.close();
await orm.close();
return;
}
if (operation === 'query') return connection.execute(args[0], args[1]);
if (operation === 'cache') {
if (!workerData.cacheStorage)
throw new Error('No cache database configured');
if (!cache) {
const { DatabaseSync } = require('node:sqlite');
mkdirSync(dirname(workerData.cacheStorage), { recursive: true });
cache = new DatabaseSync(workerData.cacheStorage);
cache.exec(
'PRAGMA busy_timeout=10000;CREATE TABLE IF NOT EXISTS keyv(key VARCHAR(255) PRIMARY KEY,value TEXT)',
);
}
const [action, key, value] = args;
switch (action) {
case 'get':
return cache.prepare('SELECT value FROM keyv WHERE key=?').get(key)
?.value;
case 'set':
cache
.prepare(
'INSERT INTO keyv(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value',
)
.run(key, value);
return true;
case 'delete':
return (
cache.prepare('DELETE FROM keyv WHERE key=?').run(key).changes > 0
);
default:
throw new Error('Unsupported cache operation');
}
}
const [name, condition, options = {}] = args as [
keyof typeof entityDefinitions,
any,
any,
];
if (!(name in entityDefinitions)) throw new Error('Unknown entity');
const em = orm.em.fork();
const entityName = entities.find((entity) => entity.meta.className === name)!;
if (operation !== 'insert' && condition == null)
throw new Error('Missing database predicate');
const query = operation === 'insert' ? {} : filter(name, condition);
const readOptions = {
...options,
orderBy: options.orderBy?.map((order: Record<string, string>) =>
Object.fromEntries(
Object.entries(order).map(([field, direction]) => [
name === 'Env' && field === 'isPinned'
? raw('COALESCE("isPinned",0)')
: field,
direction,
]),
),
),
};
switch (operation) {
case 'find':
return (await em.find(entityName, query, readOptions)).map((row) =>
plain(name, row),
);
case 'findOne':
return plain(name, await em.findOne(entityName, query, readOptions));
case 'count':
return em.count(entityName, query);
case 'insert': {
const id = await em.insert(entityName, values(name, condition, true));
return plain(name, await em.findOneOrFail(entityName, { id }));
}
case 'nativeUpdate':
return em.nativeUpdate(entityName, query, values(name, options));
case 'nativeDelete':
return em.nativeDelete(entityName, query);
default:
throw new Error('Unsupported database operation');
}
}
export function startDatabaseWorker(settings: typeof workerData) {
workerData = settings;
ready = initialize();
void ready.catch(() => {});
let queue = Promise.resolve();
process.on('message', ({ id, operation, args, token }) => {
// Continue after a failed statement so the owner can still roll back.
queue = queue.then(async () => {
try {
process.send?.({
id,
value: await execute(operation, args, token),
});
} catch (error: any) {
const code =
error.name === 'UniqueConstraintViolationException'
? 'UNIQUE_CONSTRAINT'
: error.name === 'TableNotFoundException'
? 'DATABASE_SCHEMA_MISSING'
: (
{
5: 'SQLITE_BUSY',
6: 'SQLITE_LOCKED',
8: 'SQLITE_READONLY',
10: 'SQLITE_IOERR',
11: 'SQLITE_CORRUPT',
13: 'SQLITE_FULL',
14: 'SQLITE_CANTOPEN',
26: 'SQLITE_NOTADB',
} as Record<number, string>
)[Number(error.errcode) & 255] || 'DATABASE_QUERY_FAILED';
process.send?.({
id,
error: {
code,
message:
code === 'UNIQUE_CONSTRAINT'
? 'Resource already exists'
: `Database operation failed (${code})`,
},
});
}
});
});
}
+3 -16
View File
@@ -1,5 +1,4 @@
import { sequelize } from '.';
import { DataTypes, Model, ModelDefined } from 'sequelize';
import { Repository, RecordRow } from './repository';
export class Dependence {
timestamp?: string;
@@ -47,17 +46,5 @@ export enum versionDependenceCommandTypes {
'=',
}
export interface DependenceInstance
extends Model<Dependence, Dependence>,
Dependence {}
export const DependenceModel = sequelize.define<DependenceInstance>(
'Dependence',
{
name: DataTypes.STRING,
type: DataTypes.NUMBER,
timestamp: DataTypes.STRING,
status: DataTypes.NUMBER,
log: DataTypes.JSON,
remark: DataTypes.STRING,
},
);
export type DependenceInstance = RecordRow<Dependence>;
export const DependenceModel = new Repository<Dependence>('Dependence');
+3 -13
View File
@@ -1,5 +1,4 @@
import { DataTypes, Model } from 'sequelize';
import { sequelize } from '.';
import { Repository, RecordRow } from './repository';
export class Env {
value?: string;
@@ -38,14 +37,5 @@ export const initPosition = 4500000000000000;
export const stepPosition = 10000000000;
export const minPosition = 100;
export interface EnvInstance extends Model<Env, Env>, Env {}
export const EnvModel = sequelize.define<EnvInstance>('Env', {
value: { type: DataTypes.STRING, unique: 'compositeIndex' },
timestamp: DataTypes.STRING,
status: DataTypes.NUMBER,
position: DataTypes.NUMBER,
name: { type: DataTypes.STRING, unique: 'compositeIndex' },
remarks: DataTypes.STRING,
isPinned: DataTypes.NUMBER,
labels: DataTypes.JSON,
});
export type EnvInstance = RecordRow<Env>;
export const EnvModel = new Repository<Env>('Env');
+5 -18
View File
@@ -1,23 +1,10 @@
import { Sequelize, Transaction } from 'sequelize';
import config from '../config/index';
import { join } from 'path';
import { DatabaseClient } from './databaseClient';
export const sequelize = new Sequelize({
dialect: 'sqlite',
storage: join(config.dbPath, 'database.sqlite'),
logging: false,
retry: {
max: 10,
match: ['SQLITE_BUSY: database is locked'],
},
pool: {
max: 5,
min: 2,
idle: 30000,
acquire: 30000,
evict: 10000,
},
transactionType: Transaction.TYPES.IMMEDIATE,
});
export const database = new DatabaseClient(
join(config.dbPath, 'database.sqlite'),
join(config.dbPath, 'keyv.sqlite'),
);
export type ResponseType<T> = { code: number; data?: T; message?: string };
+11 -11
View File
@@ -1,5 +1,4 @@
import { sequelize } from '.';
import { DataTypes, Model, ModelDefined } from 'sequelize';
import { Repository, RecordRow } from './repository';
export class App {
name: string;
@@ -24,13 +23,14 @@ export interface AppToken {
expiration: number;
}
export type AppScope = 'envs' | 'crons' | 'configs' | 'scripts' | 'logs' | 'system' | 'dashboard';
export type AppScope =
| 'envs'
| 'crons'
| 'configs'
| 'scripts'
| 'logs'
| 'system'
| 'dashboard';
export interface AppInstance extends Model<App, App>, App {}
export const AppModel = sequelize.define<AppInstance>('App', {
name: { type: DataTypes.STRING, unique: 'name' },
scopes: DataTypes.JSON,
client_id: DataTypes.STRING,
client_secret: DataTypes.STRING,
tokens: DataTypes.JSON,
});
export type AppInstance = RecordRow<App>;
export const AppModel = new Repository<App>('App');
+25
View File
@@ -0,0 +1,25 @@
export type Query = Record<string, any>;
export type QuerySpec = {
where: Query;
orderBy?: Record<string, string>[];
offset?: number;
limit?: number;
};
export function orderBy(order: string[][]): Record<string, string>[] {
return order.map(([field, direction]) => {
if (!/^(asc|desc)$/i.test(direction))
throw new Error('Invalid sort direction');
return { [field]: direction.toLowerCase() };
});
}
export function textSearch(fields: string[], text: string): Query {
return {
$or: fields.flatMap((field) =>
[...new Set([text, encodeURI(text)])].map((value) => ({
[field]: { $like: `%${value}%` },
})),
),
};
}
+54
View File
@@ -0,0 +1,54 @@
import { database } from '.';
import { DatabaseClient } from './databaseClient';
export type RecordRow<T> = T & { id: number; createdAt: Date; updatedAt: Date };
export type Query = Record<string, any>;
export type ReadOptions = {
fields?: string[];
orderBy?: Record<string, string>[];
limit?: number;
offset?: number;
};
// A small IPC boundary with MikroORM's native query/operation names. No ORM
// entities or request identity maps escape the single database worker.
export class Repository<T> {
constructor(
private name: string,
private client: () => DatabaseClient = () => database,
) {}
find(where: Query = {}, options: ReadOptions = {}): Promise<RecordRow<T>[]> {
return this.client().request('find', this.name, where, options);
}
findOne(
where: Query,
options: ReadOptions = {},
): Promise<RecordRow<T> | null> {
return this.client().request('findOne', this.name, where, options);
}
count(where: Query = {}): Promise<number> {
return this.client().request('count', this.name, where);
}
insert(data: Partial<T>): Promise<RecordRow<T>> {
return this.client().request('insert', this.name, data);
}
nativeUpdate(where: Query, data: Partial<T>): Promise<number> {
return this.client().request('nativeUpdate', this.name, where, data);
}
nativeDelete(where: Query): Promise<number> {
return this.client().request('nativeDelete', this.name, where);
}
async insertMany(data: Partial<T>[]): Promise<RecordRow<T>[]> {
return this.client().transaction(async () => {
const rows = [];
for (const item of data) rows.push(await this.insert(item));
return rows;
});
}
async getOrCreate(where: Query): Promise<RecordRow<T>> {
return this.client().transaction(
async () =>
(await this.findOne(where)) || this.insert(where as Partial<T>),
);
}
}
+3 -38
View File
@@ -1,5 +1,4 @@
import { sequelize } from '.';
import { DataTypes, Model } from 'sequelize';
import { Repository, RecordRow } from './repository';
export enum InstanceStatus {
'running' = 0,
@@ -41,41 +40,7 @@ export class RunningInstance {
}
}
export interface RunningInstanceModel
extends Model<RunningInstanceAttributes, RunningInstanceAttributes>,
RunningInstanceAttributes {}
export const RunningInstanceModel = sequelize.define<RunningInstanceModel>(
export type RunningInstanceModel = RecordRow<RunningInstanceAttributes>;
export const RunningInstanceModel = new Repository<RunningInstanceAttributes>(
'RunningInstance',
{
cron_id: {
type: DataTypes.NUMBER,
allowNull: false,
},
pid: {
type: DataTypes.NUMBER,
allowNull: true,
},
log_path: {
type: DataTypes.STRING,
allowNull: true,
},
started_at: {
type: DataTypes.NUMBER,
allowNull: false,
},
finished_at: {
type: DataTypes.NUMBER,
allowNull: true,
},
status: {
type: DataTypes.NUMBER,
allowNull: false,
defaultValue: InstanceStatus.running,
},
exit_code: {
type: DataTypes.NUMBER,
allowNull: true,
},
},
);
+178
View File
@@ -0,0 +1,178 @@
// Persisted SQLite names and types are part of the rollback contract.
type Field = {
type: string;
fieldName: string;
nullable?: boolean;
primary?: boolean;
autoincrement?: boolean;
};
function entity<Name extends string>(
name: Name,
tableName: string,
groups: Record<string, string[]>,
required: string[] = [],
) {
const properties: Record<string, Field> = {
id: { type: 'number', fieldName: 'id', primary: true, autoincrement: true },
};
for (const [type, fields] of Object.entries(groups))
for (const field of fields)
properties[field] = {
type,
fieldName: field,
nullable: !required.includes(field),
};
for (const field of ['createdAt', 'updatedAt'])
properties[field] = {
type: 'legacyDate',
fieldName: field,
nullable: false,
};
return { name, tableName, properties };
}
export const entityDefinitions = {
App: entity(
'App',
'Apps',
{
string: ['name', 'client_id', 'client_secret'],
json: ['scopes', 'tokens'],
},
[],
),
Auth: entity('Auth', 'Auths', { string: ['ip', 'type'], json: ['info'] }, []),
CrontabStat: entity(
'CrontabStat',
'CrontabStats',
{
number: [
'ref_id',
'run_count',
'success_count',
'fail_count',
'total_time',
'max_time',
],
string: ['date'],
},
['ref_id', 'date'],
),
CrontabView: entity(
'CrontabView',
'CrontabViews',
{
string: ['name', 'filterRelation'],
number: ['position', 'isDisabled', 'type'],
json: ['filters', 'sorts'],
},
[],
),
Crontab: entity(
'Crontab',
'Crontabs',
{
string: [
'name',
'command',
'schedule',
'timestamp',
'log_path',
'queued_token',
'task_before',
'task_after',
'log_name',
'work_dir',
],
boolean: ['saved'],
number: [
'status',
'isSystem',
'pid',
'isDisabled',
'isPinned',
'last_running_time',
'last_execution_time',
'sub_id',
'allow_multiple_instances',
],
json: ['labels', 'extra_schedules'],
},
[],
),
Dependence: entity(
'Dependence',
'Dependences',
{
string: ['name', 'timestamp', 'remark'],
number: ['type', 'status'],
json: ['log'],
},
[],
),
Env: entity(
'Env',
'Envs',
{
string: ['value', 'timestamp', 'name', 'remarks'],
number: ['status', 'position', 'isPinned'],
json: ['labels'],
},
[],
),
RunningInstance: entity(
'RunningInstance',
'RunningInstances',
{
number: [
'cron_id',
'pid',
'started_at',
'finished_at',
'status',
'exit_code',
],
string: ['log_path'],
},
['cron_id', 'started_at', 'status'],
),
Subscription: entity(
'Subscription',
'Subscriptions',
{
string: [
'name',
'url',
'schedule',
'type',
'whitelist',
'blacklist',
'dependences',
'extensions',
'sub_before',
'sub_after',
'branch',
'pull_type',
'log_path',
'schedule_type',
'alias',
'proxy',
],
json: ['interval_schedule', 'pull_option'],
number: ['status', 'pid', 'is_disabled', 'autoAddCron', 'autoDelCron'],
},
[],
),
};
export const initialSchema = [
'CREATE TABLE IF NOT EXISTS `Apps` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `scopes` JSON, `client_id` VARCHAR(255), `client_secret` VARCHAR(255), `tokens` JSON, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`name`))',
'CREATE TABLE IF NOT EXISTS `Auths` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `ip` VARCHAR(255), `type` VARCHAR(255), `info` JSON, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL)',
'CREATE TABLE IF NOT EXISTS `CrontabStats` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `ref_id` NUMBER NOT NULL, `date` VARCHAR(255) NOT NULL, `run_count` NUMBER DEFAULT 0, `success_count` NUMBER DEFAULT 0, `fail_count` NUMBER DEFAULT 0, `total_time` NUMBER DEFAULT 0, `max_time` NUMBER DEFAULT 0, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL)',
'CREATE TABLE IF NOT EXISTS `CrontabViews` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `position` NUMBER, `isDisabled` NUMBER, `filters` JSON, `sorts` JSON, `filterRelation` VARCHAR(255), `type` NUMBER, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`name`))',
'CREATE TABLE IF NOT EXISTS `Crontabs` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `command` VARCHAR(255), `schedule` VARCHAR(255), `timestamp` VARCHAR(255), `saved` TINYINT(1), `status` NUMBER, `isSystem` NUMBER, `pid` NUMBER, `isDisabled` NUMBER, `isPinned` NUMBER, `log_path` VARCHAR(255), `queued_token` VARCHAR(255), `labels` JSON, `last_running_time` NUMBER, `last_execution_time` NUMBER, `sub_id` NUMBER, `extra_schedules` JSON, `task_before` VARCHAR(255), `task_after` VARCHAR(255), `log_name` VARCHAR(255), `allow_multiple_instances` NUMBER, `work_dir` VARCHAR(255), `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`name`, `command`, `schedule`))',
'CREATE TABLE IF NOT EXISTS `Dependences` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `type` NUMBER, `timestamp` VARCHAR(255), `status` NUMBER, `log` JSON, `remark` VARCHAR(255), `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL)',
'CREATE TABLE IF NOT EXISTS `Envs` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `value` VARCHAR(255), `timestamp` VARCHAR(255), `status` NUMBER, `position` NUMBER, `name` VARCHAR(255), `remarks` VARCHAR(255), `isPinned` NUMBER, `labels` JSON, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`value`, `name`))',
'CREATE TABLE IF NOT EXISTS `RunningInstances` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `cron_id` NUMBER NOT NULL, `pid` NUMBER, `log_path` VARCHAR(255), `started_at` NUMBER NOT NULL, `finished_at` NUMBER, `status` NUMBER NOT NULL DEFAULT 0, `exit_code` NUMBER, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL)',
'CREATE TABLE IF NOT EXISTS `Subscriptions` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `url` VARCHAR(255), `schedule` VARCHAR(255), `interval_schedule` JSON, `type` VARCHAR(255), `whitelist` VARCHAR(255), `blacklist` VARCHAR(255), `status` NUMBER, `dependences` VARCHAR(255), `extensions` VARCHAR(255), `sub_before` VARCHAR(255), `sub_after` VARCHAR(255), `branch` VARCHAR(255), `pull_type` VARCHAR(255), `pull_option` JSON, `pid` NUMBER, `is_disabled` NUMBER, `log_path` VARCHAR(255), `schedule_type` VARCHAR(255), `alias` VARCHAR(255), `proxy` VARCHAR(255), `autoAddCron` NUMBER, `autoDelCron` NUMBER, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`name`, `url`, `schedule`, `interval_schedule`), UNIQUE (`alias`))',
'CREATE INDEX IF NOT EXISTS `crontab_stats_date` ON `CrontabStats` (`date`)',
'CREATE UNIQUE INDEX IF NOT EXISTS `crontab_stats_ref_id_date` ON `CrontabStats` (`ref_id`, `date`)',
];
+45
View File
@@ -0,0 +1,45 @@
import { database } from '.';
// Aggregate SQL stays explicit and bound; public input is never used as an SQL
// identifier. These queries preserve the dashboard response names.
export const statistics = {
async totals(date: string) {
const rows = await database.query(
`SELECT SUM(run_count) AS total_runs,
SUM(success_count) AS total_success, SUM(fail_count) AS total_fail,
SUM(total_time) AS total_time FROM CrontabStats WHERE date=?`,
[date],
);
return rows[0];
},
results(date: string, column: 'success_count' | 'fail_count') {
if (!['success_count', 'fail_count'].includes(column))
throw new Error('Invalid statistic');
return database.query(
`SELECT ref_id,SUM(${column}) AS result_count FROM CrontabStats
WHERE date=? AND ${column}>0 GROUP BY ref_id ORDER BY result_count DESC,ref_id ASC`,
[date],
);
},
trend(dates: string[]) {
if (!dates.length) return Promise.resolve([]);
return database.query(
`SELECT date,SUM(run_count) AS total_runs,
SUM(success_count) AS total_success,SUM(fail_count) AS total_fail FROM CrontabStats
WHERE date IN (${dates
.map(() => '?')
.join(',')}) GROUP BY date ORDER BY date ASC`,
dates,
);
},
ranking(date: string, metric: 'total_time' | 'run_count') {
if (!['total_time', 'run_count'].includes(metric))
throw new Error('Invalid statistic');
return database.query(
`SELECT ref_id,SUM(total_time) AS total_time,SUM(run_count) AS run_count,
SUM(success_count) AS success_count,MAX(max_time) AS max_time FROM CrontabStats
WHERE date=? AND run_count>0 GROUP BY ref_id ORDER BY SUM(${metric}) DESC LIMIT 5`,
[date],
);
},
};
+3 -45
View File
@@ -1,5 +1,4 @@
import { sequelize } from '.';
import { DataTypes, Model, ModelDefined } from 'sequelize';
import { Repository, RecordRow } from './repository';
import { SimpleIntervalSchedule } from 'toad-scheduler';
type SimpleIntervalScheduleUnit = keyof SimpleIntervalSchedule;
@@ -70,46 +69,5 @@ export enum SubscriptionStatus {
'queued',
}
export interface SubscriptionInstance
extends Model<Subscription, Subscription>,
Subscription {}
export const SubscriptionModel = sequelize.define<SubscriptionInstance>(
'Subscription',
{
name: {
unique: 'compositeIndex',
type: DataTypes.STRING,
},
url: {
unique: 'compositeIndex',
type: DataTypes.STRING,
},
schedule: {
unique: 'compositeIndex',
type: DataTypes.STRING,
},
interval_schedule: {
unique: 'compositeIndex',
type: DataTypes.JSON,
},
type: DataTypes.STRING,
whitelist: DataTypes.STRING,
blacklist: DataTypes.STRING,
status: DataTypes.NUMBER,
dependences: DataTypes.STRING,
extensions: DataTypes.STRING,
sub_before: DataTypes.STRING,
sub_after: DataTypes.STRING,
branch: DataTypes.STRING,
pull_type: DataTypes.STRING,
pull_option: DataTypes.JSON,
pid: DataTypes.NUMBER,
is_disabled: DataTypes.NUMBER,
log_path: DataTypes.STRING,
schedule_type: DataTypes.STRING,
alias: { type: DataTypes.STRING, unique: 'alias' },
proxy: { type: DataTypes.STRING, allowNull: true },
autoAddCron: { type: DataTypes.NUMBER, allowNull: true },
autoDelCron: { type: DataTypes.NUMBER, allowNull: true },
},
);
export type SubscriptionInstance = RecordRow<Subscription>;
export const SubscriptionModel = new Repository<Subscription>('Subscription');
+3 -13
View File
@@ -1,5 +1,4 @@
import { sequelize } from '.';
import { DataTypes, Model, ModelDefined } from 'sequelize';
import { Repository, RecordRow } from './repository';
import { NotificationInfo } from './notify';
export class SystemInfo {
@@ -106,14 +105,5 @@ export type SystemModelInfo = SystemConfigInfo &
Partial<NotifyLogInfo> &
Partial<AuthInfo>;
export interface SystemInstance
extends Model<SystemInfo, SystemInfo>,
SystemInfo {}
export const SystemModel = sequelize.define<SystemInstance>('Auth', {
ip: DataTypes.STRING,
type: DataTypes.STRING,
info: {
type: DataTypes.JSON,
allowNull: true,
},
});
export type SystemInstance = RecordRow<SystemInfo>;
export const SystemModel = new Repository<SystemInfo>('Auth');
+11
View File
@@ -0,0 +1,11 @@
{
"extends": "../tsconfig.json",
"compilerOptions": {
"module": "node20",
"moduleResolution": "node16",
"rootDir": "..",
"outDir": "../../static/build"
},
"include": ["./databaseWorker.ts", "./schema.ts"],
"exclude": ["node_modules"]
}
+6 -22
View File
@@ -1,30 +1,14 @@
import Logger from './logger';
import { EnvModel } from '../data/env';
import { CrontabModel } from '../data/cron';
import { DependenceModel } from '../data/dependence';
import { AppModel } from '../data/open';
import { SystemModel } from '../data/system';
import { SubscriptionModel } from '../data/subscription';
import { CrontabViewModel } from '../data/cronView';
import { CrontabStatModel } from '../data/cronStats';
import { RunningInstanceModel } from '../data/runningInstance';
import { sequelize } from '../data';
import { database } from '../data';
import { initialSchema } from '../data/schema';
import { migrateSchema } from '../shared/schemaMigrations';
export default async () => {
try {
await CrontabModel.sync();
await DependenceModel.sync();
await AppModel.sync();
await SystemModel.sync();
await EnvModel.sync();
await SubscriptionModel.sync();
await CrontabViewModel.sync();
await CrontabStatModel.sync();
await RunningInstanceModel.sync();
await migrateSchema(sequelize);
await database.transaction(async () => {
for (const sql of initialSchema) await database.query(sql);
await migrateSchema(database);
});
Logger.info('[boot] DB loaded');
} catch (error) {
Logger.error('[boot] DB load failed', error);
+85 -99
View File
@@ -16,7 +16,7 @@ import { t } from '../shared/i18n';
import { AppScope } from '../data/open';
import uniqueConstraintError from '../middlewares/uniqueConstraintError';
import protectedPathCase from '../middlewares/protectedPathCase';
import { withAuthAvailability } from '../middlewares/authAvailability';
function resolveTrustProxy(value = process.env.QL_TRUST_PROXY) {
const setting = value?.trim();
if (!setting) {
@@ -30,24 +30,20 @@ function resolveTrustProxy(value = process.env.QL_TRUST_PROXY) {
}
return setting;
}
export default ({ app }: { app: Application }) => {
// Security: Enable strict routing to prevent case-insensitive path bypass
app.set('case sensitive routing', true);
app.set('strict routing', true);
app.set('trust proxy', resolveTrustProxy());
app.use(cors());
// Security: Reject case variations under protected API namespaces before
// authentication checks can interpret the request differently from routing.
app.use(protectedPathCase);
// Rewrite URLs to strip baseUrl prefix if configured
// This allows the rest of the app to work without baseUrl awareness
if (config.baseUrl) {
app.use(rewrite(`${config.baseUrl}/*`, '/$1'));
}
app.get(`${config.api.prefix}/env.js`, serveEnv);
app.use(
`${config.api.prefix}/static`,
@@ -58,7 +54,6 @@ export default ({ app }: { app: Application }) => {
},
}),
);
const credentialPaths = ['/api', '/open'].flatMap((prefix) =>
['/user/login', '/user/init', '/user/two-factor/login'].map(
(route) => `${prefix}${route}`,
@@ -69,13 +64,10 @@ export default ({ app }: { app: Application }) => {
bodyParser.json({ limit: '16kb' }),
bodyParser.urlencoded({ limit: '16kb', extended: false }),
);
app.use(bodyParser.json({ limit: '50mb' }));
app.use(bodyParser.urlencoded({ limit: '50mb', extended: true }));
const frontendPath = path.join(config.rootPath, 'static/dist');
app.use(express.static(frontendPath));
app.use(
expressjwt({
secret: config.jwt.secret,
@@ -84,7 +76,6 @@ export default ({ app }: { app: Application }) => {
path: [...config.apiWhiteList, /^(\/(?!api\/).*)$/i],
}),
);
app.use((req: Request, res, next) => {
if (!req.headers) {
req.platform = 'desktop';
@@ -94,91 +85,84 @@ export default ({ app }: { app: Application }) => {
}
return next();
});
app.use(async (req: Request, res, next) => {
const pathLower = req.path.toLowerCase();
if (!['/open/', '/api/'].some((x) => pathLower.startsWith(x))) {
return next();
}
const headerToken = getToken(req);
if (pathLower.startsWith('/open/')) {
const apps = await shareStore.getApps();
const doc = apps?.filter((x) =>
x.tokens?.find((y) => y.value === headerToken),
)?.[0];
if (doc && doc.tokens && doc.tokens.length > 0) {
const currentToken = doc.tokens.find((x) => x.value === headerToken);
const keyMatch = pathLower.match(/\/open\/([a-z]+)\/*/);
const key = keyMatch && keyMatch[1];
if (!doc.scopes.includes(key as AppScope)) {
const err = new UnauthorizedError('credentials_bad_scheme', {
message: t('暂无权限'),
});
return next(err);
}
if (
!currentToken ||
currentToken.expiration < Math.round(Date.now() / 1000)
) {
const err = new UnauthorizedError('invalid_token', {
message: t('Token 已失效'),
});
return next(err);
}
app.use(
withAuthAvailability(async (req: Request, res, next) => {
const pathLower = req.path.toLowerCase();
if (!['/open/', '/api/'].some((x) => pathLower.startsWith(x))) {
return next();
}
}
const originPath = `${req.baseUrl}${pathLower === '/' ? '' : pathLower}`;
if (
!headerToken &&
originPath &&
config.apiWhiteList.includes(originPath)
) {
return next();
}
const authInfo = await shareStore.getAuthInfo();
if (isValidToken(authInfo, headerToken, req.platform, config.jwt.secret)) {
return next();
}
const errorCode = headerToken ? 'invalid_token' : 'credentials_required';
const errorMessage = headerToken ? t('Token 已失效') : t('请先登录');
const err = new UnauthorizedError(errorCode, { message: errorMessage });
next(err);
});
app.use(async (req, res, next) => {
const pathLower = req.path.toLowerCase();
if (
![
'/api/user/init',
'/api/user/notification/init',
'/open/user/init',
'/open/user/notification/init',
].includes(pathLower)
) {
return next();
}
const authInfo = (await shareStore.getAuthInfo()) || ({} as AuthInfo);
let isInitialized = !isDefaultAuthInfo(authInfo);
if (isInitialized) {
return res.send({ code: 450, message: t('未知错误') });
} else {
return next();
}
});
const headerToken = getToken(req);
if (pathLower.startsWith('/open/')) {
const apps = await shareStore.getApps();
const doc = apps?.filter((x) =>
x.tokens?.find((y) => y.value === headerToken),
)?.[0];
if (doc && doc.tokens && doc.tokens.length > 0) {
const currentToken = doc.tokens.find((x) => x.value === headerToken);
const keyMatch = pathLower.match(/\/open\/([a-z]+)\/*/);
const key = keyMatch && keyMatch[1];
if (!doc.scopes.includes(key as AppScope)) {
const err = new UnauthorizedError('credentials_bad_scheme', {
message: t('暂无权限'),
});
return next(err);
}
if (
!currentToken ||
currentToken.expiration < Math.round(Date.now() / 1000)
) {
const err = new UnauthorizedError('invalid_token', {
message: t('Token 已失效'),
});
return next(err);
}
return next();
}
}
const originPath = `${req.baseUrl}${pathLower === '/' ? '' : pathLower}`;
if (
!headerToken &&
originPath &&
config.apiWhiteList.includes(originPath)
) {
return next();
}
const authInfo = await shareStore.getAuthInfo();
if (
isValidToken(authInfo, headerToken, req.platform, config.jwt.secret)
) {
return next();
}
const errorCode = headerToken ? 'invalid_token' : 'credentials_required';
const errorMessage = headerToken ? t('Token 已失效') : t('请先登录');
const err = new UnauthorizedError(errorCode, { message: errorMessage });
next(err);
}),
);
app.use(
withAuthAvailability(async (req, res, next) => {
const pathLower = req.path.toLowerCase();
if (
![
'/api/user/init',
'/api/user/notification/init',
'/open/user/init',
'/open/user/notification/init',
].includes(pathLower)
) {
return next();
}
const authInfo = (await shareStore.getAuthInfo()) || ({} as AuthInfo);
let isInitialized = !isDefaultAuthInfo(authInfo);
if (isInitialized) {
return res.send({ code: 450, message: t('未知错误') });
} else {
return next();
}
}),
);
app.use(rewrite('/open/*', '/api/$1'));
app.use(config.api.prefix, routes());
app.get('*', (req, res, next) => {
// Don't serve index.html for API routes
if (
@@ -196,13 +180,13 @@ export default ({ app }: { app: Application }) => {
}
});
});
app.use(validationErrors);
app.use(uniqueConstraintError);
app.use(
(
err: Error & { status: number },
err: Error & {
status: number;
},
req: Request,
res: Response,
next: NextFunction,
@@ -216,15 +200,16 @@ export default ({ app }: { app: Application }) => {
return next(err);
},
);
app.use(
(
err: Error & { errors: any[] },
err: Error & {
errors: any[];
},
req: Request,
res: Response,
next: NextFunction,
) => {
if (err.name.includes('Sequelize')) {
if (err.name === 'DatabaseError') {
return res
.status(500)
.send({
@@ -237,10 +222,11 @@ export default ({ app }: { app: Application }) => {
return next(err);
},
);
app.use(
(
err: Error & { status: number },
err: Error & {
status: number;
},
req: Request,
res: Response,
next: NextFunction,
+65 -82
View File
@@ -1,3 +1,5 @@
import { textSearch } from '../data/queries';
import { Query, QuerySpec, orderBy } from '../data/queries';
import DependenceService from '../services/dependence';
import { exec } from 'child_process';
import { Container } from 'typedi';
@@ -5,7 +7,6 @@ import { Crontab, CrontabModel, CrontabStatus } from '../data/cron';
import CronService from '../services/cron';
import EnvService from '../services/env';
import { DependenceModel, DependenceStatus } from '../data/dependence';
import { Op } from 'sequelize';
import config from '../config';
import { CrontabViewModel, CronViewType } from '../data/cronView';
import { initPosition } from '../data/env';
@@ -20,7 +21,6 @@ import Logger from './logger';
import cronClient from '../schedule/client';
import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
import { setLang, systemLang } from '../shared/i18n';
export default async () => {
const cronService = Container.get(CronService);
const envService = Container.get(EnvService);
@@ -28,17 +28,14 @@ export default async () => {
const systemService = Container.get(SystemService);
const userService = Container.get(UserService);
const openService = Container.get(OpenService);
// 初始化增加系统配置
await openService.initializeSystemApp();
const [systemConfig] = await SystemModel.findOrCreate({
where: { type: AuthDataType.systemConfig },
const systemConfig = await SystemModel.getOrCreate({
type: AuthDataType.systemConfig,
});
await SystemModel.findOrCreate({
where: { type: AuthDataType.notification },
});
const [authConfig] = await SystemModel.findOrCreate({
where: { type: AuthDataType.authConfig },
await SystemModel.getOrCreate({ type: AuthDataType.notification });
const authConfig = await SystemModel.getOrCreate({
type: AuthDataType.authConfig,
});
if (!authConfig?.info || isDemoEnv()) {
let authInfo = {
@@ -54,36 +51,34 @@ export default async () => {
} catch (error) {
Logger.warn('Failed to read auth config file, using default credentials');
}
await SystemModel.upsert({
id: authConfig?.id,
info: authInfo,
type: AuthDataType.authConfig,
});
await SystemModel.nativeUpdate(
{ id: authConfig.id },
{
info: authInfo,
type: AuthDataType.authConfig,
},
);
}
const installDependencies = async () => {
const docs = await DependenceModel.findAll({
where: {},
order: [
['type', 'DESC'],
['createdAt', 'DESC'],
],
raw: true,
});
await DependenceModel.update(
const docs = await DependenceModel.find(
{},
{
orderBy: orderBy([
['type', 'DESC'],
['createdAt', 'DESC'],
]),
},
);
await DependenceModel.nativeUpdate(
{ id: docs.map((x) => x.id!) },
{ status: DependenceStatus.queued, log: [] },
{ where: { id: docs.map((x) => x.id!) } },
);
setTimeout(async () => {
await dependenceService.installDependenceOneByOne(docs);
const bootAfterLoader = await import('./bootAfter');
bootAfterLoader.default();
}, 5000);
};
// 初始化更新 linux/python/nodejs 镜像源配置
if (systemConfig.info?.pythonMirror) {
systemService.updatePythonMirror({
@@ -106,43 +101,40 @@ export default async () => {
nodeMirror: systemConfig.info?.nodeMirror,
});
}
// 初始化新增默认全部任务视图
CrontabViewModel.findAll({
where: { type: CronViewType.系统, name: '全部任务' },
raw: true,
}).then((docs) => {
if (docs.length === 0) {
CrontabViewModel.create({
name: '全部任务',
type: CronViewType.系统,
position: initPosition / 2,
});
}
});
CrontabViewModel.find({ type: CronViewType.系统, name: '全部任务' }, {}).then(
(docs) => {
if (docs.length === 0) {
CrontabViewModel.insert({
name: '全部任务',
type: CronViewType.系统,
position: initPosition / 2,
});
}
},
);
// 初始化更新所有任务状态为空闲
await CrontabModel.update({ status: CrontabStatus.idle }, { where: {} });
await CrontabModel.nativeUpdate({}, { status: CrontabStatus.idle });
// 清空所有运行中的实例记录(服务重启后进程已不存在)
await RunningInstanceModel.update(
await RunningInstanceModel.nativeUpdate(
{ status: InstanceStatus.running },
{
status: InstanceStatus.stopped,
finished_at: Math.floor(Date.now() / 1000),
// The previous process's exit code cannot be recovered after a restart.
},
{ where: { status: InstanceStatus.running } },
);
// 初始化时执行一次所有的 ql repo 任务
CrontabModel.findAll({
where: {
isDisabled: { [Op.ne]: 1 },
command: {
[Op.or]: [{ [Op.like]: `%ql repo%` }, { [Op.like]: `%ql raw%` }],
},
CrontabModel.find(
{
isDisabled: { ['$ne']: 1 },
$or: [
...textSearch(['command'], 'ql repo').$or,
...textSearch(['command'], 'ql raw').$or,
],
},
}).then((docs) => {
{},
).then((docs) => {
for (let i = 0; i < docs.length; i++) {
const doc = docs[i];
if (doc) {
@@ -150,87 +142,78 @@ export default async () => {
}
}
});
// 更新2.11.3以前的脚本路径
CrontabModel.findAll({
where: {
command: {
[Op.or]: [
{ [Op.like]: `%\/${config.rootPath}\/scripts\/%` },
{ [Op.like]: `%\/${config.rootPath}\/config\/%` },
{ [Op.like]: `%\/${config.rootPath}\/log\/%` },
{ [Op.like]: `%\/${config.rootPath}\/db\/%` },
],
},
CrontabModel.find(
{
$or: ['scripts', 'config', 'log', 'db'].flatMap(
(folder) =>
textSearch(['command'], `/${config.rootPath}/${folder}/`).$or,
),
},
}).then(async (docs) => {
{},
).then(async (docs) => {
for (let i = 0; i < docs.length; i++) {
const doc = docs[i];
if (doc) {
if (doc.command.includes(`${config.rootPath}/scripts/`)) {
await CrontabModel.update(
await CrontabModel.nativeUpdate(
{ id: doc.id },
{ command: doc.command.replace(`${config.rootPath}/scripts/`, '') },
{ where: { id: doc.id } },
);
}
if (doc.command.includes(`${config.rootPath}/log/`)) {
await CrontabModel.update(
await CrontabModel.nativeUpdate(
{ id: doc.id },
{
command: `${config.dataPath}/log/${doc.command.replace(
`${config.rootPath}/log/`,
'',
)}`,
},
{ where: { id: doc.id } },
);
}
if (doc.command.includes(`${config.rootPath}/config/`)) {
await CrontabModel.update(
await CrontabModel.nativeUpdate(
{ id: doc.id },
{
command: `${config.dataPath}/config/${doc.command.replace(
`${config.rootPath}/config/`,
'',
)}`,
},
{ where: { id: doc.id } },
);
}
if (doc.command.includes(`${config.rootPath}/db/`)) {
await CrontabModel.update(
await CrontabModel.nativeUpdate(
{ id: doc.id },
{
command: `${config.dataPath}/db/${doc.command.replace(
`${config.rootPath}/db/`,
'',
)}`,
},
{ where: { id: doc.id } },
);
}
}
}
});
// 初始化语言(必须在 autosave_crontab 之前)
const lang = systemConfig.info?.lang || systemLang();
setLang(lang);
// 确保 lang_env.sh 存在
try {
const langEnvExist = await fileExist(config.langEnvFile);
if (!langEnvExist) {
await writeFile(config.langEnvFile, `export QL_LANG='${lang}'\n`);
}
} catch { }
} catch {}
// 初始化保存一次ck和定时任务数据
cronClient.readiness.configure(async () => {
await cronService.autosave_crontab(true);
await systemService.restoreCronConcurrency();
});
await cronClient.readiness.recover();
await envService.set_envs();
const authInfo = await userService.getAuthInfo();
await shareStore.updateAuthInfo(authInfo);
await openService.refreshApps();
+1 -8
View File
@@ -9,20 +9,17 @@ import { join } from 'path';
import { t } from '../shared/i18n';
import Logger from './logger';
import { assertSubscriptionAlias } from '../shared/subscriptionPath';
export default async () => {
const systemService = Container.get(SystemService);
const scheduleService = Container.get(ScheduleService);
const subscriptionService = Container.get(SubscriptionService);
const sshKeyService = Container.get(SshKeyService);
// 生成内置token
let tokenCommand = `ts-node-transpile-only ${join(
config.rootPath,
'back/token.ts',
)}`;
const tokenFile = join(config.rootPath, 'static/build/token.js');
if (await fileExist(tokenFile)) {
tokenCommand = `node ${tokenFile}`;
}
@@ -41,7 +38,6 @@ export default async () => {
},
true,
);
// 运行删除日志任务
const data = await systemService.getSystemConfig();
if (data && data.info) {
@@ -61,20 +57,17 @@ export default async () => {
true,
);
}
systemService.updateTimezone(data.info);
// Apply global SSH key if configured
if (data.info.globalSshKey) {
await sshKeyService.addGlobalSSHKey(data.info.globalSshKey, 'global');
}
}
await subscriptionService.setSshConfig();
const subs = await subscriptionService.list();
for (const sub of subs) {
try {
const doc = sub.get({ plain: true });
const doc = sub;
assertSubscriptionAlias(doc.alias);
await subscriptionService.handleTask(doc, !sub.is_disabled);
} catch (error) {
+18
View File
@@ -0,0 +1,18 @@
import type { RequestHandler } from 'express';
import { t } from '../shared/i18n';
// Express 4 does not forward rejected async middleware promises itself.
export function withAuthAvailability(handler: RequestHandler): RequestHandler {
return (req, res, next) => {
void Promise.resolve()
.then(() => handler(req, res, next))
.catch(() => {
res.setHeader('Retry-After', '1');
next(
Object.assign(new Error(t('认证服务暂时不可用,请稍后重试')), {
status: 503,
}),
);
});
};
}
+3 -4
View File
@@ -1,14 +1,13 @@
import { ErrorRequestHandler } from 'express';
import { UniqueConstraintError } from 'sequelize';
import { DatabaseError } from '../data/databaseErrors';
import { t } from '../shared/i18n';
// Handle the database constraint itself so concurrent writes are covered too.
const uniqueConstraintError: ErrorRequestHandler = (err, req, res, next) => {
if (!(err instanceof UniqueConstraintError)) return next(err);
if (!(err instanceof DatabaseError) || err.code !== 'UNIQUE_CONSTRAINT')
return next(err);
// Constraint details can contain environment values or application credentials.
res
.status(409)
.json({ code: 409, message: t('资源已存在,请检查重复的名称或值') });
};
export default uniqueConstraintError;
+1 -39
View File
@@ -39,10 +39,7 @@ import {
RunCronsRequest,
} from '../protos/api';
import { NotificationInfo } from '../data/notify';
import { Model } from 'sequelize';
Container.set('logger', LoggerInstance);
export const getEnvs = async (
call: ServerUnaryCall<GetEnvsRequest, EnvsResponse>,
callback: sendUnaryData<EnvsResponse>,
@@ -62,7 +59,6 @@ export const getEnvs = async (
});
}
};
export const createEnv = async (
call: ServerUnaryCall<CreateEnvRequest, EnvsResponse>,
callback: sendUnaryData<EnvsResponse>,
@@ -75,7 +71,6 @@ export const createEnv = async (
callback(e);
}
};
export const updateEnv = async (
call: ServerUnaryCall<UpdateEnvRequest, EnvResponse>,
callback: sendUnaryData<EnvResponse>,
@@ -88,7 +83,6 @@ export const updateEnv = async (
message: 'id parameter is required',
});
}
const envService = Container.get(EnvService);
const data = await envService.update(
pick(call.request.env, ['id', 'name', 'value', 'remarks']) as EnvItem,
@@ -98,7 +92,6 @@ export const updateEnv = async (
callback(e);
}
};
export const deleteEnvs = async (
call: ServerUnaryCall<DeleteEnvsRequest, Response>,
callback: sendUnaryData<Response>,
@@ -110,7 +103,6 @@ export const deleteEnvs = async (
message: 'ids parameter is required',
});
}
const envService = Container.get(EnvService);
await envService.remove(call.request.ids);
callback(null, { code: 200 });
@@ -118,7 +110,6 @@ export const deleteEnvs = async (
callback(e);
}
};
export const moveEnv = async (
call: ServerUnaryCall<MoveEnvRequest, EnvResponse>,
callback: sendUnaryData<EnvResponse>,
@@ -131,7 +122,6 @@ export const moveEnv = async (
message: 'id parameter is required',
});
}
const envService = Container.get(EnvService);
const data = await envService.move(call.request.id, {
fromIndex: call.request.fromIndex,
@@ -142,7 +132,6 @@ export const moveEnv = async (
callback(e);
}
};
export const disableEnvs = async (
call: ServerUnaryCall<DisableEnvsRequest, Response>,
callback: sendUnaryData<Response>,
@@ -154,7 +143,6 @@ export const disableEnvs = async (
message: 'ids parameter is required',
});
}
const envService = Container.get(EnvService);
await envService.disabled(call.request.ids);
callback(null, { code: 200 });
@@ -162,7 +150,6 @@ export const disableEnvs = async (
callback(e);
}
};
export const enableEnvs = async (
call: ServerUnaryCall<EnableEnvsRequest, Response>,
callback: sendUnaryData<Response>,
@@ -174,7 +161,6 @@ export const enableEnvs = async (
message: 'ids parameter is required',
});
}
const envService = Container.get(EnvService);
await envService.enabled(call.request.ids);
callback(null, { code: 200 });
@@ -182,7 +168,6 @@ export const enableEnvs = async (
callback(e);
}
};
export const updateEnvNames = async (
call: ServerUnaryCall<UpdateEnvNamesRequest, Response>,
callback: sendUnaryData<Response>,
@@ -194,7 +179,6 @@ export const updateEnvNames = async (
message: 'ids parameter is required',
});
}
const envService = Container.get(EnvService);
await envService.updateNames({
ids: call.request.ids,
@@ -205,7 +189,6 @@ export const updateEnvNames = async (
callback(e);
}
};
export const getEnvById = async (
call: ServerUnaryCall<GetEnvByIdRequest, EnvResponse>,
callback: sendUnaryData<EnvResponse>,
@@ -218,7 +201,6 @@ export const getEnvById = async (
message: 'id parameter is required',
});
}
const envService = Container.get(EnvService);
const data = await envService.getDb({ id: call.request.id });
callback(null, {
@@ -229,7 +211,6 @@ export const getEnvById = async (
callback(e);
}
};
export const systemNotify = async (
call: ServerUnaryCall<SystemNotifyRequest, Response>,
callback: sendUnaryData<Response>,
@@ -247,7 +228,6 @@ export const systemNotify = async (
callback(e);
}
};
export const recordNotifyLog = async (
call: ServerUnaryCall<RecordNotifyLogRequest, Response>,
callback: sendUnaryData<Response>,
@@ -267,12 +247,9 @@ export const recordNotifyLog = async (
callback(error);
}
};
const normalizeCronData = (data: CronItem | null): CronItem | undefined => {
if (!data) return undefined;
// create() returns a Sequelize instance; spreading it omits attribute getters.
const cron =
data instanceof Model ? (data.get({ plain: true }) as CronItem) : data;
const cron = data;
return {
...cron,
labels: cron.labels ?? [],
@@ -283,7 +260,6 @@ const normalizeCronData = (data: CronItem | null): CronItem | undefined => {
task_after: cron.task_after ?? undefined,
};
};
export const getCronDetail = async (
call: ServerUnaryCall<CronDetailRequest, CronDetailResponse>,
callback: sendUnaryData<CronDetailResponse>,
@@ -305,7 +281,6 @@ export const getCronDetail = async (
callback(e);
}
};
export const createCron = async (
call: ServerUnaryCall<CreateCronRequest, CronResponse>,
callback: sendUnaryData<CronResponse>,
@@ -318,7 +293,6 @@ export const createCron = async (
callback(e);
}
};
export const updateCron = async (
call: ServerUnaryCall<UpdateCronRequest, CronResponse>,
callback: sendUnaryData<CronResponse>,
@@ -326,7 +300,6 @@ export const updateCron = async (
try {
const cronService = Container.get(CronService);
const { id, ...fields } = call.request;
const updateRequest = {
id,
...Object.entries(fields).reduce((acc: any, [key, value]) => {
@@ -336,14 +309,12 @@ export const updateCron = async (
return acc;
}, {}),
} as UpdateCronRequest;
const data = (await cronService.update(updateRequest)) as CronItem;
callback(null, { code: 200, data: normalizeCronData(data) });
} catch (e: any) {
callback(e);
}
};
export const deleteCrons = async (
call: ServerUnaryCall<DeleteCronsRequest, Response>,
callback: sendUnaryData<Response>,
@@ -356,7 +327,6 @@ export const deleteCrons = async (
callback(e);
}
};
export const getCrons = async (
call: ServerUnaryCall<GetCronsRequest, CronsResponse>,
callback: sendUnaryData<CronsResponse>,
@@ -384,7 +354,6 @@ export const getCrons = async (
});
}
};
export const getCronById = async (
call: ServerUnaryCall<GetCronByIdRequest, CronResponse>,
callback: sendUnaryData<CronResponse>,
@@ -397,7 +366,6 @@ export const getCronById = async (
message: 'id parameter is required',
});
}
const cronService = Container.get(CronService);
const data = (await cronService.getDb({ id: call.request.id })) as CronItem;
callback(null, { code: 200, data: normalizeCronData(data) });
@@ -409,7 +377,6 @@ export const getCronById = async (
});
}
};
export const enableCrons = async (
call: ServerUnaryCall<EnableCronsRequest, Response>,
callback: sendUnaryData<Response>,
@@ -421,7 +388,6 @@ export const enableCrons = async (
message: 'ids parameter is required',
});
}
const cronService = Container.get(CronService);
await cronService.enabled(call.request.ids);
callback(null, { code: 200 });
@@ -429,7 +395,6 @@ export const enableCrons = async (
callback(e);
}
};
export const disableCrons = async (
call: ServerUnaryCall<DisableCronsRequest, Response>,
callback: sendUnaryData<Response>,
@@ -441,7 +406,6 @@ export const disableCrons = async (
message: 'ids parameter is required',
});
}
const cronService = Container.get(CronService);
await cronService.disabled(call.request.ids);
callback(null, { code: 200 });
@@ -449,7 +413,6 @@ export const disableCrons = async (
callback(e);
}
};
export const runCrons = async (
call: ServerUnaryCall<RunCronsRequest, Response>,
callback: sendUnaryData<Response>,
@@ -461,7 +424,6 @@ export const runCrons = async (
message: 'ids parameter is required',
});
}
const cronService = Container.get(CronService);
await cronService.run(call.request.ids);
callback(null, { code: 200 });
+2 -1
View File
@@ -4,6 +4,7 @@ import path from 'path';
import { request } from 'undici';
import { HealthCheckRequest, HealthCheckResponse } from '../protos/health';
import config from '../config';
import { database } from '../data';
function formatError(error: unknown): string {
if (!(error instanceof Error)) {
@@ -54,7 +55,7 @@ const check = async (
switch (call.request.service) {
// Local scheduler liveness only: never call HTTP from this probe.
case 'scheduler':
return callback(null, { status: 1 });
return callback(null, { status: database.healthy ? 1 : 2 });
case 'cron': {
const healthUrl = `http://localhost:${config.port}${
config.baseUrl || ''
+233 -283
View File
File diff suppressed because it is too large Load Diff
+19 -31
View File
@@ -1,3 +1,4 @@
import { Query, QuerySpec, orderBy } from '../data/queries';
import { Service, Inject } from 'typedi';
import winston from 'winston';
import { CrontabView, CrontabViewModel } from '../data/cronView';
@@ -7,12 +8,12 @@ import {
minPosition,
stepPosition,
} from '../data/env';
import { FindOptions } from 'sequelize';
@Service()
export default class CronViewService {
constructor(@Inject('logger') private logger: winston.Logger) {}
constructor(
@Inject('logger')
private logger: winston.Logger,
) {}
public async create(payload: CrontabView): Promise<CrontabView> {
let position = initPosition;
const views = await this.list();
@@ -22,61 +23,51 @@ export default class CronViewService {
position = position / 2;
const tab = new CrontabView({ ...payload, position });
const doc = await this.insert(tab);
await this.checkPosition(tab.position!);
return doc;
}
public async insert(payload: CrontabView): Promise<CrontabView> {
return await CrontabViewModel.create(payload, { returning: true });
return await CrontabViewModel.insert(payload);
}
public async update(payload: CrontabView): Promise<CrontabView> {
const doc = await this.getDb({ id: payload.id });
const tab = new CrontabView({ ...doc, ...payload });
const newDoc = await this.updateDb(tab);
return newDoc;
}
public async updateDb(payload: CrontabView): Promise<CrontabView> {
await CrontabViewModel.update(payload, { where: { id: payload.id } });
await CrontabViewModel.nativeUpdate({ id: payload.id }, payload);
return await this.getDb({ id: payload.id });
}
public async remove(ids: number[]) {
await CrontabViewModel.destroy({ where: { id: ids } });
await CrontabViewModel.nativeDelete({ id: ids });
}
public async list(): Promise<CrontabView[]> {
try {
const result = await CrontabViewModel.findAll({
where: {},
order: [['position', 'DESC']],
});
const result = await CrontabViewModel.find(
{},
{
orderBy: orderBy([['position', 'DESC']]),
},
);
return result;
} catch (error) {
throw error;
}
}
public async getDb(
query: FindOptions<CrontabView>['where'],
): Promise<CrontabView> {
const doc: any = await CrontabViewModel.findOne({ where: { ...query } });
public async getDb(query: QuerySpec['where']): Promise<CrontabView> {
const doc: any = await CrontabViewModel.findOne({ ...query }, {});
if (!doc) {
throw new Error(`CronView ${JSON.stringify(query)} not found`);
}
return doc.get({ plain: true });
return doc;
}
public async disabled(ids: number[]) {
await CrontabViewModel.update({ isDisabled: 1 }, { where: { id: ids } });
await CrontabViewModel.nativeUpdate({ id: ids }, { isDisabled: 1 });
}
public async enabled(ids: number[]) {
await CrontabViewModel.update({ isDisabled: 0 }, { where: { id: ids } });
await CrontabViewModel.nativeUpdate({ id: ids }, { isDisabled: 0 });
}
private async checkPosition(position: number) {
const precisionPosition = parseFloat(position.toPrecision(16));
if (precisionPosition < minPosition || precisionPosition > maxPosition) {
@@ -88,11 +79,9 @@ export default class CronViewService {
}
}
}
private getPrecisionPosition(position: number): number {
return parseFloat(position.toPrecision(16));
}
public async move({
id,
fromIndex,
@@ -118,7 +107,6 @@ export default class CronViewService {
id,
position: this.getPrecisionPosition(targetPosition),
});
await this.checkPosition(targetPosition);
return newDoc;
}
+48 -93
View File
@@ -1,3 +1,5 @@
import { textSearch } from '../data/queries';
import { Query, QuerySpec, orderBy } from '../data/queries';
import { Service, Inject } from 'typedi';
import winston from 'winston';
import config from '../config';
@@ -10,7 +12,6 @@ import {
} from '../data/dependence';
import { spawn } from 'cross-spawn';
import SockService from './sock';
import { FindOptions, Op } from 'sequelize';
import {
fileExist,
getPid,
@@ -27,14 +28,13 @@ import taskLimit from '../shared/pLimit';
import { detectOS } from '../config/util';
import { LINUX_DEPENDENCE_COMMAND } from '../config/const';
import { t, tf } from '../shared/i18n';
@Service()
export default class DependenceService {
constructor(
@Inject('logger') private logger: winston.Logger,
@Inject('logger')
private logger: winston.Logger,
private sockService: SockService,
) {}
public async create(payloads: Dependence[]): Promise<Dependence[]> {
const tabs = payloads.map((x) => {
getInstallCommand(x.type, x.name);
@@ -45,14 +45,14 @@ export default class DependenceService {
this.installDependenceOneByOne(docs);
return docs;
}
public async insert(payloads: Dependence[]): Promise<Dependence[]> {
const docs = await DependenceModel.bulkCreate(payloads);
const docs = await DependenceModel.insertMany(payloads);
return docs;
}
public async update(
payload: Dependence & { id: string },
payload: Dependence & {
id: string;
},
): Promise<Dependence> {
const { id, ...other } = payload;
const doc = await this.getDb({ id });
@@ -66,14 +66,12 @@ export default class DependenceService {
this.installDependenceOneByOne([newDoc]);
return newDoc;
}
private async updateDb(payload: Dependence): Promise<Dependence> {
await DependenceModel.update(payload, { where: { id: payload.id } });
await DependenceModel.nativeUpdate({ id: payload.id }, payload);
return await this.getDb({ id: payload.id });
}
public async remove(ids: number[], force = false): Promise<Dependence[]> {
const docs = await DependenceModel.findAll({ where: { id: ids } });
const docs = await DependenceModel.find({ id: ids }, {});
for (const doc of docs) {
taskLimit.removeQueuedDependency(doc);
}
@@ -88,22 +86,18 @@ export default class DependenceService {
);
const unInstalledDeps = docs.filter((x) => !installedDeps.includes(x));
await this.removeDb(unInstalledDeps.map((x) => x.id!));
if (installedDeps.length) {
await DependenceModel.update(
await DependenceModel.nativeUpdate(
{ id: installedDeps.map((x) => x.id!) },
{ status: DependenceStatus.queued, log: [] },
{ where: { id: installedDeps.map((x) => x.id!) } },
);
this.installDependenceOneByOne(installedDeps, false, force);
}
return docs;
}
public async removeDb(ids: number[]) {
await DependenceModel.destroy({ where: { id: ids } });
await DependenceModel.nativeDelete({ id: ids });
}
public async dependencies(
{
searchValue,
@@ -130,13 +124,7 @@ export default class DependenceService {
condition.status = status.split(',').map(Number);
}
if (searchValue) {
const encodeText = encodeURI(searchValue);
condition.name = {
[Op.or]: [
{ [Op.like]: `%${searchValue}%` },
{ [Op.like]: `%${encodeText}%` },
],
};
Object.assign(condition, textSearch(['name'], searchValue));
}
try {
return await this.find(condition, sort);
@@ -144,7 +132,6 @@ export default class DependenceService {
throw error;
}
}
private async refreshInstalledStatuses(type?: DependenceTypes) {
const cacheDependenceTypes = [
DependenceTypes.nodejs,
@@ -153,13 +140,13 @@ export default class DependenceService {
if (type !== undefined && !cacheDependenceTypes.includes(type)) {
return;
}
const docs = await DependenceModel.findAll({
where: {
const docs = await DependenceModel.find(
{
status: DependenceStatus.installed,
type: type === undefined ? { [Op.in]: cacheDependenceTypes } : type,
type: type === undefined ? { ['$in']: cacheDependenceTypes } : type,
},
});
{},
);
const checks = await concurrentRun(
docs
.filter(
@@ -175,15 +162,13 @@ export default class DependenceService {
const missingIds = (checks || []).filter(
(id): id is number => id !== undefined,
);
if (missingIds.length) {
await DependenceModel.update(
await DependenceModel.nativeUpdate(
{ id: missingIds },
{ status: DependenceStatus.installFailed },
{ where: { id: missingIds } },
);
}
}
private async isDependenceInstalled(dependency: Dependence) {
let depName = dependency.name.trim();
const depVersionStr = versionDependenceCommandTypes[dependency.type];
@@ -201,7 +186,6 @@ export default class DependenceService {
depVersion = parsedVersion;
}
}
const depInfo = (
await promiseExecSuccess(getGetCommand(dependency.type, depName))
)
@@ -211,12 +195,10 @@ export default class DependenceService {
(dependency.type === DependenceTypes.nodejs &&
depInfo.split(' ')?.[0] === depName) ||
dependency.type === DependenceTypes.python3;
return Boolean(
depInfo && nameMatches && (!depVersion || depInfo.includes(depVersion)),
);
}
public installDependenceOneByOne(
docs: Dependence[],
isInstall: boolean = true,
@@ -225,32 +207,27 @@ export default class DependenceService {
docs.forEach((dep) => {
this.installOrUninstallDependency(dep, isInstall, force);
});
return taskLimit.waitDependencyQueueDone();
}
public async reInstall(ids: number[]): Promise<Dependence[]> {
await DependenceModel.update(
await DependenceModel.nativeUpdate(
{ id: ids },
{ status: DependenceStatus.queued, log: [] },
{ where: { id: ids } },
);
const docs = await DependenceModel.findAll({ where: { id: ids } });
const docs = await DependenceModel.find({ id: ids }, {});
for (const doc of docs) {
taskLimit.removeQueuedDependency(doc);
}
this.installDependenceOneByOne(docs, true, true);
return docs;
}
public async cancel(ids: number[]) {
const docs = await DependenceModel.findAll({ where: { id: ids } });
const docs = await DependenceModel.find({ id: ids }, {});
for (const doc of docs) {
taskLimit.removeQueuedDependency(doc);
let depInstallCommand = getInstallCommand(doc.type, doc.name);
let depUnInstallCommand = getUninstallCommand(doc.type, doc.name);
const isLinuxDependence = doc.type === DependenceTypes.linux;
if (isLinuxDependence) {
const osType = await detectOS();
if (!osType) {
@@ -268,44 +245,37 @@ export default class DependenceService {
pid && (await killTask(pid));
}
}
await DependenceModel.update(
await DependenceModel.nativeUpdate(
{ id: ids },
{ status: DependenceStatus.cancelled },
{ where: { id: ids } },
);
}
private async find(query: any, sort: any = []): Promise<Dependence[]> {
const docs = await DependenceModel.findAll({
where: { ...query },
order: [...sort, ['createdAt', 'DESC']],
});
const docs = await DependenceModel.find(
{ ...query },
{
orderBy: orderBy([...sort, ['createdAt', 'DESC']]),
},
);
return docs;
}
public async getDb(
query: FindOptions<Dependence>['where'],
): Promise<Dependence> {
const doc: any = await DependenceModel.findOne({ where: { ...query } });
public async getDb(query: QuerySpec['where']): Promise<Dependence> {
const doc: any = await DependenceModel.findOne({ ...query }, {});
if (!doc) {
throw new Error(`Dependency ${JSON.stringify(query)} not found`);
}
return doc.get({ plain: true });
return doc;
}
private async updateLog(ids: number[], log: string): Promise<void> {
taskLimit.updateDepLog(async () => {
const docs = await DependenceModel.findAll({ where: { id: ids } });
const docs = await DependenceModel.find({ id: ids }, {});
for (const doc of docs) {
const newLog = doc?.log ? [...doc.log, log] : [log];
await DependenceModel.update(
{ log: newLog },
{ where: { id: doc.id } },
);
await DependenceModel.nativeUpdate({ id: doc.id }, { log: newLog });
}
return null;
});
}
public installOrUninstallDependency(
dependency: Dependence,
isInstall: boolean = true,
@@ -322,14 +292,14 @@ export default class DependenceService {
if (!isInstall && force) {
return this.removeDb([dependency.id!]).then(() => null);
}
return DependenceModel.update(
return DependenceModel.nativeUpdate(
{ id: dependency.id },
{
status: isInstall
? DependenceStatus.installFailed
: DependenceStatus.removeFailed,
log: [String(error)],
},
{ where: { id: dependency.id } },
).then(() => null);
}
return new Promise(async (resolve) => {
@@ -350,9 +320,9 @@ export default class DependenceService {
taskLimit.removeQueuedDependency(dependency);
if (isLinuxDependence) {
if (!osType) {
await DependenceModel.update(
await DependenceModel.nativeUpdate(
{ id: depIds },
{ status: DependenceStatus.installFailed },
{ where: { id: depIds } },
);
const startTime = dayjs();
const message = tf(
@@ -375,12 +345,10 @@ export default class DependenceService {
}
linuxCommand = LINUX_DEPENDENCE_COMMAND[osType];
}
const status = isInstall
? DependenceStatus.installing
: DependenceStatus.removing;
await DependenceModel.update({ status }, { where: { id: depIds } });
await DependenceModel.nativeUpdate({ id: depIds }, { status });
let command = validatedCommand;
if (isLinuxDependence) {
command = isInstall
@@ -388,7 +356,6 @@ export default class DependenceService {
: `${linuxCommand.uninstall} ${depName.trim()}`;
}
const startTime = dayjs();
const message = tf(
'开始%s依赖 %s,开始时间 %s\n\n',
actionText,
@@ -402,7 +369,6 @@ export default class DependenceService {
status,
});
this.updateLog(depIds, message);
// 判断是否已经安装过依赖
// pip must resolve optional dependencies even if the base distribution
// is already present; its version alone cannot prove extras are ready.
@@ -416,7 +382,6 @@ export default class DependenceService {
if (isLinuxDependence) {
getCommand = `${linuxCommand.info} ${depName}`;
}
let depVersion = '';
if (!isPythonDependence && depName.includes(depVersionStr)) {
const symbolRegx = new RegExp(
@@ -431,7 +396,6 @@ export default class DependenceService {
const depInfo = (await promiseExecSuccess(getCommand))
.replace(/\s{2,}/, ' ')
.replace(/\s+$/, '');
if (
depInfo &&
((isNodeDependence && depInfo.split(' ')?.[0] === depName) ||
@@ -456,9 +420,9 @@ export default class DependenceService {
status: DependenceStatus.installed,
});
this.updateLog(depIds, _message);
await DependenceModel.update(
await DependenceModel.nativeUpdate(
{ id: depIds },
{ status: DependenceStatus.installed },
{ where: { id: depIds } },
);
return resolve(null);
}
@@ -472,7 +436,6 @@ export default class DependenceService {
const cp = spawn(`${proxyStr} ${command}`, {
shell: '/bin/bash',
});
cp.stdout.on('data', async (data) => {
this.sockService.sendMessage({
type: socketMessageType,
@@ -482,7 +445,6 @@ export default class DependenceService {
});
this.updateLog(depIds, data.toString());
});
cp.stderr.on('data', async (data) => {
this.sockService.sendMessage({
type: socketMessageType,
@@ -492,7 +454,6 @@ export default class DependenceService {
});
this.updateLog(depIds, data.toString());
});
cp.on('error', async (err) => {
this.sockService.sendMessage({
type: socketMessageType,
@@ -502,12 +463,10 @@ export default class DependenceService {
});
this.updateLog(depIds, JSON.stringify(err));
});
cp.on('exit', async (code) => {
const endTime = dayjs();
const isSucceed = code === 0;
const resultText = isSucceed ? t('成功') : t('失败');
const message =
'\n' +
tf(
@@ -531,24 +490,20 @@ export default class DependenceService {
status: exitStatus,
});
this.updateLog(depIds, message);
const docs = await DependenceModel.findAll({ where: { id: depIds } });
const docs = await DependenceModel.find({ id: depIds }, {});
const _docIds = docs
.filter((x) => x.status !== DependenceStatus.cancelled)
.map((x) => x.id!);
if (_docIds.length > 0) {
await DependenceModel.update(
await DependenceModel.nativeUpdate(
{ id: _docIds },
{ status: exitStatus },
{ where: { id: _docIds } },
);
}
// 如果删除依赖成功或者强制删除
if ((isSucceed || force) && !isInstall) {
this.removeDb(depIds);
}
resolve(null);
});
});
+41 -85
View File
@@ -1,5 +1,6 @@
import { textSearch } from '../data/queries';
import { Query, QuerySpec, orderBy } from '../data/queries';
import groupBy from 'lodash/groupBy';
import { FindOptions, Op } from 'sequelize';
import { Inject, Service } from 'typedi';
import winston from 'winston';
import config from '../config';
@@ -13,12 +14,13 @@ import {
stepPosition,
} from '../data/env';
import { writeFileWithLock } from '../shared/utils';
import { sequelize } from '../data';
import { database } from '../data';
@Service()
export default class EnvService {
constructor(@Inject('logger') private logger: winston.Logger) { }
constructor(
@Inject('logger')
private logger: winston.Logger,
) {}
public async create(payloads: Env[]): Promise<Env[]> {
const envs = await this.envs();
let position = initPosition;
@@ -39,16 +41,14 @@ export default class EnvService {
await this.checkPosition(tabs[tabs.length - 1].position!);
return docs;
}
public async insert(payloads: Env[]): Promise<Env[]> {
const result: Env[] = [];
for (const env of payloads) {
const doc = await EnvModel.create(env, { returning: true });
const doc = await EnvModel.insert(env);
result.push(doc);
}
return result;
}
public async update(payload: Env): Promise<Env> {
const doc = await this.getDb({ id: payload.id });
const tab = new Env({ ...doc, ...payload });
@@ -56,17 +56,14 @@ export default class EnvService {
await this.set_envs();
return newDoc;
}
private async updateDb(payload: Env): Promise<Env> {
await EnvModel.update({ ...payload }, { where: { id: payload.id } });
await EnvModel.nativeUpdate({ id: payload.id }, { ...payload });
return await this.getDb({ id: payload.id });
}
public async remove(ids: number[]) {
await EnvModel.destroy({ where: { id: ids } });
await EnvModel.nativeDelete({ id: ids });
await this.set_envs();
}
public async move(
id: number,
{
@@ -89,16 +86,13 @@ export default class EnvService {
? (envs[toIndex].position! + envs[toIndex - 1].position!) / 2
: (envs[toIndex].position! + envs[toIndex + 1].position!) / 2;
}
const newDoc = await this.update({
id,
position: this.getPrecisionPosition(targetPosition),
});
await this.checkPosition(targetPosition, envs[toIndex].position!);
return newDoc;
}
private async checkPosition(position: number, edge: number = 0) {
const precisionPosition = this.getPrecisionPosition(position);
if (
@@ -114,40 +108,20 @@ export default class EnvService {
}
}
}
private getPrecisionPosition(position: number): number {
return Math.trunc(parseFloat(position.toPrecision(16)));
}
public async envs(searchText: string = '', query: any = {}): Promise<Env[]> {
let condition = { ...query };
if (searchText) {
const encodeText = encodeURI(searchText);
const reg = {
[Op.or]: [
{ [Op.like]: `%${searchText}%` },
{ [Op.like]: `%${encodeText}%` },
],
};
condition = {
...condition,
[Op.or]: [
{
name: reg,
},
{
value: reg,
},
{
remarks: reg,
},
],
...textSearch(['name', 'value', 'remarks'], searchText),
};
}
try {
const result = await this.find(condition, [
[sequelize.literal('COALESCE(`isPinned`, 0)'), 'DESC'],
['isPinned', 'DESC'],
['position', 'DESC'],
['createdAt', 'ASC'],
]);
@@ -156,90 +130,73 @@ export default class EnvService {
throw error;
}
}
private async find(query: any, sort: any = []): Promise<Env[]> {
const docs = await EnvModel.findAll({
where: { ...query },
order: [...sort],
});
return docs.map((x) => x.get({ plain: true }));
const docs = await EnvModel.find(
{ ...query },
{
orderBy: orderBy([...sort]),
},
);
return docs.map((x) => x);
}
public async getDb(query: FindOptions<Env>['where']): Promise<Env> {
const doc: any = await EnvModel.findOne({ where: { ...query } });
public async getDb(query: QuerySpec['where']): Promise<Env> {
const doc: any = await EnvModel.findOne({ ...query }, {});
if (!doc) {
throw new Error(`Env ${JSON.stringify(query)} not found`);
}
return doc.get({ plain: true });
return doc;
}
public async disabled(ids: number[]) {
await EnvModel.update(
{ status: EnvStatus.disabled },
{ where: { id: ids } },
);
await EnvModel.nativeUpdate({ id: ids }, { status: EnvStatus.disabled });
await this.set_envs();
}
public async enabled(ids: number[]) {
await EnvModel.update({ status: EnvStatus.normal }, { where: { id: ids } });
await EnvModel.nativeUpdate({ id: ids }, { status: EnvStatus.normal });
await this.set_envs();
}
public async updateNames({ ids, name }: { ids: number[]; name: string }) {
await EnvModel.update({ name }, { where: { id: ids } });
await EnvModel.nativeUpdate({ id: ids }, { name });
await this.set_envs();
}
public async pin(ids: number[]) {
await EnvModel.update({ isPinned: 1 }, { where: { id: ids } });
await EnvModel.nativeUpdate({ id: ids }, { isPinned: 1 });
}
public async unPin(ids: number[]) {
await EnvModel.update({ isPinned: 0 }, { where: { id: ids } });
await EnvModel.nativeUpdate({ id: ids }, { isPinned: 0 });
}
public async addLabels(ids: number[], labels: string[]) {
await sequelize.transaction(async (transaction) => {
const docs = await EnvModel.findAll({
where: { id: ids },
transaction,
});
await database.transaction(async () => {
const docs = await EnvModel.find({ id: ids }, {});
for (const doc of docs) {
const env = doc.get({ plain: true });
await EnvModel.update(
const env = doc;
await EnvModel.nativeUpdate(
{ id: env.id },
{ labels: Array.from(new Set([...(env.labels || []), ...labels])) },
{ where: { id: env.id }, transaction },
);
}
});
return await this.find({ id: ids });
}
public async removeLabels(ids: number[], labels: string[]) {
await sequelize.transaction(async (transaction) => {
const docs = await EnvModel.findAll({
where: { id: ids },
transaction,
});
await database.transaction(async () => {
const docs = await EnvModel.find({ id: ids }, {});
for (const doc of docs) {
const env = doc.get({ plain: true });
await EnvModel.update(
const env = doc;
await EnvModel.nativeUpdate(
{ id: env.id },
{
labels: (env.labels || []).filter(
(label: string) => !labels.includes(label),
),
},
{ where: { id: env.id }, transaction },
);
}
});
return await this.find({ id: ids });
}
public async set_envs() {
const envs = await this.envs('', {
name: { [Op.not]: null },
name: { ['$ne']: null },
status: EnvStatus.normal,
});
const groups = groupBy(envs, 'name');
@@ -249,7 +206,6 @@ export default class EnvService {
for (const key in groups) {
if (Object.prototype.hasOwnProperty.call(groups, key)) {
const group = groups[key];
// 忽略不符合bash要求的环境变量名称
if (/^[a-zA-Z_][0-9a-zA-Z_]*$/.test(key)) {
let value = group
@@ -262,9 +218,9 @@ export default class EnvService {
.map((x) => x.value)
.join('&')
.replace(/\\/g, '\\\\')}`;
js_env_string += `process.env[${JSON.stringify(key)}]=${JSON.stringify(
group.map((x) => x.value).join('&'),
)};\n`;
js_env_string += `process.env[${JSON.stringify(
key,
)}]=${JSON.stringify(group.map((x) => x.value).join('&'))};\n`;
py_env_string += `os.environ['${key}']='''${_env_value.replace(
/\'/g,
"\\'",
+4
View File
@@ -1,3 +1,4 @@
import { database } from '../data';
import { Service } from 'typedi';
import Logger from '../loaders/logger';
import cronClient from '../schedule/client';
@@ -10,6 +11,7 @@ interface HealthStatus {
services: {
http: boolean;
grpc: boolean;
database: boolean;
};
metrics: {
uptime: number;
@@ -33,6 +35,7 @@ export class HealthService {
services: {
http: true,
grpc: true,
database: database.healthy,
},
metrics: {
uptime: Math.floor((Date.now() - this.startTime) / 1000),
@@ -66,6 +69,7 @@ export class HealthService {
Logger.error('gRPC server check failed:', err);
}
if (!status.services.database) status.status = 'error';
return status;
}
}
+21 -49
View File
@@ -1,9 +1,9 @@
import { textSearch } from '../data/queries';
import { Service, Inject } from 'typedi';
import winston from 'winston';
import { createRandomString } from '../config/util';
import { App, AppModel } from '../data/open';
import { v4 as uuidV4 } from 'uuid';
import sequelize, { Op } from 'sequelize';
import { shareStore } from '../shared/store';
import { t } from '../shared/i18n';
import { lock } from 'proper-lockfile';
@@ -11,18 +11,19 @@ import path from 'path';
import os from 'os';
import { writeFile, rename, rm, stat, chmod } from 'fs/promises';
import config from '../config';
@Service()
export default class OpenService {
constructor(@Inject('logger') private logger: winston.Logger) {}
constructor(
@Inject('logger')
private logger: winston.Logger,
) {}
// Every database mutation and full authentication-cache publication shares
// this cross-process lock. Lock a separate marker, not token.json itself.
private async withAppsLock<T>(operation: () => Promise<T>): Promise<T> {
const target = path.join(config.configPath, '.apps-state');
const release = await lock(target, {
realpath: false,
stale: 10_000,
stale: 10000,
retries: { retries: 10, factor: 2, minTimeout: 100, maxTimeout: 3000 },
});
try {
@@ -31,11 +32,10 @@ export default class OpenService {
await release();
}
}
public async initializeSystemApp(): Promise<void> {
await this.withAppsLock(async () => {
const doc = await AppModel.findOne({ where: { name: 'system' } });
const app = doc?.get({ plain: true });
const doc = await AppModel.findOne({ name: 'system' }, {});
const app = doc;
if (!app) {
await this.insert({
name: 'system',
@@ -44,26 +44,23 @@ export default class OpenService {
client_secret: createRandomString(24, 24),
});
} else if (!app.scopes.includes('dashboard')) {
await AppModel.update(
await AppModel.nativeUpdate(
{ id: app.id },
{ scopes: [...app.scopes, 'dashboard'] },
{ where: { id: app.id } },
);
}
await shareStore.updateApps(await this.find({}));
});
}
public async refreshApps(): Promise<void> {
await this.withAppsLock(async () => {
await shareStore.updateApps(await this.find({}));
});
}
public async findApps(): Promise<App[] | null> {
const docs = await this.find({});
return docs;
}
public async create(payload: App): Promise<App> {
return this.withAppsLock(async () => {
const tab = { ...payload };
@@ -75,12 +72,10 @@ export default class OpenService {
return { ...doc, tokens: [] };
});
}
private async insert(payload: App): Promise<App> {
const doc = await AppModel.create(payload, { returning: true });
return doc.get({ plain: true });
const doc = await AppModel.insert(payload);
return doc;
}
public async update(payload: App): Promise<App> {
const newDoc = await this.updateDb({
name: payload.name,
@@ -89,32 +84,28 @@ export default class OpenService {
} as App);
return { ...newDoc, tokens: [] };
}
private async updateDb(payload: Partial<App>): Promise<App> {
return this.withAppsLock(async () => {
await AppModel.update(payload, { where: { id: payload.id } });
await AppModel.nativeUpdate({ id: payload.id }, payload);
const apps = await this.find({});
await shareStore.updateApps(apps);
return apps?.find((x) => x.id === payload.id) as App;
});
}
public async getDb(query: Record<string, any>): Promise<App> {
const doc = await AppModel.findOne({ where: query });
const doc = await AppModel.findOne(query, {});
if (!doc) {
throw new Error(`App ${JSON.stringify(query)} not found`);
}
return doc.get({ plain: true });
return doc;
}
public async remove(ids: number[]) {
return this.withAppsLock(async () => {
await AppModel.destroy({ where: { id: ids } });
await AppModel.nativeDelete({ id: ids });
const apps = await this.find({});
await shareStore.updateApps(apps);
});
}
public async resetSecret(id: number): Promise<App> {
const tab: Partial<App> = {
client_secret: createRandomString(24, 24),
@@ -130,7 +121,6 @@ export default class OpenService {
const newDoc = await this.updateDb(tab);
return newDoc;
}
public async list(
searchText: string = '',
sort: any = {},
@@ -138,18 +128,7 @@ export default class OpenService {
): Promise<App[]> {
let condition = { ...query };
if (searchText) {
const encodeText = encodeURI(searchText);
const reg = {
[Op.or]: [
{ [Op.like]: `%${searchText}%` },
{ [Op.like]: `%${encodeText}%` },
],
};
condition = {
...condition,
name: reg,
};
condition = { ...condition, ...textSearch(['name'], searchText) };
}
try {
const result = await this.find(condition);
@@ -160,19 +139,16 @@ export default class OpenService {
throw error;
}
}
private async find(query: Record<string, any>, sort?: any): Promise<App[]> {
const docs = await AppModel.findAll({ where: { ...query } });
return docs.map((x) => x.get({ plain: true }));
const docs = await AppModel.find({ ...query }, {});
return docs.map((x) => x);
}
public async authToken(credentials: {
client_id: string;
client_secret: string;
}): Promise<any> {
return this.withAppsLock(() => this.issueToken(credentials));
}
// Caller holds withAppsLock through database update and cache publication.
private async issueToken({
client_id,
@@ -183,7 +159,7 @@ export default class OpenService {
}): Promise<any> {
let token = uuidV4();
const expiration = Math.round(Date.now() / 1000) + 2592000; // 2592000 30天
const doc = await AppModel.findOne({ where: { client_id, client_secret } });
const doc = await AppModel.findOne({ client_id, client_secret }, {});
if (doc) {
const timestamp = Math.round(Date.now() / 1000);
const invalidTokens = (doc.tokens || []).filter(
@@ -199,10 +175,7 @@ export default class OpenService {
} else {
tokens = [...invalidTokens, { value: token, expiration }];
}
await AppModel.update(
{ tokens },
{ where: { client_id, client_secret } },
);
await AppModel.nativeUpdate({ client_id, client_secret }, { tokens });
const apps = await this.find({});
await shareStore.updateApps(apps);
return {
@@ -217,7 +190,6 @@ export default class OpenService {
return { code: 400, message: t('client_id 或 client_seret 有误') };
}
}
public async generateSystemToken(renew = false): Promise<{
value: string;
expiration: number;
+25 -41
View File
@@ -1,11 +1,10 @@
import fs from 'fs/promises';
import path from 'path';
import dayjs from 'dayjs';
import { Op } from 'sequelize';
import { Service } from 'typedi';
import config from '../config';
import { CrontabStatModel } from '../data/cronStats';
import { sequelize } from '../data';
import { database } from '../data';
import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
import { AuthDataType, SystemModel } from '../data/system';
import {
@@ -15,54 +14,50 @@ import {
normalizeRetentionPolicy,
RetentionPolicy,
} from '../shared/retention';
export interface StorageCleanupRequest extends RetentionPolicy {
dependenceCacheTypes?: DependenceCacheType[];
compactDatabase?: boolean;
}
function runningInstanceWhere(days: number) {
const cutoff = dayjs().subtract(days, 'day').unix();
return {
status: { [Op.ne]: InstanceStatus.running },
[Op.or]: [
{ finished_at: { [Op.lt]: cutoff } },
status: { ['$ne']: InstanceStatus.running },
['$or']: [
{ finished_at: { ['$lt']: cutoff } },
{
finished_at: { [Op.is]: null },
started_at: { [Op.lt]: cutoff },
finished_at: { ['$eq']: null },
started_at: { ['$lt']: cutoff },
},
],
};
}
function cronStatWhere(days: number) {
return {
date: { [Op.lt]: dayjs().subtract(days, 'day').format('YYYY-MM-DD') },
date: { ['$lt']: dayjs().subtract(days, 'day').format('YYYY-MM-DD') },
};
}
@Service()
export default class RetentionService {
public async updatePolicy(policy: Partial<RetentionPolicy>) {
const normalized = normalizeRetentionPolicy(policy);
const systemConfig = await SystemModel.findOne({
where: { type: AuthDataType.systemConfig },
});
const systemConfig = await SystemModel.findOne(
{ type: AuthDataType.systemConfig },
{},
);
if (!systemConfig) {
throw new Error('System config not found');
}
await SystemModel.update(
await SystemModel.nativeUpdate(
{ id: systemConfig.id },
{
info: {
...systemConfig.info,
...normalized,
},
},
{ where: { id: systemConfig.id } },
);
return normalized;
}
public async preview(request: StorageCleanupRequest) {
const policy = normalizeRetentionPolicy(request);
const dependenceCacheTypes = (request.dependenceCacheTypes || []).filter(
@@ -70,14 +65,12 @@ export default class RetentionService {
);
const [runningInstances, cronStats, dependenceCaches] = await Promise.all([
policy.runningInstanceRetentionDays > 0
? RunningInstanceModel.count({
where: runningInstanceWhere(policy.runningInstanceRetentionDays),
})
? RunningInstanceModel.count(
runningInstanceWhere(policy.runningInstanceRetentionDays),
)
: 0,
policy.cronStatRetentionDays > 0
? CrontabStatModel.count({
where: cronStatWhere(policy.cronStatRetentionDays),
})
? CrontabStatModel.count(cronStatWhere(policy.cronStatRetentionDays))
: 0,
Promise.all(
dependenceCacheTypes.map(async (type) => ({
@@ -88,7 +81,6 @@ export default class RetentionService {
})),
),
]);
return {
policy,
runningInstances,
@@ -101,29 +93,23 @@ export default class RetentionService {
compactDatabase: Boolean(request.compactDatabase),
};
}
public async cleanup(request: StorageCleanupRequest) {
const preview = await this.preview(request);
const deleted = await sequelize.transaction(async (transaction) => {
const deleted = await database.transaction(async () => {
const runningInstances =
preview.policy.runningInstanceRetentionDays > 0
? await RunningInstanceModel.destroy({
where: runningInstanceWhere(
preview.policy.runningInstanceRetentionDays,
),
transaction,
})
? await RunningInstanceModel.nativeDelete(
runningInstanceWhere(preview.policy.runningInstanceRetentionDays),
)
: 0;
const cronStats =
preview.policy.cronStatRetentionDays > 0
? await CrontabStatModel.destroy({
where: cronStatWhere(preview.policy.cronStatRetentionDays),
transaction,
})
? await CrontabStatModel.nativeDelete(
cronStatWhere(preview.policy.cronStatRetentionDays),
)
: 0;
return { runningInstances, cronStats };
});
const dependenceCaches = [];
for (const cache of preview.dependenceCaches) {
await fs.rm(path.join(config.dependenceCachePath, cache.type), {
@@ -132,14 +118,12 @@ export default class RetentionService {
});
dependenceCaches.push(cache);
}
if (
request.compactDatabase &&
(deleted.runningInstances || deleted.cronStats)
) {
await sequelize.query('VACUUM');
await database.query('VACUUM');
}
return {
deleted: {
...deleted,
+70 -101
View File
@@ -1,3 +1,5 @@
import { textSearch } from '../data/queries';
import { Query, QuerySpec, orderBy } from '../data/queries';
import { Service, Inject } from 'typedi';
import winston from 'winston';
import config from '../config';
@@ -19,7 +21,6 @@ import {
rmPath,
} from '../config/util';
import fs from 'fs/promises';
import { FindOptions, Op } from 'sequelize';
import path from 'path';
import ScheduleService, { TaskCallbacks } from './schedule';
import { SimpleIntervalSchedule } from 'toad-scheduler';
@@ -42,17 +43,16 @@ import CrontabService from './cron';
import taskLimit from '../shared/pLimit';
import { logStreamManager } from '../shared/logStreamManager';
import { LogReadOptions, readLogChunk } from '../shared/logReader';
@Service()
export default class SubscriptionService {
constructor(
@Inject('logger') private logger: winston.Logger,
@Inject('logger')
private logger: winston.Logger,
private scheduleService: ScheduleService,
private sockService: SockService,
private sshKeyService: SshKeyService,
private crontabService: CrontabService,
) {}
public async list(
searchText?: string,
ids?: string,
@@ -60,37 +60,23 @@ export default class SubscriptionService {
let query = {};
const subIds = JSON.parse(ids || '[]');
if (searchText) {
const reg = {
[Op.or]: [
{ [Op.like]: `%${searchText}%` },
{ [Op.like]: `%${encodeURI(searchText)}%` },
],
};
query = {
[Op.or]: [
{
name: reg,
},
{
url: reg,
},
],
};
query = textSearch(['name', 'url'], searchText);
}
try {
const result = await SubscriptionModel.findAll({
where: { ...query, ...(ids ? { id: subIds } : undefined) },
order: [
['is_disabled', 'ASC'],
['createdAt', 'DESC'],
],
});
const result = await SubscriptionModel.find(
{ ...query, ...(ids ? { id: subIds } : undefined) },
{
orderBy: orderBy([
['is_disabled', 'ASC'],
['createdAt', 'DESC'],
]),
},
);
return result;
} catch (error) {
throw error;
}
}
public async handleTask(
doc: Subscription,
needCreate = true,
@@ -101,7 +87,6 @@ export default class SubscriptionService {
const { url } = formatUrl(doc);
doc.command = formatCommand(doc, url as string);
}
if (doc.schedule_type === 'crontab') {
this.scheduleService.cancelCronTask(doc as any);
needCreate &&
@@ -122,34 +107,34 @@ export default class SubscriptionService {
));
}
}
public async setSshConfig() {
const docs = await SubscriptionModel.findAll();
const docs = await SubscriptionModel.find({}, {});
await this.sshKeyService.setSshConfig(docs);
}
private taskCallbacks(doc: Subscription): TaskCallbacks {
return {
onBefore: async (startTime) => {
const logTime = startTime.format('YYYY-MM-DD-HH-mm-ss');
const logPath = `${doc.alias}/${logTime}.log`;
await SubscriptionModel.update(
await SubscriptionModel.nativeUpdate(
{ id: doc.id },
{
status: SubscriptionStatus.running,
log_path: logPath,
},
{ where: { id: doc.id } },
);
const absolutePath = await handleLogPath(
logPath as string,
tf('## 开始执行... %s\n', startTime.format('YYYY-MM-DD HH:mm:ss')),
);
// 执行sub_before
let beforeStr = '';
try {
if (doc.sub_before) {
await logStreamManager.write(absolutePath, `\n## ${t('执行before命令...')}\n\n`);
await logStreamManager.write(
absolutePath,
`\n## ${t('执行before命令...')}\n\n`,
);
beforeStr = await promiseExec(doc.sub_before);
}
} catch (error: any) {
@@ -161,11 +146,11 @@ export default class SubscriptionService {
}
},
onStart: async (cp: ChildProcessWithoutNullStreams, startTime) => {
await SubscriptionModel.update(
await SubscriptionModel.nativeUpdate(
{ id: doc.id },
{
pid: cp.pid,
},
{ where: { id: doc.id } },
);
},
onEnd: async (cp, endTime, diff) => {
@@ -173,7 +158,6 @@ export default class SubscriptionService {
try {
const sub = await this.getDb({ id: doc.id });
absolutePath = await handleLogPath(sub.log_path as string);
// 执行 sub_after
let afterStr = '';
try {
@@ -186,12 +170,12 @@ export default class SubscriptionService {
}
} catch (error: any) {
afterStr =
(error.stderr && error.stderr.toString()) || JSON.stringify(error);
(error.stderr && error.stderr.toString()) ||
JSON.stringify(error);
}
if (afterStr) {
await logStreamManager.write(absolutePath, `${afterStr}\n`);
}
await logStreamManager.write(
absolutePath,
'\n' +
@@ -206,10 +190,10 @@ export default class SubscriptionService {
try {
if (absolutePath) await logStreamManager.closeStream(absolutePath);
} finally {
await SubscriptionModel.update(
{ status: SubscriptionStatus.idle, pid: null } as any,
{ where: { id: doc.id } },
);
await SubscriptionModel.nativeUpdate({ id: doc.id }, {
status: SubscriptionStatus.idle,
pid: null,
} as any);
this.sockService.sendMessage({
type: 'runSubscriptionEnd',
message: t('订阅执行完成'),
@@ -230,22 +214,19 @@ export default class SubscriptionService {
},
};
}
public async create(payload: Subscription): Promise<Subscription> {
assertSubscriptionAlias(payload.alias);
return withSubscriptionMutation(async () => {
const tab = new Subscription(payload);
const doc = await this.insert(tab);
await this.handleTask(doc.get({ plain: true }));
await this.handleTask(doc);
await this.setSshConfig();
return doc;
});
}
public async insert(payload: Subscription): Promise<SubscriptionInstance> {
return await SubscriptionModel.create(payload, { returning: true });
return await SubscriptionModel.insert(payload);
}
public async update(payload: Subscription): Promise<Subscription> {
assertSubscriptionAlias(payload.alias);
return withSubscriptionMutation(async () => {
@@ -258,7 +239,6 @@ export default class SubscriptionService {
return newDoc;
});
}
private async removeSshConfigForSubscription(doc: Subscription) {
if (doc.type !== 'private-repo' || doc.pull_type !== 'ssh-key') return;
// Invalid historical rows must remain deletable without deriving unsafe paths.
@@ -269,12 +249,10 @@ export default class SubscriptionService {
}
await this.sshKeyService.removeSSHKey(doc.alias, '');
}
public async updateDb(payload: Subscription): Promise<Subscription> {
await SubscriptionModel.update(payload, { where: { id: payload.id } });
await SubscriptionModel.nativeUpdate({ id: payload.id }, payload);
return await this.getDb({ id: payload.id });
}
public async status({
ids,
status,
@@ -300,16 +278,16 @@ export default class SubscriptionService {
if (last_running_time > 0) {
options.last_running_time = last_running_time;
}
return await SubscriptionModel.update(
{ ...options },
{ where: { id: ids } },
);
return await SubscriptionModel.nativeUpdate({ id: ids }, { ...options });
}
public async remove(ids: number[], query: { force?: boolean }) {
public async remove(
ids: number[],
query: {
force?: boolean;
},
) {
return withSubscriptionMutation(async () => {
const docs = await SubscriptionModel.findAll({ where: { id: ids } });
const docs = await SubscriptionModel.find({ id: ids }, {});
const storagePaths: string[] = [];
const retainedPaths: string[] = [];
const collectPaths = (
@@ -321,7 +299,9 @@ export default class SubscriptionService {
// A URL accepted by one engine can produce an invalid name in another.
// Validate each candidate independently, retaining the safe candidate.
if (!Object.values(names).every(isSubscriptionStorageName)) continue;
const resolve = deleting ? resolveSubscriptionStoragePath : path.resolve;
const resolve = deleting
? resolveSubscriptionStoragePath
: path.resolve;
const scriptPath = resolve(config.scriptPath, names.script);
if (scriptPath) targets.push(scriptPath);
if (names.repo) {
@@ -329,7 +309,10 @@ export default class SubscriptionService {
if (repoPath) targets.push(repoPath);
}
if (names.raw) {
const rawPath = resolve(path.join(config.dataPath, 'raw'), names.raw);
const rawPath = resolve(
path.join(config.dataPath, 'raw'),
names.raw,
);
if (rawPath) targets.push(rawPath);
}
}
@@ -342,70 +325,65 @@ export default class SubscriptionService {
resolveSubscriptionStoragePath(config.repoPath, doc.alias);
collectPaths(doc, storagePaths, true);
}
const remaining = await SubscriptionModel.findAll({ where: {} });
const remaining = await SubscriptionModel.find({}, {});
for (const doc of remaining) {
if (!ids.includes(doc.id!)) collectPaths(doc, retainedPaths);
}
}
for (const doc of docs) {
await this.handleTask(doc.get({ plain: true }), false);
await this.handleTask(doc, false);
}
await SubscriptionModel.destroy({ where: { id: ids } });
await SubscriptionModel.nativeDelete({ id: ids });
for (const doc of docs) {
await this.removeSshConfigForSubscription(doc);
}
await this.setSshConfig();
if (query?.force === true) {
const crons = await CrontabModel.findAll({ where: { sub_id: ids } });
const crons = await CrontabModel.find({ sub_id: ids }, {});
if (crons?.length) {
await this.crontabService.remove(crons.map((x) => x.id!));
}
for (const storagePath of new Set(storagePaths)) {
// rm is recursive: an ancestor must also be retained if a surviving
// subscription uses a nested branch, and vice versa.
const shared = retainedPaths.some((retained) =>
retained === storagePath ||
retained.startsWith(`${storagePath}${path.sep}`) ||
storagePath.startsWith(`${retained}${path.sep}`),
const shared = retainedPaths.some(
(retained) =>
retained === storagePath ||
retained.startsWith(`${storagePath}${path.sep}`) ||
storagePath.startsWith(`${retained}${path.sep}`),
);
if (!shared) await rmPath(storagePath);
}
}
});
}
public async getDb(
query: FindOptions<Subscription>['where'],
): Promise<Subscription> {
const doc = await SubscriptionModel.findOne({ where: { ...query } });
public async getDb(query: QuerySpec['where']): Promise<Subscription> {
const doc = await SubscriptionModel.findOne({ ...query }, {});
if (!doc) {
throw new Error(`Subscription ${JSON.stringify(query)} not found`);
}
return doc.get({ plain: true });
return doc;
}
public async run(ids: number[]) {
await SubscriptionModel.update(
await SubscriptionModel.nativeUpdate(
{ id: ids },
{ status: SubscriptionStatus.queued },
{ where: { id: ids } },
);
ids.forEach((id) => {
this.runSingle(id);
});
}
public async stop(ids: number[]) {
const docs = await SubscriptionModel.findAll({ where: { id: ids } });
const docs = await SubscriptionModel.find({ id: ids }, {});
let failure: unknown;
for (const doc of docs) {
try {
if (doc.pid) {
await killTask(doc.pid, true);
}
await SubscriptionModel.update(
await SubscriptionModel.nativeUpdate(
{ id: doc.id, pid: doc.pid ?? null },
{ status: SubscriptionStatus.idle, pid: null } as any,
{ where: { id: doc.id, pid: doc.pid ?? null } },
);
} catch (error) {
this.logger.error(error);
@@ -414,15 +392,12 @@ export default class SubscriptionService {
}
if (failure) throw failure;
}
private async runSingle(subscriptionId: number) {
const subscription = await this.getDb({ id: subscriptionId });
if (subscription.status !== SubscriptionStatus.queued) {
return;
}
const command = formatCommand(subscription);
this.scheduleService.runTask(command, this.taskCallbacks(subscription), {
name: subscription.name,
schedule: subscription.schedule,
@@ -431,25 +406,22 @@ export default class SubscriptionService {
runOrigin: 'subscription',
});
}
public async disabled(ids: number[]) {
await SubscriptionModel.update({ is_disabled: 1 }, { where: { id: ids } });
const docs = await SubscriptionModel.findAll({ where: { id: ids } });
await SubscriptionModel.nativeUpdate({ id: ids }, { is_disabled: 1 });
const docs = await SubscriptionModel.find({ id: ids }, {});
await this.setSshConfig();
for (const doc of docs) {
await this.handleTask(doc.get({ plain: true }), false);
await this.handleTask(doc, false);
}
}
public async enabled(ids: number[]) {
await SubscriptionModel.update({ is_disabled: 0 }, { where: { id: ids } });
const docs = await SubscriptionModel.findAll({ where: { id: ids } });
await SubscriptionModel.nativeUpdate({ id: ids }, { is_disabled: 0 });
const docs = await SubscriptionModel.find({ id: ids }, {});
await this.setSshConfig();
for (const doc of docs) {
await this.handleTask(doc.get({ plain: true }));
await this.handleTask(doc);
}
}
public async log(id: number, options: LogReadOptions = {}) {
const doc = await this.getDb({ id });
if (!doc || !doc.log_path) {
@@ -461,17 +433,14 @@ export default class SubscriptionService {
truncated: false,
};
}
const absolutePath = await handleLogPath(doc.log_path as string);
return await readLogChunk(absolutePath, options);
}
public async logs(id: number) {
const doc = await this.getDb({ id });
if (!doc) {
return [];
}
if (doc.log_path) {
const relativeDir = path.dirname(`${doc.log_path}`);
const dir = path.dirname(resolveLogPath(config.logPath, doc.log_path));
+33 -67
View File
@@ -1,3 +1,4 @@
import { Query, QuerySpec, orderBy } from '../data/queries';
import { persistNotificationAudit } from '../shared/notificationAudit';
import { spawn } from 'cross-spawn';
import { execFile } from 'child_process';
@@ -46,18 +47,16 @@ import os from 'os';
import dayjs from 'dayjs';
import { t, setLang } from '../shared/i18n';
import { updateLinuxMirrorFile } from '../config/util';
@Service()
export default class SystemService {
@Inject((type) => NotificationService)
private notificationService!: NotificationService;
constructor(
@Inject('logger') private logger: winston.Logger,
@Inject('logger')
private logger: winston.Logger,
private scheduleService: ScheduleService,
private sockService: SockService,
) {}
public async getSystemConfig() {
const doc = await this.getDb({ type: AuthDataType.systemConfig });
return {
@@ -65,22 +64,19 @@ export default class SystemService {
info: { ...doc.info, timezone: doc.info?.timezone || 'Asia/Shanghai' },
};
}
private async updateAuthDb(payload: SystemInfo): Promise<SystemInfo> {
const { id, ...others } = payload;
await SystemModel.update(others, { where: { id } });
await SystemModel.nativeUpdate({ id }, others);
const doc = await this.getDb({ id });
return doc;
}
public async getDb(query: any): Promise<SystemInfo> {
const doc = await SystemModel.findOne({ where: query });
const doc = await SystemModel.findOne(query, {});
if (!doc) {
throw new Error(`System ${JSON.stringify(query)} not found`);
}
return doc.get({ plain: true });
return doc;
}
public async updateNotificationMode(notificationInfo: NotificationInfo) {
const code = Math.random().toString().slice(-6);
const isSuccess = await this.notificationService.testNotify(
@@ -98,7 +94,6 @@ export default class SystemService {
return { code: 400, message: t('通知发送失败,请检查参数') };
}
}
public async updateLogRemoveFrequency(info: SystemModelInfo) {
const oDoc = await this.getSystemConfig();
const result = await this.updateAuthDb({
@@ -125,7 +120,6 @@ export default class SystemService {
}
return { code: 200, data: info };
}
public async updateCronConcurrency(info: SystemModelInfo) {
const limit = info.cronConcurrency ?? 0;
if (!Number.isInteger(limit) || limit < 0 || limit > 2147483647) {
@@ -142,7 +136,6 @@ export default class SystemService {
return { code: 200, data: info };
});
}
public async restoreCronConcurrency() {
await withSchedulerMutation(async () => {
const doc = await this.getSystemConfig();
@@ -151,7 +144,6 @@ export default class SystemService {
await cronClient.setConcurrency(limit);
});
}
public async updateDependenceProxy(info: SystemModelInfo) {
const oDoc = await this.getSystemConfig();
await this.updateAuthDb({
@@ -170,7 +162,6 @@ export default class SystemService {
}
return { code: 200, data: info };
}
public async updateNodeMirror(info: SystemModelInfo, res?: Response) {
const oDoc = await this.getSystemConfig();
await this.updateAuthDb({
@@ -182,12 +173,13 @@ export default class SystemService {
cmd = `pnpm config set registry ${shellQuote(info.nodeMirror)}`;
}
let command = `cd && ${cmd}`;
const docs = await DependenceModel.findAll({
where: {
const docs = await DependenceModel.find(
{
type: DependenceTypes.nodejs,
status: DependenceStatus.installed,
},
});
{},
);
if (docs.length > 0) {
command += ` && pnpm i -g`;
}
@@ -219,7 +211,6 @@ export default class SystemService {
},
);
}
public async updatePythonMirror(info: SystemModelInfo) {
const oDoc = await this.getSystemConfig();
await this.updateAuthDb({
@@ -233,7 +224,6 @@ export default class SystemService {
await promiseExec(cmd);
return { code: 200, data: info };
}
public async updateLinuxMirror(
info: SystemModelInfo,
res?: Response,
@@ -280,13 +270,11 @@ export default class SystemService {
},
);
}
public async checkUpdate() {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 15000);
try {
const currentVersionContent = await parseVersion(config.versionFile);
const { body, statusCode } = await request(
`${config.lastVersionFile}?t=${Date.now()}`,
{ signal: controller.signal },
@@ -301,7 +289,6 @@ export default class SystemService {
) {
throw new Error('Invalid version response');
}
return {
code: 200,
data: {
@@ -323,7 +310,6 @@ export default class SystemService {
clearTimeout(timeout);
}
}
private checkHasNewVersion(curVersion: string, lastVersion: string) {
const curArr = curVersion.split('.').map((x) => parseInt(x, 10));
const lastArr = lastVersion.split('.').map((x) => parseInt(x, 10));
@@ -342,25 +328,21 @@ export default class SystemService {
}
return false;
}
public async updateSystem() {
return this.startUpgradeJob('download');
}
private upgradeJobs() {
return require(path.join(
config.rootPath,
'cli/src/internal/maintenance/panelUpgrade.cjs',
));
}
public async getUpgradeStatus() {
return {
code: 200,
data: await this.upgradeJobs().getStatus(config.tmpPath),
};
}
private async startUpgradeJob(action: 'download' | 'apply') {
try {
const data = await this.upgradeJobs().startJob(
@@ -383,7 +365,6 @@ export default class SystemService {
return { code: 400, message: t('升级任务启动失败,请检查日志后重试') };
}
}
public async reloadSystem(target?: 'system' | 'data') {
if (target && !['system', 'data'].includes(target)) {
return { code: 400, message: t('参数错误') };
@@ -401,7 +382,6 @@ export default class SystemService {
});
return { code: 200 };
}
public async notify({
title,
content,
@@ -418,35 +398,38 @@ export default class SystemService {
if (notificationInfo && typeString) {
notificationInfo.type = typeString;
}
const isSuccess = await this.notificationService.notify(
title,
content,
notificationInfo,
);
if (isSuccess) {
return { code: 200, message: t('通知发送成功') };
} else {
return { code: 400, message: t('通知发送失败,请检查系统设置/通知配置') };
}
}
public async recordNotifyLog(info: NotifyLogInfo): Promise<void> {
await persistNotificationAudit(info);
}
public async getNotifyLog(): Promise<Array<NotifyLogInfo>> {
const docs = await SystemModel.findAll({
where: { type: AuthDataType.notifyLog },
order: [['id', 'DESC']],
limit: 200,
});
const docs = await SystemModel.find(
{ type: AuthDataType.notifyLog },
{
orderBy: orderBy([['id', 'DESC']]),
limit: 200,
},
);
return docs.map((x) => ({ ...x.info, id: x.id }));
}
public async run(
{ command, logPath }: { command: string; logPath?: string },
{
command,
logPath,
}: {
command: string;
logPath?: string;
},
callback: TaskCallbacks,
) {
if (!command.startsWith(TASK_COMMAND)) {
@@ -463,17 +446,14 @@ export default class SystemService {
},
);
}
public async stop({ command, pid }: { command: string; pid: number }) {
if (!pid && !command) {
return { code: 400, message: t('参数错误') };
}
if (pid) {
await killTask(pid);
return { code: 200 };
}
if (!command.startsWith(TASK_COMMAND)) {
command = `${TASK_COMMAND} ${command}`;
}
@@ -485,7 +465,6 @@ export default class SystemService {
return { code: 400, message: t('任务未找到') };
}
}
public async exportData(res: Response, type?: string[]) {
try {
let dataDirs = ['db', 'upload'];
@@ -527,7 +506,6 @@ export default class SystemService {
return res.send({ code: 400, message: error.message });
}
}
public async importData() {
try {
await promiseExec(`rm -rf ${path.join(config.tmpPath, 'data')}`);
@@ -539,7 +517,6 @@ export default class SystemService {
return { code: 400, message: error.message };
}
}
public async getSystemLog(
res: Response,
query: {
@@ -559,7 +536,6 @@ export default class SystemService {
.reverse()
.filter((x) => x.title.endsWith('.log'))
.filter((x) => x.createTime >= startTime && x.createTime <= endTime);
const limit = Math.min(
Math.max(Number(query.limit) || 1024 * 1024, 1),
1024 * 1024,
@@ -567,7 +543,10 @@ export default class SystemService {
const total = logs.reduce((size, log) => size + (log.size || 0), 0);
let remaining = limit;
const selected: Array<
(typeof logs)[number] & { start: number; length: number }
(typeof logs)[number] & {
start: number;
length: number;
}
> = [];
for (let index = logs.length - 1; index >= 0 && remaining > 0; index--) {
const log = logs[index];
@@ -578,7 +557,6 @@ export default class SystemService {
remaining -= length;
}
}
const contentLength = selected.reduce((size, log) => size + log.length, 0);
res.set({
'Content-Length': contentLength,
@@ -590,7 +568,6 @@ export default class SystemService {
res.end();
return;
}
const currentLog = fileNames.shift();
if (currentLog) {
const currentFileStream = fs.createReadStream(
@@ -607,7 +584,6 @@ export default class SystemService {
}
})(res, selected);
}
public async deleteSystemLog() {
const result = await readDirs(config.systemLogPath, config.systemLogPath);
const logs = result.reverse().filter((x) => x.title.endsWith('.log'));
@@ -625,7 +601,6 @@ export default class SystemService {
}
}
}
public async updateTimezone(info: SystemModelInfo) {
if (!info.timezone) {
info.timezone = 'Asia/Shanghai';
@@ -642,7 +617,6 @@ export default class SystemService {
return { code: 400, message: t('设置时区失败') };
}
}
public async updateLanguage(info: SystemModelInfo) {
const oDoc = await this.getSystemConfig();
const lang = info.lang || 'zh';
@@ -662,7 +636,6 @@ export default class SystemService {
setLang(lang);
return { code: 200, data: { lang } };
}
public async updatePanelTitle(info: SystemModelInfo) {
const oDoc = await this.getSystemConfig();
const panelTitle = info.panelTitle?.trim() || '';
@@ -672,43 +645,36 @@ export default class SystemService {
});
return { code: 200, data: { panelTitle } };
}
public async updateGlobalSshKey(info: SystemModelInfo) {
const oDoc = await this.getSystemConfig();
const result = await this.updateAuthDb({
...oDoc,
info: { ...oDoc.info, ...info },
});
// Apply the global SSH key
const SshKeyService = require('./sshKey').default;
const Container = require('typedi').Container;
const sshKeyService = Container.get(SshKeyService);
if (info.globalSshKey) {
await sshKeyService.addGlobalSSHKey(info.globalSshKey, 'global');
} else {
await sshKeyService.removeGlobalSSHKey('global');
}
return { code: 200, data: result };
}
public async cleanDependence(type: 'node' | 'python3') {
if (!type || !['node', 'python3'].includes(type)) {
return { code: 400, message: t('参数错误') };
}
const finalPath = path.join(config.dependenceCachePath, type);
await fs.promises.rm(finalPath, { recursive: true, force: true });
await DependenceModel.update(
{ status: DependenceStatus.installFailed },
await DependenceModel.nativeUpdate(
{
where: {
type:
type === 'node' ? DependenceTypes.nodejs : DependenceTypes.python3,
status: DependenceStatus.installed,
},
type:
type === 'node' ? DependenceTypes.nodejs : DependenceTypes.python3,
status: DependenceStatus.installed,
},
{ status: DependenceStatus.installFailed },
);
return { code: 200 };
}
+20 -71
View File
@@ -34,27 +34,27 @@ import {
} from '../shared/password';
import { serializeAuthMutation } from '../shared/authMutation';
import { LoginThrottle } from '../shared/loginThrottle';
@Service()
export default class UserService {
private loginThrottle = new LoginThrottle();
@Inject((type) => NotificationService)
private notificationService!: NotificationService;
constructor(
@Inject('logger') private logger: winston.Logger,
@Inject('logger')
private logger: winston.Logger,
private scheduleService: ScheduleService,
private sockService: SockService,
) {}
@serializeAuthMutation
public async login(
payloads: { username: string; password: string },
payloads: {
username: string;
password: string;
},
req: Request,
): Promise<any> {
return this.authenticate(payloads, req);
}
private async authenticate(
payloads: {
username: string;
@@ -89,7 +89,6 @@ export default class UserService {
platform,
blockedIps = [],
} = content;
if (
ip &&
blockedIps.some((blockedIp) => normalizeClientIp(blockedIp) === ip)
@@ -107,7 +106,6 @@ export default class UserService {
this.getLoginLog();
return { code: 403, message: t('该 IP 已被列入黑名单') };
}
const waitTime = this.loginThrottle.retryAfter(ip, timestamp);
if (waitTime > 0) {
return {
@@ -116,10 +114,8 @@ export default class UserService {
data: waitTime,
};
}
const passwordMatches =
username === cUsername && (await verifyPassword(password, cPassword));
if (passwordMatches && twoFactorActivated && needTwoFactor) {
await this.updateAuthInfo(content, {
isTwoFactorChecking: true,
@@ -130,7 +126,6 @@ export default class UserService {
message: '',
};
}
if (passwordMatches) {
this.loginThrottle.reset(ip);
const data = createRandomString(50, 100);
@@ -139,7 +134,6 @@ export default class UserService {
expiresIn: config.jwt.expiresIn || expiration,
algorithm: 'HS384',
});
const tokenInfo: TokenInfo = {
value: token,
timestamp,
@@ -147,13 +141,11 @@ export default class UserService {
address,
platform: req.platform,
};
const updatedTokens = this.addTokenToList(
tokens,
req.platform,
tokenInfo,
);
await this.updateAuthInfo(content, {
password: isPasswordHash(cPassword)
? cPassword
@@ -252,16 +244,13 @@ export default class UserService {
}
}
}
@serializeAuthMutation
public async logout(platform: string, tokenValue: string): Promise<any> {
if (!platform || !tokenValue) {
this.logger.warn('Invalid logout parameters - empty platform or token');
return;
}
const authInfo = await this.getAuthInfo();
// Verify the token exists before attempting to remove it
const tokenExists = this.findTokenInList(
authInfo.tokens,
@@ -275,45 +264,36 @@ export default class UserService {
);
return;
}
const updatedTokens = this.removeTokenFromList(
authInfo.tokens,
platform,
tokenValue,
);
await this.updateAuthInfo(authInfo, {
token: authInfo.token === tokenValue ? '' : authInfo.token,
tokens: updatedTokens,
});
}
public async getLoginLog(): Promise<Array<SystemModelInfo | undefined>> {
const docs = await SystemModel.findAll({
where: { type: AuthDataType.loginLog },
});
const docs = await SystemModel.find({ type: AuthDataType.loginLog }, {});
if (docs && docs.length > 0) {
const result = docs.sort(
(a, b) => b.info!.timestamp! - a.info!.timestamp!,
);
if (result.length > 100) {
const ids = result.slice(100).map((x) => x.id!);
await SystemModel.destroy({
where: { id: ids },
});
await SystemModel.nativeDelete({ id: ids });
}
return result.map((x) => x.info);
}
return [];
}
public async getIpBlacklist(): Promise<string[]> {
const authInfo = await this.getAuthInfo();
return uniq((authInfo.blockedIps || []).map(normalizeClientIp)).filter(
Boolean,
);
}
@serializeAuthMutation
public async blockIp(ip: string): Promise<string[]> {
const authInfo = await this.getAuthInfo();
@@ -324,7 +304,6 @@ export default class UserService {
await this.updateAuthInfo(authInfo, { blockedIps });
return blockedIps;
}
@serializeAuthMutation
public async unblockIp(ip: string): Promise<string[]> {
const authInfo = await this.getAuthInfo();
@@ -335,12 +314,10 @@ export default class UserService {
await this.updateAuthInfo(authInfo, { blockedIps });
return blockedIps;
}
private async insertDb(payload: SystemInfo): Promise<SystemInfo> {
const doc = await SystemModel.create({ ...payload }, { returning: true });
const doc = await SystemModel.insert({ ...payload });
return doc;
}
@serializeAuthMutation
public async initializeUser({
username,
@@ -366,7 +343,6 @@ export default class UserService {
});
return { code: 200, message: t('更新成功') };
}
@serializeAuthMutation
public async updateUsernameAndPassword({
username,
@@ -389,14 +365,12 @@ export default class UserService {
});
return { code: 200, message: t('更新成功') };
}
@serializeAuthMutation
public async updateAvatar(avatar: string) {
const authInfo = await this.getAuthInfo();
await this.updateAuthInfo(authInfo, { avatar });
return { code: 200, data: avatar, message: t('更新成功') };
}
@serializeAuthMutation
public async initTwoFactor() {
const secret = authenticator.generateSecret();
@@ -408,7 +382,6 @@ export default class UserService {
await this.updateAuthInfo(authInfo, { twoFactorSecret: secret });
return { secret, url: otpauth };
}
@serializeAuthMutation
public async activeTwoFactor(code: string) {
const authInfo = await this.getAuthInfo();
@@ -427,14 +400,17 @@ export default class UserService {
}
return isValid;
}
@serializeAuthMutation
public async twoFactorLogin(
{
username,
password,
code,
}: { username: string; password: string; code: string },
}: {
username: string;
password: string;
code: string;
},
req: any,
) {
const authInfo = await this.getAuthInfo();
@@ -482,7 +458,6 @@ export default class UserService {
return { code: 430, message: t('验证失败') };
}
}
@serializeAuthMutation
public async deactivateTwoFactor() {
const authInfo = await this.getAuthInfo();
@@ -496,7 +471,6 @@ export default class UserService {
});
return true;
}
public async getAuthInfo() {
const authInfo = await shareStore.getAuthInfo();
if (authInfo) {
@@ -505,7 +479,6 @@ export default class UserService {
const doc = await this.getDb({ type: AuthDataType.authConfig });
return (doc.info || {}) as AuthInfo;
}
private async updateAuthInfo(authInfo: AuthInfo, info: Partial<AuthInfo>) {
const result = { ...authInfo, ...info };
await shareStore.updateAuthInfo(result);
@@ -517,34 +490,27 @@ export default class UserService {
this.sockService.getClients().forEach((conn) => conn.close('401'));
}
}
public async getNotificationMode(): Promise<NotificationInfo> {
const doc = await this.getDb({ type: AuthDataType.notification });
return (doc.info || {}) as NotificationInfo;
}
private async updateAuthDb(payload: SystemInfo): Promise<any> {
let doc = await SystemModel.findOne({ where: { type: payload.type } });
let doc = await SystemModel.findOne({ type: payload.type }, {});
if (doc) {
const updateResult = await SystemModel.update(payload, {
where: { id: doc.id },
returning: true,
});
doc = updateResult[1][0];
await SystemModel.nativeUpdate({ id: doc.id }, payload);
doc = await SystemModel.findOne({ id: doc.id });
} else {
doc = await SystemModel.create(payload, { returning: true });
doc = await SystemModel.insert(payload);
}
return doc;
}
public async getDb(query: any): Promise<SystemInfo> {
const doc = await SystemModel.findOne({ where: { ...query } });
const doc = await SystemModel.findOne({ ...query }, {});
if (!doc) {
throw new Error(`${JSON.stringify(query)} not found`);
}
return doc.get({ plain: true });
return doc;
}
public async updateNotificationMode(notificationInfo: NotificationInfo) {
const code = Math.random().toString().slice(-6);
const isSuccess = await this.notificationService.testNotify(
@@ -562,12 +528,10 @@ export default class UserService {
return { code: 400, message: t('通知发送失败,请检查参数') };
}
}
private normalizeTokens(
tokens: Record<string, string | TokenInfo[]>,
): Record<string, TokenInfo[]> {
const normalized: Record<string, TokenInfo[]> = {};
for (const [platform, value] of Object.entries(tokens)) {
if (typeof value === 'string') {
// Legacy format: convert string token to TokenInfo array
@@ -589,10 +553,8 @@ export default class UserService {
normalized[platform] = value || [];
}
}
return normalized;
}
private addTokenToList(
tokens: Record<string, string | TokenInfo[]>,
platform: string,
@@ -606,16 +568,12 @@ export default class UserService {
);
maxTokensPerPlatform = config.maxTokensPerPlatform;
}
const normalized = this.normalizeTokens(tokens);
if (!normalized[platform]) {
normalized[platform] = [];
}
// Add new token
normalized[platform].unshift(tokenInfo);
// Limit the number of active tokens per platform
if (normalized[platform].length > maxTokensPerPlatform) {
normalized[platform] = normalized[platform].slice(
@@ -623,40 +581,32 @@ export default class UserService {
maxTokensPerPlatform,
);
}
return normalized;
}
private removeTokenFromList(
tokens: Record<string, string | TokenInfo[]>,
platform: string,
tokenValue: string,
): Record<string, TokenInfo[]> {
const normalized = this.normalizeTokens(tokens);
if (normalized[platform]) {
normalized[platform] = normalized[platform].filter(
(t) => t.value !== tokenValue,
);
}
return normalized;
}
private findTokenInList(
tokens: Record<string, string | TokenInfo[]>,
platform: string,
tokenValue: string,
): TokenInfo | undefined {
const normalized = this.normalizeTokens(tokens);
if (normalized[platform]) {
return normalized[platform].find((t) => t.value === tokenValue);
}
return undefined;
}
@serializeAuthMutation
public async resetAuthInfo(info: Partial<AuthInfo>) {
const { retries, twoFactorActivated, password, username } = info;
@@ -673,7 +623,6 @@ export default class UserService {
},
(x) => !isNil(x),
);
if (password !== undefined) {
payload.password = await hashPassword(password);
}
+4 -2
View File
@@ -135,8 +135,10 @@ export function parseCronSchedule(schedule: unknown): CronSchedule {
};
Object.freeze(patterns);
Object.freeze(parsed);
if (scheduleCache.size >= 512)
scheduleCache.delete(scheduleCache.keys().next().value);
if (scheduleCache.size >= 512) {
const oldest = scheduleCache.keys().next().value;
if (oldest !== undefined) scheduleCache.delete(oldest);
}
scheduleCache.set(cacheKey, parsed);
return parsed;
}
+2
View File
@@ -1,6 +1,8 @@
const messages: Record<string, Record<string, string>> = {
zh: {},
en: {
'认证服务暂时不可用,请稍后重试':
'Authentication service is temporarily unavailable; try again shortly.',
'升级任务启动失败,请检查日志后重试':
'Failed to start the update. Check the logs and try again.',
'[schedule][停止已运行任务] 任务ID: %s, PID: %s':
+14 -15
View File
@@ -1,17 +1,16 @@
import { Op } from 'sequelize';
import { Query, QuerySpec, orderBy } from '../data/queries';
import {
SystemModel,
AuthDataType,
NotifyStatus,
NotifyLogInfo,
} from '../data/system';
// Both backend sends and script reports use the same bounded, best-effort sink.
export async function persistNotificationAudit(
info: NotifyLogInfo,
): Promise<void> {
try {
await SystemModel.create({
await SystemModel.insert({
type: AuthDataType.notifyLog,
info: {
timestamp: Date.now(),
@@ -27,19 +26,19 @@ export async function persistNotificationAudit(
taskId: /^\d+$/.test(info.taskId || '') ? info.taskId : undefined,
},
});
const oldest = await SystemModel.findAll({
where: { type: AuthDataType.notifyLog },
attributes: ['id'],
order: [['id', 'DESC']],
offset: 200,
limit: 1,
});
const oldest = await SystemModel.find(
{ type: AuthDataType.notifyLog },
{
fields: ['id'],
orderBy: orderBy([['id', 'DESC']]),
offset: 200,
limit: 1,
},
);
if (oldest[0]?.id) {
await SystemModel.destroy({
where: {
type: AuthDataType.notifyLog,
id: { [Op.lte]: oldest[0].id },
},
await SystemModel.nativeDelete({
type: AuthDataType.notifyLog,
id: { ['$lte']: oldest[0].id },
});
}
} catch (error) {
+14 -7
View File
@@ -137,13 +137,20 @@ class TaskLimit {
this.manualCronoLimit.concurrency = limit;
return;
}
await SystemModel.sync();
const doc = await SystemModel.findOne({
where: { type: AuthDataType.systemConfig },
});
if (doc?.info?.cronConcurrency) {
this.cronLimit.concurrency = doc.info.cronConcurrency;
this.manualCronoLimit.concurrency = doc.info.cronConcurrency;
try {
const doc = await SystemModel.findOne({
type: AuthDataType.systemConfig,
});
if (doc?.info?.cronConcurrency) {
this.cronLimit.concurrency = doc.info.cronConcurrency;
this.manualCronoLimit.concurrency = doc.info.cronConcurrency;
}
} catch (error) {
// Imports can precede the isolated initializer on a fresh installation.
// Use defaults until that initializer creates Auths; never create schema here.
if ((error as { code?: string }).code !== 'DATABASE_SCHEMA_MISSING')
throw error;
}
}
+7 -9
View File
@@ -20,7 +20,7 @@ export function runCron(
// Check if the cron is already running and stop it (only if multiple instances are not allowed)
try {
const existingCron = await CrontabModel.findOne({
where: { id: Number(cron.id) },
id: Number(cron.id),
});
// A mutation may commit while this queued run is reading the database.
@@ -45,19 +45,17 @@ export function runCron(
await killTask(existingCron.pid, true);
// Mark old running instances as stopped
const stoppedAt = dayjs().unix();
await RunningInstanceModel.update(
{ status: InstanceStatus.stopped, finished_at: stoppedAt },
await RunningInstanceModel.nativeUpdate(
{
where: {
cron_id: Number(cron.id),
status: InstanceStatus.running,
},
cron_id: Number(cron.id),
status: InstanceStatus.running,
},
{ status: InstanceStatus.stopped, finished_at: stoppedAt },
);
// Update the status to idle after killing
await CrontabModel.update(
await CrontabModel.nativeUpdate(
{ id: Number(cron.id) },
{ status: CrontabStatus.idle, pid: undefined },
{ where: { id: Number(cron.id) } },
);
}
} catch (error) {
+5 -15
View File
@@ -1,4 +1,4 @@
import { QueryTypes, Sequelize } from 'sequelize';
import { DatabaseClient } from '../data/databaseClient';
// Append new entries; IDs are persisted and must not be renumbered or reused.
const columns = [
@@ -27,25 +27,19 @@ const columns = [
{ table: 'Crontabs', column: 'queued_token', type: 'VARCHAR(255)' },
];
export async function migrateSchema(database: Sequelize): Promise<void> {
await database.transaction(async (transaction) => {
export async function migrateSchema(database: DatabaseClient): Promise<void> {
await database.transaction(async () => {
await database.query(
'CREATE TABLE IF NOT EXISTS "SchemaMigrations" ("id" TEXT PRIMARY KEY, "applied_at" TEXT NOT NULL)',
{ transaction },
);
const applied = await database.query<{ id: string }>(
'SELECT "id" FROM "SchemaMigrations"',
{ type: QueryTypes.SELECT, transaction },
);
const appliedIds = new Set(applied.map(({ id }) => id));
for (const { table, column, type } of columns) {
const id = `add-${table}-${column}`;
const fields = await database.query<{ name: string }>(
`PRAGMA table_info("${table}")`,
{
type: QueryTypes.SELECT,
transaction,
},
);
if (fields.length === 0)
throw new Error(`Migration table is missing: ${table}`);
@@ -53,16 +47,12 @@ export async function migrateSchema(database: Sequelize): Promise<void> {
// table/column/type come only from the static migration manifest above.
await database.query(
`ALTER TABLE "${table}" ADD COLUMN "${column}" ${type}`,
{ transaction },
);
}
if (!appliedIds.has(id)) {
await database.query(
'INSERT INTO "SchemaMigrations" ("id", "applied_at") VALUES (:id, :appliedAt)',
{
replacements: { id, appliedAt: new Date().toISOString() },
transaction,
},
'INSERT INTO "SchemaMigrations" ("id", "applied_at") VALUES (?, ?)',
[id, new Date().toISOString()],
);
}
}
+7 -5
View File
@@ -1,9 +1,7 @@
import { AuthInfo } from '../data/system';
import { App } from '../data/open';
import Keyv from 'keyv';
import KeyvSqlite from '@keyv/sqlite';
import config from '../config';
import path from 'path';
import { CacheStore } from '../data/cacheStore';
export enum EKeyv {
'apps' = 'apps',
@@ -17,8 +15,12 @@ export interface IKeyvStore {
lang: string;
}
const keyvSqlite = new KeyvSqlite(path.join(config.dbPath, 'keyv.sqlite'));
export const keyvStore = new Keyv<IKeyvStore>({ store: keyvSqlite });
const keyvSqlite = new CacheStore();
// A stopped database is unavailable, not a cache miss or an expired session.
export const keyvStore = new Keyv<IKeyvStore>({
store: keyvSqlite,
throwOnErrors: true,
});
export const shareStore = {
getAuthInfo() {
+12 -31
View File
@@ -2,21 +2,15 @@ import express, { Application, Request } from 'express';
import { AuthDataType, SystemModel } from '../data/system';
import { normalizeClientIp } from './clientIp';
import { t } from './i18n';
const DEFAULT_TRUST_PROXY = 'loopback';
type TrustProxyValue = boolean | number | string;
let activeApp: Application | undefined;
function getEnvironmentSetting(): string {
return process.env.QL_TRUST_PROXY?.trim() || '';
}
function normalizeSetting(value?: string): string {
return value?.trim() || DEFAULT_TRUST_PROXY;
}
export function resolveTrustProxy(value?: string): TrustProxyValue {
const setting = normalizeSetting(value);
if (setting === 'true' || setting === 'false') {
@@ -27,7 +21,6 @@ export function resolveTrustProxy(value?: string): TrustProxyValue {
}
return setting;
}
function validateTrustProxy(value: string): string {
const setting = normalizeSetting(value);
if (setting.length > 500 || /[\r\n]/.test(setting)) {
@@ -36,25 +29,22 @@ function validateTrustProxy(value: string): string {
if (/^\d+$/.test(setting) && Number(setting) > 20) {
throw new Error(t('代理层数不能超过 20'));
}
const probe = express();
probe.set('trust proxy', resolveTrustProxy(setting));
return setting;
}
async function getStoredSetting(): Promise<string> {
const doc = await SystemModel.findOne({
where: { type: AuthDataType.systemConfig },
});
const info = (doc?.get('info') || {}) as Record<string, unknown>;
const doc = await SystemModel.findOne(
{ type: AuthDataType.systemConfig },
{},
);
const info = (doc?.info || {}) as Record<string, unknown>;
return typeof info.trustProxy === 'string' ? info.trustProxy : '';
}
export async function getTrustProxyConfig() {
const environmentSetting = getEnvironmentSetting();
const storedSetting = await getStoredSetting();
const trustProxy = normalizeSetting(environmentSetting || storedSetting);
return {
trustProxy,
source: environmentSetting
@@ -65,36 +55,31 @@ export async function getTrustProxyConfig() {
editable: !environmentSetting,
};
}
export async function initializeTrustProxy(app: Application) {
activeApp = app;
const { trustProxy } = await getTrustProxyConfig();
app.set('trust proxy', resolveTrustProxy(trustProxy));
}
export async function updateTrustProxy(value: string) {
if (getEnvironmentSetting()) {
throw new Error(t('环境变量 QL_TRUST_PROXY 已生效,系统设置不可覆盖'));
}
const trustProxy = validateTrustProxy(value);
const doc = await SystemModel.findOne({
where: { type: AuthDataType.systemConfig },
});
const doc = await SystemModel.findOne(
{ type: AuthDataType.systemConfig },
{},
);
if (!doc) {
throw new Error(t('系统配置不存在'));
}
const plain = doc.get({ plain: true });
await SystemModel.update(
const plain = doc;
await SystemModel.nativeUpdate(
{ id: plain.id },
{ info: { ...(plain.info || {}), trustProxy } as any },
{ where: { id: plain.id } },
);
activeApp?.set('trust proxy', resolveTrustProxy(trustProxy));
return getTrustProxyConfig();
}
function parseForwardedFor(value: string | string[] | undefined): string[] {
const values = Array.isArray(value) ? value : value ? [value] : [];
return values
@@ -102,7 +87,6 @@ function parseForwardedFor(value: string | string[] | undefined): string[] {
.map((item) => normalizeClientIp(item))
.filter(Boolean);
}
export async function diagnoseClientIp(req: Request) {
const remoteAddress = normalizeClientIp(req.socket.remoteAddress);
const forwardedFor = parseForwardedFor(req.headers['x-forwarded-for']);
@@ -112,7 +96,6 @@ export async function diagnoseClientIp(req: Request) {
const trust = req.app.get('trust proxy fn') as
| ((ip: string, hop: number) => boolean)
| undefined;
let selectedIndex = Math.max(hopsFromApp.length - 1, 0);
for (let index = 0; index < hopsFromApp.length - 1; index += 1) {
if (!trust?.(hopsFromApp[index], index)) {
@@ -120,7 +103,6 @@ export async function diagnoseClientIp(req: Request) {
break;
}
}
const hops = hopsFromApp.map((ip, index) => ({
ip,
hop: index,
@@ -131,7 +113,6 @@ export async function diagnoseClientIp(req: Request) {
? 'client'
: 'not_checked',
}));
return {
...(await getTrustProxyConfig()),
remoteAddress,
+1 -1
View File
@@ -23,5 +23,5 @@
"esModuleInterop": true
},
"include": ["./**/*"],
"exclude": ["node_modules"]
"exclude": ["node_modules", "./data/databaseWorker.ts"]
}
+15 -2
View File
@@ -164,7 +164,10 @@ async function readState(tmp) {
}
}
async function getStatus(tmp) {
async function getStatus(
tmp,
{ root = path.resolve(__dirname, '../../../..') } = {},
) {
const state = await readState(tmp);
if (active.has(state.status)) {
let alive = false;
@@ -198,7 +201,17 @@ async function getStatus(tmp) {
await handle.close();
}
}
return state;
const pkg = JSON.parse(
await fs
.readFile(path.join(root, 'package.json'), 'utf8')
.catch((error) => {
if (error.code !== 'ENOENT') throw error;
return '{}';
}),
);
// Read the installed package on every poll: the worker's completion record
// survives replacement, while this marker disappears only with the new runtime.
return { ...state, migrationRequired: Boolean(pkg.qinglongMigrationBridge) };
}
// Manual restarts accept the currently prepared package, including one retained
+179 -30
View File
@@ -1,5 +1,6 @@
// One installed verifier handles release downloads and local offline archives.
const fs = require('node:fs/promises');
const { constants, createReadStream } = require('node:fs');
const path = require('node:path');
const crypto = require('node:crypto');
const os = require('node:os');
@@ -176,23 +177,39 @@ async function verifyPayload(selected, manifest, target) {
});
const names = Object.keys(actual).sort(),
expected = Object.keys(manifest.files).sort();
if (
JSON.stringify(names) !== JSON.stringify(expected) ||
names.some(
(name) =>
JSON.stringify(actual[name]) !== JSON.stringify(manifest.files[name]),
)
)
const differences = [...new Set([...expected, ...names])].filter(
(name) =>
!Object.hasOwn(actual, name) ||
!Object.hasOwn(manifest.files, name) ||
JSON.stringify(actual[name]) !== JSON.stringify(manifest.files[name]),
);
if (differences.length)
throw new Error(
'Platform bundle files or dependency checksums do not match',
'Platform bundle files or dependency checksums do not match: ' +
JSON.stringify({
count: differences.length,
files: differences.slice(0, 5).map((name) => ({
name,
expected: manifest.files[name] || null,
actual: actual[name] || null,
})),
}),
);
if (
sha256(await fs.readFile(path.join(selected.source, 'pnpm-lock.yaml'))) !==
manifest.lockfileSha256
)
throw new Error('Platform bundle lockfile mismatch');
const pkg = JSON.parse(
await fs.readFile(path.join(selected.source, 'package.json'), 'utf8'),
);
await fs.access(
path.join(selected.source, 'node_modules/sqlite3/package.json'),
path.join(
selected.source,
pkg.dependencies?.['@mikro-orm/sql']
? 'node_modules/@mikro-orm/sql/package.json'
: 'node_modules/sqlite3/package.json',
),
);
}
@@ -202,34 +219,58 @@ async function smokeDependencies(source, signal) {
const path = require('node:path'),fs = require('node:fs');
const root = fs.realpathSync(process.cwd());
const r = require('node:module').createRequire(path.join(root,'package.json'));
for (const name of ['sqlite3','sequelize','cron-parser','@grpc/grpc-js','undici']) {
const pkg=r('./package.json'), modern=Boolean(pkg.dependencies?.['@mikro-orm/sql']);
if (modern) {
const [major,minor]=process.versions.node.split('.').map(Number);
if(major<22 || (major===22 && minor<17)) throw new Error('Bundle requires Node >=22.17; the existing Node is retained');
}
for (const name of [...(modern?['@mikro-orm/core','@mikro-orm/sql']:['sqlite3','sequelize']),'cron-parser','@grpc/grpc-js','undici']) {
if (!fs.realpathSync(r.resolve(name)).startsWith(root + path.sep)) throw new Error('Dependency outside staged payload: '+name);
r(name);
}
if (modern) {
const {DatabaseSync}=require('node:sqlite'), db=new DatabaseSync(':memory:');
try {
db.exec('CREATE TABLE verify_bundle(value INTEGER);INSERT INTO verify_bundle VALUES(42)');
if(db.prepare('SELECT value FROM verify_bundle').get().value!==42) throw new Error('SQLite readback failed');
} finally {db.close();}
} else {
const sqlite=r('sqlite3'), db=new sqlite.Database(':memory:');
db.serialize(()=>{db.run('CREATE TABLE verify_bundle(value INTEGER)');db.run('INSERT INTO verify_bundle VALUES(42)');
db.get('SELECT value FROM verify_bundle',(err,row)=>{if(err||row.value!==42){console.error(err||'SQLite readback failed');process.exitCode=1;}db.close();});});
}
`;
// Match the server's normal runtime. Node 20.18's built-in HTTP parser needs
// WebAssembly, which --jitless disables; only archive workers need that flag.
await execute(process.execPath, [...serverRuntimeArgs(process), '-e', probe], {
cwd: source,
env: { ...process.env, NODE_PATH: '', NODE_OPTIONS: '' },
// ARM1176 devices need the same startup allowance as the backend probe.
// Keep this bounded and cancellable without forcing an unnecessary repair.
timeout: nodeRuntimeArgs().length ? 300000 : 30000,
signal,
});
await execute(
process.execPath,
[...serverRuntimeArgs(process), '-e', probe],
{
cwd: source,
env: { ...process.env, NODE_PATH: '', NODE_OPTIONS: '' },
// ARM1176 devices need the same startup allowance as the backend probe.
// Keep this bounded and cancellable without forcing an unnecessary repair.
timeout: nodeRuntimeArgs().length ? 300000 : 30000,
signal,
},
);
}
async function extractArchive(archive, destination) {
const entries = new Map(),
links = new Set();
directories = new Map([['qinglong', 0o755]]);
const header = (entry) => ({
type: entry.type,
size: entry.size,
mode: entry.mode,
linkpath: entry.linkpath,
});
let unpacked = 0,
invalid;
await tar.t({
file: archive,
strict: true,
maxReadSize: 256 * 1024,
onReadEntry(entry) {
try {
const name = entry.path.replace(/\/$/, '');
@@ -237,16 +278,19 @@ async function extractArchive(archive, destination) {
!name ||
name.includes('\\') ||
/[\x00-\x1f\x7f]/.test(name) ||
name.split('/').length > 1024 ||
name.split('/').some((p) => !p || p === '.' || p === '..') ||
!(name === 'qinglong' || name.startsWith('qinglong/')) ||
entries.has(name) ||
!['File', 'OldFile', 'Directory', 'SymbolicLink'].includes(entry.type)
)
throw new Error(`Unsafe bundle archive entry: ${name}`);
if (!Number.isSafeInteger(entry.size) || entry.size < 0)
throw new Error('Invalid bundle archive entry size');
unpacked += entry.size;
if (unpacked > 2 * 1024 ** 3 || entries.size > 200000)
throw new Error('Bundle archive exceeds extraction limits');
entries.set(name, entry.type);
entries.set(name, header(entry));
if (entry.type === 'SymbolicLink') {
if (
path.posix.isAbsolute(entry.linkpath) ||
@@ -257,7 +301,6 @@ async function extractArchive(archive, destination) {
)
)
throw new Error('Unsafe bundle symlink');
links.add(name);
}
} catch (error) {
invalid ||= error;
@@ -265,22 +308,101 @@ async function extractArchive(archive, destination) {
},
});
if (invalid) throw invalid;
for (const name of entries.keys()) {
for (const [name, entry] of entries) {
if (entry.type === 'Directory')
directories.set(name, ((entry.mode ?? 0o755) & 0o777) | 0o700);
let parent = path.posix.dirname(name);
while (parent !== '.') {
if (links.has(parent))
throw new Error('Archive writes through a symlink');
if (entries.has(parent) && entries.get(parent).type !== 'Directory')
throw new Error('Archive writes through a symlink or non-directory');
if (!directories.has(parent)) directories.set(parent, 0o755);
parent = path.posix.dirname(parent);
}
}
await tar.x({
file: archive,
// The extraction root must be new and private. Create every parent before
// links, and keep directories private/searchable until all writes finish.
const directoryNames = [...directories.keys()].sort(
(a, b) => a.split('/').length - b.split('/').length,
);
for (const name of directoryNames) {
const filename = path.join(destination, name);
await fs.mkdir(filename, { mode: 0o700 });
await fs.chmod(filename, 0o700);
}
let writes = Promise.resolve(),
failure;
const active = new Set(),
seen = new Set();
const parser = tar.t({
cwd: destination,
strict: true,
preservePaths: false,
noChown: true,
umask: 0,
noResume: true,
onReadEntry(entry) {
active.add(entry);
// A parser failure can destroy queued entries before their turn starts.
entry.on('error', () => {});
writes = writes
.then(async () => {
if (failure) return entry.resume();
const name = entry.path.replace(/\/$/, '');
if (
seen.has(name) ||
JSON.stringify(entries.get(name)) !== JSON.stringify(header(entry))
)
throw new Error('Archive changed during extraction');
seen.add(name);
const filename = path.join(destination, name);
if (entry.type === 'Directory') return entry.resume();
if (entry.type === 'SymbolicLink') {
await fs.symlink(entry.linkpath, filename);
return entry.resume();
}
const mode = (entry.mode ?? 0o644) & 0o777;
const handle = await fs.open(
filename,
constants.O_WRONLY |
constants.O_CREAT |
constants.O_EXCL |
constants.O_NOFOLLOW,
0o600,
);
try {
// tar's fs-minipass writer discards successful partial write/writev
// results. Node's writeFile loops until each chunk is fully written.
// One entry/chunk at a time bounds memory and open FDs on ARMv6.
for await (const chunk of entry) await handle.writeFile(chunk);
await handle.chmod(mode);
} finally {
await handle.close();
}
})
.catch((error) => {
failure ||= error;
entry.resume();
})
.finally(() => active.delete(entry));
},
});
const input = createReadStream(archive, { highWaterMark: 256 * 1024 });
try {
await new Promise((resolve, reject) => {
input.once('error', reject);
parser.once('error', reject);
parser.once('end', resolve);
input.pipe(parser);
});
} catch (error) {
failure ||= error;
for (const entry of active) entry.destroy(error);
} finally {
input.destroy();
}
await writes;
if (failure) throw failure;
if (seen.size !== entries.size)
throw new Error('Archive changed during extraction');
for (const name of directoryNames.reverse())
await fs.chmod(path.join(destination, name), directories.get(name));
}
async function stageBundle({
@@ -297,6 +419,8 @@ async function stageBundle({
repair = repairDependencies,
}) {
let local = env.QL_UPGRADE_FILE;
let pendingMigration = false,
requiresModern = false;
if (!local) {
const current = JSON.parse(
await fs
@@ -307,6 +431,9 @@ async function stageBundle({
}),
);
if (current.qinglongUpgradeFormat !== 2) return null; // One-time compatibility path for older releases.
pendingMigration = Boolean(current.qinglongMigrationBridge);
requiresModern =
pendingMigration || Boolean(current.dependencies?.['@mikro-orm/sql']);
}
const target = await runtimeTarget();
const branch = ['develop', 'debian-dev'].includes(env.QL_BRANCH)
@@ -361,7 +488,16 @@ async function stageBundle({
indexUrl,
]);
const index = JSON.parse(await fs.readFile(indexFile, 'utf8'));
const entry = index.packages?.[targetKey(target)];
if (pendingMigration && !Object.hasOwn(index, 'runtimePackages'))
throw new Error('更新索引尚未提供第二步运行包,请稍后重试');
requiresModern ||= Object.hasOwn(index, 'runtimePackages');
// Old format-2 clients read packages and receive an updater bridge. Once
// installed, this selector uses only the actual runtime packages. A broken
// runtime entry must never fall back to the bridge or a different platform.
const packages = Object.hasOwn(index, 'runtimePackages')
? index.runtimePackages
: index.packages;
const entry = packages?.[targetKey(target)];
if (
index.format !== 2 ||
!/^[a-f0-9]{40}$/.test(index.sourceCommit) ||
@@ -403,6 +539,10 @@ async function stageBundle({
const manifest = JSON.parse(
await fs.readFile(path.join(incoming, 'bundle.json'), 'utf8'),
);
if (manifest.migrationBridge)
throw new Error(
'This is a legacy upgrader bridge; download the runtime package instead',
);
if (expectedCommit && manifest.sourceCommit !== expectedCommit)
throw new Error('Update index and bundle commit differ');
const selected = {
@@ -413,6 +553,15 @@ async function stageBundle({
await fs.rename(path.join(incoming, 'static'), selected.static);
await report({ phase: 'verifying', percent: null });
await verifyPayload(selected, manifest, target);
if (
requiresModern &&
!JSON.parse(
await fs.readFile(path.join(selected.source, 'package.json'), 'utf8'),
).dependencies?.['@mikro-orm/sql']
)
throw new Error(
'第二步更新包仍是旧数据库后端,已保留当前服务,请稍后重试',
);
const build = await verifyStatic(selected.source, selected.static, env);
if (build.sourceCommit !== manifest.sourceCommit)
throw new Error('Bundle and build commit differ');
@@ -210,6 +210,28 @@ async function prestartBackend(
}
}
async function cleanSqliteBuildArtifacts(directory) {
const entries = await fs.readdir(directory, { withFileTypes: true }).catch((error) => {
if (error.code !== 'ENOENT') throw error;
return [];
});
if (!entries.length) return;
const pkg = JSON.parse(await fs.readFile(path.join(directory, 'package.json'), 'utf8'));
const runtimePath = path.posix.normalize(pkg.binary?.module_path || '.');
const copiedRuntime = runtimePath.startsWith('lib/binding/');
// Our legacy fork copies its runtime to lib/binding. Other sqlite3 versions
// load build/Release directly: retain that addon and remove only the compiler
// helper directory. Never traverse a build-directory symlink while cleaning.
for (const entry of entries) {
if (entry.name !== 'build' && !/^build-tmp-napi-v\d+$/.test(entry.name)) continue;
const build = path.join(directory, entry.name);
await fs.rm(
copiedRuntime || entry.isSymbolicLink() ? build : path.join(build, 'node_gyp_bins'),
{ recursive: true, force: true },
);
}
}
async function repairDependencies(source, { target, signal, env }) {
const workspace = await fs.mkdtemp(
path.join(path.dirname(source), 'repair-'),
@@ -299,7 +321,12 @@ async function repairDependencies(source, { target, signal, env }) {
'--child-concurrency=1',
'--network-concurrency=2',
]);
await run(['rebuild', 'sqlite3', '@whyour/sqlite3']);
if (pkg.dependencies?.sqlite3) {
await run(['rebuild', 'sqlite3', '@whyour/sqlite3']);
await cleanSqliteBuildArtifacts(
path.join(workspace, 'node_modules/sqlite3'),
);
}
if (
!(await fs.readFile(path.join(source, 'pnpm-lock.yaml'))).equals(
await fs.readFile(path.join(workspace, 'pnpm-lock.yaml')),
@@ -390,5 +417,6 @@ module.exports = {
prestartBackend,
dependencyFailure,
repairDependencies,
cleanSqliteBuildArtifacts,
probeLimits,
};
+8 -7
View File
@@ -1,6 +1,6 @@
# Run Node package installation natively on the builder. Node/npm can spin at
# 100% CPU when Alpine s390x is emulated through QEMU.
FROM --platform=$BUILDPLATFORM node:18-alpine3.18 AS builder
FROM --platform=$BUILDPLATFORM node:24-alpine3.24 AS builder
ARG TARGETARCH
ENV NPM_CONFIG_PREFIX=/opt/node-global
@@ -20,15 +20,17 @@ RUN set -x \
&& npm_config_target_platform=linux \
npm_config_target_arch="${NODE_ARCH}" \
npm_config_target_libc=musl \
pnpm install --prod --frozen-lockfile
QL_BUILD_ONLY=1 pnpm install --prod --frozen-lockfile
# Pin amd64 to upstream's musl runtime after native URL parser crashes with APK Node.
# The amd64 APK runtime has crashed in native URL parsing on dev. Keep its
# URL implementation bundled with the pinned upstream musl Node binary.
FROM --platform=linux/amd64 node:24.18.1-alpine3.24@sha256:f70403e87646dc51b45295f4b8b70cdad0b63d2297c4c9899119b03f7af7a6b3 AS node-runtime-amd64
FROM python:3.11-alpine
ARG TARGETARCH
ARG TARGETVARIANT
COPY --from=node-runtime-amd64 /usr/local/bin/node /tmp/node-runtime-amd64
ARG QL_MAINTAINER="whyour"
LABEL maintainer="${QL_MAINTAINER}"
@@ -50,8 +52,7 @@ EXPOSE 5700
COPY --from=builder /opt/node-global/lib/node_modules/. /usr/local/lib/node_modules/
COPY --from=builder /opt/node-global/bin/. /usr/local/bin/
RUN --mount=type=bind,from=node-runtime-amd64,source=/usr/local/bin/node,target=/tmp/node-runtime-amd64 \
set -x \
RUN set -x \
&& apk update -f \
&& apk upgrade \
&& apk --no-cache add -f bash \
@@ -70,9 +71,9 @@ RUN --mount=type=bind,from=node-runtime-amd64,source=/usr/local/bin/node,target=
unzip \
npm \
&& if [ "$TARGETARCH" = amd64 ]; then \
cp /tmp/node-runtime-amd64 /usr/local/bin/node; \
mv /tmp/node-runtime-amd64 /usr/local/bin/node; \
ln -sf /usr/local/bin/node /usr/bin/node; \
fi \
else rm /tmp/node-runtime-amd64; fi \
&& rm -rf /var/cache/apk/* \
&& apk update \
&& ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
+8 -7
View File
@@ -1,6 +1,6 @@
# Run Node package installation natively on the builder. Node/npm can spin at
# 100% CPU when Alpine s390x is emulated through QEMU.
FROM --platform=$BUILDPLATFORM node:18-alpine3.18 AS builder
FROM --platform=$BUILDPLATFORM node:24-alpine3.24 AS builder
ARG TARGETARCH
ENV NPM_CONFIG_PREFIX=/opt/node-global
@@ -20,15 +20,17 @@ RUN set -x \
&& npm_config_target_platform=linux \
npm_config_target_arch="${NODE_ARCH}" \
npm_config_target_libc=musl \
pnpm install --prod --frozen-lockfile
QL_BUILD_ONLY=1 pnpm install --prod --frozen-lockfile
# Pin amd64 to upstream's musl runtime after native URL parser crashes with APK Node.
# The amd64 APK runtime has crashed in native URL parsing on dev. Keep its
# URL implementation bundled with the pinned upstream musl Node binary.
FROM --platform=linux/amd64 node:24.18.1-alpine3.24@sha256:f70403e87646dc51b45295f4b8b70cdad0b63d2297c4c9899119b03f7af7a6b3 AS node-runtime-amd64
FROM python:3.10-alpine
ARG TARGETARCH
ARG TARGETVARIANT
COPY --from=node-runtime-amd64 /usr/local/bin/node /tmp/node-runtime-amd64
ARG QL_MAINTAINER="whyour"
LABEL maintainer="${QL_MAINTAINER}"
@@ -50,8 +52,7 @@ EXPOSE 5700
COPY --from=builder /opt/node-global/lib/node_modules/. /usr/local/lib/node_modules/
COPY --from=builder /opt/node-global/bin/. /usr/local/bin/
RUN --mount=type=bind,from=node-runtime-amd64,source=/usr/local/bin/node,target=/tmp/node-runtime-amd64 \
set -x \
RUN set -x \
&& apk update -f \
&& apk upgrade \
&& apk --no-cache add -f bash \
@@ -70,9 +71,9 @@ RUN --mount=type=bind,from=node-runtime-amd64,source=/usr/local/bin/node,target=
unzip \
npm \
&& if [ "$TARGETARCH" = amd64 ]; then \
cp /tmp/node-runtime-amd64 /usr/local/bin/node; \
mv /tmp/node-runtime-amd64 /usr/local/bin/node; \
ln -sf /usr/local/bin/node /usr/bin/node; \
fi \
else rm /tmp/node-runtime-amd64; fi \
&& rm -rf /var/cache/apk/* \
&& apk update \
&& ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
+7 -4
View File
@@ -1,6 +1,9 @@
# Node 20 Bookworm is the latest official Node image variant that covers the
# full Debian build matrix, including arm/v7, ppc64le, and s390x.
FROM node:20-bookworm-slim AS nodebuilder
FROM python:3.11.14-slim-bookworm AS nodebuilder
ARG TARGETARCH
ARG TARGETVARIANT
COPY docker/install-node-runtime.sh /tmp/install-node-runtime.sh
RUN apt-get update && apt-get install --no-install-recommends -y ca-certificates curl xz-utils libatomic1 libstdc++6 && \
sh /tmp/install-node-runtime.sh /usr/local
FROM python:3.11.14-slim-bookworm AS builder
COPY package.json .npmrc pnpm-lock.yaml /tmp/build/
@@ -13,7 +16,7 @@ RUN set -x && \
npm i -g pnpm@8.3.1 && \
test "$(pnpm --version)" = "8.3.1" && \
cd /tmp/build && \
pnpm install --prod --frozen-lockfile
QL_BUILD_ONLY=1 pnpm install --prod --frozen-lockfile
FROM python:3.11.14-slim-bookworm
+7 -4
View File
@@ -1,6 +1,9 @@
# Node 20 Bookworm is the latest official Node image variant that covers the
# full Debian build matrix, including arm/v7, ppc64le, and s390x.
FROM node:20-bookworm-slim AS nodebuilder
FROM python:3.10.19-slim-bookworm@sha256:23f63358922e79a794f71be8f3723c84e5ccca9638af3f74456dc73d6184499e AS nodebuilder
ARG TARGETARCH
ARG TARGETVARIANT
COPY docker/install-node-runtime.sh /tmp/install-node-runtime.sh
RUN apt-get update && apt-get install --no-install-recommends -y ca-certificates curl xz-utils libatomic1 libstdc++6 && \
sh /tmp/install-node-runtime.sh /usr/local
# Keep the full release matrix, including s390x, on the same immutable base.
# The floating 3.10-slim-bookworm tag no longer includes s390x.
@@ -14,7 +17,7 @@ RUN set -x && \
apt-get install --no-install-recommends -y libatomic1 && \
npm i -g pnpm@8.3.1 && \
cd /tmp/build && \
pnpm install --prod --frozen-lockfile
QL_BUILD_ONLY=1 pnpm install --prod --frozen-lockfile
FROM python:3.10.19-slim-bookworm@sha256:23f63358922e79a794f71be8f3723c84e5ccca9638af3f74456dc73d6184499e
+22
View File
@@ -0,0 +1,22 @@
ARG LEGACY_IMAGE
ARG BASE_IMAGE
FROM ${LEGACY_IMAGE} AS legacy
FROM ${BASE_IMAGE} AS bridge
ARG BASE_IMAGE
ENV QL_BUNDLE_BASE_IMAGE=${BASE_IMAGE}
ARG TARGETARCH
ARG TARGETVARIANT
USER root
COPY --from=legacy /ql/ /legacy/
RUN ulimit -c 0; if [ "$TARGETARCH/$TARGETVARIANT" = arm/v6 ]; then \
export QEMU_CPU=arm1176; \
apk add --no-cache build-base nodejs-dev && \
node --jitless /ql/scripts/build-migration-bridge.cjs /legacy /ql /out && \
for archive in /out/qinglong-*-linux-armv6-musl.tar.gz; do \
case "$archive" in */qinglong-migration-*) continue ;; esac; \
node /ql/shell/install-bundle.cjs --root /tmp/ql-bundle-offline-smoke --file "$archive" --install && \
rm -rf /tmp/ql-bundle-offline-smoke || exit 1; \
done; \
else node /ql/scripts/build-migration-bridge.cjs /legacy /ql /out; fi
FROM scratch
COPY --from=bridge /out/ /
+1 -12
View File
@@ -6,18 +6,7 @@ ENV QL_BUNDLE_BASE_IMAGE=${BASE_IMAGE}
ARG TARGETARCH
ARG TARGETVARIANT
USER root
# The generic "arm" sqlite release is ARMv7. ARMv6 must compile its own addon.
RUN ulimit -c 0; if [ "$TARGETARCH/$TARGETVARIANT" = arm/v6 ]; then \
apk add --no-cache --virtual .ql-bundle-build build-base binutils python3 nodejs-dev && \
cd "$(readlink -f /ql/node_modules/sqlite3)" && \
CFLAGS='-march=armv6 -mfpu=vfp -mfloat-abi=hard' \
CXXFLAGS='-march=armv6 -mfpu=vfp -mfloat-abi=hard' \
npm_config_nodedir=/usr npm_config_build_from_source=true QEMU_CPU=arm1176 \
node -e 'require("node:child_process").execFileSync(process.execPath,[require.resolve("@mapbox/node-pre-gyp/bin/node-pre-gyp"),"rebuild","--napi_build_version=6"],{stdio:"inherit"})' && \
readelf -A lib/binding/napi-v6-linux-arm/node_sqlite3.node | grep -E 'Tag_CPU_arch: v6($|K)' && \
rm -rf build && \
apk del .ql-bundle-build; \
fi
# SQLite comes from the retained Node runtime; no architecture-specific addon.
RUN ulimit -c 0; if [ "$TARGETARCH/$TARGETVARIANT" = arm/v6 ]; then \
export QEMU_CPU=arm1176; \
node --jitless /ql/scripts/build-upgrade-bundle.cjs /ql /out && \
+30
View File
@@ -0,0 +1,30 @@
#!/bin/sh
set -eu
# Official binaries, pinned to their release checksums. Node 24 no longer
# publishes ARMv7 binaries, so that existing Debian target uses Node 22 LTS.
node_version=24.18.1
case "${TARGETARCH:?}/${TARGETVARIANT:-}" in
amd64/*) node_arch=x64; node_sha=d6c664df3f3f61458e8c277585571328522d705166723a7c7823a9253a4d15a0 ;;
arm64/*) node_arch=arm64; node_sha=7201e3a09dc825bac57867c81913e2b8f0ef87d04cb9082af4cda82f6ff3d88c ;;
ppc64le/*) node_arch=ppc64le; node_sha=c47812c13862be7f6e71194c434c4d78a99096e72efb9c31a2627437e056d669 ;;
s390x/*) node_arch=s390x; node_sha=acf57bd726f35afc2ba8475194e5a8bd532c00e1a3b2436295b2fbdde199d0e9 ;;
arm/v7) node_version=22.23.2; node_arch=armv7l; node_sha=ef8f26a3de19acd8c23548e6c3cfc2052610b0e67abb5fd64dbd92c8b1c1245b ;;
*) echo "Unsupported Debian Node architecture: ${TARGETARCH}/${TARGETVARIANT:-}" >&2; exit 1 ;;
esac
if [ "${1:-}" = --print ]; then
printf '%s %s %s\n' "$node_version" "$node_arch" "$node_sha"
exit 0
fi
node_prefix=${1:-/usr/local}
node_work_dir=$(mktemp -d)
trap 'rm -rf "$node_work_dir"' EXIT
node_archive="node-v${node_version}-linux-${node_arch}.tar.xz"
curl --fail --location --retry 3 --connect-timeout 20 --max-time 300 \
"https://nodejs.org/dist/v${node_version}/${node_archive}" -o "$node_work_dir/$node_archive"
printf '%s %s\n' "$node_sha" "$node_work_dir/$node_archive" | sha256sum -c -
mkdir -p "$node_prefix"
tar -xJf "$node_work_dir/$node_archive" -C "$node_prefix" --strip-components=1 --no-same-owner
"$node_prefix/bin/node" -e 'const {DatabaseSync}=require("node:sqlite"); const db=new DatabaseSync(":memory:"); try {db.exec("CREATE TABLE node_probe(value INTEGER);INSERT INTO node_probe VALUES(42)");if(db.prepare("SELECT value FROM node_probe").get().value!==42)throw Error("Node SQLite probe failed");}finally{db.close();}'
+5
View File
@@ -0,0 +1,5 @@
{
"sourceCommit": "99a4d0eff8431898d7638fefff305591fe326ed5",
"musl": "ghcr.io/whyour/qinglong@sha256:0847b0035083cd439c00ff3087392f67035b26842209c110ee976f51ec13fbcd",
"glibc": "ghcr.io/whyour/qinglong@sha256:d2775474223793a5dfe0060910acdcd9bab4c2b70ff6b2ef770ec536afb206f6"
}
+25 -11
View File
@@ -1,12 +1,18 @@
# 平台完整升级包
CI 从同次构建的不可变镜像摘要导出完整 tar.gz,包含源码中的运行脚本和模板、前后端编译产物、锁文件和已经安装的生产 `node_modules`。不包含 Node 可执行文件、用户数据、`.env`、Git 历史或 npm 缓存。保留现有 Node 和用户脚本依赖。
CI 从同次构建的不可变镜像摘要导出完整 tar.gz,包含源码中的运行脚本和模板、前后端编译产物、锁文件和已经安装的生产 `node_modules`。不包含 Node 可执行文件、用户数据、`.env`、Git 历史或 npm 缓存。保留现有 Node 和用户脚本依赖。面板最低需要 Node 22.17,推荐 Node 24;Node 20 安装先升级运行时,应用更新包不会替换 Node。
## 在线升级
`package.json` 中 `qinglongUpgradeFormat: 2` 表示使用完整包。旧版升级器仍可通过原归档路径完成首次迁移;完整包版本不会在平台包缺失、损坏或不兼容时退回设备端 npm 安装。仅当完整包已通过校验、依赖加载或后端预启动明确报告依赖错误时,在线升级会按同一冻结锁文件修复一次生产依赖,然后再次预启动;第二次失败即停止,保留正在运行的旧服务。
`package.json` 中 `qinglongUpgradeFormat: 2` 表示使用完整包。满足 Node 22.17 的旧格式 2 面板可在面板内分两步迁移,不需要手动运行安装器或更换镜像:第一次下载独立的兼容升级器包,预启动通过后手动重启;此时仍运行旧数据库后端,界面提示“升级器已更新,请继续升级新版”。点击“继续升级新版”完成第二次下载和新版后端预启动,再手动重启,健康与版本检查通过后才显示“升级成功”并隐藏侧栏状态。关闭页面、刷新或换浏览器不会丢失第二步提示;Node 不满足最低要求会在第一步准备阶段被旧校验器拒绝,旧服务继续运行。
面板下载任务与本地离线入口共用 `platformBundle.cjs`:平台匹配 → 归档安全检查 → 全文件/链接/权限和锁文件校验 → SQLite/Sequelize/cron-parser/gRPC/undici 加载及 SQLite 内存库读写 → 隔离后端预启动 → 发布就绪指针。预启动使用现有 Node、临时数据目录和 loopback 随机端口,运行真实数据库建表/迁移和事务读写、gRPC 服务,以及与正常启动共用的 HTTP 中间件、路由和 WebSocket 初始化,再通过 `/api/health` 确认双服务与目标版本、提交和后端摘要;不运行任务、订阅、通知、用户启动钩子和系统依赖安装。临时库不是用户数据库副本,因此实际应用后的健康检查和回退仍不可省略。点击重启时,接口只检查就绪记录并提交后台任务;完整文件与依赖复验由独立工作进程执行,复验通过后才停止旧服务,避免小设备的预检耗时超过 HTTP 请求期限。离线升级同步面板任务状态,并与在线任务互斥;中断准备时保留之前的可用就绪包。原有服务停止、文件备份、启动健康检查及失败恢复仍由共享升级引擎处理。
兼容包单独命名为 `qinglong-migration-<commit>-<platform>.tar.gz`,来自固定摘要、固定源码提交的旧镜像,保留真实旧 SQLite/Sequelize 依赖和旧数据库后端,只替换升级器、前端和构建身份;它不伪造 SQLite 模块,也不执行用户数据迁移。正常运行包仍为 `qinglong-<commit>-<platform>.tar.gz`,只使用 node:sqlite。索引的 `packages` 为旧客户端保留兼容包,`runtimePackages` 指向正式运行包;新版升级器优先且只使用完整的 `runtimePackages`,不因损坏或缺失而退回兼容包。新版在线/离线入口会拒绝误用兼容包。生产索引发布前,每个平台都必须使用未经修改的旧升级器真实下载、校验、预启动第一步,再使用包内新版升级器下载、校验、预启动第二步。两步失败均沿用原有回滚流程。
固定旧镜像中的 generic ARM SQLite addon 实际使用 ARMv7 指令。ARMv6 musl 兼容包会在 CI 构建时从同一旧依赖源码重新编译该 addon,使用匹配构建 Node 的本地头文件、单并发及 ARMv6/VFP 硬浮点参数,并检查 ELF 指令集,拒绝 ARMv7、Thumb-2 或 VFPv3 产物。旧锁文件及后端保持不变,其他架构不执行此步骤;正常运行包仍不包含旧 SQLite,目标设备无需为第一步迁移联网编译。
旧源码归档升级器继续保留原兼容路径;完整包版本不会在平台包缺失、损坏或不兼容时退回设备端 npm 安装。仅当完整包已通过校验、依赖加载或后端预启动明确报告依赖错误时,在线升级会按同一冻结锁文件修复一次生产依赖,然后再次预启动;第二次失败即停止,保留正在运行的旧服务。第一步旧后端的检查仍使用真实旧依赖,第二步才切换到 MikroORM 和 Node 内置 SQLite。
面板下载任务与本地离线入口共用 `platformBundle.cjs`:平台匹配 → 归档安全检查 → 全文件/链接/权限和锁文件校验 → MikroORM/cron-parser/gRPC/undici 加载及 node:sqlite 内存库读写(旧包继续检查 SQLite/Sequelize) → 隔离后端预启动 → 发布就绪指针。预启动使用现有 Node、临时数据目录和 loopback 随机端口,运行真实数据库建表/迁移和事务读写、gRPC 服务,以及与正常启动共用的 HTTP 中间件、路由和 WebSocket 初始化,再通过 `/api/health` 确认双服务与目标版本、提交和后端摘要;不运行任务、订阅、通知、用户启动钩子和系统依赖安装。临时库不是用户数据库副本,因此实际应用后的健康检查和回退仍不可省略。点击重启时,接口只检查就绪记录并提交后台任务;完整文件与依赖复验由独立工作进程执行,复验通过后才停止旧服务,避免小设备的预检耗时超过 HTTP 请求期限。离线升级同步面板任务状态,并与在线任务互斥;中断准备时保留之前的可用就绪包。原有服务停止、文件备份、启动健康检查及失败恢复仍由共享升级引擎处理。
正式升级包跟随 master 同一次版本镜像构建;develop 的普通提交也自动构建开发包。两者都直接使用该次 Alpine/Debian 镜像的不可变 digest。Git tag 工作流和 `static_sync_only` 不生成完整包。需要重建当前开发包时,可手动运行同一工作流:
@@ -24,7 +30,7 @@ gh workflow run build-docker-image.yml --ref develop
## 手动离线安装和升级
从 `upgrade-v<version>` 的说明或 `latest.json` 中取得当前批次的下载链接,下载匹配设备的 `.tar.gz` 和独立 `install-bundle.cjs`;两者都来自同一个 `upgrade-build-v<version>-<run-id>-<run-attempt>` release。独立安装器已内置解析依赖,在目标设备正常情况无需 npm、pnpm 或重新安装 SQLite。离线准备和预启动默认不联网;依赖不符合要求时直接报错。可对照同批次 release 的 `latest.json` 核对归档 SHA256。
从 `upgrade-v<version>` 的说明或 `latest.json` 的 `runtimePackages` 中取得当前批次的下载链接,下载匹配设备的正常运行 `.tar.gz` 和独立 `install-bundle.cjs`;两者都来自同一个 `upgrade-build-v<version>-<run-id>-<run-attempt>` release。不要下载 `qinglong-migration-` 包作为离线安装包。独立安装器已内置解析依赖,在目标设备正常情况无需 npm、pnpm 或重新安装 SQLite。离线准备和预启动默认不联网;依赖不符合要求时直接报错。可对照同批次 release 的 `latest.json` 核对归档 SHA256。
已有安装(默认应用并重启):
@@ -32,7 +38,7 @@ gh workflow run build-docker-image.yml --ref develop
node install-bundle.cjs --root /ql --file /absolute/path/qinglong-<commit>-linux-arm64-musl.tar.gz
```
只准备,稍后手动重启:
已迁移的安装只准备,稍后手动重启(首次从旧格式 2 迁移应直接执行上一条新版独立安装器命令,不能调用旧版 `ql reload system`):
```sh
node install-bundle.cjs --root /ql --file /absolute/path/package.tar.gz --download-only
@@ -45,7 +51,7 @@ ql reload system
node install-bundle.cjs --root /ql --file /absolute/path/package.tar.gz --download-only --repair-deps
```
修复需要已有 pnpm 8.3.1 及必要编译工具。依赖在私有目录按 `--prod --frozen-lockfile` 重装,仅重建 SQLite;ARMv6 强制源码编译和 ARMv6 硬浮点指令,禁止下载 generic ARM 的 ARMv7 二进制。修复仅执行一次;不安装或替换 Node、不改锁文件、不改已校验源码和静态产物。应用前以原包校验清单和独立记录的修复依赖摘要复验。
修复需要已有 pnpm 8.3.1。新包在私有目录按 `--prod --frozen-lockfile` 重装生产 JavaScript 依赖,不编译 SQLite。只有旧包显式依赖 sqlite3 时才需要编译工具并重建 SQLite;旧 ARMv6 包强制源码编译和 ARMv6 硬浮点指令,禁止下载 generic ARM 的 ARMv7 二进制。修复仅执行一次;不安装或替换 Node、不改锁文件、不改已校验源码和静态产物。应用前以原包校验清单和独立记录的修复依赖摘要复验。
修复继承调用环境中的标准 HTTP/HTTPS/ALL/NO_PROXY(含小写形式),其余 Node 预加载和编译参数仍隔离。修复安装或 SQLite 重建失败时,清理私有目录,保留当前服务、当前依赖及上一次就绪包;不会标记本次更新就绪。预启动或修复主进程退出时立即终止其进程组,防止持有输出管道的子进程继续写入已清理目录。
@@ -74,9 +80,9 @@ npm 会排除根目录的 `pnpm-lock.yaml`。主包的原生 `npm pack/publish`
`package.json#files` 是 npm 包内容的唯一项目规则,不再维护根 `.npmignore`:npm 在根包使用 `files` 时会停用该目录的 `.npmignore` 和 `.gitignore` 规则。打包测试在没有 `.npmignore` 的独立源码目录中注入私有配置、密钥占位文件、缓存及生成的预加载配置,执行真实 `npm pack` 和安装,检查运行文件完整、私有文件及其内容不进入归档。此规则只用于 npm 包,完整平台 tar.gz 仍由平台包构建器处理。
干净源码完成 `pnpm build:front`、`pnpm build:back`、`pnpm build:info` 后,运行 `node scripts/verify-panel-package.cjs pack /tmp/qinglong-npm` 生成并检查真实 npm 归档。CI 在一次性 Linux 容器中,以 Node 20.18.1 和 Node 24 安装同一个归档,使用真实生产依赖和已安装的 `qinglong`、`ql` 入口,验证启动、损坏下载保留旧服务、在线准备与重启、离线准备与 Bash 重启及账号、配置、环境变量、任务和脚本保留。测试还安装完整性摘要固定的已发布 `@whyour/qinglong@2.23.0`,在旧服务中建立真实数据,再用新 `.tgz` 执行 npm 覆盖安装,验证账号、配置、环境变量、任务和脚本保留。之后升级到同次 Debian 镜像不可变 digest 通过正式 `Dockerfile.upgrade-bundle` 导出的原始平台包,确认运行目录变为 pnpm 布局。该单平台验收包仅上传短期 CI artifact,不发布 Release,也不推进开发或正式更新索引。
干净源码完成 `pnpm build:front`、`pnpm build:back`、`pnpm build:info` 后,运行 `node scripts/verify-panel-package.cjs pack /tmp/qinglong-npm` 生成并检查真实 npm 归档。CI 在一次性 Linux 容器中,以 Node 22.17.0 和 Node 24 安装同一个归档,使用真实生产依赖和已安装的 `qinglong`、`ql` 入口,验证启动、损坏下载保留旧服务、在线准备与重启、离线准备与 Bash 重启及账号、配置、环境变量、任务和脚本保留。测试还安装完整性摘要固定的已发布 `@whyour/qinglong@2.23.0`,在旧服务中建立真实数据,再用新 `.tgz` 执行 npm 覆盖安装,验证账号、配置、环境变量、任务和脚本保留。之后升级到同次 Debian 镜像不可变 digest 通过正式 `Dockerfile.upgrade-bundle` 导出的原始平台包,确认运行目录变为 pnpm 布局。该单平台验收包仅上传短期 CI artifact,不发布 Release,也不推进开发或正式更新索引。
Umi 4.4.4 的构建依赖仍使用 Node 24 已移除的 `http_parser`。前端源码构建使用 `.nvmrc` 中的 Node 22,CI 从该文件读取版本,Vercel 前后端项目均配置为 Node 22.x。`package.json` 不声明 `engines.node`,避免 Vercel 用开放的最低运行版本范围覆盖项目的构建版本;`pnpm build:front` 直接执行构建,不下载或切换 Node。面板运行时仍支持 Node 20.18.1 及以上,平台包的清单与安装器独立校验最低 Node 要求,npm 实装验收继续覆盖 Node 20/24。
Umi 4.4.4 的构建依赖仍使用 Node 24 已移除的 `http_parser`。前端源码构建使用 `.nvmrc` 中的 Node 22,CI 从该文件读取版本,Vercel 前后端项目均配置为 Node 22.x。`package.json` 不声明 `engines.node`,避免 Vercel 用开放的最低运行版本范围覆盖项目的构建版本;`pnpm build:front` 直接执行构建,不下载或切换 Node。面板运行时最低为 Node 22.17,平台包清单、npm preinstall 和数据库进程均检查最低 Node 要求。CI 的 Node 20.18.1 项使用实际新 npm 归档验证拒绝安装,以及本地/全局旧包、`.env`、外置数据和旧服务保留;它不启动新后端。构建专用环境可跳过安装阶段的运行时探测,后端启动不跳过版本检查。
服务集成测试只能在专用容器中运行:设置 `QL_PANEL_PACKAGE_TEST_ISOLATED=1`、指向隔离 PM2 的 `QL_PANEL_TEST_PM2` 和指向正式构建脚本输出目录的 `QL_PANEL_TEST_BUNDLE_DIR`,然后执行 `node scripts/verify-panel-package.cjs verify /tmp/qinglong-npm /tmp/qinglong-npm-results`。容器需已有 Bash、curl、OpenSSL、Python、jq、nginx 和 procps。旧正式包会重置 PATH,因此验收容器将 PM2 6.0.14 安装到 `/usr/local/bin/pm2`,并以此设置 `QL_PANEL_TEST_PM2`;隔离通过专属容器和独立 `PM2_HOME` 保证。测试创建独立的 npm prefix、HOME、PM2_HOME 和数据目录,结束后清理。
@@ -84,13 +90,21 @@ Umi 4.4.4 的构建依赖仍使用 Node 24 已移除的 `http_parser`。前端
master 发布任务等待上述测试通过,直接发布已验证的 `.tgz`,不重新构建或打包。develop 只生成测试归档与验收记录,不发布 npm。
## 数据库兼容与回滚
正常运行包使用 MikroORM 7 的官方 NodeSqliteDialect。Sequelize 和 sqlite3 只作为开发测试依赖,用于验证旧库读写,不进入正常运行包的生产依赖;独立的第一步兼容包仍携带固定旧镜像的真实旧依赖和后端,以满足已安装旧升级器的检查。九张业务表及既有迁移记录保留原名和结构,不运行 ORM 自动改表;日期保持旧版 UTC 文本格式,JSON、ID、排序精度及 Keyv 缓存格式保持兼容,旧版可读新版写入的数据。
每个后端进程延迟启动一个有界数据库子进程,避免同步 node:sqlite 阻塞 HTTP 和调度事件循环。事务独占请求队列,嵌套事务使用 SAVEPOINT;事务异常回滚。数据库进程退出或超时后拒绝后续请求、健康检查失败,不自动重放写入;恢复由服务重启完成。后端正常退出会终止其数据库子进程,预启动监督器还会终止整个临时进程组。数据库进程在打开 SQLite 前启动一个仅监视父进程的轻量线程;即使主 SQL 线程阻塞,父进程被强杀后也会终止整个数据库进程并释放事务锁,避免阻止旧服务重启。
## 平台兼容
- 最低 Node 为 20.18.1,Node-API 至少 6;安装器只检查,不替换 Node。
- 最低 Node 为 22.17,Node-API 至少 6;安装器只检查,不替换 Node。Debian amd64、arm64、ppc64le、s390x 使用 Node 24;官方 Node 24 不提供 Linux ARMv7 二进制,Debian ARMv7 使用 Node 22 LTS。
- 以 Node 实际运行架构、ARM 指令集和 libc 类型/版本选择包;不依据 Docker 架构别名猜测。
- glibc、musl 分包,并记录构建系统 libc 下限。系统库仍以目标设备加载测试作为最后检查。
- ARMv6 与 ARMv7 使用不同包名。ARMv6 在目标镜像中从源码编译 SQLite,禁止复用 generic `arm` 的 ARMv7 预编译件;构建需通过加载和内存库读写,失败则不发布渠道。
- ARMv6 的归档构建、在线更新工作进程及独立离线安装器使用 `--jitless`,避开 ARM1176 上 V8 的优化编译崩溃;保留现有 Node 二进制,Node 24 及以上的 ARMv6 面板进程通过共享 `nodeRuntime` 使用 `--no-opt --no-sparkplug --no-maglev`,其他架构和较低 Node 版本保持原启动参数。ARMv6 的 gRPC 工作进程首次启动及重建最多等待 2 分钟;升级后的整体健康检查最多等待 5 分钟,覆盖慢设备按顺序启动数据库、调度与 HTTP 工作进程的耗时;其他平台仍为 60 秒,超时或版本校验失败仍执行回退。依赖加载与后端预启动使用现有 Node 的正常模式,使 Node 20.18.1 的内置 HTTP/WebAssembly 组件保持可用。预启动只有一个后端进程,按顺序初始化数据库、gRPC、HTTP,不启动 PM2/cluster;ARMv6 V8 堆上限 128 MiB、期限 5 分钟,其他平台为 192 MiB/60 秒。堆上限不是 RSS 总内存上限,系统仍需为现有服务和 OpenSSL 留出内存。依赖修复的包管理器及编译使用单并发、低网络并发和有界时长;ARMv6 包管理器采用 `--jitless`、256 MiB 堆上限,并优先复用版本匹配的现有 Node 头文件与实际运行时编译配置;超时、中断会终止整个进程组并清理临时目录。
- ARMv6 与 ARMv7 使用不同包名。当前生产依赖没有 SQLite 原生 addon,使用设备 Node 自带的 node:sqlite。ARMv6 Node 本身仍须兼容 ARM1176 和硬浮点;构建需通过内置 SQLite 的加载和内存库读写,失败则不发布渠道。
- ARMv6 的归档构建、在线更新工作进程及独立离线安装器使用 `--jitless`,避开 ARM1176 上 V8 的优化编译崩溃;保留现有 Node 二进制,Node 24 及以上的 ARMv6 面板进程通过共享 `nodeRuntime` 使用 `--no-opt --no-sparkplug --no-maglev`,其他架构和较低 Node 版本保持原启动参数。ARMv6 的 gRPC 工作进程首次启动及重建最多等待 2 分钟;升级后的整体健康检查最多等待 5 分钟,覆盖慢设备按顺序启动数据库、调度与 HTTP 工作进程的耗时;其他平台仍为 60 秒,超时或版本校验失败仍执行回退。依赖加载与后端预启动保留 HTTP/WebAssembly 所需能力。预启动使用一个后端和一个数据库子进程,按顺序初始化数据库、gRPC、HTTP,不启动 PM2/cluster;后端的 ARMv6 V8 堆上限 128 MiB、期限 5 分钟,其他平台为 192 MiB/60 秒。数据库执行主线程的 V8 堆上限 128 MiB,ARMv6 使用 `--jitless`;监视线程不加载 ORM、HTTP 或 ts-node,并保留进程的 V8 模式。主线程和监视线程的 V8 限制不是 RSS 总内存上限,系统仍需为现有服务和 OpenSSL 留出内存。依赖修复的包管理器及编译使用单并发、低网络并发和有界时长;ARMv6 包管理器采用 `--jitless`、256 MiB 堆上限,并优先复用版本匹配的现有 Node 头文件与实际运行时编译配置;超时、中断会终止整个进程组并清理临时目录。
- tar 中允许 pnpm 的包内符号链接;拒绝绝对路径、越界路径、包外链接、硬链接、设备条目和经符号链接写入的条目。重启前再次校验依赖,暂存失败保留上一次可用的就绪包。
归档先完整预检路径、类型、大小与链接,再向新建的私有目录逐个流式写入文件。文件写入使用 Node 的完整写入接口,处理短写后才关闭文件;目录权限在写入结束后恢复,避免受调用进程的 umask 影响。读取、解析或写入失败时,等候待写文件关闭后再清理,不发布就绪状态。校验失败会列出最多五个文件的预期和实际权限、哈希或链接信息,不输出文件内容。
自托管镜像可设置 `QL_UPGRADE_INDEX_URL` 指向兼容索引;要求 HTTPS,测试时仅放行 loopback HTTP。索引和所有平台归档应部署到同一可访问源,避免依赖额外的 raw 域名。
+14 -11
View File
@@ -49,6 +49,7 @@
"version.yaml"
],
"scripts": {
"preinstall": "node -e \"if (process.env.QL_BUILD_ONLY !== '1' && process.env.VERCEL !== '1') require('./shell/runtime-guard.cjs').assertNodeRuntime()\"",
"build:cli": "node cli/scripts/build.cjs",
"check:cli": "npm --prefix cli run check",
"test:cli": "npm run build:cli && node cli/scripts/test.cjs",
@@ -57,7 +58,7 @@
"start:back": "nodemon ./back/app.ts",
"start:front": "max dev",
"build:front": "max build",
"build:back": "tsc -p back/tsconfig.json",
"build:back": "tsc -p back/tsconfig.json && tsc -p back/data/tsconfig.worker.json",
"test": "TS_NODE_PROJECT=back/tsconfig.json node -r ts-node/register/transpile-only --test test/back/*.test.cjs test/front/*.test.cjs",
"benchmark:execution": "TS_NODE_PROJECT=back/tsconfig.json node -r ts-node/register/transpile-only scripts/benchmark-execution.cjs",
"build:info": "node scripts/write-build-info.cjs",
@@ -121,11 +122,11 @@
}
},
"dependencies": {
"@ant-design/plots": "^2.6.8",
"@bufbuild/protobuf": "^2.10.0",
"@grpc/grpc-js": "1.14.5",
"@grpc/proto-loader": "^0.8.0",
"@keyv/sqlite": "^4.0.1",
"@mikro-orm/core": "7.2.5",
"@mikro-orm/sql": "7.2.5",
"@otplib/preset-default": "^12.0.1",
"body-parser": "^1.20.3",
"celebrate": "^15.0.3",
@@ -147,7 +148,7 @@
"ip2region": "2.3.0",
"js-yaml": "^4.1.0",
"jsonwebtoken": "^9.0.2",
"keyv": "^5.2.3",
"keyv": "5.6.0",
"lodash": "^4.17.21",
"multer": "2.4.0",
"node-cron": "4.6.0",
@@ -157,19 +158,18 @@
"ps-tree": "^1.2.0",
"reflect-metadata": "^0.2.2",
"request-ip": "3.3.0",
"sequelize": "6.37.8",
"sockjs": "^0.3.24",
"sqlite3": "npm:@whyour/sqlite3@1.1.2",
"tar": "7.5.20",
"toad-scheduler": "^3.0.1",
"typedi": "^0.10.0",
"undici": "^7.9.0",
"uuid": "^11.0.3",
"winston": "^3.17.0",
"winston-daily-rotate-file": "^5.0.0",
"tar": "7.5.20"
"winston-daily-rotate-file": "^5.0.0"
},
"devDependencies": {
"@ant-design/icons": "^5.0.1",
"@ant-design/plots": "^2.6.8",
"@ant-design/pro-layout": "6.38.22",
"@codemirror/state": "6.5.4",
"@codemirror/view": "6.39.16",
@@ -187,7 +187,7 @@
"@types/jsonwebtoken": "^8.5.8",
"@types/lodash": "^4.14.185",
"@types/multer": "^1.4.7",
"@types/node": "^17.0.21",
"@types/node": "^22.20.5",
"@types/nodemailer": "^6.4.4",
"@types/proper-lockfile": "^4.1.4",
"@types/ps-tree": "^1.1.6",
@@ -233,13 +233,16 @@
"react-intl-universal": "^2.12.0",
"react-router-dom": "6.26.1",
"react-split-pane": "^0.1.92",
"sequelize": "6.37.8",
"sockjs-client": "^1.6.0",
"sqlite3": "npm:@whyour/sqlite3@1.1.2",
"ts-node": "^10.9.2",
"ts-proto": "^2.6.1",
"tslib": "^2.4.0",
"typescript": "5.2.2",
"typescript": "5.9.3",
"vh-check": "^2.0.5",
"virtualizedtableforantd4": "1.3.0"
},
"qinglongUpgradeFormat": 2
"qinglongUpgradeFormat": 2,
"qinglongRuntimeMinimum": "22.17.0"
}
+539 -646
View File
File diff suppressed because it is too large Load Diff
+272
View File
@@ -0,0 +1,272 @@
// A separate first-hop package for installed format-2 verifiers. Its database
// backend and dependencies remain the pinned legacy implementation. Only the
// updater and frontend are replaced; the next download uses runtimePackages.
const fs = require('node:fs/promises');
const { createReadStream } = require('node:fs');
const path = require('node:path');
const http = require('node:http');
const assert = require('node:assert/strict');
const { pipeline } = require('node:stream/promises');
const { once } = require('node:events');
const { collectBuildFiles } = require('../docker/build-manifest.cjs');
const { buildBundle } = require('./build-upgrade-bundle.cjs');
const artifacts = require('../cli/src/internal/maintenance/upgradeArtifacts.cjs');
const bundle = require('../cli/src/internal/maintenance/platformBundle.cjs');
const baseline = require('../docker/migration-bridge-images.json');
const { rebuildMigrationSqlite } = require('./rebuild-migration-sqlite.cjs');
const read = async (file) => JSON.parse(await fs.readFile(file, 'utf8'));
async function prepareBridge(legacy, current, destination) {
const oldBuild = await artifacts.verifyUpgrade(
legacy,
path.join(legacy, 'static'),
);
const build = await artifacts.verifyUpgrade(
current,
path.join(current, 'static'),
);
const pkg = await read(path.join(legacy, 'package.json'));
const next = await read(path.join(current, 'package.json'));
if (
oldBuild.sourceCommit !== baseline.sourceCommit ||
pkg.qinglongUpgradeFormat !== 2 ||
!pkg.dependencies?.sqlite3 ||
!pkg.dependencies?.sequelize ||
!next.dependencies?.['@mikro-orm/sql']
)
throw new Error(
'Migration bridge requires the pinned legacy backend and a modern runtime',
);
// Never copy user state, even if this builder is used outside Docker.
const excluded = new Set(['.git', '.tmp', '.env', 'data', 'output']);
await fs.cp(legacy, destination, {
recursive: true,
verbatimSymlinks: true,
filter: (file) =>
!excluded.has(path.relative(legacy, file).split(path.sep)[0]),
});
const maintenance = 'cli/src/internal/maintenance';
await fs.rm(path.join(destination, maintenance), {
recursive: true,
force: true,
});
await fs.cp(
path.join(current, maintenance),
path.join(destination, maintenance),
{ recursive: true },
);
await fs.rm(path.join(destination, 'static/dist'), {
recursive: true,
force: true,
});
await fs.cp(
path.join(current, 'static/dist'),
path.join(destination, 'static/dist'),
{ recursive: true },
);
const migrationBridge = {
format: 1,
legacySourceCommit: oldBuild.sourceCommit,
sourceCommit: build.sourceCommit,
targetVersion: next.version,
};
await fs.writeFile(
path.join(destination, 'package.json'),
JSON.stringify(
{
...pkg,
qinglongRuntimeMinimum: '22.17.0',
qinglongMigrationBridge: migrationBridge,
},
null,
2,
),
);
await fs.writeFile(
path.join(destination, 'static/build-info.json'),
JSON.stringify({
...oldBuild,
sourceCommit: build.sourceCommit,
files: collectBuildFiles(path.join(destination, 'static')),
}),
);
// No backend bytes or dependency lock may change in the updater hop.
assert.equal(
bundle.sha256(
await fs.readFile(path.join(destination, 'static/build/app.js')),
),
oldBuild.files['build/app.js'],
);
assert.equal(
bundle.sha256(await fs.readFile(path.join(destination, 'pnpm-lock.yaml'))),
oldBuild.lockfileSha256,
);
return migrationBridge;
}
async function verifyBothDownloads(legacy, current, directory, records) {
let index, workspace;
const files = new Map(
records.map((r) => ['/' + r.filename, path.join(directory, r.filename)]),
);
const server = http.createServer((req, res) => {
if (!index) {
res.writeHead(503);
res.end();
return;
}
if (req.url === '/latest.json') {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(index));
} else if (files.has(req.url)) {
const record = records.find((r) => '/' + r.filename === req.url);
res.setHeader('Content-Length', record.size);
pipeline(createReadStream(files.get(req.url)), res).catch(() =>
res.destroy(),
);
} else {
res.writeHead(404);
res.end();
}
});
try {
server.listen(0, '127.0.0.1');
await once(server, 'listening');
const base = `http://127.0.0.1:${server.address().port}`;
const entry = (r) => ({
size: r.size,
sha256: r.sha256,
url: base + '/' + r.filename,
});
const [bridge, runtime] = records;
index = {
format: 2,
sourceCommit: runtime.sourceCommit,
packages: { [bridge.target]: entry(bridge) },
runtimePackages: { [runtime.target]: entry(runtime) },
};
workspace = await fs.mkdtemp(path.join(directory, '.legacy-client-'));
const env = {
...process.env,
QL_BRANCH: 'develop',
QL_UPGRADE_INDEX_URL: base + '/latest.json',
QL_UPGRADE_FILE: '',
NO_PROXY: '127.0.0.1',
no_proxy: '127.0.0.1',
};
const old = require(path.join(
legacy,
'cli/src/internal/maintenance/upgradeArtifacts.cjs',
));
const firstTmp = path.join(workspace, 'first');
await fs.mkdir(firstTmp);
// These are the unchanged, genuinely installed old download, dependency
// smoke, prestart and readiness verifiers. No injected probe or verifier.
const selected = await old.stageUpgrade({
root: legacy,
tmp: firstTmp,
env,
run: (program, args, capture) =>
old.runProcess(env, program, args, capture),
});
await old.selectedUpgrade(firstTmp, 'develop', env);
assert(
(await read(path.join(selected.source, 'package.json')))
.qinglongMigrationBridge,
);
const upgraded = require(path.join(
selected.source,
'cli/src/internal/maintenance/upgradeArtifacts.cjs',
));
const secondTmp = path.join(workspace, 'second');
await fs.mkdir(secondTmp);
const next = await upgraded.stageUpgrade({
root: selected.source,
tmp: secondTmp,
env,
run: (program, args, capture) =>
upgraded.runProcess(env, program, args, capture),
});
await upgraded.selectedUpgrade(secondTmp, 'develop', env);
const pkg = await read(path.join(next.source, 'package.json'));
assert(pkg.dependencies['@mikro-orm/sql']);
assert.equal(pkg.qinglongMigrationBridge, undefined);
await assert.rejects(
fs.access(path.join(next.source, 'node_modules/sqlite3')),
{ code: 'ENOENT' },
);
await assert.rejects(
fs.access(path.join(next.source, 'node_modules/sequelize')),
{ code: 'ENOENT' },
);
const oldBundle = require(path.join(
legacy,
'cli/src/internal/maintenance/platformBundle.cjs',
));
const manifest = await read(
path.join(path.dirname(selected.source), 'bundle.json'),
);
assert.throws(
() =>
oldBundle.assertCompatible(
manifest,
{ ...manifest.target, libcVersion: manifest.target.libcMinimum },
'20.18.1',
),
/requires Node/,
);
return {
legacyOnlineVerified: true,
modernOnlineVerified: true,
legacyNode20Rejected: true,
};
} finally {
server.closeAllConnections();
if (server.listening) await new Promise((resolve) => server.close(resolve));
if (workspace) await fs.rm(workspace, { recursive: true, force: true });
}
}
async function buildMigrationBridge(legacy, current, output) {
legacy = path.resolve(legacy);
current = path.resolve(current);
output = path.resolve(output);
await fs.mkdir(output, { recursive: true });
const workspace = await fs.mkdtemp(path.join(output, '.migration-'));
let bridge;
try {
const root = path.join(workspace, 'root');
await prepareBridge(legacy, current, root);
await rebuildMigrationSqlite(root);
const runtime = await buildBundle(current, output);
bridge = await buildBundle(root, output);
const verification = await verifyBothDownloads(legacy, current, output, [
bridge,
runtime,
]);
bridge = { ...bridge, migrationVerified: true, ...verification };
await fs.writeFile(
path.join(output, bridge.filename + '.json'),
JSON.stringify(bridge) + '\n',
);
console.log(JSON.stringify(bridge));
return bridge;
} catch (error) {
if (bridge)
await fs.rm(path.join(output, bridge.filename + '.json'), {
force: true,
});
throw error;
} finally {
await fs.rm(workspace, { recursive: true, force: true });
}
}
module.exports = { prepareBridge, verifyBothDownloads, buildMigrationBridge };
if (require.main === module)
buildMigrationBridge(process.argv[2], process.argv[3], process.argv[4]).catch(
(error) => {
console.error(error);
process.exitCode = 1;
},
);
+31 -18
View File
@@ -1,10 +1,15 @@
const fs = require('node:fs/promises');
const { createWriteStream } = require('node:fs');
const path = require('node:path');
const { pipeline } = require('node:stream/promises');
const tar = require('tar');
const bundle = require('../cli/src/internal/maintenance/platformBundle.cjs');
const {
verifyUpgrade,
} = require('../cli/src/internal/maintenance/upgradeArtifacts.cjs');
const {
prestartBackend,
} = require('../cli/src/internal/maintenance/upgradePrestart.cjs');
async function normalizeModes(directory) {
for (const entry of await fs.readdir(directory, { withFileTypes: true })) {
@@ -19,6 +24,17 @@ async function normalizeModes(directory) {
}
}
async function writeBundleArchive(workspace, filename) {
// Core streams complete partial writes; tar's file option uses fs-minipass,
// whose writer can silently discard a successful short write/writev result.
await pipeline(
tar.c({ gzip: true, portable: true, noMtime: true, cwd: workspace }, [
'qinglong',
]),
createWriteStream(filename, { highWaterMark: 256 * 1024 }),
);
}
async function buildBundle(root, output) {
root = path.resolve(root);
output = path.resolve(output);
@@ -26,6 +42,10 @@ async function buildBundle(root, output) {
if (process.platform !== 'linux')
throw new Error('Release bundles must be built and probed on Linux');
const build = await verifyUpgrade(root, path.join(root, 'static'));
const pkg = JSON.parse(
await fs.readFile(path.join(root, 'package.json'), 'utf8'),
);
const migrationBridge = pkg.qinglongMigrationBridge;
await bundle.smokeDependencies(root);
await fs.mkdir(output, { recursive: true });
const workspace = await fs.mkdtemp(path.join(output, '.bundle-'));
@@ -64,9 +84,8 @@ async function buildBundle(root, output) {
const manifest = {
format: 2,
sourceCommit: build.sourceCommit,
version: JSON.parse(
await fs.readFile(path.join(root, 'package.json'), 'utf8'),
).version,
version: pkg.version,
...(migrationBridge ? { migrationBridge } : {}),
lockfileSha256: build.lockfileSha256,
target: {
platform: target.platform,
@@ -76,7 +95,7 @@ async function buildBundle(root, output) {
endianness: target.endianness,
armFloatAbi: target.armFloatAbi,
},
nodeMinimum: '20.18.1',
nodeMinimum: '22.17.0',
nodeApi: 6,
files: await bundle.inventory({ source, static: staticRoot }),
};
@@ -85,19 +104,10 @@ async function buildBundle(root, output) {
JSON.stringify(manifest),
);
console.log('Bundle inventory verified; writing archive');
const filename = `qinglong-${build.sourceCommit}-${bundle.targetKey(
target,
)}.tar.gz`;
await tar.c(
{
gzip: true,
portable: true,
noMtime: true,
cwd: workspace,
file: path.join(output, filename),
},
['qinglong'],
);
const filename = `qinglong-${migrationBridge ? 'migration-' : ''}${
build.sourceCommit
}-${bundle.targetKey(target)}.tar.gz`;
await writeBundleArchive(workspace, path.join(output, filename));
// Verify the bytes users receive, not just the pre-archive build directory.
const roundtrip = path.join(workspace, 'roundtrip');
await fs.mkdir(roundtrip);
@@ -111,6 +121,8 @@ async function buildBundle(root, output) {
await verifyUpgrade(extracted.source, extracted.static);
await bundle.smokeDependencies(extracted.source);
console.log('Archived production dependencies and SQLite verified');
await prestartBackend(extracted, { target });
console.log('Archived backend database, gRPC and HTTP startup verified');
const bytes = await fs.readFile(path.join(output, filename));
const record = {
format: 2,
@@ -119,6 +131,7 @@ async function buildBundle(root, output) {
target: bundle.targetKey(target),
filename,
baseImage: process.env.QL_BUNDLE_BASE_IMAGE,
...(migrationBridge ? { migrationBridge } : {}),
size: bytes.length,
sha256: bundle.sha256(bytes),
};
@@ -132,7 +145,7 @@ async function buildBundle(root, output) {
await fs.rm(workspace, { recursive: true, force: true });
}
}
module.exports = { buildBundle, normalizeModes };
module.exports = { buildBundle, normalizeModes, writeBundleArchive };
if (require.main === module)
buildBundle(process.argv[2] || '/ql', process.argv[3] || '/out').catch(
(e) => {
+65 -36
View File
@@ -19,39 +19,63 @@ const platforms = [
'linux-s390x-glibc',
];
function releaseIndex(directory, commit, repository, options = {}) {
const { version, releaseTag = `upgrade-${commit}` } = options;
const {
version,
releaseTag = `upgrade-${commit}`,
migration = false,
} = options;
if (
!/^[a-f0-9]{40}$/.test(commit) ||
!/^[\w.-]+\/[\w.-]+$/.test(repository) ||
!/^[\w.+-]+$/.test(releaseTag)
)
throw new Error('Invalid release source');
const packages = {};
for (const target of platforms) {
const filename = `qinglong-${commit}-${target}.tar.gz`;
const record = JSON.parse(
fs.readFileSync(path.join(directory, filename + '.json')),
);
const bytes = fs.readFileSync(path.join(directory, filename));
if (
record.format !== 2 ||
record.sourceCommit !== commit ||
(version !== undefined && record.version !== version) ||
record.target !== target ||
record.filename !== filename ||
record.size !== bytes.length ||
record.sha256 !== sha256(bytes)
)
throw new Error(`Unverified platform bundle: ${target}`);
packages[target] = {
size: record.size,
sha256: record.sha256,
url: `https://github.com/${repository}/releases/download/${encodeURIComponent(
releaseTag,
)}/${filename}`,
};
}
return { format: 2, sourceCommit: commit, packages };
const readPackages = (bridge) => {
const packages = {};
for (const target of platforms) {
const filename = `qinglong-${
bridge ? 'migration-' : ''
}${commit}-${target}.tar.gz`;
const record = JSON.parse(
fs.readFileSync(path.join(directory, filename + '.json')),
);
const bytes = fs.readFileSync(path.join(directory, filename));
if (
record.format !== 2 ||
record.sourceCommit !== commit ||
(version !== undefined &&
(bridge ? record.migrationBridge?.targetVersion : record.version) !==
version) ||
(bridge &&
(record.migrationVerified !== true ||
record.migrationBridge?.sourceCommit !== commit ||
record.migrationBridge?.legacySourceCommit !==
require('../docker/migration-bridge-images.json')
.sourceCommit)) ||
(!bridge && record.migrationBridge) ||
record.target !== target ||
record.filename !== filename ||
record.size !== bytes.length ||
record.sha256 !== sha256(bytes)
)
throw new Error(`Unverified platform bundle: ${target}`);
packages[target] = {
size: record.size,
sha256: record.sha256,
url: `https://github.com/${repository}/releases/download/${encodeURIComponent(
releaseTag,
)}/${filename}`,
};
}
return packages;
};
const runtimePackages = readPackages(false);
return {
format: 2,
sourceCommit: commit,
packages: migration ? readPackages(true) : runtimePackages,
...(migration ? { runtimePackages } : {}),
};
}
function publishBundles({
directory,
@@ -83,6 +107,7 @@ function publishBundles({
...releaseIndex(directory, commit, repository, {
version,
releaseTag: tag,
migration: true,
}),
version,
buildId,
@@ -115,10 +140,12 @@ function publishBundles({
};
const notes =
`Verified platform bundles for v${version}, source ${commit}, build ${buildId}. Existing Node runtime is retained.\n\n` +
Object.entries(index.packages)
Object.entries(index.runtimePackages)
.map(([target, entry]) => `- [${target}](${entry.url})`)
.join('\n') +
`\n- [Standalone offline installer](${index.installer.url})`;
`\n- [Standalone offline installer](${index.installer.url})\n\n` +
'Legacy format-2 panels update the upgrader first, then download the runtime. ' +
'The packages index contains the separate legacy bridge; runtimePackages contains the normal online/offline packages.';
const ensureRelease = (ref, title) => {
if (!lookup(ref))
gh(
@@ -150,13 +177,15 @@ function publishBundles({
}
ensureRelease(tag, `Platform upgrade v${version} (${buildId})`);
const assets = platforms
.flatMap((target) => {
const archive = path.join(
directory,
`qinglong-${commit}-${target}.tar.gz`,
);
return [archive, archive + '.json'];
})
.flatMap((target) =>
[false, true].flatMap((bridge) => {
const archive = path.join(
directory,
`qinglong-${bridge ? 'migration-' : ''}${commit}-${target}.tar.gz`,
);
return [archive, archive + '.json'];
}),
)
.concat(
path.join(directory, 'install-bundle.cjs'),
path.join(directory, 'latest.json'),
+64
View File
@@ -0,0 +1,64 @@
// The legacy image's generic ARM prebuild contains ARMv7 instructions. Rebuild
// only the first-hop bridge on the builder; never add sqlite3 to normal bundles
// or ask an ARMv6 device to compile it during the migration.
const fs = require('node:fs/promises');
const path = require('node:path');
const { createRequire } = require('node:module');
const { execFile } = require('node:child_process');
const { promisify } = require('node:util');
const bundle = require('../cli/src/internal/maintenance/platformBundle.cjs');
const { repairDependencies, cleanSqliteBuildArtifacts } = require('../cli/src/internal/maintenance/upgradePrestart.cjs');
function assertArmv6Addon(attributes) {
// Alpine's ARM CRT objects are ARMv6KZ (the ARM1176 architecture). Linking
// them raises a base-v6 object's ELF tag without introducing ARMv7/Thumb-2.
if (
!/Tag_CPU_arch:\s+v6(?:K|KZ|ZK)?\s*$/m.test(attributes) ||
/Tag_THUMB_ISA_use:\s+Thumb-2|Tag_FP_arch:\s+VFPv[3-9]|Tag_Advanced_SIMD_arch:\s+\S/.test(attributes) ||
!/Tag_ABI_VFP_args:\s+VFP registers/.test(attributes)
)
throw new Error('Legacy migration SQLite must use ARMv6 hard-float instructions: ' +
String(attributes).split('\n').filter((line) =>
/Tag_(CPU_arch|THUMB_ISA_use|FP_arch|ABI_VFP_args|Advanced_SIMD_arch):/.test(line),
).slice(0, 5).map((line) => line.trim()).join('; '));
}
async function rebuildMigrationSqlite(root, {
target,
run = promisify(execFile),
runtime = process,
repair = repairDependencies,
} = {}) {
target ||= await bundle.runtimeTarget();
if (target.arch !== 'armv6' || target.libc !== 'musl') return false;
const pkg = JSON.parse(await fs.readFile(path.join(root, 'package.json')));
if (!pkg.qinglongMigrationBridge || !pkg.dependencies?.sqlite3)
throw new Error('Only a legacy migration bridge may rebuild SQLite');
const prefix = path.resolve(path.dirname(runtime.execPath), '..');
const header = await fs.readFile(path.join(prefix, 'include/node/node_version.h'), 'utf8');
const version = ['MAJOR', 'MINOR', 'PATCH'].map((part) =>
header.match(new RegExp(`#define\\s+NODE_${part}_VERSION\\s+(\\d+)`))?.[1],
).join('.');
if (version !== runtime.versions.node)
throw new Error('Migration SQLite requires headers matching the existing Node');
const lock = await fs.readFile(path.join(root, 'pnpm-lock.yaml'));
// The Node 18 legacy builder skipped node-gyp 12's optional dependency.
// The existing frozen repair installs that locked graph using the current
// Node, then forces an ARMv6 source build with local headers and one job.
await repair(root, { target, env: runtime.env });
// Repair replaces node_modules, so resolve its new layout after completion.
const rebuilt = path.dirname(createRequire(path.join(root, 'package.json')).resolve('sqlite3/package.json'));
const addon = path.join(rebuilt, 'lib/binding/napi-v6-linux-arm/node_sqlite3.node');
const { stdout } = await run('readelf', ['-A', addon], { timeout: 30000 });
assertArmv6Addon(stdout);
if (!lock.equals(await fs.readFile(path.join(root, 'pnpm-lock.yaml'))))
throw new Error('Migration SQLite rebuild changed the frozen lockfile');
// node-pre-gyp copies the addon to lib/binding and then renames build to
// build-tmp-napi-v<N>. Both layouts contain compiler objects and an absolute
// node_gyp_bins/python3 link; none belongs in a portable runtime archive.
await cleanSqliteBuildArtifacts(rebuilt);
console.log('Legacy bridge SQLite rebuilt and verified for ARMv6 hard-float');
return true;
}
module.exports = { rebuildMigrationSqlite, assertArmv6Addon };
-44
View File
@@ -1,44 +0,0 @@
"""Check serialized Docker layers, including files hidden by later deletions."""
import hashlib
import json
import sys
import tarfile
archive, architecture = sys.argv[1:]
assert architecture in {"amd64", "arm"}
nodes = []
with tarfile.open(archive) as image:
manifests = json.load(image.extractfile("manifest.json"))
assert len(manifests) == 1, "Expected one platform image"
manifest = manifests[0]
config = json.load(image.extractfile(manifest["Config"]))
assert config["architecture"] == architecture
for name in manifest["Layers"]:
with tarfile.open(fileobj=image.extractfile(name), mode="r|*") as layer:
for member in layer:
filename = member.name.removeprefix("./")
assert not (
filename == "tmp/node-runtime-amd64" and member.size > 0
), "Temporary amd64 Node remains in an immutable layer"
if filename not in {"usr/local/bin/node", "usr/bin/node"}:
continue
if not member.isfile():
continue
with layer.extractfile(member) as binary:
header = binary.read(20)
assert header[:4] == b"\x7fELF", "Expected a native Node binary"
digest = hashlib.sha256(header)
while chunk := binary.read(1024 * 1024):
digest.update(chunk)
endian = "little" if header[5] == 1 else "big"
machine = int.from_bytes(header[18:20], endian)
assert machine == {"amd64": 62, "arm": 40}[architecture], (
"Wrong Node architecture in an image layer"
)
nodes.append({"path": filename, "size": member.size, "sha256": digest.hexdigest()})
assert len(nodes) == 1, "Expected exactly one retained Node binary across all layers"
assert nodes[0]["path"] == (
"usr/local/bin/node" if architecture == "amd64" else "usr/bin/node"
)
print(json.dumps({"pass": True, "architecture": architecture, "layers": len(manifest["Layers"]), "nodes": nodes}))
+519 -114
View File
@@ -16,61 +16,134 @@ const execute = promisify(execFile);
const repository = path.resolve(__dirname, '..');
const hash = (bytes) => createHash('sha256').update(bytes).digest('hex');
const read = async (file) => JSON.parse(await fs.readFile(file, 'utf8'));
const run = (program, args, options = {}) => execute(program, args, {
encoding: 'utf8', maxBuffer: 8 * 1024 * 1024, timeout: 120000,
...options,
});
const run = (program, args, options = {}) =>
execute(program, args, {
encoding: 'utf8',
maxBuffer: 8 * 1024 * 1024,
timeout: 120000,
...options,
});
const requiredFiles = [
'.env.example', 'back/protos/api.proto', 'ecosystem.config.js', 'version.yaml',
'shell/upgrade.cjs', 'shell/install-bundle.cjs', 'shell/install-package.cjs',
'shell/upgrade-lock.yaml', 'shell/node-runtime-probe.cjs', 'shell/start.sh', 'shell/start-simplify.sh',
'shell/task.sh', 'shell/preload/client.js', 'shell/preload/client.py',
'shell/preload/sitecustomize.js', 'shell/preload/sitecustomize.py',
'shell/preload/esm-loader.mjs', 'shell/preload/notificationAudit.js',
'sample/config.sample.sh', 'sample/auth.sample.json',
'static/build-info.json', 'static/build/app.js', 'static/build/token.js',
'.env.example',
'back/protos/api.proto',
'ecosystem.config.js',
'version.yaml',
'shell/upgrade.cjs',
'shell/install-bundle.cjs',
'shell/install-package.cjs',
'shell/upgrade-lock.yaml',
'shell/runtime-guard.cjs',
'shell/node-runtime-probe.cjs',
'shell/start.sh',
'shell/start-simplify.sh',
'shell/task.sh',
'shell/preload/client.js',
'shell/preload/client.py',
'shell/preload/sitecustomize.js',
'shell/preload/sitecustomize.py',
'shell/preload/esm-loader.mjs',
'shell/preload/notificationAudit.js',
'sample/config.sample.sh',
'sample/auth.sample.json',
'static/build-info.json',
'static/build/app.js',
'static/build/token.js',
'static/build/bootstrap/upgradeProbe.js',
...[
'databaseClient',
'databaseErrors',
'databaseProcess',
'databaseWorker',
'repository',
'schema',
'queries',
'cacheStore',
'statistics',
].map((name) => `static/build/data/${name}.js`),
'static/dist/index.html',
...['upgradeArtifacts', 'platformBundle', 'upgradePrestart', 'panelUpgrade', 'nodeRuntime']
.map((name) => `cli/src/internal/maintenance/${name}.cjs`),
...[
'upgradeArtifacts',
'platformBundle',
'upgradePrestart',
'panelUpgrade',
'nodeRuntime',
].map((name) => `cli/src/internal/maintenance/${name}.cjs`),
];
function checkPackedFiles(files) {
const names = new Set(files.map((file) => file.path));
for (const name of requiredFiles) assert.ok(names.has(name), `Missing npm runtime: ${name}`);
for (const name of requiredFiles)
assert.ok(names.has(name), `Missing npm runtime: ${name}`);
for (const name of names) {
assert.ok(/^(?:package\.json$|LICENSE$|README[^/]*\.md$|\.env\.example$|ecosystem\.config\.js$|version\.yaml$|back\/protos\/api\.proto$|cli\/(?:README[^/]*\.md$|src\/internal\/maintenance\/[^/]+\.cjs$)|sample\/|shell\/|static\/)/.test(name), `Unexpected npm file: ${name}`);
assert.ok(!/^shell\/(?:pub\.sh$|preload\/(?:env\.|notify\.|__ql_notify__\.|lang_env\.sh|.*-notify\.json))/.test(name), `Generated or release-only runtime file: ${name}`);
assert.ok(!/(?:^|\/)__pycache__\/|\.pyc$|(?:^|\/)\.DS_Store$/.test(name), name);
assert.ok(
/^(?:package\.json$|LICENSE$|README[^/]*\.md$|\.env\.example$|ecosystem\.config\.js$|version\.yaml$|back\/protos\/api\.proto$|cli\/(?:README[^/]*\.md$|src\/internal\/maintenance\/[^/]+\.cjs$)|sample\/|shell\/|static\/)/.test(
name,
),
`Unexpected npm file: ${name}`,
);
assert.ok(
!/^shell\/(?:pub\.sh$|preload\/(?:env\.|notify\.|__ql_notify__\.|lang_env\.sh|.*-notify\.json))/.test(
name,
),
`Generated or release-only runtime file: ${name}`,
);
assert.ok(
!/(?:^|\/)__pycache__\/|\.pyc$|(?:^|\/)\.DS_Store$/.test(name),
name,
);
assert.notEqual(name, 'sample/tool.ts');
}
}
async function packPanel(output) {
const git = (...args) => run('git', args, { cwd: repository });
assert.equal((await git('status', '--porcelain', '--untracked-files=all')).stdout.trim(), '', 'Pack only from a clean source checkout');
await run(process.execPath, [path.join(repository, 'docker/verify-build.cjs')], { cwd: repository });
assert.equal(
(await git('status', '--porcelain', '--untracked-files=all')).stdout.trim(),
'',
'Pack only from a clean source checkout',
);
await run(
process.execPath,
[path.join(repository, 'docker/verify-build.cjs')],
{ cwd: repository },
);
await fs.mkdir(output, { recursive: true });
const cache = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-npm-pack-cache-'));
try {
const result = await run('npm', ['pack', '--json', '--cache', cache, '--pack-destination', output], { cwd: repository });
const result = await run(
'npm',
['pack', '--json', '--cache', cache, '--pack-destination', output],
{ cwd: repository },
);
const [manifest] = JSON.parse(result.stdout);
checkPackedFiles(manifest.files);
assert.equal(manifest.name, '@whyour/qinglong');
const build = await read(path.join(repository, 'static/build-info.json'));
const names = new Set(manifest.files.map((file) => file.path));
for (const name of Object.keys(build.files)) assert.ok(names.has(`static/${name}`), `npm filtered a verified build artifact: ${name}`);
for (const name of Object.keys(build.files))
assert.ok(
names.has(`static/${name}`),
`npm filtered a verified build artifact: ${name}`,
);
const receipt = {
filename: manifest.filename, sha256: hash(await fs.readFile(path.join(output, manifest.filename))),
sourceCommit: build.sourceCommit, size: manifest.size, fileCount: manifest.files.length,
filename: manifest.filename,
sha256: hash(await fs.readFile(path.join(output, manifest.filename))),
sourceCommit: build.sourceCommit,
size: manifest.size,
fileCount: manifest.files.length,
};
await fs.writeFile(path.join(output, 'package-receipt.json'), JSON.stringify(receipt, null, 2));
await fs.writeFile(
path.join(output, 'package-receipt.json'),
JSON.stringify(receipt, null, 2),
);
console.log(JSON.stringify(receipt));
return receipt;
} finally {
await fs.rm(cache, { recursive: true, force: true });
// Also recover an interrupted npm pack whose postpack hook did not execute.
await fs.rm(path.join(repository, 'shell/upgrade-lock.yaml'), { force: true });
await fs.rm(path.join(repository, 'shell/upgrade-lock.yaml'), {
force: true,
});
}
}
@@ -84,8 +157,13 @@ async function unusedPort() {
// Consume the unmodified output of Dockerfile.upgrade-bundle / buildBundle.
async function platformFixture(root, directory, expectedCommit) {
assert.ok(directory, 'Set QL_PANEL_TEST_BUNDLE_DIR to the official platform bundle output');
const names = (await fs.readdir(directory)).filter((name) => name.endsWith('.tar.gz'));
assert.ok(
directory,
'Set QL_PANEL_TEST_BUNDLE_DIR to the official platform bundle output',
);
const names = (await fs.readdir(directory)).filter((name) =>
name.endsWith('.tar.gz'),
);
assert.equal(names.length, 1, 'Expected one official platform package');
const archive = path.resolve(directory, names[0]);
const record = await read(archive + '.json');
@@ -105,64 +183,140 @@ async function platformFixture(root, directory, expectedCommit) {
async function legacyArchive(temp, env) {
const directory = path.join(temp, 'legacy');
await fs.mkdir(directory);
const result = await run('npm', ['pack', '@whyour/qinglong@2.23.0', '--ignore-scripts', '--json', '--registry=https://registry.npmjs.org', '--pack-destination', directory, '--cache', path.join(temp, 'cache')], { cwd: temp, env, timeout: 300000 });
const result = await run(
'npm',
[
'pack',
'@whyour/qinglong@2.23.0',
'--ignore-scripts',
'--json',
'--registry=https://registry.npmjs.org',
'--pack-destination',
directory,
'--cache',
path.join(temp, 'cache'),
],
{ cwd: temp, env, timeout: 300000 },
);
const [record] = JSON.parse(result.stdout);
assert.equal(record.name, '@whyour/qinglong');
assert.equal(record.version, '2.23.0');
assert.equal(record.filename, 'whyour-qinglong-2.23.0.tgz');
const archive = path.join(directory, record.filename);
const integrity = 'sha512-' + createHash('sha512').update(await fs.readFile(archive)).digest('base64');
assert.equal(integrity, 'sha512-x6B9s1/SRKkHe7jRQtb3LzMluJ3O3PAAssrr/5jaFMMl7DZvABSEiqkkZMu0r1BNtiCF72uj8OqXN9pCnF2WVw==', 'Published legacy package integrity changed');
const integrity =
'sha512-' +
createHash('sha512')
.update(await fs.readFile(archive))
.digest('base64');
assert.equal(
integrity,
'sha512-x6B9s1/SRKkHe7jRQtb3LzMluJ3O3PAAssrr/5jaFMMl7DZvABSEiqkkZMu0r1BNtiCF72uj8OqXN9pCnF2WVw==',
'Published legacy package integrity changed',
);
return { archive, integrity, version: record.version };
}
async function verifyPanel(input, output) {
assert.equal(process.platform, 'linux', 'Service tests require Linux');
assert.equal(process.env.QL_PANEL_PACKAGE_TEST_ISOLATED, '1', 'Run only inside a disposable Linux container: QL_PANEL_PACKAGE_TEST_ISOLATED=1');
assert.equal(
process.env.QL_PANEL_PACKAGE_TEST_ISOLATED,
'1',
'Run only inside a disposable Linux container: QL_PANEL_PACKAGE_TEST_ISOLATED=1',
);
const stat = await fs.stat(input);
if (stat.isDirectory()) {
const names = (await fs.readdir(input)).filter((name) => name.endsWith('.tgz'));
const names = (await fs.readdir(input)).filter((name) =>
name.endsWith('.tgz'),
);
assert.equal(names.length, 1, 'Expected one exact npm archive');
input = path.join(input, names[0]);
}
const expected = await read(path.join(path.dirname(input), 'package-receipt.json'));
const expected = await read(
path.join(path.dirname(input), 'package-receipt.json'),
);
assert.equal(path.basename(input), expected.filename);
assert.equal(hash(await fs.readFile(input)), expected.sha256, 'npm archive differs from its build receipt');
assert.equal(
hash(await fs.readFile(input)),
expected.sha256,
'npm archive differs from its build receipt',
);
assert.equal((await fs.stat(input)).size, expected.size);
assert.match(expected.sourceCommit, /^[a-f0-9]{40}$/);
if (process.env.GITHUB_SHA) assert.equal(expected.sourceCommit, process.env.GITHUB_SHA);
if (process.env.GITHUB_SHA)
assert.equal(expected.sourceCommit, process.env.GITHUB_SHA);
const nodeSha = hash(await fs.readFile(process.execPath));
const pm2 = process.env.QL_PANEL_TEST_PM2;
assert.ok(pm2 && path.isAbsolute(pm2), 'Set QL_PANEL_TEST_PM2 to an isolated PM2 executable');
const temp = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'ql-npm-install-')));
const home = path.join(temp, 'home'), prefix = path.join(temp, 'prefix'), data = path.join(temp, 'data');
assert.ok(
pm2 && path.isAbsolute(pm2),
'Set QL_PANEL_TEST_PM2 to an isolated PM2 executable',
);
const temp = await fs.realpath(
await fs.mkdtemp(path.join(os.tmpdir(), 'ql-npm-install-')),
);
const home = path.join(temp, 'home'),
prefix = path.join(temp, 'prefix'),
data = path.join(temp, 'data');
await fs.mkdir(home);
await fs.mkdir(data);
const port = await unusedPort();
let grpcPort = await unusedPort();
while (grpcPort === port) grpcPort = await unusedPort();
const env = { ...process.env, HOME: home, PM2_HOME: path.join(temp, '.pm2'),
QL_DATA_DIR: data, QL_BRANCH: 'develop', NODE_ENV: 'production', QL_CONTAINER: 'false',
BACK_PORT: String(port), GRPC_PORT: String(grpcPort), QlPort: String(port), QlGrpcPort: String(grpcPort),
BIND_HOST: '127.0.0.1', BIND_HOST_GRPC: '127.0.0.1',
NODE_PATH: '', NODE_OPTIONS: '',
PATH: [path.join(prefix, 'bin'), path.dirname(pm2), process.env.PATH].join(path.delimiter),
const env = {
...process.env,
HOME: home,
PM2_HOME: path.join(temp, '.pm2'),
QL_DATA_DIR: data,
QL_BRANCH: 'develop',
NODE_ENV: 'production',
QL_CONTAINER: 'false',
BACK_PORT: String(port),
GRPC_PORT: String(grpcPort),
QlPort: String(port),
QlGrpcPort: String(grpcPort),
BIND_HOST: '127.0.0.1',
BIND_HOST_GRPC: '127.0.0.1',
NODE_PATH: '',
NODE_OPTIONS: '',
PATH: [path.join(prefix, 'bin'), path.dirname(pm2), process.env.PATH].join(
path.delimiter,
),
};
for (const key of ['QL_CLI_ROOT', 'QL_DIR', 'QL_UPGRADE_FILE', 'QL_UPGRADE_REPAIR_DEPS', 'QL_UPGRADE_LAUNCHER_PID', 'QL_UPGRADE_INDEX_URL']) delete env[key];
for (const key of [
'QL_CLI_ROOT',
'QL_DIR',
'QL_UPGRADE_FILE',
'QL_UPGRADE_REPAIR_DEPS',
'QL_UPGRADE_LAUNCHER_PID',
'QL_UPGRADE_INDEX_URL',
])
delete env[key];
let root, fixtureServer, token;
const api = async (route, method = 'GET', body, anonymous = false) => {
const response = await fetch(`http://127.0.0.1:${port}/api/${route}`, {
method, headers: { 'Content-Type': 'application/json', ...(!anonymous && token ? { Authorization: `Bearer ${token}` } : {}) },
body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(5000),
method,
headers: {
'Content-Type': 'application/json',
...(!anonymous && token ? { Authorization: `Bearer ${token}` } : {}),
},
body: body === undefined ? undefined : JSON.stringify(body),
signal: AbortSignal.timeout(5000),
});
const result = await response.json();
assert.equal(result.code, 200, `${method} ${route}: ${result.message}`);
try {
assert.equal(result.code, 200, `${method} ${route}: ${result.message}`);
} catch (error) {
error.status = response.status;
throw error;
}
return result.data;
};
const healthy = async () => {
const deadline = Date.now() + 120000;
while (Date.now() < deadline) {
try { const value = await api('health', 'GET', undefined, true); if (value.status === 'ok') return value; } catch {}
try {
const value = await api('health', 'GET', undefined, true);
if (value.status === 'ok') return value;
} catch {}
await delay(300);
}
throw new Error('Installed npm backend did not become healthy');
@@ -174,32 +328,81 @@ async function verifyPanel(input, output) {
const value = await api('system/update-status');
assert.equal(value.id, id);
if (value.status === status) return value;
if (value.status === 'failed' && status !== 'failed') throw new Error(value.error);
if (value.status === 'failed' && status !== 'failed')
throw new Error(value.error);
} catch (error) {
if (!/fetch failed|ECONNRESET|ECONNREFUSED|aborted|timeout/i.test(error.message)) throw error;
if (
error.status !== 503 &&
!/fetch failed|ECONNRESET|ECONNREFUSED|aborted|timeout/i.test(
error.message,
)
)
throw error;
}
await delay(250);
}
throw new Error(`Installed npm update never reached ${status}`);
};
try {
const install = (archive) => run('npm', ['install', '--global', '--prefix', prefix, '--force', '--omit=dev', '--no-audit', '--no-fund', '--cache', path.join(temp, 'cache'), archive], { cwd: temp, env, timeout: 600000 });
const start = () => run(path.join(prefix, 'bin/qinglong'), ['reload'], { cwd: temp, env, timeout: 180000 });
const stop = () => run(pm2, ['delete', 'ecosystem.config.js'], { cwd: root, env });
console.log(`Installing actual npm archive with production lifecycle scripts on ${process.version}`);
const install = (archive) =>
run(
'npm',
[
'install',
'--global',
'--prefix',
prefix,
'--force',
'--omit=dev',
'--no-audit',
'--no-fund',
'--cache',
path.join(temp, 'cache'),
archive,
],
{ cwd: temp, env, timeout: 600000 },
);
const start = () =>
run(path.join(prefix, 'bin/qinglong'), ['reload'], {
cwd: temp,
env,
timeout: 180000,
});
const stop = () =>
run(pm2, ['delete', 'ecosystem.config.js'], { cwd: root, env });
console.log(
`Installing actual npm archive with production lifecycle scripts on ${process.version}`,
);
await install(input);
const globalRoot = (await run('npm', ['root', '--global', '--prefix', prefix], { cwd: temp, env })).stdout.trim();
const globalRoot = (
await run('npm', ['root', '--global', '--prefix', prefix], {
cwd: temp,
env,
})
).stdout.trim();
root = path.join(globalRoot, '@whyour/qinglong');
env.QL_DIR = root;
const runtimeProbes = [];
const checkRuntime = async () => {
const result = await run(process.execPath, [path.join(root, 'shell/node-runtime-probe.cjs')], { cwd: temp, env, timeout: 540000 });
const result = await run(
process.execPath,
[path.join(root, 'shell/node-runtime-probe.cjs')],
{ cwd: temp, env },
);
runtimeProbes.push(JSON.parse(result.stdout.trim()));
};
await checkRuntime();
// Use a subprocess: cached candidate modules must not mask missing files
// after old-package installation and the real npm overwrite below.
await run(process.execPath, ['-e', `const root=process.argv[1],p=require('node:path');const m=require(root+'/package.json');for(const target of Object.values(m.bin))require('node:fs').accessSync(p.join(root,target));for(const name of ['platformBundle','upgradePrestart','panelUpgrade'])require(root+'/cli/src/internal/maintenance/'+name+'.cjs');require(root+'/shell/upgrade.cjs').verifyUpgrade(root,p.join(root,'static')).catch(e=>{console.error(e);process.exitCode=1});`, root], { env, cwd: temp });
await run(
process.execPath,
[
'-e',
`const root=process.argv[1],p=require('node:path');const m=require(root+'/package.json');for(const target of Object.values(m.bin))require('node:fs').accessSync(p.join(root,target));for(const name of ['platformBundle','upgradePrestart','panelUpgrade'])require(root+'/cli/src/internal/maintenance/'+name+'.cjs');require(root+'/shell/upgrade.cjs').verifyUpgrade(root,p.join(root,'static')).catch(e=>{console.error(e);process.exitCode=1});`,
root,
],
{ env, cwd: temp },
);
env.QL_DATA_DIR = path.join(temp, 'fresh/data');
await fs.mkdir(env.QL_DATA_DIR, { recursive: true });
console.log('Starting the freshly installed npm package');
@@ -207,40 +410,75 @@ async function verifyPanel(input, output) {
const fresh = await healthy();
assert.equal(fresh.build.sourceCommit, expected.sourceCommit);
assert.equal(fresh.services.grpc, true);
assert.match(await fetch(`http://127.0.0.1:${port}/`).then((r) => r.text()), /<html/i);
assert.match(
await fetch(`http://127.0.0.1:${port}/`).then((r) => r.text()),
/<html/i,
);
await stop();
await fs.rm(path.join(temp, 'fresh'), { recursive: true, force: true });
env.QL_DATA_DIR = data;
console.log('Installing the integrity-pinned published legacy npm package');
const legacy = await legacyArchive(temp, env);
await install(legacy.archive);
assert.equal((await read(path.join(root, 'package.json'))).version, legacy.version);
await assert.rejects(fs.access(path.join(root, 'cli/src/internal/maintenance/platformBundle.cjs')), { code: 'ENOENT' });
const legacyBackendSha = hash(await fs.readFile(path.join(root, 'static/build/app.js')));
assert.equal(
(await read(path.join(root, 'package.json'))).version,
legacy.version,
);
await assert.rejects(
fs.access(
path.join(root, 'cli/src/internal/maintenance/platformBundle.cjs'),
),
{ code: 'ENOENT' },
);
const legacyBackendSha = hash(
await fs.readFile(path.join(root, 'static/build/app.js')),
);
await start();
await healthy();
const credentials = { username: 'npm_package_verification', password: `Test_${randomUUID()}!` };
const credentials = {
username: 'npm_package_verification',
password: `Test_${randomUUID()}!`,
};
await api('user/init', 'PUT', credentials, true);
token = (await api('user/login', 'POST', credentials, true)).token;
const nonce = randomUUID();
await api('envs', 'POST', [{ name: 'NPM_PACKAGE_PRESERVE', value: nonce }]);
const script = path.join(data, 'scripts/npm-package-check.js');
await fs.writeFile(script, `console.log(${JSON.stringify(nonce)});\n`);
const cron = await api('crons', 'POST', { name: 'npm package retained task', command: `node ${script}`, schedule: '0 0 1 1 *' });
const cron = await api('crons', 'POST', {
name: 'npm package retained task',
command: `node ${script}`,
schedule: '0 0 1 1 *',
});
const userConfig = path.join(data, 'config/config.sh');
await fs.appendFile(userConfig, `\n# retained npm package setting ${nonce}\n`);
await fs.appendFile(
userConfig,
`\n# retained npm package setting ${nonce}\n`,
);
const userConfigSha = hash(await fs.readFile(userConfig));
const preservedData = async () => {
assert.equal((await api('user')).username, credentials.username);
assert.ok((await api('envs')).some((x) => x.name === 'NPM_PACKAGE_PRESERVE' && x.value === nonce));
assert.ok(
(await api('envs')).some(
(x) => x.name === 'NPM_PACKAGE_PRESERVE' && x.value === nonce,
),
);
assert.equal((await api(`crons/${cron.id}`)).command, cron.command);
assert.equal(hash(await fs.readFile(userConfig)), userConfigSha);
assert.equal(await fs.readFile(script, 'utf8'), `console.log(${JSON.stringify(nonce)});\n`);
assert.equal(
await fs.readFile(script, 'utf8'),
`console.log(${JSON.stringify(nonce)});\n`,
);
};
await preservedData();
console.log('Stopping legacy service and verifying npm automatically preserves its .env during overwrite');
console.log(
'Stopping legacy service and verifying npm automatically preserves its .env during overwrite',
);
await stop();
await fs.appendFile(path.join(root, '.env'), `\n# retained custom npm environment ${nonce}\n`);
await fs.appendFile(
path.join(root, '.env'),
`\n# retained custom npm environment ${nonce}\n`,
);
const savedEnv = await fs.readFile(path.join(root, '.env'));
await install(input);
assert.deepEqual(await fs.readFile(path.join(root, '.env')), savedEnv);
@@ -248,25 +486,62 @@ async function verifyPanel(input, output) {
const configSha = hash(savedEnv);
const local = createRequire(path.join(root, 'package.json'));
const artifacts = local('./shell/upgrade.cjs');
const build = await artifacts.verifyUpgrade(root, path.join(root, 'static'));
const build = await artifacts.verifyUpgrade(
root,
path.join(root, 'static'),
);
assert.equal(build.sourceCommit, expected.sourceCommit);
assert.notEqual(hash(await fs.readFile(path.join(root, 'static/build/app.js'))), legacyBackendSha);
assert.equal(hash(await fs.readFile(path.join(root, 'pnpm-lock.yaml'))), build.lockfileSha256);
const fixture = await platformFixture(root, process.env.QL_PANEL_TEST_BUNDLE_DIR, expected.sourceCommit);
let badChecksum = true, requests = 0;
assert.notEqual(
hash(await fs.readFile(path.join(root, 'static/build/app.js'))),
legacyBackendSha,
);
assert.equal(
hash(await fs.readFile(path.join(root, 'pnpm-lock.yaml'))),
build.lockfileSha256,
);
const fixture = await platformFixture(
root,
process.env.QL_PANEL_TEST_BUNDLE_DIR,
expected.sourceCommit,
);
let badChecksum = true,
requests = 0;
fixtureServer = http.createServer((request, response) => {
requests++;
if (request.url === '/latest.json') {
response.setHeader('Content-Type', 'application/json');
response.end(JSON.stringify({ format: 2, sourceCommit: fixture.sourceCommit, packages: { [fixture.target]: { size: fixture.size, sha256: badChecksum ? '0'.repeat(64) : fixture.sha256, url: `http://127.0.0.1:${fixtureServer.address().port}/bundle.tar.gz` } } }));
response.end(
JSON.stringify({
format: 2,
sourceCommit: fixture.sourceCommit,
packages: {
[fixture.target]: {
size: fixture.size,
sha256: badChecksum ? '0'.repeat(64) : fixture.sha256,
url: `http://127.0.0.1:${
fixtureServer.address().port
}/bundle.tar.gz`,
},
},
}),
);
} else if (request.url === '/bundle.tar.gz') {
response.setHeader('Content-Length', fixture.size);
sync.createReadStream(fixture.archive).pipe(response);
} else { response.statusCode = 404; response.end(); }
} else {
response.statusCode = 404;
response.end();
}
});
await new Promise((resolve) => fixtureServer.listen(0, '127.0.0.1', resolve));
env.QL_UPGRADE_INDEX_URL = `http://127.0.0.1:${fixtureServer.address().port}/latest.json`;
console.log('Starting the overwritten npm package against legacy user data');
await new Promise((resolve) =>
fixtureServer.listen(0, '127.0.0.1', resolve),
);
env.QL_UPGRADE_INDEX_URL = `http://127.0.0.1:${
fixtureServer.address().port
}/latest.json`;
console.log(
'Starting the overwritten npm package against legacy user data',
);
await start();
const original = await healthy();
assert.equal(original.build.sourceCommit, build.sourceCommit);
@@ -283,13 +558,22 @@ async function verifyPanel(input, output) {
assert.equal((await healthy()).build.instanceId, original.build.instanceId);
await preserved();
badChecksum = false;
console.log('Testing installed panel download, prestart and explicit application');
console.log(
'Testing installed panel download, prestart and explicit application',
);
const accepted = await api('system/update', 'PUT');
const ready = await job(accepted.id, 'ready');
assert.equal((await healthy()).build.instanceId, original.build.instanceId);
const pointer = await read(path.join(root, '.tmp/upgrade-ready-develop.json'));
const receipt = await read(path.join(root, '.tmp', pointer.directory, 'prestart.json'));
assert.ok(receipt.completedAt <= ready.updatedAt, 'Readiness was recorded before backend prestart');
const pointer = await read(
path.join(root, '.tmp/upgrade-ready-develop.json'),
);
const receipt = await read(
path.join(root, '.tmp', pointer.directory, 'prestart.json'),
);
assert.ok(
receipt.completedAt <= ready.updatedAt,
'Readiness was recorded before backend prestart',
);
const applying = await api('update/system', 'PUT');
await job(applying.id, 'completed');
const afterOnline = await healthy();
@@ -297,20 +581,59 @@ async function verifyPanel(input, output) {
assert.notEqual(afterOnline.build.instanceId, original.build.instanceId);
assert.equal(afterOnline.build.sourceCommit, fixture.sourceCommit);
await fs.access(path.join(root, 'node_modules/.pnpm'));
assert.match(await fs.realpath(path.join(root, 'node_modules/sqlite3')), /[/\\]node_modules[/\\]\.pnpm[/\\]/);
assert.match(
await fs.realpath(path.join(root, 'node_modules/@mikro-orm/sql')),
/[/\\]node_modules[/\\]\.pnpm[/\\]/,
);
await assert.rejects(fs.access(path.join(root, 'node_modules/sqlite3')), {
code: 'ENOENT',
});
await assert.rejects(fs.access(path.join(root, 'node_modules/sequelize')), {
code: 'ENOENT',
});
await fs.access(path.join(root, 'scripts/build-upgrade-bundle.cjs'));
await artifacts.verifyUpgrade(root, path.join(root, 'static'));
assert.match(await fetch(`http://127.0.0.1:${port}/`).then((r) => r.text()), /<html/i);
assert.match(
await fetch(`http://127.0.0.1:${port}/`).then((r) => r.text()),
/<html/i,
);
await preserved();
console.log('Testing offline installer and Bash reload from the installed npm package');
console.log(
'Testing offline installer and Bash reload from the installed npm package',
);
const requestsBeforeOffline = requests;
await run(process.execPath, [path.join(root, 'shell/install-bundle.cjs'), '--root', root, '--file', fixture.archive, '--download-only'], { cwd: temp, env, timeout: 240000 });
assert.equal(requests, requestsBeforeOffline, 'Offline preparation contacted the index');
assert.equal((await healthy()).build.instanceId, afterOnline.build.instanceId);
await run(path.join(prefix, 'bin/ql'), ['reload', 'system'], { cwd: temp, env, timeout: 240000 });
await run(
process.execPath,
[
path.join(root, 'shell/install-bundle.cjs'),
'--root',
root,
'--file',
fixture.archive,
'--download-only',
],
{ cwd: temp, env, timeout: 240000 },
);
assert.equal(
requests,
requestsBeforeOffline,
'Offline preparation contacted the index',
);
assert.equal(
(await healthy()).build.instanceId,
afterOnline.build.instanceId,
);
await run(path.join(prefix, 'bin/ql'), ['reload', 'system'], {
cwd: temp,
env,
timeout: 240000,
});
const afterOffline = await healthy();
await checkRuntime();
assert.notEqual(afterOffline.build.instanceId, afterOnline.build.instanceId);
assert.notEqual(
afterOffline.build.instanceId,
afterOnline.build.instanceId,
);
assert.equal(afterOffline.build.sourceCommit, fixture.sourceCommit);
await fs.access(path.join(root, 'node_modules/.pnpm'));
assert.equal(hash(await fs.readFile(process.execPath)), nodeSha);
@@ -320,36 +643,66 @@ async function verifyPanel(input, output) {
let taskExecuted = false;
for (let i = 0; i < 100; i++) {
const log = String(await api(`crons/${cron.id}/log`));
if (log.includes(nonce) && /完成|执行结束|finished|completed/i.test(log)) { taskExecuted = true; break; }
if (
log.includes(nonce) &&
/完成|执行结束|finished|completed/i.test(log)
) {
taskExecuted = true;
break;
}
await delay(300);
}
assert.ok(taskExecuted, 'Retained task did not execute after npm package upgrades');
assert.ok(
taskExecuted,
'Retained task did not execute after npm package upgrades',
);
const result = {
node: process.version, platform: fixture.target,
archiveSha256: hash(await fs.readFile(input)), sourceCommit: build.sourceCommit,
installedProductionDependencies: true, restoredLockVerified: true,
legacyNpmVersion: legacy.version, legacyNpmIntegrity: legacy.integrity,
legacyNpmOverwriteInstalled: true, legacyExternalDataPreserved: true,
node: process.version,
platform: fixture.target,
archiveSha256: hash(await fs.readFile(input)),
sourceCommit: build.sourceCommit,
installedProductionDependencies: true,
restoredLockVerified: true,
legacyNpmVersion: legacy.version,
legacyNpmIntegrity: legacy.integrity,
legacyNpmOverwriteInstalled: true,
legacyExternalDataPreserved: true,
customRootEnvironmentPreservedAutomatically: true,
officialPlatformBundle: { baseImage: fixture.baseImage, sourceCommit: fixture.sourceCommit, sha256: fixture.sha256 },
officialPlatformBundle: {
baseImage: fixture.baseImage,
sourceCommit: fixture.sourceCommit,
sha256: fixture.sha256,
},
migratedFromNpmToOfficialPnpmLayout: true,
existingNodePreserved: true, nativeUrlRuntimeProbes: runtimeProbes,
frontendAndBackendStarted: true, checksumFailureKeptOldService: true,
prestartBeforeReady: true, onlineActualRestart: true,
offlinePreparationDidNotContactIndex: true, offlineBashActualRestart: true,
accountEnvironmentConfigurationAndScriptsPreserved: true, retainedTaskExecuted: true,
initialBuild: original.build, afterOnline: afterOnline.build, afterOffline: afterOffline.build,
existingNodePreserved: true,
nativeUrlRuntimeProbes: runtimeProbes,
frontendAndBackendStarted: true,
checksumFailureKeptOldService: true,
prestartBeforeReady: true,
onlineActualRestart: true,
offlinePreparationDidNotContactIndex: true,
offlineBashActualRestart: true,
accountEnvironmentConfigurationAndScriptsPreserved: true,
retainedTaskExecuted: true,
initialBuild: original.build,
afterOnline: afterOnline.build,
afterOffline: afterOffline.build,
};
if (output) {
await fs.mkdir(output, { recursive: true });
await fs.writeFile(path.join(output, `verification-${process.version}.json`), JSON.stringify(result, null, 2));
await fs.writeFile(
path.join(output, `verification-${process.version}.json`),
JSON.stringify(result, null, 2),
);
}
console.log(JSON.stringify(result, null, 2));
return result;
} catch (error) {
if (root) {
for (const file of ['qinglong-error.log', 'qinglong-out.log']) {
const text = await fs.readFile(path.join(env.PM2_HOME, 'logs', file), 'utf8').catch(() => '');
const text = await fs
.readFile(path.join(env.PM2_HOME, 'logs', file), 'utf8')
.catch(() => '');
if (text) console.error(text.slice(-6000));
}
}
@@ -359,19 +712,71 @@ async function verifyPanel(input, output) {
fixtureServer.closeAllConnections();
await new Promise((resolve) => fixtureServer.close(resolve));
}
// A failed assertion must not remove a prefix that its detached updater is
// still extracting into. Identify only this private installation's job.
if (root) {
const state = await read(
path.join(root, '.tmp/panel-upgrade.json'),
).catch(() => undefined);
if (
Number.isSafeInteger(state?.pid) &&
state.pid > 1 &&
typeof state.id === 'string'
) {
const command = (
await fs
.readFile(`/proc/${state.pid}/cmdline`, 'utf8')
.catch(() => '')
).split('\0');
if (
command.includes(
path.join(root, 'cli/src/internal/maintenance/panelUpgrade.cjs'),
) &&
command.includes('worker') &&
command.includes(state.id)
) {
const signal = (name) => {
try {
process.kill(-state.pid, name);
return true;
} catch (error) {
if (error.code !== 'ESRCH') throw error;
return false;
}
};
signal('SIGTERM');
const deadline = Date.now() + 30000;
while (signal(0) && Date.now() < deadline) await delay(100);
if (signal(0)) signal('SIGKILL');
}
}
}
await run(pm2, ['kill'], { env, cwd: temp }).catch(() => {});
await run('nginx', ['-s', 'quit'], { env, cwd: temp }).catch(() => {});
await fs.rm(temp, { recursive: true, force: true });
await fs.rm(temp, {
recursive: true,
force: true,
maxRetries: 3,
retryDelay: 100,
});
}
}
module.exports = { checkPackedFiles, packPanel, verifyPanel };
if (require.main === module) {
const [action, input, output] = process.argv.slice(2);
const operation = action === 'pack' && input
? packPanel(path.resolve(input))
: action === 'verify' && input
const operation =
action === 'pack' && input
? packPanel(path.resolve(input))
: action === 'verify' && input
? verifyPanel(path.resolve(input), output && path.resolve(output))
: Promise.reject(new Error('Usage: node scripts/verify-panel-package.cjs pack OUTPUT | verify ARCHIVE_OR_DIRECTORY [RESULT_DIRECTORY]'));
operation.catch((error) => { console.error(error); process.exitCode = 1; });
: Promise.reject(
new Error(
'Usage: node scripts/verify-panel-package.cjs pack OUTPUT | verify ARCHIVE_OR_DIRECTORY [RESULT_DIRECTORY]',
),
);
operation.catch((error) => {
console.error(error);
process.exitCode = 1;
});
}
+180
View File
@@ -0,0 +1,180 @@
// Verify the actual archive under an unsupported Node without touching any
// existing installation. This is also the Node 20 negative CI matrix entry.
const fs = require('node:fs/promises');
const path = require('node:path');
const os = require('node:os');
const assert = require('node:assert/strict');
const { execFile, spawn } = require('node:child_process');
const { promisify } = require('node:util');
const execute = promisify(execFile);
async function verifyRuntimeFloor(input, output) {
assert.equal(
Number(process.versions.node.split('.')[0]),
20,
'Run this rejection test with Node 20',
);
const names = (await fs.readdir(input)).filter((name) =>
name.endsWith('.tgz'),
);
assert.equal(names.length, 1, 'Expected one actual npm archive');
const archive = path.resolve(input, names[0]);
const temp = await fs.realpath(
await fs.mkdtemp(path.join(os.tmpdir(), 'ql-node-floor-')),
);
const env = {
PATH: process.env.PATH,
HOME: temp,
TMPDIR: temp,
npm_config_cache: path.join(temp, 'cache'),
npm_config_userconfig: path.join(temp, 'npmrc'),
npm_config_globalconfig: path.join(temp, 'global-npmrc'),
};
const npm = (args) =>
execute('npm', args, {
cwd: temp,
env,
encoding: 'utf8',
timeout: 600000,
maxBuffer: 4 * 1024 * 1024,
});
try {
await fs.writeFile(env.npm_config_userconfig, '');
await fs.writeFile(env.npm_config_globalconfig, '');
const legacy = path.join(temp, 'legacy');
await fs.mkdir(legacy);
await fs.writeFile(
path.join(legacy, 'package.json'),
JSON.stringify({ name: '@whyour/qinglong', version: '2.22.0' }),
);
await fs.writeFile(path.join(legacy, 'legacy-marker'), 'old application');
const [old] = JSON.parse(
(
await execute('npm', ['pack', '--json', '--ignore-scripts'], {
cwd: legacy,
env,
})
).stdout,
);
for (const global of [false, true]) {
const prefix = path.join(temp, global ? 'global' : 'local');
const installed = path.join(
prefix,
global
? 'lib/node_modules/@whyour/qinglong'
: 'node_modules/@whyour/qinglong',
);
const args = [
'install',
...(global ? ['--global'] : []),
'--prefix',
prefix,
'--omit=dev',
'--no-audit',
'--no-fund',
];
await npm([...args, path.join(legacy, old.filename)]);
const environment = Buffer.from(
'QL_DATA_DIR=/external/data\r\nSYNTHETIC_SECRET=keep\n',
);
const data = path.join(temp, global ? 'global-data' : 'local-data');
await fs.writeFile(path.join(installed, '.env'), environment);
await fs.writeFile(data, 'synthetic old data');
const child = spawn(
process.execPath,
[
'-e',
`
const fs=require('node:fs'),http=require('node:http');
const value=fs.readFileSync(process.argv[1],'utf8')+fs.readFileSync(process.argv[2],'utf8');
const server=http.createServer((q,r)=>r.end(value));
server.listen(0,'127.0.0.1',()=>process.send(server.address().port));
`,
path.join(installed, '.env'),
data,
],
{ env, stdio: ['ignore', 'ignore', 'pipe', 'ipc'] },
);
const stopped = new Promise((resolve) => child.once('exit', resolve));
try {
const port = await new Promise((resolve, reject) => {
const timer = setTimeout(
() => reject(new Error('Old service startup timed out')),
10000,
);
child.once('message', (value) => {
clearTimeout(timer);
resolve(value);
});
child.once('error', (error) => {
clearTimeout(timer);
reject(error);
});
child.once('exit', () => {
clearTimeout(timer);
reject(new Error('Old service exited'));
});
});
const url = `http://127.0.0.1:${port}`;
const before = await (await fetch(url)).text();
await assert.rejects(npm([...args, archive]), (error) => {
assert.equal(
error.killed,
false,
`npm did not reach the runtime guard: ${error.message}`,
);
assert.match(error.stderr + error.stdout, /requires Node >=22\.17/);
return true;
});
assert.equal(
JSON.parse(await fs.readFile(path.join(installed, 'package.json')))
.version,
'2.22.0',
);
assert.deepEqual(
await fs.readFile(path.join(installed, '.env')),
environment,
);
assert.equal(await fs.readFile(data, 'utf8'), 'synthetic old data');
assert.equal(child.exitCode, null);
assert.equal(await (await fetch(url)).text(), before);
} finally {
child.kill('SIGKILL');
await stopped;
}
}
const result = {
node: process.version,
unsupportedRuntimeRejected: true,
localAndGlobalOldPackagePreserved: true,
environmentAndDataPreserved: true,
oldServiceStayedHealthy: true,
};
if (output) {
await fs.mkdir(output, { recursive: true });
await fs.writeFile(
path.join(output, `verification-${process.version}.json`),
JSON.stringify(result, null, 2),
);
}
console.log(JSON.stringify(result, null, 2));
return result;
} finally {
await fs.rm(temp, { recursive: true, force: true });
}
}
module.exports = { verifyRuntimeFloor };
if (require.main === module) {
const [input, output] = process.argv.slice(2);
if (!input)
throw new Error(
'Usage: node scripts/verify-panel-runtime-floor.cjs ARCHIVE_DIRECTORY [RESULT_DIRECTORY]',
);
verifyRuntimeFloor(path.resolve(input), output && path.resolve(output)).catch(
(error) => {
console.error(error);
process.exitCode = 1;
},
);
}
+17
View File
@@ -0,0 +1,17 @@
const minimum = '22.17.0';
function assertNodeRuntime(version = process.versions.node) {
const [major, minor] = version.split('.').map(Number);
if (!Number.isInteger(major) || !Number.isInteger(minor) || major < 22 || (major === 22 && minor < 17)) {
throw Object.assign(new Error(`Qinglong requires Node >=${minimum}; upgrade Node before installing. Application updates retain the existing Node.`), { code: 'UNSUPPORTED_NODE' });
}
if (version === process.versions.node) {
try {
if (typeof require('node:sqlite').DatabaseSync !== 'function') throw new Error();
} catch {
throw Object.assign(new Error('This Node build does not provide node:sqlite; use a Node build with SQLite support.'), { code: 'UNSUPPORTED_NODE_SQLITE' });
}
}
}
module.exports = { minimum, assertNodeRuntime };
+4 -2
View File
@@ -20,14 +20,16 @@ export default function UpgradeNotice({
const showNotice =
activeStatuses.includes(job.status) ||
job.status === 'ready' ||
job.status === 'failed';
job.status === 'failed' ||
Boolean(job.migrationRequired);
if (!showNotice && !open) return null;
const text = !connected
? intl.get('连接已断开,正在重新连接')
: intl.get(upgradeStatusLabel(job));
const label =
activeStatuses.includes(job.status) &&
job.phase === 'downloading' && job.percent != null
job.phase === 'downloading' &&
job.percent != null
? `${text} ${job.percent}%`
: text;
return (
+11 -2
View File
@@ -15,6 +15,7 @@ export interface UpgradeJob {
version?: string;
error?: string;
log?: string;
migrationRequired?: boolean;
}
export const activeStatuses = ['starting', 'running', 'restarting'];
@@ -34,6 +35,8 @@ export const phaseLabels: Record<string, string> = {
// Status is authoritative; old progress fields must not claim completion.
export function upgradeStatusLabel(job: UpgradeJob): string {
if (job.migrationRequired && ['idle', 'completed'].includes(job.status))
return '升级器已更新,请继续升级新版';
if (job.status === 'ready') return '下载完成,等待重启';
if (job.status === 'completed') return '升级成功';
if (job.status === 'failed') return '更新失败';
@@ -94,8 +97,14 @@ export function createUpgradeStore() {
) {
sessionStorage.removeItem(completionKey);
notification.success({
message: intl.get('升级成功'),
description: intl.get('已重新启动并运行新版本,无需再次重启。'),
message: intl.get(
result.data.migrationRequired ? '升级器已更新' : '升级成功',
),
description: intl.get(
result.data.migrationRequired
? '当前仍运行旧版,请继续下载并重启以完成新版升级。'
: '已重新启动并运行新版本,无需再次重启。',
),
duration: 8,
});
}
+8
View File
@@ -758,6 +758,14 @@
"暂存更新已变化,请重新下载更新。": "The staged update changed. Download the update again.",
"启动版本与更新包不一致": "The running build does not match the update package.",
"下载完成,等待重启": "Download complete, restart to update",
"升级器已更新,请继续升级新版": "Upgrader updated; continue to the new version",
"升级器已更新": "Upgrader updated",
"当前仍运行旧版,请继续下载并重启以完成新版升级。": "The previous version is still running. Download the new version and restart to finish upgrading.",
"继续升级新版": "Continue upgrading",
"升级器已更新。第二步将下载并预启动新版,准备成功后再重启应用。": "The upgrader is updated. Step 2 downloads and tests the new backend before you restart to apply it.",
"下载新版": "Download new version",
"更新索引尚未提供第二步运行包,请稍后重试": "The update index does not yet provide the second-step runtime. Please try again later.",
"第二步更新包仍是旧数据库后端,已保留当前服务,请稍后重试": "The second-step package still uses the legacy database backend. Your current service is unchanged. Please try again later.",
"升级成功": "Update successful",
"已重新启动并运行新版本,无需再次重启。": "Restarted and running the new version. No further restart is needed."
}
+8
View File
@@ -758,6 +758,14 @@
"暂存更新已变化,请重新下载更新。": "暂存更新已变化,请重新下载更新。",
"启动版本与更新包不一致": "启动版本与更新包不一致",
"下载完成,等待重启": "下载完成,等待重启",
"升级器已更新,请继续升级新版": "升级器已更新,请继续升级新版",
"升级器已更新": "升级器已更新",
"当前仍运行旧版,请继续下载并重启以完成新版升级。": "当前仍运行旧版,请继续下载并重启以完成新版升级。",
"继续升级新版": "继续升级新版",
"升级器已更新。第二步将下载并预启动新版,准备成功后再重启应用。": "升级器已更新。第二步将下载并预启动新版,准备成功后再重启应用。",
"下载新版": "下载新版",
"更新索引尚未提供第二步运行包,请稍后重试": "更新索引尚未提供第二步运行包,请稍后重试",
"第二步更新包仍是旧数据库后端,已保留当前服务,请稍后重试": "第二步更新包仍是旧数据库后端,已保留当前服务,请稍后重试",
"升级成功": "升级成功",
"已重新启动并运行新版本,无需再次重启。": "已重新启动并运行新版本,无需再次重启。"
}
+26 -4
View File
@@ -35,6 +35,18 @@ const CheckUpdate = ({ systemInfo }: any) => {
};
const checkUpgrade = async () => {
if (job.migrationRequired) {
Modal.confirm({
title: intl.get('继续升级新版'),
content: intl.get(
'升级器已更新。第二步将下载并预启动新版,准备成功后再重启应用。',
),
okText: intl.get('下载新版'),
cancelText: intl.get('以后再说'),
onOk: startUpdate,
});
return;
}
if (!upgradeStore.beginMutation()) return;
try {
const { code, data } = await request.put(
@@ -101,7 +113,9 @@ const CheckUpdate = ({ systemInfo }: any) => {
});
const label = intl.get(upgradeStatusLabel(job));
const hasJob = job.status !== 'idle' && job.status !== 'loading';
const hasJob =
Boolean(job.migrationRequired) ||
(job.status !== 'idle' && job.status !== 'loading');
const actions = !active && (
<Space wrap>
<Button
@@ -110,9 +124,15 @@ const CheckUpdate = ({ systemInfo }: any) => {
disabled={!connected || job.status === 'loading'}
onClick={ready ? showReloadModal : checkUpgrade}
>
{intl.get(ready ? '重启并更新' : '检查更新')}
{intl.get(
ready
? '重启并更新'
: job.migrationRequired
? '继续升级新版'
: '检查更新',
)}
</Button>
{!ready && !completed && (
{!ready && !completed && !job.migrationRequired && (
<Button
disabled={busy || !connected || job.status === 'loading'}
onClick={() => reloadSystem(false)}
@@ -148,7 +168,9 @@ const CheckUpdate = ({ systemInfo }: any) => {
? intl.get(job.error || '更新失败,请检查网络及日志或稍后再试') ||
job.error
: intl.get(
completed
job.migrationRequired && !ready && !active
? '当前仍运行旧版,请继续下载并重启以完成新版升级。'
: completed
? '已重新启动并运行新版本,无需再次重启。'
: ready
? '更新已就绪,重启后应用新版本'
+16 -15
View File
@@ -14,7 +14,6 @@ function temporary(t) {
t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
return dir;
}
test('environment values remain literal when the generated JavaScript loads', async () => {
const output = {};
const Env = load(path.resolve('back/services/env.ts'), {
@@ -42,7 +41,6 @@ test('environment values remain literal when the generated JavaScript loads', as
assert.equal(context.process.env[`VALUE_${i}`], value),
);
});
test('subscription shell parameters round trip without expansion or command substitution', () => {
const { formatCommand } = load(path.resolve('back/config/subscription.ts'));
const payload =
@@ -83,7 +81,6 @@ test('subscription shell parameters round trip without expansion or command subs
}
}
});
test('SSH alias traversal and symlinks cannot write or remove outside files', async (t) => {
const base = temporary(t);
const root = path.join(base, 'ssh');
@@ -141,14 +138,17 @@ test('SSH alias traversal and symlinks cannot write or remove outside files', as
service.generateSingleSshConfig('safe', 'host', '$(printf BAD)'),
);
});
function cronService(config, updates = []) {
const Cron = load(path.resolve('back/services/cron.ts'), {
typedi: decorators,
'../config': config,
'../data/cron': {
CrontabStatus: { running: 0, idle: 1, queued: 3 },
CrontabModel: { update: async (v) => updates.push(v) },
CrontabModel: {
nativeUpdate: async (where, v) => {
return updates.push(v);
},
},
},
'../data/runningInstance': { RunningInstanceModel: {}, InstanceStatus: {} },
'../config/util': { fileExist: async (p) => fs.existsSync(p) },
@@ -183,7 +183,6 @@ function subscriptionService(config, model = {}) {
}).default;
return new Subscription(logger, {}, {}, {}, {});
}
test('stored cron/subscription paths cannot read or enumerate outside the log root', async (t) => {
const base = temporary(t);
const root = path.join(base, 'log');
@@ -224,7 +223,6 @@ test('stored cron/subscription paths cannot read or enumerate outside the log ro
(await subscription.logs(1)).some((x) => x.filename === 'normal.log'),
);
});
test('subscription aliases are checked before persistence and forced deletion', async (t) => {
const base = temporary(t);
for (const dir of ['scripts', 'repo']) fs.mkdirSync(path.join(base, dir));
@@ -235,8 +233,12 @@ test('subscription aliases are checked before persistence and forced deletion',
repoPath: path.join(base, 'repo'),
},
{
findAll: async () => [{ alias: '../victim' }],
destroy: async () => writes++,
find: async (where, read = {}) => {
return [{ alias: '../victim' }];
},
nativeDelete: async (where, read = {}) => {
return writes++;
},
},
);
service.insert = service.updateDb = async () => writes++;
@@ -254,14 +256,17 @@ test('subscription aliases are checked before persistence and forced deletion',
);
assert.equal(writes, 0);
});
function systemService(config, overrides = {}) {
const System = load(path.resolve('back/services/system.ts'), {
typedi: decorators,
'../config': config,
'../config/const': {},
'../data/dependence': {
DependenceModel: { findAll: async () => [] },
DependenceModel: {
find: async (where, read = {}) => {
return [];
},
},
DependenceTypes: {},
DependenceStatus: {},
},
@@ -277,7 +282,6 @@ function systemService(config, overrides = {}) {
}).default;
return new System(logger, {}, {});
}
test('backup rejects unrecognized paths before spawning; valid export uses argv without a shell', async (t) => {
const base = temporary(t);
const calls = [];
@@ -333,7 +337,6 @@ test('backup rejects unrecognized paths before spawning; valid export uses argv
assert.equal(calls[0][2].shell, undefined);
assert.equal(downloaded, true);
});
test('proxy, language and mirror settings stay literal at their shell boundaries', async (t) => {
const base = temporary(t);
const commands = [];
@@ -373,7 +376,6 @@ test('proxy, language and mirror settings stay literal at their shell boundaries
}
assert.equal((await service.reloadSystem('$(printf BAD)')).code, 400);
});
test('the ignored-log sentinel remains supported without exposing device files', async (t) => {
const root = temporary(t);
const cron = cronService({ logPath: root });
@@ -385,7 +387,6 @@ test('the ignored-log sentinel remains supported without exposing device files',
cron.status({ ids: [], status: 0, log_path: '/dev/zero' }),
);
});
test('normal backup creates an archive containing the selected data directories', async (t) => {
const base = temporary(t);
const dataPath = path.join(base, 'data');
+129
View File
@@ -0,0 +1,129 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const path = require('node:path');
const os = require('node:os');
const fs = require('node:fs/promises');
const express = require('express');
const jwt = require('jsonwebtoken');
const load = require('../helpers/load-security-module.cjs');
test('authentication cache propagates database read and write failures', async () => {
const failure = new Error('synthetic unavailable database');
const CacheStore = load(path.resolve('back/data/cacheStore.ts'), {
'.': {
database: {
request: async () => {
throw failure;
},
},
},
}).CacheStore;
const { shareStore } = load(path.resolve('back/shared/store.ts'), {
'../data/cacheStore': { CacheStore },
});
await assert.rejects(shareStore.getAuthInfo(), (error) => error === failure);
await assert.rejects(
shareStore.updateAuthInfo({ username: 'qa' }),
(error) => error === failure,
);
});
test('HTTP authentication distinguishes unavailable storage from an invalid session', async (t) => {
const root = await fs.mkdtemp(
path.join(os.tmpdir(), 'ql-auth-availability-'),
);
const secret = 'synthetic-auth-availability';
const token = jwt.sign({}, secret, { algorithm: 'HS384', expiresIn: '1h' });
const auth = { username: 'qa', password: 'configured', token };
let unavailable = false;
let initialized = 0;
const read = async (value) => {
if (unavailable) throw new Error(`SQL failure must not expose ${token}`);
return value;
};
const router = express.Router();
router.get('/system/update-status', (_req, res) => res.json({ code: 200 }));
router.get('/envs', (_req, res) => res.json({ code: 200 }));
router.get('/health', (_req, res) => res.json({ code: 200 }));
router.put('/user/init', (_req, res) => {
initialized++;
res.json({ code: 200 });
});
const app = express();
load(path.resolve('back/loaders/express.ts'), {
'../config': {
rootPath: root,
uploadPath: path.join(root, 'upload'),
configPath: path.join(root, 'config'),
api: { prefix: '/api' },
apiWhiteList: ['/api/health', '/api/user/init'],
jwt: { secret },
},
'../api': () => router,
'../config/util': {
getToken: (req) =>
(req.headers.authorization || '').replace(/^Bearer /, ''),
getPlatform: () => 'desktop',
},
'../shared/store': {
shareStore: {
getAuthInfo: () => read(auth),
getApps: () =>
read([
{
scopes: ['envs'],
tokens: [
{ value: 'app-token', expiration: Date.now() / 1000 + 3600 },
],
},
]),
},
},
'../shared/i18n': { t: (value) => value },
'../config/serverEnv': { serveEnv: (_req, res) => res.end() },
'../data/open': {},
'../data/system': {},
}).default({ app });
const server = app.listen(0, '127.0.0.1');
t.after(async () => {
await new Promise((resolve) => server.close(resolve));
await fs.rm(root, { recursive: true, force: true });
});
await new Promise((resolve) => server.once('listening', resolve));
const request = async (route, value, method = 'GET') => {
const response = await fetch(
`http://127.0.0.1:${server.address().port}${route}`,
{
method,
headers: value ? { Authorization: `Bearer ${value}` } : {},
signal: AbortSignal.timeout(3000),
},
);
return { status: response.status, body: await response.text() };
};
assert.equal((await request('/api/system/update-status', token)).status, 200);
unavailable = true;
for (const [route, value, method] of [
['/api/system/update-status', token],
['/open/envs', 'app-token'],
['/api/user/init', undefined, 'PUT'],
]) {
const response = await request(route, value, method);
assert.equal(response.status, 503);
assert.equal(JSON.parse(response.body).code, 503);
assert.ok(!response.body.includes(token));
assert.ok(!response.body.includes('SQL failure'));
}
assert.equal(initialized, 0);
assert.equal((await request('/api/health')).status, 200);
unavailable = false;
assert.equal((await request('/api/system/update-status', token)).status, 200);
const invalid = jwt.sign({ unrecognized: true }, secret, {
algorithm: 'HS384',
expiresIn: '1h',
});
assert.equal(
(await request('/api/system/update-status', invalid)).status,
401,
);
});
+69
View File
@@ -0,0 +1,69 @@
const assert = require('node:assert/strict');
const test = require('node:test');
const express = require('express');
const { DatabaseClient } = require('../../back/data/databaseClient');
const { Repository } = require('../../back/data/repository');
const { initialize } = require('../helpers/repository-fixture.cjs');
function mockModule(modulePath, exports) {
const filename = require.resolve(modulePath);
require.cache[filename] = {
id: filename,
filename,
loaded: true,
exports,
children: [],
paths: [],
};
}
test('instance API retains newest-first ordering independently of insertion order', async (t) => {
const db = new DatabaseClient(':memory:');
t.after(() => db.close());
await initialize(db);
const instances = new Repository('RunningInstance', () => db);
for (const [cron_id, started_at] of [
[7, 200],
[7, 300],
[8, 900],
[7, 100],
]) {
await instances.insert({ cron_id, started_at, status: 2 });
}
for (const service of ['cron', 'cronView']) {
mockModule(`../../back/services/${service}`, {
__esModule: true,
default: class {},
});
}
mockModule('../../back/schedule/client', { __esModule: true, default: {} });
mockModule('../../back/shared/i18n', { t: (message) => message });
mockModule('../../back/data/runningInstance', {
RunningInstanceModel: instances,
InstanceStatus: { running: 0 },
});
const app = express.Router();
require('../../back/api/cron').default(app);
const router = app.stack.find((layer) => layer.name === 'router').handle;
const handler = router.stack
.find((layer) => layer.route?.path === '/:id/instances')
.route.stack.at(-1).handle;
let response;
await handler(
{ params: { id: 7 } },
{
send: (body) => {
response = body;
},
},
(error) => {
throw error;
},
);
assert.equal(response.code, 200);
assert.deepEqual(
response.data.map((row) => row.started_at),
[300, 200, 100],
);
assert(response.data.every((row) => row.cron_id === 7));
});
+69 -12
View File
@@ -10,15 +10,40 @@ test('system and node modes route extended cron once and keep portable rules in
const end = source.indexOf(' private async getLogName', begin);
const saveBegin = source.indexOf(' private async setCrontab(');
const saveEnd = source.indexOf(' public importCrontab(', saveBegin);
const js = ts.transpileModule(`class Fixture {${source.slice(begin, end)}${source.slice(saveBegin, saveEnd)}}; module.exports = Fixture;`, {
compilerOptions: { target: ts.ScriptTarget.ES2020 },
}).outputText;
const js = ts.transpileModule(
`class Fixture {${source.slice(begin, end)}${source.slice(
saveBegin,
saveEnd,
)}}; module.exports = Fixture;`,
{
compilerOptions: { target: ts.ScriptTarget.ES2020 },
},
).outputText;
for (const mode of ['system', 'node']) {
const module = { exports: {} };
let file = '', installs = 0;
new Function('module', 'process', 'execSync', 'ScheduleType', 'config', 'writeFileWithLock', 'CrontabModel', js)(
module, { env: { QL_SCHEDULER: mode } }, () => { installs++; }, ScheduleType,
{ crontabFile: '/unused' }, async (_, data) => { file = data; }, { update: async () => {} },
let file = '',
installs = 0;
new Function(
'module',
'process',
'execSync',
'ScheduleType',
'config',
'writeFileWithLock',
'CrontabModel',
js,
)(
module,
{ env: { QL_SCHEDULER: mode } },
() => {
installs++;
},
ScheduleType,
{ crontabFile: '/unused' },
async (_, data) => {
file = data;
},
{ nativeUpdate: async () => {} },
);
const service = new module.exports();
service.makeCommand = (row) => `task ${row.id}.js`;
@@ -40,20 +65,52 @@ test('system and node modes route extended cron once and keep portable rules in
{ id: 'named', schedule: '0 0 * JAN MON' },
{ id: 'macro', schedule: '@daily' },
{ id: 'short', schedule: '*' },
{ id: 'extra', schedule: '* * * * *', extra_schedules: [{ schedule: '0 * * * *' }] },
{
id: 'extra',
schedule: '* * * * *',
extra_schedules: [{ schedule: '0 * * * *' }],
},
{ id: 'once', schedule: '@once' },
{ id: 'boot', schedule: '@boot' },
];
for (const row of rows) {
const special = ['once', 'boot'].includes(row.id);
const portable = ['plain', 'spaces'].includes(row.id);
assert.equal(service.shouldUseCronClient(row), !special && (mode === 'node' || !portable), `${mode}: ${row.id}`);
assert.equal(
service.shouldUseCronClient(row),
!special && (mode === 'node' || !portable),
`${mode}: ${row.id}`,
);
}
await service.setCrontab({ data: rows, total: rows.length });
assert.equal(installs, mode === 'system' ? 1 : 0);
for (const id of ['numeric-step', 'sunday-seven', 'day-step', 'restricted-days', 'restricted-week', 'annually', 'midnight', 'minutely', 'seconds', 'question', 'last', 'nth', 'named', 'macro', 'short', 'extra', 'once', 'boot']) {
const line = file.split('\n').find((line) => line.endsWith(`task ${id}.js`));
assert.ok(line.startsWith('# '), `${mode}: ${id} cannot also run from system crontab`);
for (const id of [
'numeric-step',
'sunday-seven',
'day-step',
'restricted-days',
'restricted-week',
'annually',
'midnight',
'minutely',
'seconds',
'question',
'last',
'nth',
'named',
'macro',
'short',
'extra',
'once',
'boot',
]) {
const line = file
.split('\n')
.find((line) => line.endsWith(`task ${id}.js`));
assert.ok(
line.startsWith('# '),
`${mode}: ${id} cannot also run from system crontab`,
);
}
}
});
+12 -21
View File
@@ -1,28 +1,19 @@
const { DatabaseClient } = require('../../back/data/databaseClient');
const { repository, initialize } = require('../helpers/repository-fixture.cjs');
const { orderBy } = require('../../back/data/queries');
require('ts-node/register/transpile-only');
const assert = require('node:assert/strict');
const { test } = require('node:test');
const { Sequelize, DataTypes } = require('sequelize');
const dayjs = require('dayjs');
const express = require('express');
test('today result lists include mixed and deleted tasks and exclude other dates', async (t) => {
const db = new Sequelize({
dialect: 'sqlite',
storage: ':memory:',
logging: false,
});
const db = new DatabaseClient(':memory:');
t.after(() => db.close());
const crons = db.define('Cron', {
name: DataTypes.STRING,
command: DataTypes.STRING,
});
const stats = db.define('Stat', {
ref_id: DataTypes.INTEGER,
date: DataTypes.STRING,
fail_count: DataTypes.INTEGER,
success_count: DataTypes.INTEGER,
});
const crons = repository(db, 'Crontab');
const stats = repository(db, 'CrontabStat');
const replacements = {
'../../back/data': { database: db },
'../../back/data/cron': { CrontabModel: crons },
'../../back/data/cronStats': { CrontabStatModel: stats },
'../../back/data/runningInstance': {},
@@ -42,14 +33,14 @@ test('today result lists include mixed and deleted tasks and exclude other dates
else delete require.cache[resolved];
});
}
await db.sync();
await crons.bulkCreate([
await initialize(db);
await crons.insertMany([
{ id: 1, name: 'Recovered task', command: 'task recovered.js' },
{ id: 2, name: '', command: 'task unnamed.js' },
{ id: 3, name: 'Successful task', command: 'task success.js' },
]);
const today = dayjs().format('YYYY-MM-DD');
await stats.bulkCreate([
await stats.insertMany([
{ ref_id: 1, date: today, fail_count: 2, success_count: 1 },
{ ref_id: 2, date: today, fail_count: 1, success_count: 0 },
{ ref_id: 3, date: today, fail_count: 0, success_count: 4 },
@@ -135,7 +126,7 @@ test('today result lists include mixed and deleted tasks and exclude other dates
},
],
});
await stats.destroy({ where: {} });
await stats.nativeDelete({});
await successHandler(
{},
{
@@ -160,7 +151,7 @@ test('today result lists include mixed and deleted tasks and exclude other dates
},
);
assert.deepEqual(response, { code: 200, data: [] });
await stats.drop();
await db.query('DROP TABLE CrontabStats');
let forwarded;
await handler(
{},
+222
View File
@@ -0,0 +1,222 @@
const test = require('node:test'),
assert = require('node:assert/strict');
const fs = require('node:fs/promises'),
path = require('node:path'),
os = require('node:os');
const {
Sequelize,
DataTypes,
Op,
QueryTypes,
where,
col,
} = require('sequelize');
const { DatabaseClient } = require('../../back/data/databaseClient');
const { Repository } = require('../../back/data/repository');
const { initialize } = require('../helpers/repository-fixture.cjs');
const names = {
App: 'Apps',
Auth: 'Auths',
Crontab: 'Crontabs',
CrontabStat: 'CrontabStats',
CrontabView: 'CrontabViews',
Dependence: 'Dependences',
Env: 'Envs',
RunningInstance: 'RunningInstances',
Subscription: 'Subscriptions',
};
async function legacy(file) {
const database = new Sequelize({
dialect: 'sqlite',
storage: file,
logging: false,
});
const models = {};
for (const [name, tableName] of Object.entries(names)) {
const columns = await database.query(`PRAGMA table_info("${tableName}")`, {
type: QueryTypes.SELECT,
});
const attrs = Object.fromEntries(
columns.map((c) => [
c.name,
{
type:
c.type === 'JSON'
? DataTypes.JSON
: c.type === 'DATETIME'
? DataTypes.DATE
: c.type === 'TINYINT(1)'
? DataTypes.BOOLEAN
: /INT|NUMBER/.test(c.type)
? DataTypes.NUMBER
: DataTypes.STRING,
...(c.pk ? { primaryKey: true, autoIncrement: true } : {}),
},
]),
);
models[name] = database.define(name, attrs, {
tableName,
freezeTableName: true,
});
}
return { database, models };
}
test('all nine legacy tables remain readable in both versions without changing schema, IDs, dates or .env', async (t) => {
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-orm-legacy-'));
t.after(() => fs.rm(dir, { recursive: true, force: true }));
const file = path.join(dir, 'database.sqlite'),
envFile = path.join(dir, '.env');
const environment = 'QL_DATA_DIR=/synthetic/old-data\nSYNTHETIC_ENV=keep\n';
await fs.writeFile(envFile, environment);
const seed = new Sequelize({
dialect: 'sqlite',
storage: file,
logging: false,
});
try {
const sql = await fs.readFile(
path.resolve('test/fixtures/sequelize-2.23-schema.sql'),
'utf8',
);
for (const statement of sql
.replace(/^--.*$/gm, '')
.split(';')
.map((x) => x.trim())
.filter(Boolean))
await seed.query(statement);
} finally {
await seed.close();
}
let old = await legacy(file);
const payloads = {
App: { name: 'legacy', scopes: ['envs'], client_id: 'synthetic-client' },
Auth: {
type: 'authConfig',
info: {
username: 'synthetic-user',
password: 'synthetic-hash',
tokens: { desktop: ['keep'] },
},
},
Crontab: {
name: 'legacy',
command: 'echo synthetic',
schedule: '0 0 * * *',
saved: true,
labels: ['中文'],
},
CrontabStat: { ref_id: 7, date: '2026-10-21', run_count: 2 },
CrontabView: {
name: 'legacy',
filters: [{ property: 'status', operation: 'In', value: [1] }],
},
Dependence: { name: 'legacy', type: 0, log: ['旧日志'] },
Env: {
name: 'legacy',
value: '旧值',
labels: ['中文'],
position: 4500000000000000,
},
RunningInstance: { cron_id: 7, started_at: 100, status: 2, exit_code: 143 },
Subscription: {
name: 'legacy',
alias: 'legacy',
pull_option: { username: 'synthetic', password: 'synthetic-only' },
},
};
const date = new Date('2020-01-02T03:04:05.006Z');
for (const [name, payload] of Object.entries(payloads))
await old.models[name].create({
id: 7,
...payload,
createdAt: date,
updatedAt: date,
});
const schema = await old.database.query(
"SELECT type,name,sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' ORDER BY type,name",
{ type: QueryTypes.SELECT },
);
await old.database.close();
const client = new DatabaseClient(file);
t.after(() => client.close());
await initialize(client);
for (const [name, payload] of Object.entries(payloads)) {
const repository = new Repository(name, () => client),
row = await repository.findOne({ id: 7 });
for (const [field, value] of Object.entries(payload))
assert.deepEqual(row[field], value, `${name}.${field}`);
assert.equal(row.createdAt.toISOString(), date.toISOString());
const added = await repository.insert({
...payload,
id: undefined,
...(payload.name ? { name: 'new' } : {}),
...(payload.alias ? { alias: 'new' } : {}),
...(payload.date ? { date: '2026-10-22' } : {}),
});
assert(added.id > 7, name);
assert(added.createdAt instanceof Date);
}
assert.deepEqual(
await client.query(
"SELECT type,name,sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' ORDER BY type,name",
),
schema,
);
await client.close();
old = await legacy(file);
try {
for (const name of Object.keys(payloads)) {
const rows = await old.models[name].findAll({ order: [['id', 'ASC']] });
assert.equal(rows.length, 2);
assert.equal(rows[0].createdAt.toISOString(), date.toISOString());
assert(rows[1].createdAt instanceof Date);
assert(!Number.isNaN(+rows[1].createdAt));
}
assert.equal(await fs.readFile(envFile, 'utf8'), environment);
} finally {
await old.database.close();
}
});
test('native filters retain legacy NULL, IN, nested groups and JSON text matching', async (t) => {
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-orm-filters-'));
t.after(() => fs.rm(dir, { recursive: true, force: true }));
const file = path.join(dir, 'database.sqlite');
const client = new DatabaseClient(file);
t.after(() => client.close());
await initialize(client);
const repository = new Repository('Env', () => client);
for (const row of [
{ name: 'one', value: 'a', status: 0, labels: ['中文', 'keep'] },
{ name: 'two', value: 'b', status: 1, labels: [] },
{ name: null, value: 'c', status: 0, labels: null },
])
await repository.insert(row);
const old = await legacy(file);
t.after(() => old.database.close());
const cases = [
[{ status: [0, 1] }, { status: [0, 1] }],
[{ name: { [Op.not]: null } }, { name: { $ne: null } }],
[
{ [Op.or]: [{ name: 'missing' }, {}] },
{ $or: [{ name: 'missing' }, {}] },
],
[{ [Op.and]: [{ status: 0 }, {}] }, { $and: [{ status: 0 }, {}] }],
[{ [Op.or]: [] }, { $or: [] }],
[
where(col('labels'), { [Op.like]: '%keep%' }),
{ labels: { $like: '%keep%' } },
],
[{ name: { [Op.notIn]: ['one'] } }, { name: { $nin: ['one'] } }],
];
for (const [before, after] of cases) {
const expected = (
await old.models.Env.findAll({ where: before, order: [['id', 'ASC']] })
).map((x) => x.id);
const actual = (
await repository.find(after, { orderBy: [{ id: 'asc' }] })
).map((x) => x.id);
assert.deepEqual(actual, expected, JSON.stringify(after));
}
});
@@ -0,0 +1,77 @@
const test = require('node:test'),
assert = require('node:assert/strict');
const fs = require('node:fs/promises'),
path = require('node:path'),
os = require('node:os'),
{ spawnSync } = require('node:child_process');
const { minimum, assertNodeRuntime } = require('../../shell/runtime-guard.cjs');
const { databaseStartupFailure } = require('../../back/data/databaseErrors');
const {
dependencyFailure,
} = require('../../cli/src/internal/maintenance/upgradePrestart.cjs');
test('runtime policy matches package metadata and accepts the exact Node minimum', () => {
assert.equal(minimum, require('../../package.json').qinglongRuntimeMinimum);
for (const version of ['22.17.0', '22.23.2', '24.18.1', '26.0.0'])
assert.doesNotThrow(() => assertNodeRuntime(version));
for (const version of ['20.20.2', '22.16.9', 'invalid'])
assert.throws(() => assertNodeRuntime(version), {
code: 'UNSUPPORTED_NODE',
});
});
test('database startup repairs only third-party dependency import failures', () => {
for (const message of [
"Cannot find module '@mikro-orm/sql'",
"Cannot find module '/candidate/node_modules/@mikro-orm/core/missing.js'",
"Cannot find module './missing.js'\nRequire stack:\n- /candidate/node_modules/@mikro-orm/sql/index.js",
]) {
const result = databaseStartupFailure(
Object.assign(new Error(message), { code: 'MODULE_NOT_FOUND' }),
);
assert.equal(dependencyFailure(result), true);
}
for (const value of [
new Error('heap out of memory'),
undefined,
Object.assign(
new Error(
"Cannot find module './schema'\nRequire stack:\n- /candidate/static/build/data/databaseWorker.js",
),
{ code: 'MODULE_NOT_FOUND' },
),
Object.assign(new Error("Cannot find module 'node:sqlite'"), {
code: 'MODULE_NOT_FOUND',
}),
])
assert.equal(dependencyFailure(databaseStartupFailure(value)), false);
});
test('Node 20 rejects backend initialization before changing its old database or .env', async (t) => {
const node20 = process.env.QL_TEST_NODE20;
if (!node20) {
t.skip('Set QL_TEST_NODE20 to verify an existing Node 20 runtime');
return;
}
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-runtime-floor-'));
t.after(() => fs.rm(dir, { recursive: true, force: true }));
const data = path.join(dir, 'data'),
db = path.join(data, 'db/database.sqlite'),
env = path.join(dir, '.env');
await fs.mkdir(path.dirname(db), { recursive: true });
await fs.writeFile(db, 'old synthetic database marker');
await fs.writeFile(env, 'SYNTHETIC_ENV=keep\n');
const result = spawnSync(
node20,
[path.resolve('static/build/bootstrap/database.js')],
{
env: { ...process.env, QL_DIR: dir, QL_DATA_DIR: data },
encoding: 'utf8',
timeout: 10000,
},
);
assert.equal(result.status, 1, result.stderr);
assert.match(result.stderr, /Node >=22\.17/);
assert.equal(await fs.readFile(db, 'utf8'), 'old synthetic database marker');
assert.equal(await fs.readFile(env, 'utf8'), 'SYNTHETIC_ENV=keep\n');
});
+515
View File
@@ -0,0 +1,515 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs/promises');
const path = require('node:path');
const os = require('node:os');
const { DatabaseClient } = require('../../back/data/databaseClient');
const { Repository } = require('../../back/data/repository');
const { initialSchema } = require('../../back/data/schema');
const { migrateSchema } = require('../../back/shared/schemaMigrations');
test('requests buffered before bootstrap and during owner-monitor startup retain their order', async (t) => {
const { fork } = require('node:child_process');
const directory = await fs.mkdtemp(
path.join(os.tmpdir(), 'ql-db-bootstrap-queue-'),
);
const script = path.join(directory, 'delayed.ts');
await fs.writeFile(
script,
`setTimeout(()=>require(${JSON.stringify(
require.resolve('../../back/data/databaseProcess'),
)}),100)`,
);
const child = fork(script, [], {
execArgv: [
'--require',
require.resolve('ts-node/register/transpile-only'),
'--max-old-space-size=128',
],
env: {
...process.env,
TS_NODE_PROJECT: path.resolve('back/data/tsconfig.worker.json'),
},
stdio: ['ignore', 'ignore', 'pipe', 'ipc'],
});
const stopped = new Promise((resolve) => child.once('exit', resolve));
let stderr = '';
child.stderr.on('data', (chunk) => {
stderr += chunk;
});
t.after(async () => {
child.kill('SIGKILL');
await stopped;
await fs.rm(directory, { recursive: true, force: true });
});
const responses = new Promise((resolve, reject) => {
const replies = [];
const timer = setTimeout(
() => reject(new Error('Buffered startup requests were lost')),
30000,
);
child.on('message', (reply) => {
if (reply.error) {
clearTimeout(timer);
reject(new Error(reply.error.message));
return;
}
replies.push(reply);
if (replies.length === 2) {
clearTimeout(timer);
resolve(replies);
}
});
child.once('exit', () => {
clearTimeout(timer);
reject(new Error('Database exited during bootstrap: ' + stderr));
});
child.once('error', reject);
});
child.send({ storage: ':memory:', ownerPid: process.pid });
child.send({ id: 1, operation: 'query', args: ['SELECT 42 AS value'] });
child.send({ id: 2, operation: 'query', args: ['SELECT 43 AS value'] });
assert.deepEqual(await responses, [
{ id: 1, value: [{ value: 42 }] },
{ id: 2, value: [{ value: 43 }] },
]);
});
test('killing the owner during synchronous SQL releases the write lock and rolls back before restart', async (t) => {
const { fork } = require('node:child_process');
const { DatabaseSync } = require('node:sqlite');
const directory = await fs.mkdtemp(
path.join(os.tmpdir(), 'ql-db-owner-exit-'),
);
const file = path.join(directory, 'data.sqlite');
const script = path.join(directory, 'owner.cjs');
await fs.writeFile(
script,
`
const {DatabaseClient}=require(${JSON.stringify(
require.resolve('../../back/data/databaseClient'),
)});
const db=new DatabaseClient(process.argv[2]);
(async()=>{
await db.query('CREATE TABLE probe(value INTEGER)');
await db.query('INSERT INTO probe VALUES(1)');
await db.transaction(async()=>{
await db.query('UPDATE probe SET value=2');
const running=db.query('WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<200000000000) SELECT SUM(x) FROM n');
setTimeout(()=>process.send({databasePid:db.worker.pid}),300);
await running;
});
})().catch(error=>{console.error(error);process.exitCode=1});
`,
);
const owner = fork(script, [file], {
execArgv: ['--require', require.resolve('ts-node/register/transpile-only')],
env: {
...process.env,
TS_NODE_PROJECT: path.resolve('back/tsconfig.json'),
},
stdio: ['ignore', 'ignore', 'pipe', 'ipc'],
});
const exited = new Promise((resolve) => owner.once('exit', resolve));
let databasePid,
probe,
stderr = '',
unlocked = false;
owner.stderr.on('data', (chunk) => {
stderr += chunk;
});
t.after(async () => {
probe?.close();
owner.kill('SIGKILL');
await exited;
if (databasePid && !unlocked) {
try {
process.kill(databasePid, 'SIGKILL');
} catch (error) {
if (error.code !== 'ESRCH') throw error;
}
}
await fs.rm(directory, { recursive: true, force: true });
});
databasePid = await new Promise((resolve, reject) => {
const timer = setTimeout(
() => reject(new Error('Owner startup timed out: ' + stderr)),
30000,
);
owner.once('message', (message) => {
clearTimeout(timer);
resolve(message.databasePid);
});
owner.once('exit', () => {
clearTimeout(timer);
reject(new Error('Owner exited before SQL: ' + stderr));
});
owner.once('error', reject);
});
probe = new DatabaseSync(file);
probe.exec('PRAGMA busy_timeout=50');
assert.throws(() => probe.exec('BEGIN IMMEDIATE'), /locked/);
owner.kill('SIGKILL');
await exited;
const deadline = Date.now() + 5000;
do {
try {
probe.exec('BEGIN IMMEDIATE;ROLLBACK');
unlocked = true;
break;
} catch (error) {
if (!/locked/.test(error.message)) throw error;
}
await new Promise((resolve) => setTimeout(resolve, 50));
} while (Date.now() < deadline);
assert.equal(
unlocked,
true,
'The orphan database process still blocks the old service',
);
assert.equal(probe.prepare('SELECT value FROM probe').get().value, 1);
});
async function fixture(t, file = ':memory:') {
const db = new DatabaseClient(file);
t.after(() => db.close());
await db.transaction(async () => {
for (const sql of initialSchema) await db.query(sql);
await migrateSchema(db);
});
return {
db,
env: new Repository('Env', () => db),
auth: new Repository('Auth', () => db),
};
}
test('direct ts-node CLI startup registers the isolated database worker compiler', async () => {
const execute = require('node:util').promisify(
require('node:child_process').execFile,
);
const result = await execute(
process.execPath,
[
path.resolve('node_modules/ts-node/dist/bin.js'),
'--transpile-only',
'--project',
'back/tsconfig.json',
'-e',
`const {DatabaseClient}=require('./back/data/databaseClient');const db=new DatabaseClient(':memory:');db.query('SELECT 42 AS value').then(async rows=>{console.log(JSON.stringify(rows));await db.close()}).catch(async error=>{console.error(error);process.exitCode=1;await db.close().catch(()=>{})})`,
],
{ cwd: path.resolve('.'), timeout: 30000 },
);
assert.deepEqual(JSON.parse(result.stdout), [{ value: 42 }]);
});
test('worker round-trips legacy field names, JSON, dates, null and precise positions', async (t) => {
const { db, env } = await fixture(t);
const row = await env.insert({
name: '中文',
value: 'keep',
position: 4500000000000000,
labels: ['old', '中文'],
remarks: null,
});
assert.deepEqual((await env.findOne({ id: row.id })).labels, row.labels);
assert.equal(row.position, 4500000000000000);
assert.equal(row.remarks, null);
assert(row.createdAt instanceof Date);
const stored = (
await db.query('SELECT typeof(createdAt) AS type,createdAt FROM Envs')
)[0];
assert.equal(stored.type, 'text');
assert.match(
stored.createdAt,
/^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\.\d{3} \+00:00$/,
);
assert.equal(
(await db.query('SELECT count(*) AS n FROM SchemaMigrations'))[0].n,
15,
);
});
test('JSON LIKE uses serialized text without quoting the pattern', async (t) => {
const { env } = await fixture(t);
await env.insert({ name: 'a', value: 'b', labels: ['中文', 'old'] });
assert.equal((await env.find({ labels: { $like: '%old%' } })).length, 1);
assert.equal(
(await env.find({ $not: { labels: { $like: '%old%' } } })).length,
0,
);
});
test('transaction commit, failure and nested savepoints preserve ownership', async (t) => {
const { db, env } = await fixture(t);
await env.insert({ name: 'a', value: 'old' });
await db.transaction(async () => {
await env.nativeUpdate({ name: 'a' }, { value: 'committed' });
await assert.rejects(
db.transaction(async () => {
await env.nativeUpdate({ name: 'a' }, { value: 'rollback' });
throw Error('injected');
}),
/injected/,
);
assert.equal((await env.findOne({ name: 'a' })).value, 'committed');
});
await assert.rejects(
db.transaction(async () => {
await env.nativeUpdate({ name: 'a' }, { value: 'rollback all' });
throw Error('outer failure');
}),
/outer failure/,
);
assert.equal((await env.findOne({ name: 'a' })).value, 'committed');
});
test('unrelated reads wait for rollback and cannot see uncommitted data', async (t) => {
const { db, env } = await fixture(t);
await env.insert({ name: 'a', value: 'old' });
let release, writing;
const started = new Promise((r) => {
writing = r;
});
const gate = new Promise((r) => {
release = r;
});
const transaction = db.transaction(async () => {
await env.nativeUpdate({ name: 'a' }, { value: 'uncommitted' });
writing();
await gate;
throw Error('rollback');
});
const rejected = assert.rejects(transaction, /rollback/);
await started;
const read = env.findOne({ name: 'a' });
release();
await rejected;
assert.equal((await read).value, 'old');
});
test('expired async transaction scopes cannot write after commit', async (t) => {
const { db, env } = await fixture(t);
// Capture an asynchronous resource inside the scope, then release it afterwards.
let release, result;
const gate = new Promise((r) => {
release = r;
});
await db.transaction(async () => {
result = gate.then(() => env.insert({ name: 'late' }));
});
const rejected = assert.rejects(result, { code: 'TRANSACTION_SCOPE' });
release();
await rejected;
assert.equal(await env.count(), 0);
});
test('conditional updates keep compare-and-swap counts and do not clear undefined data', async (t) => {
const { env } = await fixture(t);
const row = await env.insert({ name: 'a', value: 'old' });
assert.equal(
await env.nativeUpdate({ id: row.id, value: 'newer' }, { value: 'bad' }),
0,
);
assert.equal(
await env.nativeUpdate(
{ id: row.id, value: 'old' },
{ value: 'new', remarks: undefined },
),
1,
);
assert.equal((await env.findOne({ id: row.id })).value, 'new');
await assert.rejects(env.nativeUpdate({ id: undefined }, { value: 'bad' }));
});
test('missing predicates cannot become unrestricted writes or disappear inside an IN filter', async (t) => {
const { env } = await fixture(t);
const row = await env.insert({ name: 'retained', value: 'old' });
for (const where of [undefined, null, { id: { $in: [row.id, undefined] } }]) {
await assert.rejects(env.nativeUpdate(where, { value: 'must not write' }));
await assert.rejects(env.nativeDelete(where));
}
assert.equal(await env.count(), 1);
assert.equal((await env.findOne({ id: row.id })).value, 'old');
});
test('worker failures reject writes, never replay them, and roll back SQLite', async (t) => {
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-db-worker-'));
t.after(() => fs.rm(dir, { recursive: true, force: true }));
const file = path.join(dir, 'database.sqlite');
const { db, env } = await fixture(t, file);
await env.insert({ name: 'a', value: 'old' });
await assert.rejects(
db.transaction(async () => {
await env.nativeUpdate({ name: 'a' }, { value: 'uncommitted' });
db.worker.kill('SIGKILL');
await db.stopped;
await env.count();
}),
);
assert.equal(db.healthy, false);
await assert.rejects(env.insert({ name: 'replayed' }));
const reopened = new DatabaseClient(file);
t.after(() => reopened.close());
const records = new Repository('Env', () => reopened);
assert.equal((await records.findOne({ name: 'a' })).value, 'old');
assert.equal(await records.count(), 1);
});
test('a long database query leaves main-thread timers responsive', async (t) => {
const { db } = await fixture(t);
const started = performance.now();
const expensive = db.query(
'WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<2000000) SELECT SUM(x) AS n FROM n',
);
const timer = await new Promise((r) =>
setTimeout(() => r(performance.now() - started), 10),
);
assert(timer < 150, `main-thread timer delayed ${timer}ms`);
assert.equal((await expensive)[0].n, 2000001000000);
});
test('existing Keyv authentication cache stays byte-compatible with legacy readers', async (t) => {
const { DatabaseSync } = require('node:sqlite');
const Keyv = require('keyv').default;
const { CacheStore } = require('../../back/data/cacheStore');
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-cache-compat-'));
t.after(() => fs.rm(dir, { recursive: true, force: true }));
const cacheFile = path.join(dir, 'keyv.sqlite');
const old = new DatabaseSync(cacheFile);
old.exec('CREATE TABLE keyv(key VARCHAR(255) PRIMARY KEY,value TEXT)');
const value = {
username: 'synthetic-user',
password: 'synthetic-hash',
tokens: { desktop: ['keep'] },
};
old
.prepare('INSERT INTO keyv VALUES(?,?)')
.run('keyv:authInfo', JSON.stringify({ value, expires: null }));
old.close();
const db = new DatabaseClient(':memory:', cacheFile);
t.after(() => db.close());
const cache = new Keyv({ store: new CacheStore(() => db) });
assert.deepEqual(await cache.get('authInfo'), value);
const next = { ...value, username: 'new-synthetic-user' };
await cache.set('authInfo', next);
const legacy = new DatabaseSync(cacheFile, { readOnly: true });
try {
assert.deepEqual(
JSON.parse(
legacy
.prepare('SELECT value FROM keyv WHERE key=?')
.get('keyv:authInfo').value,
).value,
next,
);
} finally {
legacy.close();
}
});
test('optional DTO fields and nested JSON values may be undefined on insert', async (t) => {
const { env, auth } = await fixture(t);
const created = await env.insert({
name: 'optional',
value: 'keep',
id: undefined,
remarks: undefined,
});
assert.equal(created.value, 'keep');
const row = await auth.insert({
type: 'notifyLog',
info: { title: 'keep', taskId: undefined },
});
assert.deepEqual(row.info, { title: 'keep' });
});
test('legacy null pin flags sort like zero flags before comparing positions', async (t) => {
const { env } = await fixture(t);
await env.insert({ name: 'null', position: 20, isPinned: null });
await env.insert({ name: 'zero', position: 10, isPinned: 0 });
await env.insert({ name: 'pinned', position: 1, isPinned: 1 });
assert.deepEqual(
(
await env.find(
{},
{ orderBy: [{ isPinned: 'desc' }, { position: 'desc' }] },
)
).map((x) => x.name),
['pinned', 'null', 'zero'],
);
});
test('a stalled transaction callback times out and rolls back before unrelated reads', async (t) => {
const { db, env } = await fixture(t);
db.timeoutMs = 100;
await env.insert({ name: 'a', value: 'old' });
let release;
const gate = new Promise((r) => {
release = r;
});
await assert.rejects(
db.transaction(async () => {
await env.nativeUpdate({ name: 'a' }, { value: 'uncommitted' });
await gate;
await env.insert({ name: 'too-late' });
}),
{ code: 'DATABASE_TRANSACTION_TIMEOUT' },
);
release();
assert.equal((await env.findOne({ name: 'a' })).value, 'old');
assert.equal(await env.count(), 1);
assert.equal(db.healthy, true);
});
test('a stuck database worker is terminated, rejects its queue and never replays writes', async (t) => {
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-db-timeout-'));
t.after(() => fs.rm(dir, { recursive: true, force: true }));
const file = path.join(dir, 'database.sqlite');
const { db, env } = await fixture(t, file);
await env.insert({ name: 'a', value: 'old' });
db.timeoutMs = 100;
await assert.rejects(
db.transaction(async () => {
await env.nativeUpdate({ name: 'a' }, { value: 'uncommitted' });
await db.query(
'WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<200000000) SELECT SUM(x) FROM n',
);
}),
(e) =>
['DATABASE_TRANSACTION_TIMEOUT', 'DATABASE_WORKER_TIMEOUT'].includes(
e.code,
),
);
assert.equal(db.healthy, false);
await assert.rejects(env.insert({ name: 'replayed' }));
await db.close();
const next = new DatabaseClient(file);
t.after(() => next.close());
const records = new Repository('Env', () => next);
assert.equal((await records.findOne({ name: 'a' })).value, 'old');
assert.equal(await records.count(), 1);
});
test('the database queue is bounded while a transaction owns its connection', async (t) => {
const { db, env } = await fixture(t);
db.queueLimit = 2;
db.timeoutMs = 3000;
let entered, release;
const started = new Promise((r) => {
entered = r;
});
const gate = new Promise((r) => {
release = r;
});
const transaction = db.transaction(async () => {
entered();
await gate;
});
await started;
const waiting = env.count();
await assert.rejects(env.count(), { code: 'DATABASE_BUSY' });
release();
await transaction;
assert.equal(await waiting, 0);
});
+2 -9
View File
@@ -1,6 +1,5 @@
const assert = require('node:assert/strict');
const test = require('node:test');
test('dependency listing marks cache entries missing from disk for reinstall', async (t) => {
const moduleStubs = new Map();
const stubModule = (modulePath, exports) => {
@@ -15,7 +14,6 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
paths: [],
};
};
const DependenceStatus = {
installing: 0,
installed: 1,
@@ -28,7 +26,7 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
];
const updates = [];
const DependenceModel = {
findAll: async ({ where }) => {
find: async (where, read = {}) => {
if (where.status === DependenceStatus.installed) {
return docs;
}
@@ -39,11 +37,10 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
: doc.status,
}));
},
update: async ({ status }, { where }) => {
nativeUpdate: async (where, { status }) => {
updates.push({ status, ids: where.id });
},
};
stubModule('../../back/config', {
__esModule: true,
default: {},
@@ -83,7 +80,6 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
__esModule: true,
default: class SockService {},
});
const servicePath = require.resolve('../../back/services/dependence');
const originalService = require.cache[servicePath];
delete require.cache[servicePath];
@@ -101,7 +97,6 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
}
}
});
const DependenceService = require('../../back/services/dependence').default;
const service = new DependenceService({}, {});
const result = await service.dependencies({
@@ -109,7 +104,6 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
type: 'nodejs',
status: '',
});
assert.deepEqual(updates, [
{ status: DependenceStatus.installFailed, ids: [1] },
]);
@@ -120,7 +114,6 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
{ id: 2, status: DependenceStatus.installed },
],
);
await service.dependencies({ searchValue: '', type: 'linux', status: '' });
assert.equal(updates.length, 1);
});
+5 -7
View File
@@ -1,7 +1,6 @@
const assert = require('node:assert/strict');
const test = require('node:test');
const express = require('express');
function mockModule(modulePath, exports) {
const filename = require.resolve(modulePath);
require.cache[filename] = {
@@ -13,7 +12,6 @@ function mockModule(modulePath, exports) {
paths: [],
};
}
mockModule('../../back/config', {
__esModule: true,
default: {
@@ -49,15 +47,17 @@ for (const service of ['config', 'script', 'log']) {
}
mockModule('../../back/data/runningInstance', {
InstanceStatus: { running: 'running' },
RunningInstanceModel: { findOne: async () => null },
RunningInstanceModel: {
findOne: async (where, read = {}) => {
return null;
},
},
});
const deprecatedRoutes = [
['config', '/configs/detail'],
['script', '/scripts/detail'],
['log', '/logs/detail'],
];
for (const [moduleName, replacement] of deprecatedRoutes) {
test(`${moduleName} filename route points callers to its detail API`, () => {
const app = express.Router();
@@ -68,12 +68,10 @@ for (const [moduleName, replacement] of deprecatedRoutes) {
);
const handler = deprecatedRoute.route.stack.at(-1).handle;
let responseBody;
handler(
{ params: { file: 'Example.js' }, query: {} },
{ send: (body) => (responseBody = body) },
);
assert.deepEqual(responseBody, {
code: 410,
message: `接口已下线,请使用 ${replacement} 接口`,
+20 -16
View File
@@ -5,7 +5,6 @@ const os = require('node:os');
const path = require('node:path');
const { execFileSync } = require('node:child_process');
const load = require('../helpers/load-security-module.cjs');
const types = { nodejs: 0, python3: 1, linux: 2 };
const statuses = { installed: 1, installFailed: 2, queued: 6, cancelled: 7 };
const util = load('back/config/util.ts', {
@@ -15,7 +14,6 @@ const util = load('back/config/util.ts', {
'../shared/utils': {},
'../data/dependence': { DependenceTypes: types },
});
function serviceFor(docs) {
const probes = [],
removed = [],
@@ -28,18 +26,21 @@ function serviceFor(docs) {
DependenceStatus: statuses,
versionDependenceCommandTypes: { 0: '@', 1: '==', 2: '=' },
DependenceModel: {
findAll: async ({ where }) =>
docs.filter(
find: async (where, read = {}) => {
return docs.filter(
(d) =>
(where.status === undefined || d.status === where.status) &&
(!where.id || where.id.includes(d.id)),
),
update: async (values, { where }) => {
);
},
nativeUpdate: async (where, values) => {
docs
.filter((d) => where.id.includes(d.id))
.forEach((d) => Object.assign(d, values));
},
destroy: async ({ where }) => removed.push(...where.id),
nativeDelete: async (where, read = {}) => {
return removed.push(...where.id);
},
},
},
'../config/util': {
@@ -62,7 +63,6 @@ function serviceFor(docs) {
};
return { service, probes, removed, uninstalls };
}
test('listing preserves Python source results and still detects missing named packages', async () => {
const docs = [
' requirements with spaces/requirements.txt ',
@@ -88,7 +88,6 @@ test('listing preserves Python source results and still detects missing named pa
// Source installs must continue to execute pip rather than treating the source as a package name.
assert.equal(util.getGetCommand(1, docs[0].name), 'echo ""');
});
test('deletion cleans previously misclassified Python sources and only queues intended records', async () => {
const docs = [
{ id: 1, name: 'requirements.txt', type: 1, status: 2 },
@@ -106,7 +105,6 @@ test('deletion cleans previously misclassified Python sources and only queues in
]);
assert.equal(docs[2].status, 2);
});
test('Alpine container mirror command runs without sudo and writes valid repositories', async (t) => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-alpine-mirror-'));
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
@@ -151,7 +149,6 @@ test('Alpine container mirror command runs without sudo and writes valid reposit
),
);
});
test('force deletion removes a source record even when its requirements file is gone', async () => {
const removed = [],
updates = [];
@@ -167,8 +164,12 @@ test('force deletion removes a source record even when its requirements file is
DependenceTypes: types,
DependenceStatus: { ...statuses, removeFailed: 5 },
DependenceModel: {
destroy: async ({ where }) => removed.push(...where.id),
update: async (values) => updates.push(values),
nativeDelete: async (where, read = {}) => {
return removed.push(...where.id);
},
nativeUpdate: async (where, values) => {
return updates.push(values);
},
},
},
'../shared/pLimit': {
@@ -186,7 +187,6 @@ test('force deletion removes a source record even when its requirements file is
await service.installOrUninstallDependency(dep, false, true);
assert.deepEqual(removed, [1]);
});
test('Python extras run pip even when the base package version is already installed', async (t) => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-extra-install-'));
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
@@ -217,8 +217,12 @@ test('Python extras run pip even when the base package version is already instal
DependenceStatus: { ...statuses, installing: 0, cancelled: 7 },
versionDependenceCommandTypes: { 0: '@', 1: '==', 2: '=' },
DependenceModel: {
findAll: async () => [dep],
update: async (values) => Object.assign(dep, values),
find: async (where, read = {}) => {
return [dep];
},
nativeUpdate: async (where, values) => {
return Object.assign(dep, values);
},
},
},
'../shared/pLimit': {
+5 -10
View File
@@ -9,10 +9,8 @@ const path = require('node:path');
const load = require('../helpers/load-security-module.cjs');
const { observeChildProcess } = require('../../back/shared/childProcess');
const { LogStreamManager } = require('../../back/shared/logStreamManager');
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
const logger = { info() {}, error() {} };
test(
'spawn failure settles without exit, and the next scheduled run can proceed',
{ timeout: 3000 },
@@ -36,7 +34,11 @@ test(
},
'../loaders/logger': logger,
'../data/cron': {
CrontabModel: { findOne: async () => ({ isDisabled: 0 }) },
CrontabModel: {
findOne: async (where, read = {}) => {
return { isDisabled: 0 };
},
},
CrontabStatus: {},
},
'../data/runningInstance': {
@@ -51,7 +53,6 @@ test(
assert.equal(releases, 2);
},
);
test(
'completion waits for slow log consumers and preserves the final output',
{ timeout: 5000 },
@@ -79,7 +80,6 @@ test(
assert.match(chunks.join(''), /错误/);
},
);
test('exit is not completion, and UTF-8 split across writes remains intact', async () => {
const child = new EventEmitter();
child.stdout = new PassThrough();
@@ -105,7 +105,6 @@ test('exit is not completion, and UTF-8 split across writes remains intact', asy
await observed.completed;
assert.equal(chunks.join(''), '末尾中文');
});
test(
'failed log sink drains large output instead of blocking the child',
{ timeout: 5000 },
@@ -124,7 +123,6 @@ test(
assert.equal(result.error.message, 'ENOSPC');
},
);
function scheduleFixture() {
let active = 0;
const limit = async (_params, fn) => {
@@ -150,7 +148,6 @@ function scheduleFixture() {
}).default;
return { service: new Schedule(logger), active: () => active };
}
test(
'PID response keeps queue capacity occupied until cleanup completes',
{ timeout: 4000 },
@@ -178,7 +175,6 @@ test(
assert.equal(fixture.active(), 0);
},
);
test(
'before/error/end callback failures settle and release capacity',
{ timeout: 3000 },
@@ -207,7 +203,6 @@ test(
assert.equal(fixture.active(), 0);
},
);
test(
'concurrent writes and closes preserve all bytes, and failed files can be closed',
{ timeout: 5000 },
+5 -5
View File
@@ -5,7 +5,6 @@ const os = require('node:os');
const path = require('node:path');
const express = require('express');
const load = require('../helpers/load-security-module.cjs');
function fixture(t, customData = false) {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-access-scope-'));
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
@@ -46,12 +45,15 @@ function fixture(t, customData = false) {
'../config/const': {},
'../data/runningInstance': {
InstanceStatus: { running: 'running' },
RunningInstanceModel: { findOne: async () => null },
RunningInstanceModel: {
findOne: async (where, read = {}) => {
return null;
},
},
},
};
return { config, mocks, secret };
}
function api(name, service, mocks) {
mocks.typedi = { Container: { get: () => service } };
const app = express.Router();
@@ -79,7 +81,6 @@ function api(name, service, mocks) {
return result;
};
}
for (const customData of [false, true]) {
test(`config APIs reject script compatibility paths (custom=${customData})`, async (t) => {
const { config, mocks, secret } = fixture(t, customData);
@@ -144,7 +145,6 @@ for (const customData of [false, true]) {
assert.equal(fs.readFileSync(secret, 'utf8'), 'PANEL-SECRET');
});
}
test('log APIs allow configuration-like names while retaining directory boundaries', async (t) => {
const { config, mocks, secret } = fixture(t);
const Log = load(path.resolve('back/services/log.ts'), mocks).default;
+5 -20
View File
@@ -2,7 +2,7 @@ const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const ts = require('typescript');
const { Sequelize, DataTypes, Model } = require('sequelize');
const load = require('../helpers/load-security-module.cjs');
const { CronResponse } = load('back/protos/api.ts');
const source = fs.readFileSync('back/schedule/api.ts', 'utf8');
@@ -11,31 +11,16 @@ const end = source.indexOf('export const getCronDetail', start);
const code = ts.transpileModule(source.slice(start, end), {
compilerOptions: { target: ts.ScriptTarget.ES2020 },
}).outputText;
const normalize = new Function('Model', `${code}\nreturn normalizeCronData;`)(
Model,
);
const normalize = new Function(`${code}\nreturn normalizeCronData;`)();
test('gRPC cron response preserves Sequelize model fields through protobuf serialization', async (t) => {
const db = new Sequelize({
dialect: 'sqlite',
storage: ':memory:',
logging: false,
});
t.after(() => db.close());
const Cron = db.define('Cron', {
name: DataTypes.STRING,
command: DataTypes.STRING,
schedule: DataTypes.STRING,
labels: DataTypes.JSON,
extra_schedules: DataTypes.JSON,
});
const model = Cron.build({
test('gRPC cron response preserves plain repository fields through protobuf serialization', async (t) => {
const model = {
id: 7,
name: 'smoke',
command: 'true',
schedule: '* * * * *',
labels: ['test'],
});
};
const decoded = CronResponse.decode(
CronResponse.encode({ code: 200, data: normalize(model) }).finish(),
);
+1 -5
View File
@@ -5,7 +5,6 @@ const os = require('node:os');
const path = require('node:path');
const load = require('../helpers/load-security-module.cjs');
const { LogStreamManager } = require('../../back/shared/logStreamManager');
async function fixture(t) {
const base = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-log-boundary-'));
t.after(() => fs.rm(base, { recursive: true, force: true }));
@@ -31,7 +30,6 @@ async function fixture(t) {
];
return { root, outside, victim, invalid };
}
test('log streams reject traversal, sibling prefixes and escaping symlinks before writing', async (t) => {
const { root, outside, victim, invalid } = await fixture(t);
const manager = new LogStreamManager(root);
@@ -58,7 +56,6 @@ test('log streams reject traversal, sibling prefixes and escaping symlinks befor
await manager.closeAll();
assert.equal(await fs.readFile(log, 'utf8'), '开始\n结束\n');
});
test('log initialization rejects unsafe paths before mkdir or file writes', async (t) => {
const { root, outside, victim, invalid } = await fixture(t);
const { handleLogPath } = load(path.resolve('back/config/util.ts'), {
@@ -87,7 +84,6 @@ test('log initialization rejects unsafe paths before mkdir or file writes', asyn
assert.equal(await handleLogPath(log, 'ignored'), log);
assert.equal(await fs.readFile(log, 'utf8'), 'initial');
});
test('manual execution rejects escaping log names before creating directories or spawning', async (t) => {
const { root, outside, victim } = await fixture(t);
let spawned = 0;
@@ -97,7 +93,7 @@ test('manual execution rejects escaping log names before creating directories or
'../config': { logPath: root },
'../data/cron': {
CrontabStatus: { queued: 3, idle: 1 },
CrontabModel: { update: async () => {} },
CrontabModel: { nativeUpdate: async (where, data) => {} },
},
'../data/runningInstance': { RunningInstanceModel: {}, InstanceStatus: {} },
'../config/util': {},
+5 -3
View File
@@ -6,7 +6,6 @@ const os = require('node:os');
const { spawn } = require('node:child_process');
const load = require('../helpers/load-security-module.cjs');
const { LogStreamManager } = require('../../back/shared/logStreamManager');
test(
'manual execution captures a short child before slow status storage and flushes before release',
{ timeout: 5000 },
@@ -22,10 +21,13 @@ test(
'../data/cron': {
CrontabStatus: { queued: 3, running: 0, idle: 1 },
CrontabModel: {
update: async (data, options) => {
nativeUpdate: async (where, data) => {
const options = {
where: where,
};
await new Promise((resolve) => setTimeout(resolve, 40));
updates.push({ data, options });
return [1];
return 1;
},
},
},
+38 -43
View File
@@ -1,3 +1,6 @@
const { DatabaseClient } = require('../../back/data/databaseClient');
const { repository, initialize } = require('../helpers/repository-fixture.cjs');
const { orderBy } = require('../../back/data/queries');
const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs/promises');
@@ -9,36 +12,21 @@ const { Sequelize, DataTypes } = require('sequelize');
const load = require('../helpers/load-security-module.cjs');
const { killTask } = require('../../back/config/util');
const { LogStreamManager } = require('../../back/shared/logStreamManager');
for (const conflict of ['stop', 'newer-queue', 'stop-requeue']) {
test(
`manual startup loses its conditional claim after ${conflict} and terminates the late child`,
{ timeout: 10000 },
async (t) => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-manual-claim-'));
const db = new Sequelize({
dialect: 'sqlite',
storage: ':memory:',
logging: false,
});
const db = new DatabaseClient(':memory:');
t.after(async () => {
await db.close();
await fs.rm(root, { recursive: true, force: true });
});
const crons = db.define('Cron', {
status: DataTypes.INTEGER,
pid: DataTypes.INTEGER,
log_path: DataTypes.STRING,
queued_token: DataTypes.STRING,
command: DataTypes.STRING,
});
const instances = db.define('Instance', {
cron_id: DataTypes.INTEGER,
status: DataTypes.INTEGER,
pid: DataTypes.INTEGER,
});
await db.sync();
await crons.create({
const crons = repository(db, 'Crontab');
const instances = repository(db, 'RunningInstance');
await initialize(db);
await crons.insert({
id: 1,
status: 3,
pid: null,
@@ -102,7 +90,9 @@ for (const conflict of ['stop', 'newer-queue', 'stop-requeue']) {
}).default;
const service = new CronService({ info() {}, error() {} });
service.getDb = async () =>
(await crons.findByPk(1)).get({ plain: true });
await crons.findOne({
id: 1,
});
service.makeCommand = () => 'ignored';
const running = service.runSingle(1);
await pending;
@@ -115,16 +105,22 @@ for (const conflict of ['stop', 'newer-queue', 'stop-requeue']) {
assert.equal(pendingRuns.length, 1);
assert.equal(
pendingRuns[0].token,
(await crons.findByPk(1)).queued_token,
(
await crons.findOne({
id: 1,
})
).queued_token,
);
} else if (conflict === 'stop') await service.stop([1]);
else await crons.update({ log_path: 'newer.log' }, { where: { id: 1 } });
else await crons.nativeUpdate({ id: 1 }, { log_path: 'newer.log' });
release();
const result = await running;
assert.match(result.error.message, /stopped or superseded/);
assert.ok(child.signalCode || child.exitCode !== null);
assert.throws(() => process.kill(child.pid, 0), { code: 'ESRCH' });
const row = await crons.findByPk(1);
const row = await crons.findOne({
id: 1,
});
assert.equal(row.status, conflict === 'stop' ? 1 : 3);
assert.equal(
row.log_path,
@@ -135,7 +131,6 @@ for (const conflict of ['stop', 'newer-queue', 'stop-requeue']) {
},
);
}
test(
'verified termination waits for a SIGTERM-resistant target to exit',
{ timeout: 5000 },
@@ -158,22 +153,12 @@ test(
assert.equal(child.signalCode, 'SIGKILL');
},
);
test('a runner waiting for a concurrency slot cannot adopt a newer queued generation', async (t) => {
const db = new Sequelize({
dialect: 'sqlite',
storage: ':memory:',
logging: false,
});
const db = new DatabaseClient(':memory:');
t.after(() => db.close());
const crons = db.define('Cron', {
status: DataTypes.INTEGER,
pid: DataTypes.INTEGER,
log_path: DataTypes.STRING,
queued_token: DataTypes.STRING,
});
await db.sync();
await crons.create({ id: 1, status: 1, log_path: 'same.log' });
const crons = repository(db, 'Crontab');
await initialize(db);
await crons.insert({ id: 1, status: 1, log_path: 'same.log' });
const waiting = [];
const CronService = load('back/services/cron.ts', {
'../config': {},
@@ -182,7 +167,7 @@ test('a runner waiting for a concurrency slot cannot adopt a newer queued genera
CrontabStatus: { queued: 3, idle: 1, running: 0 },
},
'../data/runningInstance': {
RunningInstanceModel: { findAll: async () => [] },
RunningInstanceModel: { find: async () => [] },
InstanceStatus: { running: 0 },
},
'../config/util': {
@@ -205,14 +190,24 @@ test('a runner waiting for a concurrency slot cannot adopt a newer queued genera
const errors = [];
const service = new CronService({ error: (...args) => errors.push(args) });
await service.run([1]);
const first = (await crons.findByPk(1)).queued_token;
const first = (
await crons.findOne({
id: 1,
})
).queued_token;
await service.stop([1]);
await service.run([1]);
const second = (await crons.findByPk(1)).queued_token;
const second = (
await crons.findOne({
id: 1,
})
).queued_token;
assert.notEqual(first, second);
assert.equal(waiting.length, 2);
await waiting[0]();
const row = await crons.findByPk(1);
const row = await crons.findOne({
id: 1,
});
assert.equal(row.status, 3);
assert.equal(row.queued_token, second);
assert.equal(row.log_path, 'same.log');

Some files were not shown because too many files have changed in this diff Show More