Compare commits

..
Author SHA1 Message Date
whyour af93fb6cf0 fix: preserve prerelease version in health build identity (#3112) 2026-10-25 20:04:28 +08:00
whyour 2218c2c175 build: mark develop platform bundles as 2.24.0-dev.0 (#3111) 2026-10-25 19:43:26 +08:00
whyour 32c20ba556 build: unify container Node 24 and build ARM64 bundles natively
Unify Alpine and Debian images on Node 24, remove Debian arm/v7 from the published matrix, and build ARM64 upgrade bundles on native runners to avoid QEMU SIGILL.

Combines #3109 and #3110.
2026-10-25 17:27:38 +08:00
whyour b92621bdb8 build: upgrade sqlite3 to 1.1.4 (#3108) 2026-10-25 01:52:58 +08:00
da9c6a6bbb fix: skip new scheduled triggers while a single-instance task is active (#3101)
* fix: skip active single-instance scheduled executions

* fix: recover stale single-instance scheduled tasks

* test: remove obsolete single-instance replacement assertions

---------

Co-authored-by: Patrick Gu <username@users.noreply.github.com>
Co-authored-by: whyour <imwhyour@gmail.com>
2026-10-24 23:11:36 +08:00
whyour d895fbba36 build: use one bundle validation path for all architectures (#3106) 2026-10-24 19:59:41 +08:00
whyour 6862ad4f6c build: use sqlite3 1.1.3 ARM musl prebuilds (#3105) 2026-10-24 17:40:47 +08:00
whyour b8110c52eb fix: isolate Alpine Node runtime and bound URL probes (#3104)
Pin Alpine amd64 to the verified upstream Node runtime without retaining temporary binaries in image layers. Bound fresh-process URL probes and verify serialized amd64/ARMv6 layers in CI. Keep existing Sequelize, SQLite and Debian runtime behavior unchanged.
2026-10-23 23:34:24 +08:00
141 changed files with 3996 additions and 8571 deletions
+20 -33
View File
@@ -247,7 +247,7 @@ jobs:
version: "8.3.1"
- uses: actions/setup-node@v6
with:
node-version: "20"
node-version: "24"
cache: "pnpm"
cache-dependency-path: pnpm-lock.yaml
@@ -336,7 +336,7 @@ jobs:
version: "8.3.1"
- uses: actions/setup-node@v6
with:
node-version: "20"
node-version: "24"
cache: "pnpm"
cache-dependency-path: pnpm-lock.yaml
@@ -394,7 +394,7 @@ jobs:
SOURCE_COMMIT=${{ github.sha }}
network: host
# Keep s390x on Debian: npm can hang under QEMU with Alpine (nodejs/docker-node#1973).
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
platforms: linux/amd64,linux/arm64,linux/ppc64le,linux/s390x
context: .
file: ./docker/Dockerfile.debian
push: true
@@ -423,7 +423,7 @@ jobs:
version: "8.3.1"
- uses: actions/setup-node@v6
with:
node-version: "20"
node-version: "24"
cache: "pnpm"
cache-dependency-path: pnpm-lock.yaml
@@ -495,7 +495,7 @@ jobs:
version: "8.3.1"
- uses: actions/setup-node@v6
with:
node-version: "20"
node-version: "24"
cache: "pnpm"
cache-dependency-path: pnpm-lock.yaml
@@ -538,7 +538,7 @@ jobs:
SOURCE_COMMIT=${{ github.sha }}
network: host
# Keep s390x on Debian: npm can hang under QEMU with Alpine (nodejs/docker-node#1973).
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
platforms: linux/amd64,linux/arm64,linux/ppc64le,linux/s390x
context: .
file: ./docker/Dockerfile.debian310
push: true
@@ -550,14 +550,10 @@ jobs:
cache-from: type=registry,ref=whyour/qinglong:cache-debian-python3.10
cache-to: type=registry,ref=whyour/qinglong:cache-debian-python3.10,mode=max
# Unmodified amd64 packages for glibc and musl migration acceptance. This does not
# One unmodified amd64 package for migration acceptance only. This does not
# publish bundles or advance development/formal channel indexes.
build-npm-migration-bundle:
needs: [build-debian, build-alpine]
strategy:
fail-fast: false
matrix:
libc: [glibc, musl]
needs: build-debian
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
@@ -565,7 +561,7 @@ jobs:
- uses: docker/setup-buildx-action@v4
- name: Export the official platform package from this run's image digest
env:
IMAGE_DIGEST: ${{ matrix.libc == 'musl' && needs.build-alpine.outputs.image-digest || needs.build-debian.outputs.image-digest }}
IMAGE_DIGEST: ${{ needs.build-debian.outputs.image-digest }}
run: |
set -euo pipefail
[[ "$IMAGE_DIGEST" =~ ^sha256:[a-f0-9]{64}$ ]]
@@ -573,21 +569,20 @@ jobs:
docker buildx build --platform linux/amd64 --build-arg "BASE_IMAGE=$image" --file docker/Dockerfile.upgrade-bundle --output "type=local,dest=$RUNNER_TEMP/npm-migration-bundle" .
- uses: actions/upload-artifact@v6
with:
name: npm-migration-bundle-${{ matrix.libc }}-${{ github.sha }}
name: npm-migration-bundle-${{ github.sha }}
path: ${{ runner.temp }}/npm-migration-bundle/*
if-no-files-found: error
retention-days: 3
verify-npm-package:
needs: [build-static, build-npm-migration-bundle, build-alpine]
needs: [build-static, build-npm-migration-bundle]
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
node: ['20.18.1', '22.17.0', '24', 'alpine']
container:
image: ${{ matrix.node == 'alpine' && format('ghcr.io/{0}@{1}', github.repository, needs.build-alpine.outputs.image-digest) || format('node:{0}-bookworm', matrix.node) }}
node: ['20.18.1', '24']
container: node:${{ matrix.node }}-bookworm
steps:
- uses: actions/checkout@v6
- uses: actions/download-artifact@v6
@@ -596,30 +591,22 @@ jobs:
path: ${{ runner.temp }}/npm-panel/
- uses: actions/download-artifact@v6
with:
name: npm-migration-bundle-${{ matrix.node == 'alpine' && 'musl' || 'glibc' }}-${{ github.sha }}
name: npm-migration-bundle-${{ github.sha }}
path: ${{ runner.temp }}/npm-migration-bundle/
- name: Install isolated service test tools
run: |
if [ -f /etc/alpine-release ]; then
apk add --no-cache python3 openssl curl jq nginx procps make g++
else
apt-get update
apt-get install -y --no-install-recommends python3 openssl curl jq nginx procps
fi
apt-get update
apt-get install -y --no-install-recommends python3 openssl curl jq nginx procps
# The published legacy launcher resets PATH to standard directories.
# /usr/local is confined to this disposable job container; PM2_HOME
# remains private to each verification run.
npm install --global --prefix /usr/local --no-audit --no-fund pm2@6.0.14
- name: Install and upgrade the exact npm archive
if: matrix.node != '20.18.1'
env:
QL_PANEL_PACKAGE_TEST_ISOLATED: '1'
QL_PANEL_TEST_PM2: /usr/local/bin/pm2
QL_PANEL_TEST_BUNDLE_DIR: ${{ runner.temp }}/npm-migration-bundle
run: node scripts/verify-panel-package.cjs verify "$RUNNER_TEMP/npm-panel" "$RUNNER_TEMP/npm-panel-results"
- name: Reject an unsupported runtime while preserving the old installation
if: matrix.node == '20.18.1'
run: node scripts/verify-panel-runtime-floor.cjs "$RUNNER_TEMP/npm-panel" "$RUNNER_TEMP/npm-panel-results"
- uses: actions/upload-artifact@v6
with:
name: qinglong-npm-verification-${{ matrix.node }}-${{ github.sha }}
@@ -675,7 +662,8 @@ jobs:
# Both channels publish packages from this run's immutable image digests.
if: ${{ !inputs.static_sync_only && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop') }}
needs: [build-alpine, build-debian]
runs-on: ubuntu-22.04
# Node 24 can SIGILL while archiving ARM64 musl under x64 QEMU.
runs-on: ${{ matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-22.04' }}
timeout-minutes: 90
permissions:
contents: read
@@ -692,13 +680,13 @@ jobs:
- { platform: linux/s390x, libc: musl, name: s390x-musl }
- { platform: linux/386, libc: musl, name: ia32-musl }
- { platform: linux/amd64, libc: glibc, name: x64-glibc }
- { platform: linux/arm/v7, libc: glibc, name: armv7-glibc }
- { platform: linux/arm64, libc: glibc, name: arm64-glibc }
- { platform: linux/ppc64le, libc: glibc, name: ppc64-glibc }
- { platform: linux/s390x, libc: glibc, name: s390x-glibc }
steps:
- uses: actions/checkout@v6
- uses: docker/setup-qemu-action@v4
if: ${{ matrix.platform != 'linux/arm64' }}
with:
cache-image: false
- uses: docker/setup-buildx-action@v4
@@ -714,8 +702,7 @@ jobs:
if [ "$BUNDLE_LIBC" = glibc ]; then digest="$DEBIAN_DIGEST"; fi
test -n "$digest"
image="ghcr.io/${GITHUB_REPOSITORY}@${digest}"
legacy=$(node -p "require('./docker/migration-bridge-images.json')['$BUNDLE_LIBC']")
docker buildx build --platform "$BUNDLE_PLATFORM" --build-arg "BASE_IMAGE=$image" --build-arg "LEGACY_IMAGE=$legacy" --file docker/Dockerfile.upgrade-bridge --output type=local,dest=output/upgrade .
docker buildx build --platform "$BUNDLE_PLATFORM" --build-arg "BASE_IMAGE=$image" --file docker/Dockerfile.upgrade-bundle --output type=local,dest=output/upgrade .
- uses: actions/upload-artifact@v6
with:
name: upgrade-${{ matrix.name }}-${{ github.sha }}
+57 -14
View File
@@ -6,6 +6,11 @@ on:
- '.github/workflows/release-readiness.yml'
- 'package.json'
- 'pnpm-lock.yaml'
- 'docker/Dockerfile*'
- 'shell/node-runtime-probe.cjs'
- 'scripts/verify-panel-package.cjs'
- 'scripts/verify-image-node-layers.py'
- 'test/back/node-runtime-probe.test.cjs'
workflow_dispatch:
permissions:
@@ -31,7 +36,7 @@ jobs:
version: '8.3.1'
- uses: actions/setup-node@v6
with:
node-version: '22'
node-version: '20'
cache: pnpm
- run: pnpm install --frozen-lockfile
- run: pnpm build:front && pnpm build:back && pnpm build:info
@@ -41,6 +46,13 @@ jobs:
path: static/
include-hidden-files: true
if-no-files-found: error
- name: Pack the verified npm archive for isolated runtime validation
run: node scripts/verify-panel-package.cjs pack "$RUNNER_TEMP/npm-panel"
- uses: actions/upload-artifact@v6
with:
name: release-readiness-npm-panel
path: ${{ runner.temp }}/npm-panel/
if-no-files-found: error
images:
needs: [regression, static]
@@ -57,7 +69,7 @@ jobs:
- image: debian
file: docker/Dockerfile.debian
cache: debian
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
platforms: linux/amd64,linux/arm64,linux/ppc64le,linux/s390x
- image: alpine-python3.10
file: docker/Dockerfile.310
cache: alpine-python3.10
@@ -65,7 +77,7 @@ jobs:
- image: debian-python3.10
file: docker/Dockerfile.debian310
cache: debian-python3.10
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
platforms: linux/amd64,linux/arm64,linux/ppc64le,linux/s390x
steps:
- uses: actions/checkout@v6
with:
@@ -120,17 +132,48 @@ jobs:
run: |
docker run --rm --network none --entrypoint node \
qinglong-readiness:${{ matrix.image }} \
-e 'const assert=require("node:assert/strict"); const {createRequire}=require("node:module"); for(const [name,want] of Object.entries({"@grpc/grpc-js":"1.14.5",compression:"1.8.2",multer:"2.4.0"}))assert.equal(require(name+"/package.json").version,want); const expressRequire=createRequire(require.resolve("express")); assert.equal(expressRequire("proxy-addr/package.json").version,"2.0.8"); const sockRequire=createRequire(require.resolve("sockjs")); assert.equal(sockRequire("websocket-driver/package.json").version,"0.7.5"); assert.equal(typeof require("@mikro-orm/sql").NodeSqliteDialect,"function"); const db=new (require("node:sqlite").DatabaseSync)(":memory:"); assert.equal(db.prepare("SELECT 42 AS value").get().value,42); db.close(); require("nodemailer"); require("undici"); console.log("Minimal production patches and runtime imports verified")'
- name: Export native Debian image for isolated dev validation
if: matrix.image == 'debian'
run: |
docker save qinglong-readiness:debian | gzip -1 > release-readiness-debian-image.tar.gz
sha256sum release-readiness-debian-image.tar.gz > release-readiness-debian-image.sha256
docker image inspect qinglong-readiness:debian > release-readiness-debian-image.json
- uses: actions/upload-artifact@v6
if: matrix.image == 'debian'
-e 'const assert=require("node:assert/strict"); const {createRequire}=require("node:module"); for(const [name,want] of Object.entries({"@grpc/grpc-js":"1.14.5",compression:"1.8.2",multer:"2.4.0",sequelize:"6.37.8"}))assert.equal(require(name+"/package.json").version,want); const expressRequire=createRequire(require.resolve("express")); assert.equal(expressRequire("proxy-addr/package.json").version,"2.0.8"); const sockRequire=createRequire(require.resolve("sockjs")); assert.equal(sockRequire("websocket-driver/package.json").version,"0.7.5"); require("sqlite3"); require("nodemailer"); require("undici"); console.log("Minimal production patches and runtime imports verified")'
- name: Load ARMv6 image for SQLite and Node regression checks
if: startsWith(matrix.image, 'alpine')
uses: docker/build-push-action@v7
with:
name: release-readiness-debian-image
path: release-readiness-debian-image.*
context: .
file: ${{ matrix.file }}
platforms: linux/arm/v6
build-args: |
QL_URL=${{ steps.source.outputs.url }}
QL_BRANCH=${{ steps.source.outputs.branch }}
SOURCE_COMMIT=${{ steps.source.outputs.sha }}
tags: qinglong-readiness:${{ matrix.image }}-armv6
load: true
push: false
provenance: false
sbom: false
- name: Verify ARMv6 SQLite on an ARM1176 CPU without networking
if: startsWith(matrix.image, 'alpine')
run: |
docker run --rm --network none --platform linux/arm/v6 \
-e QEMU_CPU=arm1176 --entrypoint node \
qinglong-readiness:${{ matrix.image }}-armv6 --jitless \
-e 'const assert=require("node:assert/strict"); const sqlite3=require("sqlite3"); const db=new sqlite3.Database(":memory:"); db.get("SELECT 42 AS n",(error,row)=>{assert.ifError(error); assert.equal(row.n,42); db.close()});'
- name: Check Node binaries in every serialized image layer
if: startsWith(matrix.image, 'alpine')
run: |
docker save qinglong-readiness:${{ matrix.image }} -o "$RUNNER_TEMP/node-layers-amd64.tar"
docker save qinglong-readiness:${{ matrix.image }}-armv6 -o "$RUNNER_TEMP/node-layers-arm.tar"
python3 scripts/verify-image-node-layers.py "$RUNNER_TEMP/node-layers-amd64.tar" amd64
python3 scripts/verify-image-node-layers.py "$RUNNER_TEMP/node-layers-arm.tar" arm
rm "$RUNNER_TEMP/node-layers-amd64.tar" "$RUNNER_TEMP/node-layers-arm.tar"
- name: Export native image for isolated dev validation
if: matrix.image == 'debian' || matrix.image == 'alpine'
run: |
docker save qinglong-readiness:${{ matrix.image }} | gzip -1 > release-readiness-${{ matrix.image }}-image.tar.gz
sha256sum release-readiness-${{ matrix.image }}-image.tar.gz > release-readiness-${{ matrix.image }}-image.sha256
docker image inspect qinglong-readiness:${{ matrix.image }} > release-readiness-${{ matrix.image }}-image.json
- uses: actions/upload-artifact@v6
if: matrix.image == 'debian' || matrix.image == 'alpine'
with:
name: release-readiness-${{ matrix.image }}-image
path: release-readiness-${{ matrix.image }}-image.*
compression-level: 0
if-no-files-found: error
@@ -1,201 +0,0 @@
name: SQLite migration candidate
on:
pull_request:
branches: [develop]
workflow_dispatch:
permissions:
contents: read
concurrency:
group: sqlite-migration-candidate-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
SOURCE_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
jobs:
build-static:
if: ${{ (github.event_name == 'pull_request' && github.head_ref == 'codex/sqlite-runtime-migration' && github.event.pull_request.head.repo.full_name == github.repository) || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/codex/sqlite-runtime-migration') }}
runs-on: ubuntu-latest
timeout-minutes: 25
steps:
- uses: actions/checkout@v6
with:
ref: ${{ env.SOURCE_COMMIT }}
- uses: pnpm/action-setup@v6
with:
version: '8.3.1'
- uses: actions/setup-node@v6
with:
node-version: '22.17.0'
cache: pnpm
- run: pnpm install --frozen-lockfile
- run: cp .env.example .env
- run: npm test
- run: npm run build:front && npm run build:back && npm run build:info
- uses: actions/upload-artifact@v6
with:
name: migration-static-${{ env.SOURCE_COMMIT }}
path: static/
include-hidden-files: true
if-no-files-found: error
retention-days: 7
test-node24:
needs: build-static
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v6
with:
ref: ${{ env.SOURCE_COMMIT }}
- uses: pnpm/action-setup@v6
with:
version: '8.3.1'
- uses: actions/setup-node@v6
with:
node-version: '24'
cache: pnpm
- run: pnpm install --frozen-lockfile
- run: cp .env.example .env
- run: npm test
build-images:
needs: build-static
runs-on: ubuntu-22.04
timeout-minutes: 90
permissions:
contents: read
packages: write
strategy:
fail-fast: false
matrix:
include:
- libc: musl
file: docker/Dockerfile
platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x,linux/386
- libc: glibc
file: docker/Dockerfile.debian
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
steps:
- uses: actions/checkout@v6
with:
ref: ${{ env.SOURCE_COMMIT }}
- uses: actions/download-artifact@v6
with:
name: migration-static-${{ env.SOURCE_COMMIT }}
path: static/
- uses: docker/setup-qemu-action@v4
with:
cache-image: false
- uses: docker/setup-buildx-action@v4
- uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ github.token }}
- uses: docker/build-push-action@v7
id: image
with:
context: .
file: ${{ matrix.file }}
platforms: ${{ matrix.platforms }}
build-args: |
QL_BRANCH=codex/sqlite-runtime-migration
SOURCE_COMMIT=${{ env.SOURCE_COMMIT }}
push: true
tags: ghcr.io/${{ github.repository }}:migration-candidate-${{ env.SOURCE_COMMIT }}-${{ matrix.libc }}
cache-from: type=registry,ref=whyour/qinglong:cache-${{ matrix.libc == 'musl' && 'alpine' || 'debian' }}
- name: Record the immutable candidate image
env:
IMAGE_DIGEST: ${{ steps.image.outputs.digest }}
BUNDLE_LIBC: ${{ matrix.libc }}
run: |
[[ "$IMAGE_DIGEST" =~ ^sha256:[a-f0-9]{64}$ ]]
printf '%s\n' "ghcr.io/$GITHUB_REPOSITORY@$IMAGE_DIGEST" > "$BUNDLE_LIBC-image.txt"
- uses: actions/upload-artifact@v6
with:
name: migration-${{ matrix.libc }}-image-${{ env.SOURCE_COMMIT }}
path: ${{ matrix.libc }}-image.txt
retention-days: 7
build-bundles:
needs: build-images
runs-on: ubuntu-22.04
timeout-minutes: 90
strategy:
fail-fast: false
matrix:
include:
- { platform: linux/amd64, libc: musl, name: x64-musl }
- { platform: linux/arm/v6, libc: musl, name: armv6-musl }
- { platform: linux/arm/v7, libc: musl, name: armv7-musl }
- { platform: linux/arm64, libc: musl, name: arm64-musl }
- { platform: linux/ppc64le, libc: musl, name: ppc64-musl }
- { platform: linux/s390x, libc: musl, name: s390x-musl }
- { platform: linux/386, libc: musl, name: ia32-musl }
- { platform: linux/amd64, libc: glibc, name: x64-glibc }
- { platform: linux/arm/v7, libc: glibc, name: armv7-glibc }
- { platform: linux/arm64, libc: glibc, name: arm64-glibc }
- { platform: linux/ppc64le, libc: glibc, name: ppc64-glibc }
- { platform: linux/s390x, libc: glibc, name: s390x-glibc }
steps:
- uses: actions/checkout@v6
with:
ref: ${{ env.SOURCE_COMMIT }}
- uses: actions/download-artifact@v6
with:
name: migration-${{ matrix.libc }}-image-${{ env.SOURCE_COMMIT }}
- uses: docker/setup-qemu-action@v4
with:
cache-image: false
- uses: docker/setup-buildx-action@v4
- name: Build and verify both online hops using the unchanged legacy client
env:
BUNDLE_PLATFORM: ${{ matrix.platform }}
BUNDLE_LIBC: ${{ matrix.libc }}
run: |
set -euo pipefail
image=$(cat "$BUNDLE_LIBC-image.txt")
legacy=$(node -p "require('./docker/migration-bridge-images.json')['$BUNDLE_LIBC']")
docker buildx build --platform "$BUNDLE_PLATFORM" --build-arg "BASE_IMAGE=$image" --build-arg "LEGACY_IMAGE=$legacy" --file docker/Dockerfile.upgrade-bridge --output type=local,dest=output/upgrade .
- uses: actions/upload-artifact@v6
with:
name: migration-bundles-${{ matrix.name }}-${{ env.SOURCE_COMMIT }}
path: output/upgrade/*
if-no-files-found: error
retention-days: 7
verify-index:
needs: [build-bundles, test-node24]
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v6
with:
ref: ${{ env.SOURCE_COMMIT }}
- uses: actions/setup-node@v6
with:
node-version: '24'
- uses: pnpm/action-setup@v6
with:
version: '8.3.1'
- run: pnpm install --prod --frozen-lockfile && npm --prefix cli ci
- uses: actions/download-artifact@v6
with:
pattern: migration-bundles-*-${{ env.SOURCE_COMMIT }}
path: output/upgrade/
merge-multiple: true
- run: node scripts/build-bundle-installer.cjs output/upgrade/install-bundle.cjs
- name: Validate every platform without publishing a channel
run: |
node -e "const fs=require('fs'),{releaseIndex}=require('./scripts/publish-upgrade-bundles.cjs'); const index=releaseIndex('output/upgrade',process.env.SOURCE_COMMIT,process.env.GITHUB_REPOSITORY,{migration:true}); fs.writeFileSync('output/upgrade/candidate-index.json',JSON.stringify(index));"
- uses: actions/upload-artifact@v6
with:
name: migration-candidate-index-${{ env.SOURCE_COMMIT }}
path: |
output/upgrade/candidate-index.json
output/upgrade/install-bundle.cjs
retention-days: 7
+1 -1
View File
@@ -18,7 +18,7 @@ jobs:
version: '8.3.1'
- uses: actions/setup-node@v6
with:
node-version: '24'
node-version: '20'
cache: pnpm
- run: pnpm install --frozen-lockfile
- run: cp .env.example .env
-2
View File
@@ -1,7 +1,5 @@
# Node.js 原生部署与 Python venv
面板要求 Node.js >=22.17,推荐 Node.js 24,并要求该 Node 构建提供 `node:sqlite`。应用升级保留现有 Node;旧 Node 20 环境需先升级运行时。远程管理 CLI 的最低版本要求独立于面板。
设置 `QL_DIR` 为安装目录,`QL_DATA_DIR` 为以 `/data` 结尾的绝对持久化数据路径。
如需隔离 Python,请在启动进程的环境中显式设置 `PYTHON_VENV_DIR`(绝对路径或相对 `QL_DIR` 的路径),例如 `export PYTHON_VENV_DIR=/srv/qinglong/.venv`。未设置时保留现有 Python prefix 行为。
+14 -21
View File
@@ -5,7 +5,10 @@ import CronService from '../services/cron';
import CronViewService from '../services/cronView';
import { celebrate, Joi } from 'celebrate';
import { commonCronSchema } from '../validation/schedule';
import { RunningInstanceModel, InstanceStatus } from '../data/runningInstance';
import {
RunningInstanceModel,
InstanceStatus,
} from '../data/runningInstance';
import { t } from '../shared/i18n';
import cronClient from '../schedule/client';
@@ -16,17 +19,8 @@ export default (app: Router) => {
route.use(async (req, res, next) => {
// Keep stop/status callbacks available even when the scheduler is down.
if (
['POST', 'PUT', 'DELETE'].includes(req.method) &&
[
'/',
'/run',
'/enable',
'/disable',
'/views/enable',
'/views/disable',
].includes(req.path)
) {
if (['POST', 'PUT', 'DELETE'].includes(req.method) &&
['/', '/run', '/enable', '/disable', '/views/enable', '/views/disable'].includes(req.path)) {
try {
await cronClient.readiness.ensureReady();
} catch (error) {
@@ -503,10 +497,13 @@ export default (app: Router) => {
}),
async (req: Request<{ id: number }>, res: Response, next: NextFunction) => {
try {
const instances = await RunningInstanceModel.find(
{ cron_id: req.params.id },
{ orderBy: [{ started_at: 'desc' }] },
);
const instances = await RunningInstanceModel.findAll({
where: {
cron_id: req.params.id,
},
order: [['started_at', 'DESC']],
raw: true,
});
return res.send({ code: 200, data: instances });
} catch (e) {
return next(e);
@@ -522,11 +519,7 @@ export default (app: Router) => {
instanceId: Joi.number().required(),
}),
}),
async (
req: Request<{ id: number; instanceId: number }>,
res: Response,
next: NextFunction,
) => {
async (req: Request<{ id: number; instanceId: number }>, res: Response, next: NextFunction) => {
try {
const cronService = Container.get(CronService);
const data = await cronService.stopInstance(req.params.instanceId);
+157 -93
View File
@@ -1,32 +1,38 @@
import { statistics } from '../data/statistics';
import { Query, QuerySpec, orderBy } from '../data/queries';
import { Router, Request, Response, NextFunction } from 'express';
import { Container } from 'typedi';
import { fn, col, where, Op } from 'sequelize';
import { CrontabModel } from '../data/cron';
import { CrontabStatModel } from '../data/cronStats';
import { RunningInstanceModel, InstanceStatus } from '../data/runningInstance';
import {
RunningInstanceModel,
InstanceStatus,
} from '../data/runningInstance';
import dayjs from 'dayjs';
import os from 'os';
import { isEmpty } from 'lodash';
import { t, tf } from '../shared/i18n';
const route = Router();
export default (app: Router) => {
app.use('/dashboard', route);
route.post('/record', async (req: Request, res: Response) => {
try {
const { ref_id, code, elapsed } = req.body;
if (!ref_id) return res.send({ code: 400, message: 'ref_id required' });
const today = dayjs().format('YYYY-MM-DD');
const isSuccess = code === 0 ? 1 : 0;
const isFail = code !== 0 ? 1 : 0;
const elapsedMs = (Number(elapsed) || 0) * 1000;
const existing = await CrontabStatModel.findOne(
{ ref_id: Number(ref_id), date: today },
{},
);
const existing = await CrontabStatModel.findOne({
where: { ref_id: Number(ref_id), date: today },
});
if (existing) {
await CrontabStatModel.nativeUpdate(
{ id: existing.id },
await CrontabStatModel.update(
{
run_count: (existing.run_count || 0) + 1,
success_count: (existing.success_count || 0) + isSuccess,
@@ -34,9 +40,10 @@ export default (app: Router) => {
total_time: (existing.total_time || 0) + elapsedMs,
max_time: Math.max(existing.max_time || 0, elapsedMs),
},
{ where: { id: existing.id } },
);
} else {
await CrontabStatModel.insert({
await CrontabStatModel.create({
ref_id: Number(ref_id),
date: today,
run_count: 1,
@@ -46,27 +53,41 @@ export default (app: Router) => {
max_time: elapsedMs,
});
}
res.send({ code: 200 });
} catch (e) {
res.send({ code: 500 });
}
});
route.get(
'/overview',
async (req: Request, res: Response, next: NextFunction) => {
try {
const today = dayjs().format('YYYY-MM-DD');
const [total, enabled, disabled, stats] = await Promise.all([
CrontabModel.count({}),
CrontabModel.count({ isDisabled: 0 }),
CrontabModel.count({ isDisabled: 1 }),
statistics.totals(today),
CrontabModel.count(),
CrontabModel.count({ where: { isDisabled: 0 } }),
CrontabModel.count({ where: { isDisabled: 1 } }),
CrontabStatModel.findOne({
attributes: [
[fn('SUM', col('run_count')), 'total_runs'],
[fn('SUM', col('success_count')), 'total_success'],
[fn('SUM', col('fail_count')), 'total_fail'],
[fn('SUM', col('total_time')), 'total_time'],
],
where: { date: today },
raw: true,
}),
]);
const row = stats as any;
const totalRuns = Number(row?.total_runs) || 0;
const totalSuccess = Number(row?.total_success) || 0;
const totalFail = Number(row?.total_fail) || 0;
const totalTime = Number(row?.total_time) || 0;
res.send({
code: 200,
data: {
@@ -76,10 +97,7 @@ export default (app: Router) => {
todayRuns: totalRuns,
todaySuccess: totalSuccess,
todayFail: totalFail,
successRate:
totalRuns > 0
? ((totalSuccess / totalRuns) * 100).toFixed(1)
: '0',
successRate: totalRuns > 0 ? ((totalSuccess / totalRuns) * 100).toFixed(1) : '0',
avgTime: totalRuns > 0 ? Math.round(totalTime / totalRuns) : 0,
},
});
@@ -88,25 +106,25 @@ export default (app: Router) => {
}
},
);
for (const isSuccess of [false, true]) {
route.get(
isSuccess ? '/successes' : '/failures',
async (req: Request, res: Response, next: NextFunction) => {
try {
const countColumn = isSuccess ? 'success_count' : 'fail_count';
const rows = (await statistics.results(
dayjs().format('YYYY-MM-DD'),
countColumn,
)) as any[];
const crons =
rows.length > 0
? await CrontabModel.find(
{ id: { ['$in']: rows.map((row) => Number(row.ref_id)) } },
{
fields: ['id', 'name', 'command'],
},
)
: [];
const rows = (await CrontabStatModel.findAll({
attributes: ['ref_id', [fn('SUM', col(countColumn)), 'result_count']],
where: { date: dayjs().format('YYYY-MM-DD'), [countColumn]: { [Op.gt]: 0 } },
group: ['ref_id'],
order: [[fn('SUM', col(countColumn)), 'DESC'], ['ref_id', 'ASC']],
raw: true,
})) as any[];
const crons = rows.length > 0 ? await CrontabModel.findAll({
attributes: ['id', 'name', 'command'],
where: { id: { [Op.in]: rows.map((row) => Number(row.ref_id)) } },
raw: true,
}) : [];
const cronMap = new Map(crons.map((cron) => [cron.id, cron]));
res.send({
code: 200,
@@ -117,9 +135,7 @@ export default (app: Router) => {
id,
name: cron?.name || cron?.command || tf('任务#%s', id),
command: cron?.command || '',
[isSuccess ? 'successCount' : 'failCount']: Number(
row.result_count,
),
[isSuccess ? 'successCount' : 'failCount']: Number(row.result_count),
deleted: !cron,
};
}),
@@ -130,6 +146,7 @@ export default (app: Router) => {
},
);
}
route.get(
'/trend',
async (req: Request, res: Response, next: NextFunction) => {
@@ -139,7 +156,22 @@ export default (app: Router) => {
for (let i = days - 1; i >= 0; i--) {
dates.push(dayjs().subtract(i, 'day').format('YYYY-MM-DD'));
}
const rows = (await statistics.trend(dates)) as any[];
const rows = (await CrontabStatModel.findAll({
attributes: [
'date',
[fn('SUM', col('run_count')), 'total_runs'],
[fn('SUM', col('success_count')), 'total_success'],
[fn('SUM', col('fail_count')), 'total_fail'],
],
where: {
date: { [Op.in]: dates },
},
group: ['date'],
order: [['date', 'ASC']],
raw: true,
})) as any[];
const dataMap: Record<string, any> = {};
rows.forEach((r: any) => {
dataMap[r.date] = {
@@ -148,52 +180,89 @@ export default (app: Router) => {
fail: Number(r.total_fail) || 0,
};
});
const data = dates.map((d) => ({
date: dayjs(d).format('MM-DD'),
...(dataMap[d] || { total: 0, success: 0, fail: 0 }),
}));
res.send({ code: 200, data });
} catch (e) {
next(e);
}
},
);
route.get(
'/top-time',
async (req: Request, res: Response, next: NextFunction) => {
try {
const today = dayjs().format('YYYY-MM-DD');
const rows = (await statistics.ranking(today, 'total_time')) as any[];
const rows = (await CrontabStatModel.findAll({
attributes: [
'ref_id',
[fn('SUM', col('total_time')), 'total_time'],
[fn('SUM', col('run_count')), 'run_count'],
[fn('MAX', col('max_time')), 'max_time'],
],
where: { date: today, run_count: { [Op.gt]: 0 } },
group: ['ref_id'],
order: [[fn('SUM', col('total_time')), 'DESC']],
limit: 5,
raw: true,
})) as any[];
const ids = rows.map((r) => Number(r.ref_id));
const crons = await CrontabModel.find({ id: { ['$in']: ids } }, {});
const nameMap: Record<number, string> = {};
crons.forEach((c: any) => {
nameMap[c.id] = c.name || c.command;
const crons = await CrontabModel.findAll({
where: { id: { [Op.in]: ids } },
raw: true,
});
const nameMap: Record<number, string> = {};
crons.forEach((c: any) => { nameMap[c.id] = c.name || c.command; });
const data = rows.map((r: any, i) => ({
rank: i + 1,
name: nameMap[Number(r.ref_id)] || tf('任务#%s', r.ref_id),
avgTime: Math.round(Number(r.total_time) / Number(r.run_count)),
maxTime: Number(r.max_time),
}));
res.send({ code: 200, data });
} catch (e) {
next(e);
}
},
);
route.get(
'/top-count',
async (req: Request, res: Response, next: NextFunction) => {
try {
const today = dayjs().format('YYYY-MM-DD');
const rows = (await statistics.ranking(today, 'run_count')) as any[];
const rows = (await CrontabStatModel.findAll({
attributes: [
'ref_id',
[fn('SUM', col('run_count')), 'run_count'],
[fn('SUM', col('total_time')), 'total_time'],
[fn('SUM', col('success_count')), 'success_count'],
],
where: { date: today, run_count: { [Op.gt]: 0 } },
group: ['ref_id'],
order: [[fn('SUM', col('run_count')), 'DESC']],
limit: 5,
raw: true,
})) as any[];
const ids = rows.map((r) => Number(r.ref_id));
const crons = await CrontabModel.find({ id: { ['$in']: ids } }, {});
const nameMap: Record<number, string> = {};
crons.forEach((c: any) => {
nameMap[c.id] = c.name || c.command;
const crons = await CrontabModel.findAll({
where: { id: { [Op.in]: ids } },
raw: true,
});
const nameMap: Record<number, string> = {};
crons.forEach((c: any) => { nameMap[c.id] = c.name || c.command; });
const data = rows.map((r: any, i) => ({
rank: i + 1,
name: nameMap[Number(r.ref_id)] || tf('任务#%s', r.ref_id),
@@ -201,42 +270,48 @@ export default (app: Router) => {
avgTime: Math.round(Number(r.total_time) / Number(r.run_count)),
successRate:
Number(r.run_count) > 0
? ((Number(r.success_count) / Number(r.run_count)) * 100).toFixed(
1,
)
? ((Number(r.success_count) / Number(r.run_count)) * 100).toFixed(1)
: '0',
}));
res.send({ code: 200, data });
} catch (e) {
next(e);
}
},
);
route.get(
'/runtime',
async (req: Request, res: Response, next: NextFunction) => {
try {
const runningInstances = await RunningInstanceModel.find(
{
const runningInstances = await RunningInstanceModel.findAll({
where: {
status: InstanceStatus.running,
},
{},
);
const queuedCrons = await CrontabModel.find(
{
raw: true,
});
const queuedCrons = await CrontabModel.findAll({
where: {
status: 3, // queued
},
{},
);
raw: true,
});
// Fetch cron names for running instances
const cronIds = [
...new Set(runningInstances.map((i: any) => i.cron_id)),
];
const crons =
cronIds.length > 0
? await CrontabModel.find({ id: cronIds }, {})
? await CrontabModel.findAll({
where: { id: cronIds },
raw: true,
})
: [];
const cronMap = new Map(crons.map((c: any) => [c.id, c]));
const now = dayjs().unix();
const running = runningInstances.map((inst: any) => {
const cron = cronMap.get(inst.cron_id);
@@ -249,18 +324,19 @@ export default (app: Router) => {
logPath: inst.log_path,
};
});
const dayAgo = dayjs().subtract(24, 'hour').unix();
const idleTasks = await CrontabModel.find(
{
const idleTasks = await CrontabModel.findAll({
where: {
isDisabled: 0,
status: 1,
last_execution_time: { ['$lt']: dayAgo },
last_execution_time: { [Op.lt]: dayAgo },
},
{
orderBy: orderBy([['last_execution_time', 'ASC']]),
limit: 5,
},
);
order: [['last_execution_time', 'ASC']],
limit: 5,
raw: true,
});
res.send({
code: 200,
data: {
@@ -281,45 +357,33 @@ export default (app: Router) => {
}
},
);
route.get(
'/labels',
async (req: Request, res: Response, next: NextFunction) => {
try {
const today = dayjs().format('YYYY-MM-DD');
const [crons, stats] = await Promise.all([
CrontabModel.find({ isDisabled: 0 }, {}),
CrontabStatModel.find({ date: today }, {}),
]);
const [crons, stats] = (await Promise.all([
CrontabModel.findAll({ where: { isDisabled: 0 }, raw: true }),
CrontabStatModel.findAll({ where: { date: today }, raw: true }),
]));
const statMap: Record<number, any> = {};
stats.forEach((s: any) => {
statMap[s.ref_id] = s;
});
const labelMap: Record<
string,
{
count: number;
runs: number;
success: number;
totalTime: number;
}
> = {};
stats.forEach((s: any) => { statMap[s.ref_id] = s; });
const labelMap: Record<string, { count: number; runs: number; success: number; totalTime: number }> = {};
crons.forEach((c) => {
let rawLabels = c.labels;
if (typeof rawLabels === 'string') rawLabels = JSON.parse(rawLabels);
const labels: string[] = Array.isArray(rawLabels)
? [
...new Set(
(rawLabels as string[]).filter((l: string) => !isEmpty(l)),
),
]
? [...new Set((rawLabels as string[]).filter((l: string) => !isEmpty(l)))]
: [];
if (labels.length === 0) {
labels.push(t('未分类'));
}
const st = statMap[c.id!];
labels.forEach((label: string) => {
if (!labelMap[label])
labelMap[label] = { count: 0, runs: 0, success: 0, totalTime: 0 };
if (!labelMap[label]) labelMap[label] = { count: 0, runs: 0, success: 0, totalTime: 0 };
labelMap[label].count += 1;
if (st) {
labelMap[label].runs += Number(st.run_count) || 0;
@@ -328,22 +392,24 @@ export default (app: Router) => {
}
});
});
const data = Object.entries(labelMap)
.map(([label, v]) => ({
label,
count: v.count,
todayRuns: v.runs,
successRate:
v.runs > 0 ? ((v.success / v.runs) * 100).toFixed(1) : '0',
successRate: v.runs > 0 ? ((v.success / v.runs) * 100).toFixed(1) : '0',
avgTime: v.runs > 0 ? Math.round(v.totalTime / v.runs) : 0,
}))
.sort((a, b) => b.todayRuns - a.todayRuns);
res.send({ code: 200, data });
} catch (e) {
next(e);
}
},
);
route.get(
'/system',
async (req: Request, res: Response, next: NextFunction) => {
@@ -356,9 +422,7 @@ export default (app: Router) => {
uptime: Math.floor(process.uptime()),
memTotal: os.totalmem(),
memFree: os.freemem(),
memUsagePercent: ((1 - os.freemem() / os.totalmem()) * 100).toFixed(
1,
),
memUsagePercent: ((1 - os.freemem() / os.totalmem()) * 100).toFixed(1),
heapUsed: Math.round(memUsage.heapUsed / 1024 / 1024),
heapTotal: Math.round(memUsage.heapTotal / 1024 / 1024),
loadAvg: os.loadavg().map((v) => Number(v.toFixed(2))),
+5 -2
View File
@@ -1,16 +1,19 @@
import { errStack } from '../shared/errors';
// Also terminate if the primary disappears without forwarding a signal.
const onParentDisconnect = () => process.exit(1);
process.once('disconnect', onParentDisconnect);
async function initializeDatabase() {
const { database } = await import('../data');
const { sequelize } = await import('../data');
try {
const { default: loadDatabase } = await import('../loaders/db');
await loadDatabase();
} finally {
await database.close();
await sequelize.close();
}
}
initializeDatabase()
.catch((error) => {
console.error(`[boot] Database initialization failed:\n${errStack(error)}`);
+19 -21
View File
@@ -3,12 +3,11 @@
import 'reflect-metadata';
import dns from 'dns';
import http from 'http';
// The first scheduler connection can outlive its short RPC deadline. Let the
// normal single-flight recovery finish within the supervised startup budget.
async function waitForUpgradeScheduler(
readiness: {
recover(): Promise<boolean>;
},
readiness: { recover(): Promise<boolean> },
timeoutMs: number,
retryMs = 250,
) {
@@ -23,12 +22,13 @@ async function waitForUpgradeScheduler(
} while (Date.now() < deadline);
throw new Error('Upgrade scheduler readiness failed');
}
async function probeUpgrade() {
if (!process.send || process.env.QL_UPGRADE_PROBE !== '1')
throw new Error('Upgrade probe requires its isolated supervisor');
dns.setDefaultResultOrder('ipv4first');
const { default: config } = await import('../config');
const { database } = await import('../data');
const { sequelize } = await import('../data');
const { default: loadDatabase } = await import('../loaders/db');
const { Container } = await import('typedi');
const { GrpcServerService } = await import('../services/grpc');
@@ -43,17 +43,17 @@ async function probeUpgrade() {
try {
await loadDatabase();
// Exercise an actual on-disk transaction as well as schema initialization.
await database.transaction(async () => {
await database.query('CREATE TABLE upgrade_probe (value INTEGER)');
await database.query('INSERT INTO upgrade_probe VALUES (42)');
const rows = await database.query('SELECT value FROM upgrade_probe');
if (
(
rows[0] as {
value: number;
}
)?.value !== 42
)
await sequelize.transaction(async (transaction) => {
await sequelize.query('CREATE TABLE upgrade_probe (value INTEGER)', {
transaction,
});
await sequelize.query('INSERT INTO upgrade_probe VALUES (42)', {
transaction,
});
const [rows] = await sequelize.query('SELECT value FROM upgrade_probe', {
transaction,
});
if ((rows[0] as { value: number })?.value !== 42)
throw new Error('Upgrade database readback failed');
});
inject();
@@ -64,11 +64,8 @@ async function probeUpgrade() {
cronClient.readiness,
process.arch === 'arm' &&
Number(
(
process.config.variables as {
arm_version?: string | number;
}
).arm_version,
(process.config.variables as { arm_version?: string | number })
.arm_version,
) === 6
? 60000
: 10000,
@@ -122,9 +119,10 @@ async function probeUpgrade() {
} finally {
await server.shutdown();
grpc.getServer().forceShutdown();
await database.close();
await sequelize.close();
}
}
process.once('disconnect', () => process.exit(1));
probeUpgrade()
.then(() => process.exit(0))
-28
View File
@@ -1,28 +0,0 @@
import { EventEmitter } from 'events';
import { database } from '.';
import { DatabaseClient } from './databaseClient';
// Keep the existing Keyv serialization and namespace. Only its SQLite transport
// changes; old cached authentication records remain readable in either version.
export class CacheStore extends EventEmitter {
readonly opts = { dialect: 'sqlite' };
namespace?: string;
constructor(private client: () => DatabaseClient = () => database) {
super();
}
get(key: string): Promise<string | undefined> {
return this.client().request('cache', 'get', key);
}
async set(key: string, value: string) {
return this.client().request('cache', 'set', key, value);
}
async delete(key: string): Promise<boolean> {
return this.client().request('cache', 'delete', key);
}
async clear(): Promise<void> {
throw new Error('Clearing all authentication namespaces is not supported');
}
async disconnect(): Promise<void> {
/* The shared database client owns the worker. */
}
}
+36 -3
View File
@@ -1,4 +1,5 @@
import { Repository, RecordRow } from './repository';
import { sequelize } from '.';
import { DataTypes, Model, ModelDefined } from 'sequelize';
export class Crontab {
name?: string;
@@ -61,5 +62,37 @@ export enum CrontabStatus {
'queued' = 3,
}
export type CronInstance = RecordRow<Crontab>;
export const CrontabModel = new Repository<Crontab>('Crontab');
export interface CronInstance extends Model<Crontab, Crontab>, Crontab {}
export const CrontabModel = sequelize.define<CronInstance>('Crontab', {
name: {
unique: 'compositeIndex',
type: DataTypes.STRING,
},
command: {
unique: 'compositeIndex',
type: DataTypes.STRING,
},
schedule: {
unique: 'compositeIndex',
type: DataTypes.STRING,
},
timestamp: DataTypes.STRING,
saved: DataTypes.BOOLEAN,
status: DataTypes.NUMBER,
isSystem: DataTypes.NUMBER,
pid: DataTypes.NUMBER,
isDisabled: DataTypes.NUMBER,
isPinned: DataTypes.NUMBER,
log_path: DataTypes.STRING,
queued_token: DataTypes.STRING,
labels: DataTypes.JSON,
last_running_time: DataTypes.NUMBER,
last_execution_time: DataTypes.NUMBER,
sub_id: { type: DataTypes.NUMBER, allowNull: true },
extra_schedules: DataTypes.JSON,
task_before: DataTypes.STRING,
task_after: DataTypes.STRING,
log_name: DataTypes.STRING,
allow_multiple_instances: DataTypes.NUMBER,
work_dir: DataTypes.STRING,
});
+48 -3
View File
@@ -1,4 +1,5 @@
import { Repository, RecordRow } from './repository';
import { DataTypes, Model } from 'sequelize';
import { sequelize } from '.';
export class CrontabStat {
id?: number;
@@ -22,5 +23,49 @@ export class CrontabStat {
}
}
export type CrontabStatInstance = RecordRow<CrontabStat>;
export const CrontabStatModel = new Repository<CrontabStat>('CrontabStat');
export interface CrontabStatInstance extends Model<CrontabStat, CrontabStat>, CrontabStat {}
export const CrontabStatModel = sequelize.define<CrontabStatInstance>(
'CrontabStat',
{
id: {
type: DataTypes.INTEGER,
primaryKey: true,
autoIncrement: true,
},
ref_id: {
type: DataTypes.NUMBER,
allowNull: false,
},
date: {
type: DataTypes.STRING,
allowNull: false,
},
run_count: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
success_count: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
fail_count: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
total_time: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
max_time: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
},
{
indexes: [
{ unique: true, fields: ['ref_id', 'date'] },
{ fields: ['date'] },
],
},
);
+23 -3
View File
@@ -1,4 +1,5 @@
import { Repository, RecordRow } from './repository';
import { sequelize } from '.';
import { DataTypes, Model } from 'sequelize';
export enum CronViewType {
'系统' = 1,
@@ -38,5 +39,24 @@ export class CrontabView {
}
}
export type CronViewInstance = RecordRow<CrontabView>;
export const CrontabViewModel = new Repository<CrontabView>('CrontabView');
export interface CronViewInstance
extends Model<CrontabView, CrontabView>,
CrontabView {}
export const CrontabViewModel = sequelize.define<CronViewInstance>(
'CrontabView',
{
name: {
unique: 'name',
type: DataTypes.STRING,
},
position: DataTypes.NUMBER,
isDisabled: DataTypes.NUMBER,
filters: DataTypes.JSON,
sorts: DataTypes.JSON,
filterRelation: {
type: DataTypes.STRING,
allowNull: true,
},
type: DataTypes.NUMBER,
},
);
-325
View File
@@ -1,325 +0,0 @@
import { AsyncLocalStorage } from 'async_hooks';
import { ChildProcess, fork } from 'child_process';
import { join } from 'path';
import { setTimeout, clearTimeout } from 'timers';
import { DatabaseError } from './databaseErrors';
type Scope = { active: boolean; child: boolean; token: number };
type Reply = {
id: number;
value?: any;
error?: { code: string; message: string };
};
type Limits = { timeoutMs?: number; queueLimit?: number };
// One worker per backend process; SQLite itself coordinates independent processes.
// A transaction owns this client's queue until its callback and commit finish.
export class DatabaseClient {
private worker?: ChildProcess;
private stopped: Promise<void> = Promise.resolve();
private failed?: Error;
private closed = false;
private nextId = 0;
private queued = 0;
private tail: Promise<void> = Promise.resolve();
private pending = new Map<
number,
{
resolve(value: any): void;
reject(error: Error): void;
timer: NodeJS.Timeout;
}
>();
private scope = new AsyncLocalStorage<Scope>();
private readonly timeoutMs: number;
private readonly queueLimit: number;
constructor(
public readonly storage: string,
private cacheStorage?: string,
limits: Limits = {},
) {
this.timeoutMs =
limits.timeoutMs ?? (process.arch === 'arm' ? 300000 : 60000);
this.queueLimit = limits.queueLimit ?? 256;
if (
!Number.isSafeInteger(this.timeoutMs) ||
this.timeoutMs < 1 ||
!Number.isSafeInteger(this.queueLimit) ||
this.queueLimit < 1
)
throw new Error('Invalid database worker limits');
}
private abort(error: Error) {
this.failed ||= error;
for (const request of this.pending.values()) {
clearTimeout(request.timer);
request.reject(this.failed);
}
this.pending.clear();
this.unref();
this.worker?.kill('SIGKILL');
}
private unref() {
this.worker?.unref();
this.worker?.channel?.unref();
}
private start() {
if (this.worker) return this.worker;
const [major, minor] = process.versions.node.split('.').map(Number);
if (major < 22 || (major === 22 && minor < 17))
throw new DatabaseError(
'Qinglong requires Node >=22.17; keep the previous installation until Node is upgraded',
'UNSUPPORTED_NODE',
);
// ts-node's CLI places its executable in execArgv, rather than using -r.
// Source workers register their own compiler; compiled workers need none.
const preloads = __filename.endsWith('.ts')
? ['--require', require.resolve('ts-node/register/transpile-only')]
: [];
const armv6 =
process.arch === 'arm' &&
Number(
(process.config.variables as { arm_version?: string | number })
.arm_version,
) === 6;
const worker = fork(require.resolve('./databaseProcess'), [], {
serialization: 'advanced',
stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
execArgv: [
...preloads,
...(armv6 ? ['--jitless'] : []),
'--max-old-space-size=128',
],
env: {
...process.env,
NODE_OPTIONS: '',
...(__filename.endsWith('.ts')
? { TS_NODE_PROJECT: join(__dirname, 'tsconfig.worker.json') }
: {}),
},
});
this.stopped = new Promise((resolve) =>
worker.once('close', () => resolve()),
);
const onOwnerExit = () => worker.kill('SIGKILL');
process.once('exit', onOwnerExit);
worker.once('close', () => process.removeListener('exit', onOwnerExit));
this.worker = worker;
worker.on('message', (reply: Reply & { type?: string }) => {
if (reply.type === 'fatal' && reply.error) {
this.abort(new DatabaseError(reply.error.message, reply.error.code));
return;
}
const request = this.pending.get(reply.id);
if (!request) return;
this.pending.delete(reply.id);
clearTimeout(request.timer);
if (!this.pending.size) this.unref();
if (reply.error)
request.reject(
new DatabaseError(reply.error.message, reply.error.code),
);
else request.resolve(reply.value);
});
const fail = () =>
this.abort(
new DatabaseError(
'Database worker stopped; restart the service before retrying a write',
'DATABASE_WORKER_EXIT',
),
);
worker.on('error', fail);
worker.on('exit', () => {
if (!this.closed) fail();
});
worker.send({
storage: this.storage,
cacheStorage: this.cacheStorage,
ownerPid: process.pid,
});
this.unref();
return worker;
}
private send(
operation: string,
args: any[] = [],
token?: number,
): Promise<any> {
if (this.failed) return Promise.reject(this.failed);
if (this.closed)
return Promise.reject(
new DatabaseError('Database is closed', 'DATABASE_CLOSED'),
);
return new Promise((resolve, reject) => {
try {
const worker = this.start();
const id = ++this.nextId;
const timer = setTimeout(
() =>
this.abort(
new DatabaseError(
'Database worker timed out; writes are never automatically replayed',
'DATABASE_WORKER_TIMEOUT',
),
),
this.timeoutMs,
);
timer.unref();
this.pending.set(id, { resolve, reject, timer });
worker.ref();
worker.channel?.ref();
try {
worker.send({ id, operation, args, token }, (error) => {
if (error)
this.abort(
new DatabaseError(
'Database IPC disconnected',
'DATABASE_WORKER_EXIT',
),
);
});
} catch (error) {
this.pending.delete(id);
clearTimeout(timer);
if (!this.pending.size) this.unref();
reject(error);
}
} catch (error) {
reject(error);
}
});
}
private exclusive<T>(
operation: () => Promise<T>,
shutdown = false,
): Promise<T> {
if (!shutdown && this.queued >= this.queueLimit)
return Promise.reject(
new DatabaseError('Database request queue is full', 'DATABASE_BUSY'),
);
this.queued++;
const previous = this.tail;
let release!: () => void;
this.tail = new Promise<void>((resolve) => {
release = resolve;
});
return previous.then(operation).finally(() => {
this.queued--;
release();
});
}
request<T = any>(operation: string, ...args: any[]): Promise<T> {
const scope = this.scope.getStore();
if (scope) {
if (!scope.active || scope.child)
return Promise.reject(
new DatabaseError(
'Transaction scope is no longer available',
'TRANSACTION_SCOPE',
),
);
if (this.pending.size >= this.queueLimit)
return Promise.reject(
new DatabaseError('Database request queue is full', 'DATABASE_BUSY'),
);
return this.send(operation, args, scope.token);
}
return this.exclusive(() => this.send(operation, args));
}
async transaction<T>(operation: () => Promise<T>): Promise<T> {
const parent = this.scope.getStore();
if (parent && (!parent.active || parent.child))
throw new DatabaseError(
'Concurrent or expired nested transaction',
'TRANSACTION_SCOPE',
);
const run = async () => {
const token = ++this.nextId;
if (parent) parent.child = true;
const scope: Scope = { active: true, child: false, token };
let began = false;
let timer: NodeJS.Timeout | undefined;
try {
await this.send('begin', [], token);
began = true;
const deadline = new Promise<never>((_, reject) => {
timer = setTimeout(
() =>
reject(
new DatabaseError(
'Database transaction timed out',
'DATABASE_TRANSACTION_TIMEOUT',
),
),
this.timeoutMs,
);
});
const value = await Promise.race([
this.scope.run(scope, operation),
deadline,
]);
scope.active = false;
await this.send('commit', [], token);
return value;
} catch (error) {
scope.active = false;
if (began) {
try {
await this.send('rollback', [], token);
} catch {
this.abort(
new DatabaseError(
'Transaction rollback failed; restart the service',
'DATABASE_ROLLBACK_FAILED',
),
);
}
}
throw error;
} finally {
clearTimeout(timer);
if (parent) parent.child = false;
}
};
return parent ? run() : this.exclusive(run);
}
query<T = Record<string, unknown>>(
sql: string,
params: unknown[] = [],
): Promise<T[]> {
return this.request('query', sql, params);
}
get healthy() {
return !this.failed && !this.closed;
}
async close() {
if (this.scope.getStore())
throw new DatabaseError(
'Cannot close within a transaction',
'TRANSACTION_SCOPE',
);
await this.exclusive(async () => {
if (this.closed) return;
try {
if (this.worker && !this.failed) await this.send('close');
} finally {
this.closed = true;
this.worker?.ref();
this.worker?.kill('SIGKILL');
await this.stopped;
}
}, true);
}
}
-36
View File
@@ -1,36 +0,0 @@
// Error details (SQL, bindings, credentials) stay inside the database worker.
export class DatabaseError extends Error {
constructor(message: string, public readonly code: string) {
super(message);
this.name = 'DatabaseError';
}
}
export function databaseStartupFailure(value: unknown) {
const error = (value && typeof value === 'object' ? value : {}) as Error & { code?: string };
const missing = /Cannot find (?:module|package) ['"]([^'"]+)['"]/.exec(
error.message || '',
)?.[1];
const importer = /Require stack:\r?\n-\s+([^\r\n]+)/.exec(
error.message || '',
)?.[1];
const dependency =
/MODULE_NOT_FOUND/.test(error.code || '') &&
Boolean(
missing &&
((!missing.startsWith('.') &&
!missing.startsWith('/') &&
!missing.startsWith('node:')) ||
/node_modules[/\\]/.test(missing) ||
/node_modules[/\\]/.test(importer || '')),
);
return dependency
? {
code: 'MODULE_NOT_FOUND',
message: "Cannot find module '@mikro-orm/sql'",
}
: {
code: 'DATABASE_WORKER_EXIT',
message: 'Database process failed to start',
};
}
-61
View File
@@ -1,61 +0,0 @@
import { databaseStartupFailure } from './databaseErrors';
import { Worker } from 'worker_threads';
// IPC disconnect cannot run while DatabaseSync is executing SQL. Start this
// independent, allocation-light monitor before opening SQLite so a SIGKILLed
// backend cannot leave an orphan holding the old service's database write lock.
async function monitorOwner(ownerPid: number) {
if (!Number.isSafeInteger(ownerPid) || ownerPid < 1)
throw new Error('Invalid database owner');
const monitor = new Worker(
`
const {parentPort,workerData}=require('node:worker_threads');
process.on('uncaughtException',()=>process.kill(process.pid,'SIGKILL'));
const check=()=>{
if(process.ppid!==workerData.ownerPid)process.kill(process.pid,'SIGKILL');
};
check();
setInterval(check,500);
parentPort.postMessage('ready');
`,
{
eval: true,
workerData: { ownerPid },
// V8's process-wide --jitless remains active on ARMv6. Do not load the
// source worker's ts-node compiler into this tiny, plain-JavaScript monitor.
execArgv: [],
resourceLimits: {
maxOldGenerationSizeMb: 8,
maxYoungGenerationSizeMb: 2,
stackSizeMb: 1,
},
},
);
monitor.once('exit', () => process.exit(1));
await new Promise<void>((resolve, reject) => {
monitor.once('message', () => resolve());
monitor.once('error', reject);
});
monitor.unref();
}
// The bootstrap catches dependency import errors before the ORM module loads.
// The advanced IPC serializer preserves Date, Buffer and undefined values.
process.once('disconnect', () => process.exit(1));
process.once('message', async (settings) => {
const pending: unknown[] = [];
const buffer = (request: unknown) => pending.push(request);
// Keep a listener throughout asynchronous monitor startup. Node may flush
// several buffered IPC messages while delivering the initial settings.
process.on('message', buffer);
try {
await monitorOwner((settings as { ownerPid: number }).ownerPid);
require('./databaseWorker').startDatabaseWorker(settings);
process.removeListener('message', buffer);
for (const request of pending) process.emit('message', request, undefined);
} catch (error) {
process.send?.({ type: 'fatal', error: databaseStartupFailure(error) });
} finally {
process.removeListener('message', buffer);
}
});
-280
View File
@@ -1,280 +0,0 @@
import { mkdirSync } from 'fs';
import { dirname } from 'path';
import { EntitySchema, Type } from '@mikro-orm/core';
import { MikroORM, SqliteDriver, NodeSqliteDialect, raw } from '@mikro-orm/sql';
import { entityDefinitions } from './schema';
// Sequelize DATE readers expect TEXT. Numeric epoch dates would break rollback.
class LegacyDateType extends Type<Date, string> {
convertToDatabaseValue(value: Date) {
return value == null
? value
: value.toISOString().replace('T', ' ').replace('Z', ' +00:00');
}
convertToJSValue(value: string | Date) {
return value == null
? value
: value instanceof Date
? value
: new Date(value);
}
getColumnType() {
return 'DATETIME';
}
compareAsType() {
return 'string' as const;
}
get runtimeType() {
return 'Date';
}
}
const entities = Object.values(entityDefinitions).map(
(definition) =>
new EntitySchema({
...definition,
properties: Object.fromEntries(
Object.entries(definition.properties).map(([name, field]) => [
name,
{
...field,
type:
field.type === 'legacyDate' ? new LegacyDateType() : field.type,
},
]),
),
} as any),
);
let orm: Awaited<ReturnType<typeof MikroORM.init<SqliteDriver>>>;
let cache: any;
const transactions: number[] = [];
let ready: Promise<void>;
let workerData: { storage: string; cacheStorage?: string; ownerPid: number };
async function initialize() {
if (workerData.storage !== ':memory:')
mkdirSync(dirname(workerData.storage), { recursive: true });
orm = await MikroORM.init<SqliteDriver>({
driver: SqliteDriver,
driverOptions: new NodeSqliteDialect(workerData.storage),
dbName: workerData.storage,
entities,
debug: false,
allowGlobalContext: false,
discovery: { warnWhenNoEntities: false },
});
await orm.em.getConnection().execute('PRAGMA busy_timeout=10000');
}
function filter(name: keyof typeof entityDefinitions, value: any): any {
if (value === undefined) throw new Error('Undefined database predicate');
if (value == null || typeof value !== 'object' || value instanceof Date)
return value;
if (Array.isArray(value)) return value.map((part) => filter(name, part));
const properties = entityDefinitions[name].properties;
return Object.fromEntries(
Object.entries(value).map(([key, item]: [string, any]) => {
if (item === undefined) throw new Error('Undefined database predicate');
// LIKE searches JSON's serialized text, matching the pre-migration behavior.
if (
key in properties &&
(properties as any)[key].type === 'json' &&
item &&
typeof item === 'object' &&
'$like' in item
)
return [raw(`cast("${key}" as text)`), item];
return [key, filter(name, item)];
}),
);
}
function values(
name: keyof typeof entityDefinitions,
payload: Record<string, any>,
insert = false,
) {
const properties = entityDefinitions[name].properties;
const data = Object.fromEntries(
Object.entries(payload).filter(
([key, value]) => key in properties && value !== undefined,
),
);
for (const [key, value] of Object.entries(data)) {
if ((properties as any)[key].type === 'json' && value !== null)
data[key] = JSON.parse(JSON.stringify(value));
}
const now = new Date();
if (insert && !data.createdAt) data.createdAt = now;
data.updatedAt = now;
return data;
}
function plain(name: keyof typeof entityDefinitions, row: any) {
if (!row) return null;
return Object.fromEntries(
Object.keys(entityDefinitions[name].properties)
.filter((key) => row[key] !== undefined)
.map((key) => [key, row[key]]),
);
}
async function execute(
operation: string,
args: any[],
token?: number,
): Promise<any> {
await ready;
// Reject queued work after owner death; the monitor interrupts running SQL.
if (process.ppid !== workerData.ownerPid) process.exit(1);
const connection = orm.em.getConnection();
if (operation === 'begin') {
await connection.execute(
transactions.length ? `SAVEPOINT "ql_${token}"` : 'BEGIN IMMEDIATE',
);
transactions.push(token!);
return;
}
if (token !== transactions[transactions.length - 1])
throw new Error('Database transaction ownership mismatch');
if (operation === 'commit' || operation === 'rollback') {
if (!transactions.length) throw new Error('No active database transaction');
const nested = transactions.length > 1;
if (operation === 'rollback' && nested)
await connection.execute(`ROLLBACK TO SAVEPOINT "ql_${token}"`);
await connection.execute(
nested ? `RELEASE SAVEPOINT "ql_${token}"` : operation.toUpperCase(),
);
transactions.pop();
return;
}
if (operation === 'close') {
if (transactions.length)
throw new Error('Cannot close an active transaction');
cache?.close();
await orm.close();
return;
}
if (operation === 'query') return connection.execute(args[0], args[1]);
if (operation === 'cache') {
if (!workerData.cacheStorage)
throw new Error('No cache database configured');
if (!cache) {
const { DatabaseSync } = require('node:sqlite');
mkdirSync(dirname(workerData.cacheStorage), { recursive: true });
cache = new DatabaseSync(workerData.cacheStorage);
cache.exec(
'PRAGMA busy_timeout=10000;CREATE TABLE IF NOT EXISTS keyv(key VARCHAR(255) PRIMARY KEY,value TEXT)',
);
}
const [action, key, value] = args;
switch (action) {
case 'get':
return cache.prepare('SELECT value FROM keyv WHERE key=?').get(key)
?.value;
case 'set':
cache
.prepare(
'INSERT INTO keyv(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value',
)
.run(key, value);
return true;
case 'delete':
return (
cache.prepare('DELETE FROM keyv WHERE key=?').run(key).changes > 0
);
default:
throw new Error('Unsupported cache operation');
}
}
const [name, condition, options = {}] = args as [
keyof typeof entityDefinitions,
any,
any,
];
if (!(name in entityDefinitions)) throw new Error('Unknown entity');
const em = orm.em.fork();
const entityName = entities.find((entity) => entity.meta.className === name)!;
if (operation !== 'insert' && condition == null)
throw new Error('Missing database predicate');
const query = operation === 'insert' ? {} : filter(name, condition);
const readOptions = {
...options,
orderBy: options.orderBy?.map((order: Record<string, string>) =>
Object.fromEntries(
Object.entries(order).map(([field, direction]) => [
name === 'Env' && field === 'isPinned'
? raw('COALESCE("isPinned",0)')
: field,
direction,
]),
),
),
};
switch (operation) {
case 'find':
return (await em.find(entityName, query, readOptions)).map((row) =>
plain(name, row),
);
case 'findOne':
return plain(name, await em.findOne(entityName, query, readOptions));
case 'count':
return em.count(entityName, query);
case 'insert': {
const id = await em.insert(entityName, values(name, condition, true));
return plain(name, await em.findOneOrFail(entityName, { id }));
}
case 'nativeUpdate':
return em.nativeUpdate(entityName, query, values(name, options));
case 'nativeDelete':
return em.nativeDelete(entityName, query);
default:
throw new Error('Unsupported database operation');
}
}
export function startDatabaseWorker(settings: typeof workerData) {
workerData = settings;
ready = initialize();
void ready.catch(() => {});
let queue = Promise.resolve();
process.on('message', ({ id, operation, args, token }) => {
// Continue after a failed statement so the owner can still roll back.
queue = queue.then(async () => {
try {
process.send?.({
id,
value: await execute(operation, args, token),
});
} catch (error: any) {
const code =
error.name === 'UniqueConstraintViolationException'
? 'UNIQUE_CONSTRAINT'
: error.name === 'TableNotFoundException'
? 'DATABASE_SCHEMA_MISSING'
: (
{
5: 'SQLITE_BUSY',
6: 'SQLITE_LOCKED',
8: 'SQLITE_READONLY',
10: 'SQLITE_IOERR',
11: 'SQLITE_CORRUPT',
13: 'SQLITE_FULL',
14: 'SQLITE_CANTOPEN',
26: 'SQLITE_NOTADB',
} as Record<number, string>
)[Number(error.errcode) & 255] || 'DATABASE_QUERY_FAILED';
process.send?.({
id,
error: {
code,
message:
code === 'UNIQUE_CONSTRAINT'
? 'Resource already exists'
: `Database operation failed (${code})`,
},
});
}
});
});
}
+16 -3
View File
@@ -1,4 +1,5 @@
import { Repository, RecordRow } from './repository';
import { sequelize } from '.';
import { DataTypes, Model, ModelDefined } from 'sequelize';
export class Dependence {
timestamp?: string;
@@ -46,5 +47,17 @@ export enum versionDependenceCommandTypes {
'=',
}
export type DependenceInstance = RecordRow<Dependence>;
export const DependenceModel = new Repository<Dependence>('Dependence');
export interface DependenceInstance
extends Model<Dependence, Dependence>,
Dependence {}
export const DependenceModel = sequelize.define<DependenceInstance>(
'Dependence',
{
name: DataTypes.STRING,
type: DataTypes.NUMBER,
timestamp: DataTypes.STRING,
status: DataTypes.NUMBER,
log: DataTypes.JSON,
remark: DataTypes.STRING,
},
);
+13 -3
View File
@@ -1,4 +1,5 @@
import { Repository, RecordRow } from './repository';
import { DataTypes, Model } from 'sequelize';
import { sequelize } from '.';
export class Env {
value?: string;
@@ -37,5 +38,14 @@ export const initPosition = 4500000000000000;
export const stepPosition = 10000000000;
export const minPosition = 100;
export type EnvInstance = RecordRow<Env>;
export const EnvModel = new Repository<Env>('Env');
export interface EnvInstance extends Model<Env, Env>, Env {}
export const EnvModel = sequelize.define<EnvInstance>('Env', {
value: { type: DataTypes.STRING, unique: 'compositeIndex' },
timestamp: DataTypes.STRING,
status: DataTypes.NUMBER,
position: DataTypes.NUMBER,
name: { type: DataTypes.STRING, unique: 'compositeIndex' },
remarks: DataTypes.STRING,
isPinned: DataTypes.NUMBER,
labels: DataTypes.JSON,
});
+18 -5
View File
@@ -1,10 +1,23 @@
import { Sequelize, Transaction } from 'sequelize';
import config from '../config/index';
import { join } from 'path';
import { DatabaseClient } from './databaseClient';
export const database = new DatabaseClient(
join(config.dbPath, 'database.sqlite'),
join(config.dbPath, 'keyv.sqlite'),
);
export const sequelize = new Sequelize({
dialect: 'sqlite',
storage: join(config.dbPath, 'database.sqlite'),
logging: false,
retry: {
max: 10,
match: ['SQLITE_BUSY: database is locked'],
},
pool: {
max: 5,
min: 2,
idle: 30000,
acquire: 30000,
evict: 10000,
},
transactionType: Transaction.TYPES.IMMEDIATE,
});
export type ResponseType<T> = { code: number; data?: T; message?: string };
+11 -11
View File
@@ -1,4 +1,5 @@
import { Repository, RecordRow } from './repository';
import { sequelize } from '.';
import { DataTypes, Model, ModelDefined } from 'sequelize';
export class App {
name: string;
@@ -23,14 +24,13 @@ export interface AppToken {
expiration: number;
}
export type AppScope =
| 'envs'
| 'crons'
| 'configs'
| 'scripts'
| 'logs'
| 'system'
| 'dashboard';
export type AppScope = 'envs' | 'crons' | 'configs' | 'scripts' | 'logs' | 'system' | 'dashboard';
export type AppInstance = RecordRow<App>;
export const AppModel = new Repository<App>('App');
export interface AppInstance extends Model<App, App>, App {}
export const AppModel = sequelize.define<AppInstance>('App', {
name: { type: DataTypes.STRING, unique: 'name' },
scopes: DataTypes.JSON,
client_id: DataTypes.STRING,
client_secret: DataTypes.STRING,
tokens: DataTypes.JSON,
});
-25
View File
@@ -1,25 +0,0 @@
export type Query = Record<string, any>;
export type QuerySpec = {
where: Query;
orderBy?: Record<string, string>[];
offset?: number;
limit?: number;
};
export function orderBy(order: string[][]): Record<string, string>[] {
return order.map(([field, direction]) => {
if (!/^(asc|desc)$/i.test(direction))
throw new Error('Invalid sort direction');
return { [field]: direction.toLowerCase() };
});
}
export function textSearch(fields: string[], text: string): Query {
return {
$or: fields.flatMap((field) =>
[...new Set([text, encodeURI(text)])].map((value) => ({
[field]: { $like: `%${value}%` },
})),
),
};
}
-54
View File
@@ -1,54 +0,0 @@
import { database } from '.';
import { DatabaseClient } from './databaseClient';
export type RecordRow<T> = T & { id: number; createdAt: Date; updatedAt: Date };
export type Query = Record<string, any>;
export type ReadOptions = {
fields?: string[];
orderBy?: Record<string, string>[];
limit?: number;
offset?: number;
};
// A small IPC boundary with MikroORM's native query/operation names. No ORM
// entities or request identity maps escape the single database worker.
export class Repository<T> {
constructor(
private name: string,
private client: () => DatabaseClient = () => database,
) {}
find(where: Query = {}, options: ReadOptions = {}): Promise<RecordRow<T>[]> {
return this.client().request('find', this.name, where, options);
}
findOne(
where: Query,
options: ReadOptions = {},
): Promise<RecordRow<T> | null> {
return this.client().request('findOne', this.name, where, options);
}
count(where: Query = {}): Promise<number> {
return this.client().request('count', this.name, where);
}
insert(data: Partial<T>): Promise<RecordRow<T>> {
return this.client().request('insert', this.name, data);
}
nativeUpdate(where: Query, data: Partial<T>): Promise<number> {
return this.client().request('nativeUpdate', this.name, where, data);
}
nativeDelete(where: Query): Promise<number> {
return this.client().request('nativeDelete', this.name, where);
}
async insertMany(data: Partial<T>[]): Promise<RecordRow<T>[]> {
return this.client().transaction(async () => {
const rows = [];
for (const item of data) rows.push(await this.insert(item));
return rows;
});
}
async getOrCreate(where: Query): Promise<RecordRow<T>> {
return this.client().transaction(
async () =>
(await this.findOne(where)) || this.insert(where as Partial<T>),
);
}
}
+38 -3
View File
@@ -1,4 +1,5 @@
import { Repository, RecordRow } from './repository';
import { sequelize } from '.';
import { DataTypes, Model } from 'sequelize';
export enum InstanceStatus {
'running' = 0,
@@ -40,7 +41,41 @@ export class RunningInstance {
}
}
export type RunningInstanceModel = RecordRow<RunningInstanceAttributes>;
export const RunningInstanceModel = new Repository<RunningInstanceAttributes>(
export interface RunningInstanceModel
extends Model<RunningInstanceAttributes, RunningInstanceAttributes>,
RunningInstanceAttributes {}
export const RunningInstanceModel = sequelize.define<RunningInstanceModel>(
'RunningInstance',
{
cron_id: {
type: DataTypes.NUMBER,
allowNull: false,
},
pid: {
type: DataTypes.NUMBER,
allowNull: true,
},
log_path: {
type: DataTypes.STRING,
allowNull: true,
},
started_at: {
type: DataTypes.NUMBER,
allowNull: false,
},
finished_at: {
type: DataTypes.NUMBER,
allowNull: true,
},
status: {
type: DataTypes.NUMBER,
allowNull: false,
defaultValue: InstanceStatus.running,
},
exit_code: {
type: DataTypes.NUMBER,
allowNull: true,
},
},
);
-178
View File
@@ -1,178 +0,0 @@
// Persisted SQLite names and types are part of the rollback contract.
type Field = {
type: string;
fieldName: string;
nullable?: boolean;
primary?: boolean;
autoincrement?: boolean;
};
function entity<Name extends string>(
name: Name,
tableName: string,
groups: Record<string, string[]>,
required: string[] = [],
) {
const properties: Record<string, Field> = {
id: { type: 'number', fieldName: 'id', primary: true, autoincrement: true },
};
for (const [type, fields] of Object.entries(groups))
for (const field of fields)
properties[field] = {
type,
fieldName: field,
nullable: !required.includes(field),
};
for (const field of ['createdAt', 'updatedAt'])
properties[field] = {
type: 'legacyDate',
fieldName: field,
nullable: false,
};
return { name, tableName, properties };
}
export const entityDefinitions = {
App: entity(
'App',
'Apps',
{
string: ['name', 'client_id', 'client_secret'],
json: ['scopes', 'tokens'],
},
[],
),
Auth: entity('Auth', 'Auths', { string: ['ip', 'type'], json: ['info'] }, []),
CrontabStat: entity(
'CrontabStat',
'CrontabStats',
{
number: [
'ref_id',
'run_count',
'success_count',
'fail_count',
'total_time',
'max_time',
],
string: ['date'],
},
['ref_id', 'date'],
),
CrontabView: entity(
'CrontabView',
'CrontabViews',
{
string: ['name', 'filterRelation'],
number: ['position', 'isDisabled', 'type'],
json: ['filters', 'sorts'],
},
[],
),
Crontab: entity(
'Crontab',
'Crontabs',
{
string: [
'name',
'command',
'schedule',
'timestamp',
'log_path',
'queued_token',
'task_before',
'task_after',
'log_name',
'work_dir',
],
boolean: ['saved'],
number: [
'status',
'isSystem',
'pid',
'isDisabled',
'isPinned',
'last_running_time',
'last_execution_time',
'sub_id',
'allow_multiple_instances',
],
json: ['labels', 'extra_schedules'],
},
[],
),
Dependence: entity(
'Dependence',
'Dependences',
{
string: ['name', 'timestamp', 'remark'],
number: ['type', 'status'],
json: ['log'],
},
[],
),
Env: entity(
'Env',
'Envs',
{
string: ['value', 'timestamp', 'name', 'remarks'],
number: ['status', 'position', 'isPinned'],
json: ['labels'],
},
[],
),
RunningInstance: entity(
'RunningInstance',
'RunningInstances',
{
number: [
'cron_id',
'pid',
'started_at',
'finished_at',
'status',
'exit_code',
],
string: ['log_path'],
},
['cron_id', 'started_at', 'status'],
),
Subscription: entity(
'Subscription',
'Subscriptions',
{
string: [
'name',
'url',
'schedule',
'type',
'whitelist',
'blacklist',
'dependences',
'extensions',
'sub_before',
'sub_after',
'branch',
'pull_type',
'log_path',
'schedule_type',
'alias',
'proxy',
],
json: ['interval_schedule', 'pull_option'],
number: ['status', 'pid', 'is_disabled', 'autoAddCron', 'autoDelCron'],
},
[],
),
};
export const initialSchema = [
'CREATE TABLE IF NOT EXISTS `Apps` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `scopes` JSON, `client_id` VARCHAR(255), `client_secret` VARCHAR(255), `tokens` JSON, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`name`))',
'CREATE TABLE IF NOT EXISTS `Auths` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `ip` VARCHAR(255), `type` VARCHAR(255), `info` JSON, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL)',
'CREATE TABLE IF NOT EXISTS `CrontabStats` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `ref_id` NUMBER NOT NULL, `date` VARCHAR(255) NOT NULL, `run_count` NUMBER DEFAULT 0, `success_count` NUMBER DEFAULT 0, `fail_count` NUMBER DEFAULT 0, `total_time` NUMBER DEFAULT 0, `max_time` NUMBER DEFAULT 0, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL)',
'CREATE TABLE IF NOT EXISTS `CrontabViews` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `position` NUMBER, `isDisabled` NUMBER, `filters` JSON, `sorts` JSON, `filterRelation` VARCHAR(255), `type` NUMBER, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`name`))',
'CREATE TABLE IF NOT EXISTS `Crontabs` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `command` VARCHAR(255), `schedule` VARCHAR(255), `timestamp` VARCHAR(255), `saved` TINYINT(1), `status` NUMBER, `isSystem` NUMBER, `pid` NUMBER, `isDisabled` NUMBER, `isPinned` NUMBER, `log_path` VARCHAR(255), `queued_token` VARCHAR(255), `labels` JSON, `last_running_time` NUMBER, `last_execution_time` NUMBER, `sub_id` NUMBER, `extra_schedules` JSON, `task_before` VARCHAR(255), `task_after` VARCHAR(255), `log_name` VARCHAR(255), `allow_multiple_instances` NUMBER, `work_dir` VARCHAR(255), `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`name`, `command`, `schedule`))',
'CREATE TABLE IF NOT EXISTS `Dependences` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `type` NUMBER, `timestamp` VARCHAR(255), `status` NUMBER, `log` JSON, `remark` VARCHAR(255), `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL)',
'CREATE TABLE IF NOT EXISTS `Envs` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `value` VARCHAR(255), `timestamp` VARCHAR(255), `status` NUMBER, `position` NUMBER, `name` VARCHAR(255), `remarks` VARCHAR(255), `isPinned` NUMBER, `labels` JSON, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`value`, `name`))',
'CREATE TABLE IF NOT EXISTS `RunningInstances` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `cron_id` NUMBER NOT NULL, `pid` NUMBER, `log_path` VARCHAR(255), `started_at` NUMBER NOT NULL, `finished_at` NUMBER, `status` NUMBER NOT NULL DEFAULT 0, `exit_code` NUMBER, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL)',
'CREATE TABLE IF NOT EXISTS `Subscriptions` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `url` VARCHAR(255), `schedule` VARCHAR(255), `interval_schedule` JSON, `type` VARCHAR(255), `whitelist` VARCHAR(255), `blacklist` VARCHAR(255), `status` NUMBER, `dependences` VARCHAR(255), `extensions` VARCHAR(255), `sub_before` VARCHAR(255), `sub_after` VARCHAR(255), `branch` VARCHAR(255), `pull_type` VARCHAR(255), `pull_option` JSON, `pid` NUMBER, `is_disabled` NUMBER, `log_path` VARCHAR(255), `schedule_type` VARCHAR(255), `alias` VARCHAR(255), `proxy` VARCHAR(255), `autoAddCron` NUMBER, `autoDelCron` NUMBER, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`name`, `url`, `schedule`, `interval_schedule`), UNIQUE (`alias`))',
'CREATE INDEX IF NOT EXISTS `crontab_stats_date` ON `CrontabStats` (`date`)',
'CREATE UNIQUE INDEX IF NOT EXISTS `crontab_stats_ref_id_date` ON `CrontabStats` (`ref_id`, `date`)',
];
-45
View File
@@ -1,45 +0,0 @@
import { database } from '.';
// Aggregate SQL stays explicit and bound; public input is never used as an SQL
// identifier. These queries preserve the dashboard response names.
export const statistics = {
async totals(date: string) {
const rows = await database.query(
`SELECT SUM(run_count) AS total_runs,
SUM(success_count) AS total_success, SUM(fail_count) AS total_fail,
SUM(total_time) AS total_time FROM CrontabStats WHERE date=?`,
[date],
);
return rows[0];
},
results(date: string, column: 'success_count' | 'fail_count') {
if (!['success_count', 'fail_count'].includes(column))
throw new Error('Invalid statistic');
return database.query(
`SELECT ref_id,SUM(${column}) AS result_count FROM CrontabStats
WHERE date=? AND ${column}>0 GROUP BY ref_id ORDER BY result_count DESC,ref_id ASC`,
[date],
);
},
trend(dates: string[]) {
if (!dates.length) return Promise.resolve([]);
return database.query(
`SELECT date,SUM(run_count) AS total_runs,
SUM(success_count) AS total_success,SUM(fail_count) AS total_fail FROM CrontabStats
WHERE date IN (${dates
.map(() => '?')
.join(',')}) GROUP BY date ORDER BY date ASC`,
dates,
);
},
ranking(date: string, metric: 'total_time' | 'run_count') {
if (!['total_time', 'run_count'].includes(metric))
throw new Error('Invalid statistic');
return database.query(
`SELECT ref_id,SUM(total_time) AS total_time,SUM(run_count) AS run_count,
SUM(success_count) AS success_count,MAX(max_time) AS max_time FROM CrontabStats
WHERE date=? AND run_count>0 GROUP BY ref_id ORDER BY SUM(${metric}) DESC LIMIT 5`,
[date],
);
},
};
+45 -3
View File
@@ -1,4 +1,5 @@
import { Repository, RecordRow } from './repository';
import { sequelize } from '.';
import { DataTypes, Model, ModelDefined } from 'sequelize';
import { SimpleIntervalSchedule } from 'toad-scheduler';
type SimpleIntervalScheduleUnit = keyof SimpleIntervalSchedule;
@@ -69,5 +70,46 @@ export enum SubscriptionStatus {
'queued',
}
export type SubscriptionInstance = RecordRow<Subscription>;
export const SubscriptionModel = new Repository<Subscription>('Subscription');
export interface SubscriptionInstance
extends Model<Subscription, Subscription>,
Subscription {}
export const SubscriptionModel = sequelize.define<SubscriptionInstance>(
'Subscription',
{
name: {
unique: 'compositeIndex',
type: DataTypes.STRING,
},
url: {
unique: 'compositeIndex',
type: DataTypes.STRING,
},
schedule: {
unique: 'compositeIndex',
type: DataTypes.STRING,
},
interval_schedule: {
unique: 'compositeIndex',
type: DataTypes.JSON,
},
type: DataTypes.STRING,
whitelist: DataTypes.STRING,
blacklist: DataTypes.STRING,
status: DataTypes.NUMBER,
dependences: DataTypes.STRING,
extensions: DataTypes.STRING,
sub_before: DataTypes.STRING,
sub_after: DataTypes.STRING,
branch: DataTypes.STRING,
pull_type: DataTypes.STRING,
pull_option: DataTypes.JSON,
pid: DataTypes.NUMBER,
is_disabled: DataTypes.NUMBER,
log_path: DataTypes.STRING,
schedule_type: DataTypes.STRING,
alias: { type: DataTypes.STRING, unique: 'alias' },
proxy: { type: DataTypes.STRING, allowNull: true },
autoAddCron: { type: DataTypes.NUMBER, allowNull: true },
autoDelCron: { type: DataTypes.NUMBER, allowNull: true },
},
);
+13 -3
View File
@@ -1,4 +1,5 @@
import { Repository, RecordRow } from './repository';
import { sequelize } from '.';
import { DataTypes, Model, ModelDefined } from 'sequelize';
import { NotificationInfo } from './notify';
export class SystemInfo {
@@ -105,5 +106,14 @@ export type SystemModelInfo = SystemConfigInfo &
Partial<NotifyLogInfo> &
Partial<AuthInfo>;
export type SystemInstance = RecordRow<SystemInfo>;
export const SystemModel = new Repository<SystemInfo>('Auth');
export interface SystemInstance
extends Model<SystemInfo, SystemInfo>,
SystemInfo {}
export const SystemModel = sequelize.define<SystemInstance>('Auth', {
ip: DataTypes.STRING,
type: DataTypes.STRING,
info: {
type: DataTypes.JSON,
allowNull: true,
},
});
-11
View File
@@ -1,11 +0,0 @@
{
"extends": "../tsconfig.json",
"compilerOptions": {
"module": "node20",
"moduleResolution": "node16",
"rootDir": "..",
"outDir": "../../static/build"
},
"include": ["./databaseWorker.ts", "./schema.ts"],
"exclude": ["node_modules"]
}
+22 -6
View File
@@ -1,14 +1,30 @@
import Logger from './logger';
import { database } from '../data';
import { initialSchema } from '../data/schema';
import { EnvModel } from '../data/env';
import { CrontabModel } from '../data/cron';
import { DependenceModel } from '../data/dependence';
import { AppModel } from '../data/open';
import { SystemModel } from '../data/system';
import { SubscriptionModel } from '../data/subscription';
import { CrontabViewModel } from '../data/cronView';
import { CrontabStatModel } from '../data/cronStats';
import { RunningInstanceModel } from '../data/runningInstance';
import { sequelize } from '../data';
import { migrateSchema } from '../shared/schemaMigrations';
export default async () => {
try {
await database.transaction(async () => {
for (const sql of initialSchema) await database.query(sql);
await migrateSchema(database);
});
await CrontabModel.sync();
await DependenceModel.sync();
await AppModel.sync();
await SystemModel.sync();
await EnvModel.sync();
await SubscriptionModel.sync();
await CrontabViewModel.sync();
await CrontabStatModel.sync();
await RunningInstanceModel.sync();
await migrateSchema(sequelize);
Logger.info('[boot] DB loaded');
} catch (error) {
Logger.error('[boot] DB load failed', error);
+98 -84
View File
@@ -16,7 +16,7 @@ import { t } from '../shared/i18n';
import { AppScope } from '../data/open';
import uniqueConstraintError from '../middlewares/uniqueConstraintError';
import protectedPathCase from '../middlewares/protectedPathCase';
import { withAuthAvailability } from '../middlewares/authAvailability';
function resolveTrustProxy(value = process.env.QL_TRUST_PROXY) {
const setting = value?.trim();
if (!setting) {
@@ -30,20 +30,24 @@ function resolveTrustProxy(value = process.env.QL_TRUST_PROXY) {
}
return setting;
}
export default ({ app }: { app: Application }) => {
// Security: Enable strict routing to prevent case-insensitive path bypass
app.set('case sensitive routing', true);
app.set('strict routing', true);
app.set('trust proxy', resolveTrustProxy());
app.use(cors());
// Security: Reject case variations under protected API namespaces before
// authentication checks can interpret the request differently from routing.
app.use(protectedPathCase);
// Rewrite URLs to strip baseUrl prefix if configured
// This allows the rest of the app to work without baseUrl awareness
if (config.baseUrl) {
app.use(rewrite(`${config.baseUrl}/*`, '/$1'));
}
app.get(`${config.api.prefix}/env.js`, serveEnv);
app.use(
`${config.api.prefix}/static`,
@@ -54,6 +58,7 @@ export default ({ app }: { app: Application }) => {
},
}),
);
const credentialPaths = ['/api', '/open'].flatMap((prefix) =>
['/user/login', '/user/init', '/user/two-factor/login'].map(
(route) => `${prefix}${route}`,
@@ -64,10 +69,13 @@ export default ({ app }: { app: Application }) => {
bodyParser.json({ limit: '16kb' }),
bodyParser.urlencoded({ limit: '16kb', extended: false }),
);
app.use(bodyParser.json({ limit: '50mb' }));
app.use(bodyParser.urlencoded({ limit: '50mb', extended: true }));
const frontendPath = path.join(config.rootPath, 'static/dist');
app.use(express.static(frontendPath));
app.use(
expressjwt({
secret: config.jwt.secret,
@@ -76,6 +84,7 @@ export default ({ app }: { app: Application }) => {
path: [...config.apiWhiteList, /^(\/(?!api\/).*)$/i],
}),
);
app.use((req: Request, res, next) => {
if (!req.headers) {
req.platform = 'desktop';
@@ -85,84 +94,91 @@ export default ({ app }: { app: Application }) => {
}
return next();
});
app.use(
withAuthAvailability(async (req: Request, res, next) => {
const pathLower = req.path.toLowerCase();
if (!['/open/', '/api/'].some((x) => pathLower.startsWith(x))) {
return next();
}
const headerToken = getToken(req);
if (pathLower.startsWith('/open/')) {
const apps = await shareStore.getApps();
const doc = apps?.filter((x) =>
x.tokens?.find((y) => y.value === headerToken),
)?.[0];
if (doc && doc.tokens && doc.tokens.length > 0) {
const currentToken = doc.tokens.find((x) => x.value === headerToken);
const keyMatch = pathLower.match(/\/open\/([a-z]+)\/*/);
const key = keyMatch && keyMatch[1];
if (!doc.scopes.includes(key as AppScope)) {
const err = new UnauthorizedError('credentials_bad_scheme', {
message: t('暂无权限'),
});
return next(err);
}
if (
!currentToken ||
currentToken.expiration < Math.round(Date.now() / 1000)
) {
const err = new UnauthorizedError('invalid_token', {
message: t('Token 已失效'),
});
return next(err);
}
return next();
app.use(async (req: Request, res, next) => {
const pathLower = req.path.toLowerCase();
if (!['/open/', '/api/'].some((x) => pathLower.startsWith(x))) {
return next();
}
const headerToken = getToken(req);
if (pathLower.startsWith('/open/')) {
const apps = await shareStore.getApps();
const doc = apps?.filter((x) =>
x.tokens?.find((y) => y.value === headerToken),
)?.[0];
if (doc && doc.tokens && doc.tokens.length > 0) {
const currentToken = doc.tokens.find((x) => x.value === headerToken);
const keyMatch = pathLower.match(/\/open\/([a-z]+)\/*/);
const key = keyMatch && keyMatch[1];
if (!doc.scopes.includes(key as AppScope)) {
const err = new UnauthorizedError('credentials_bad_scheme', {
message: t('暂无权限'),
});
return next(err);
}
}
const originPath = `${req.baseUrl}${pathLower === '/' ? '' : pathLower}`;
if (
!headerToken &&
originPath &&
config.apiWhiteList.includes(originPath)
) {
if (
!currentToken ||
currentToken.expiration < Math.round(Date.now() / 1000)
) {
const err = new UnauthorizedError('invalid_token', {
message: t('Token 已失效'),
});
return next(err);
}
return next();
}
const authInfo = await shareStore.getAuthInfo();
if (
isValidToken(authInfo, headerToken, req.platform, config.jwt.secret)
) {
return next();
}
const errorCode = headerToken ? 'invalid_token' : 'credentials_required';
const errorMessage = headerToken ? t('Token 已失效') : t('请先登录');
const err = new UnauthorizedError(errorCode, { message: errorMessage });
next(err);
}),
);
app.use(
withAuthAvailability(async (req, res, next) => {
const pathLower = req.path.toLowerCase();
if (
![
'/api/user/init',
'/api/user/notification/init',
'/open/user/init',
'/open/user/notification/init',
].includes(pathLower)
) {
return next();
}
const authInfo = (await shareStore.getAuthInfo()) || ({} as AuthInfo);
let isInitialized = !isDefaultAuthInfo(authInfo);
if (isInitialized) {
return res.send({ code: 450, message: t('未知错误') });
} else {
return next();
}
}),
);
}
const originPath = `${req.baseUrl}${pathLower === '/' ? '' : pathLower}`;
if (
!headerToken &&
originPath &&
config.apiWhiteList.includes(originPath)
) {
return next();
}
const authInfo = await shareStore.getAuthInfo();
if (isValidToken(authInfo, headerToken, req.platform, config.jwt.secret)) {
return next();
}
const errorCode = headerToken ? 'invalid_token' : 'credentials_required';
const errorMessage = headerToken ? t('Token 已失效') : t('请先登录');
const err = new UnauthorizedError(errorCode, { message: errorMessage });
next(err);
});
app.use(async (req, res, next) => {
const pathLower = req.path.toLowerCase();
if (
![
'/api/user/init',
'/api/user/notification/init',
'/open/user/init',
'/open/user/notification/init',
].includes(pathLower)
) {
return next();
}
const authInfo = (await shareStore.getAuthInfo()) || ({} as AuthInfo);
let isInitialized = !isDefaultAuthInfo(authInfo);
if (isInitialized) {
return res.send({ code: 450, message: t('未知错误') });
} else {
return next();
}
});
app.use(rewrite('/open/*', '/api/$1'));
app.use(config.api.prefix, routes());
app.get('*', (req, res, next) => {
// Don't serve index.html for API routes
if (
@@ -180,13 +196,13 @@ export default ({ app }: { app: Application }) => {
}
});
});
app.use(validationErrors);
app.use(uniqueConstraintError);
app.use(
(
err: Error & {
status: number;
},
err: Error & { status: number },
req: Request,
res: Response,
next: NextFunction,
@@ -200,16 +216,15 @@ export default ({ app }: { app: Application }) => {
return next(err);
},
);
app.use(
(
err: Error & {
errors: any[];
},
err: Error & { errors: any[] },
req: Request,
res: Response,
next: NextFunction,
) => {
if (err.name === 'DatabaseError') {
if (err.name.includes('Sequelize')) {
return res
.status(500)
.send({
@@ -222,11 +237,10 @@ export default ({ app }: { app: Application }) => {
return next(err);
},
);
app.use(
(
err: Error & {
status: number;
},
err: Error & { status: number },
req: Request,
res: Response,
next: NextFunction,
+82 -65
View File
@@ -1,5 +1,3 @@
import { textSearch } from '../data/queries';
import { Query, QuerySpec, orderBy } from '../data/queries';
import DependenceService from '../services/dependence';
import { exec } from 'child_process';
import { Container } from 'typedi';
@@ -7,6 +5,7 @@ import { Crontab, CrontabModel, CrontabStatus } from '../data/cron';
import CronService from '../services/cron';
import EnvService from '../services/env';
import { DependenceModel, DependenceStatus } from '../data/dependence';
import { Op } from 'sequelize';
import config from '../config';
import { CrontabViewModel, CronViewType } from '../data/cronView';
import { initPosition } from '../data/env';
@@ -21,6 +20,7 @@ import Logger from './logger';
import cronClient from '../schedule/client';
import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
import { setLang, systemLang } from '../shared/i18n';
export default async () => {
const cronService = Container.get(CronService);
const envService = Container.get(EnvService);
@@ -28,14 +28,17 @@ export default async () => {
const systemService = Container.get(SystemService);
const userService = Container.get(UserService);
const openService = Container.get(OpenService);
// 初始化增加系统配置
await openService.initializeSystemApp();
const systemConfig = await SystemModel.getOrCreate({
type: AuthDataType.systemConfig,
const [systemConfig] = await SystemModel.findOrCreate({
where: { type: AuthDataType.systemConfig },
});
await SystemModel.getOrCreate({ type: AuthDataType.notification });
const authConfig = await SystemModel.getOrCreate({
type: AuthDataType.authConfig,
await SystemModel.findOrCreate({
where: { type: AuthDataType.notification },
});
const [authConfig] = await SystemModel.findOrCreate({
where: { type: AuthDataType.authConfig },
});
if (!authConfig?.info || isDemoEnv()) {
let authInfo = {
@@ -51,34 +54,36 @@ export default async () => {
} catch (error) {
Logger.warn('Failed to read auth config file, using default credentials');
}
await SystemModel.nativeUpdate(
{ id: authConfig.id },
{
info: authInfo,
type: AuthDataType.authConfig,
},
);
await SystemModel.upsert({
id: authConfig?.id,
info: authInfo,
type: AuthDataType.authConfig,
});
}
const installDependencies = async () => {
const docs = await DependenceModel.find(
{},
{
orderBy: orderBy([
['type', 'DESC'],
['createdAt', 'DESC'],
]),
},
);
await DependenceModel.nativeUpdate(
{ id: docs.map((x) => x.id!) },
const docs = await DependenceModel.findAll({
where: {},
order: [
['type', 'DESC'],
['createdAt', 'DESC'],
],
raw: true,
});
await DependenceModel.update(
{ status: DependenceStatus.queued, log: [] },
{ where: { id: docs.map((x) => x.id!) } },
);
setTimeout(async () => {
await dependenceService.installDependenceOneByOne(docs);
const bootAfterLoader = await import('./bootAfter');
bootAfterLoader.default();
}, 5000);
};
// 初始化更新 linux/python/nodejs 镜像源配置
if (systemConfig.info?.pythonMirror) {
systemService.updatePythonMirror({
@@ -101,40 +106,43 @@ export default async () => {
nodeMirror: systemConfig.info?.nodeMirror,
});
}
// 初始化新增默认全部任务视图
CrontabViewModel.find({ type: CronViewType.系统, name: '全部任务' }, {}).then(
(docs) => {
if (docs.length === 0) {
CrontabViewModel.insert({
name: '全部任务',
type: CronViewType.系统,
position: initPosition / 2,
});
}
},
);
CrontabViewModel.findAll({
where: { type: CronViewType.系统, name: '全部任务' },
raw: true,
}).then((docs) => {
if (docs.length === 0) {
CrontabViewModel.create({
name: '全部任务',
type: CronViewType.系统,
position: initPosition / 2,
});
}
});
// 初始化更新所有任务状态为空闲
await CrontabModel.nativeUpdate({}, { status: CrontabStatus.idle });
await CrontabModel.update({ status: CrontabStatus.idle }, { where: {} });
// 清空所有运行中的实例记录(服务重启后进程已不存在)
await RunningInstanceModel.nativeUpdate(
{ status: InstanceStatus.running },
await RunningInstanceModel.update(
{
status: InstanceStatus.stopped,
finished_at: Math.floor(Date.now() / 1000),
// The previous process's exit code cannot be recovered after a restart.
},
{ where: { status: InstanceStatus.running } },
);
// 初始化时执行一次所有的 ql repo 任务
CrontabModel.find(
{
isDisabled: { ['$ne']: 1 },
$or: [
...textSearch(['command'], 'ql repo').$or,
...textSearch(['command'], 'ql raw').$or,
],
CrontabModel.findAll({
where: {
isDisabled: { [Op.ne]: 1 },
command: {
[Op.or]: [{ [Op.like]: `%ql repo%` }, { [Op.like]: `%ql raw%` }],
},
},
{},
).then((docs) => {
}).then((docs) => {
for (let i = 0; i < docs.length; i++) {
const doc = docs[i];
if (doc) {
@@ -142,78 +150,87 @@ export default async () => {
}
}
});
// 更新2.11.3以前的脚本路径
CrontabModel.find(
{
$or: ['scripts', 'config', 'log', 'db'].flatMap(
(folder) =>
textSearch(['command'], `/${config.rootPath}/${folder}/`).$or,
),
CrontabModel.findAll({
where: {
command: {
[Op.or]: [
{ [Op.like]: `%\/${config.rootPath}\/scripts\/%` },
{ [Op.like]: `%\/${config.rootPath}\/config\/%` },
{ [Op.like]: `%\/${config.rootPath}\/log\/%` },
{ [Op.like]: `%\/${config.rootPath}\/db\/%` },
],
},
},
{},
).then(async (docs) => {
}).then(async (docs) => {
for (let i = 0; i < docs.length; i++) {
const doc = docs[i];
if (doc) {
if (doc.command.includes(`${config.rootPath}/scripts/`)) {
await CrontabModel.nativeUpdate(
{ id: doc.id },
await CrontabModel.update(
{ command: doc.command.replace(`${config.rootPath}/scripts/`, '') },
{ where: { id: doc.id } },
);
}
if (doc.command.includes(`${config.rootPath}/log/`)) {
await CrontabModel.nativeUpdate(
{ id: doc.id },
await CrontabModel.update(
{
command: `${config.dataPath}/log/${doc.command.replace(
`${config.rootPath}/log/`,
'',
)}`,
},
{ where: { id: doc.id } },
);
}
if (doc.command.includes(`${config.rootPath}/config/`)) {
await CrontabModel.nativeUpdate(
{ id: doc.id },
await CrontabModel.update(
{
command: `${config.dataPath}/config/${doc.command.replace(
`${config.rootPath}/config/`,
'',
)}`,
},
{ where: { id: doc.id } },
);
}
if (doc.command.includes(`${config.rootPath}/db/`)) {
await CrontabModel.nativeUpdate(
{ id: doc.id },
await CrontabModel.update(
{
command: `${config.dataPath}/db/${doc.command.replace(
`${config.rootPath}/db/`,
'',
)}`,
},
{ where: { id: doc.id } },
);
}
}
}
});
// 初始化语言(必须在 autosave_crontab 之前)
const lang = systemConfig.info?.lang || systemLang();
setLang(lang);
// 确保 lang_env.sh 存在
try {
const langEnvExist = await fileExist(config.langEnvFile);
if (!langEnvExist) {
await writeFile(config.langEnvFile, `export QL_LANG='${lang}'\n`);
}
} catch {}
} catch { }
// 初始化保存一次ck和定时任务数据
cronClient.readiness.configure(async () => {
await cronService.autosave_crontab(true);
await systemService.restoreCronConcurrency();
});
await cronClient.readiness.recover();
await envService.set_envs();
const authInfo = await userService.getAuthInfo();
await shareStore.updateAuthInfo(authInfo);
await openService.refreshApps();
+8 -1
View File
@@ -9,17 +9,20 @@ import { join } from 'path';
import { t } from '../shared/i18n';
import Logger from './logger';
import { assertSubscriptionAlias } from '../shared/subscriptionPath';
export default async () => {
const systemService = Container.get(SystemService);
const scheduleService = Container.get(ScheduleService);
const subscriptionService = Container.get(SubscriptionService);
const sshKeyService = Container.get(SshKeyService);
// 生成内置token
let tokenCommand = `ts-node-transpile-only ${join(
config.rootPath,
'back/token.ts',
)}`;
const tokenFile = join(config.rootPath, 'static/build/token.js');
if (await fileExist(tokenFile)) {
tokenCommand = `node ${tokenFile}`;
}
@@ -38,6 +41,7 @@ export default async () => {
},
true,
);
// 运行删除日志任务
const data = await systemService.getSystemConfig();
if (data && data.info) {
@@ -57,17 +61,20 @@ export default async () => {
true,
);
}
systemService.updateTimezone(data.info);
// Apply global SSH key if configured
if (data.info.globalSshKey) {
await sshKeyService.addGlobalSSHKey(data.info.globalSshKey, 'global');
}
}
await subscriptionService.setSshConfig();
const subs = await subscriptionService.list();
for (const sub of subs) {
try {
const doc = sub;
const doc = sub.get({ plain: true });
assertSubscriptionAlias(doc.alias);
await subscriptionService.handleTask(doc, !sub.is_disabled);
} catch (error) {
-18
View File
@@ -1,18 +0,0 @@
import type { RequestHandler } from 'express';
import { t } from '../shared/i18n';
// Express 4 does not forward rejected async middleware promises itself.
export function withAuthAvailability(handler: RequestHandler): RequestHandler {
return (req, res, next) => {
void Promise.resolve()
.then(() => handler(req, res, next))
.catch(() => {
res.setHeader('Retry-After', '1');
next(
Object.assign(new Error(t('认证服务暂时不可用,请稍后重试')), {
status: 503,
}),
);
});
};
}
+4 -3
View File
@@ -1,13 +1,14 @@
import { ErrorRequestHandler } from 'express';
import { DatabaseError } from '../data/databaseErrors';
import { UniqueConstraintError } from 'sequelize';
import { t } from '../shared/i18n';
// Handle the database constraint itself so concurrent writes are covered too.
const uniqueConstraintError: ErrorRequestHandler = (err, req, res, next) => {
if (!(err instanceof DatabaseError) || err.code !== 'UNIQUE_CONSTRAINT')
return next(err);
if (!(err instanceof UniqueConstraintError)) return next(err);
// Constraint details can contain environment values or application credentials.
res
.status(409)
.json({ code: 409, message: t('资源已存在,请检查重复的名称或值') });
};
export default uniqueConstraintError;
+39 -1
View File
@@ -39,7 +39,10 @@ import {
RunCronsRequest,
} from '../protos/api';
import { NotificationInfo } from '../data/notify';
import { Model } from 'sequelize';
Container.set('logger', LoggerInstance);
export const getEnvs = async (
call: ServerUnaryCall<GetEnvsRequest, EnvsResponse>,
callback: sendUnaryData<EnvsResponse>,
@@ -59,6 +62,7 @@ export const getEnvs = async (
});
}
};
export const createEnv = async (
call: ServerUnaryCall<CreateEnvRequest, EnvsResponse>,
callback: sendUnaryData<EnvsResponse>,
@@ -71,6 +75,7 @@ export const createEnv = async (
callback(e);
}
};
export const updateEnv = async (
call: ServerUnaryCall<UpdateEnvRequest, EnvResponse>,
callback: sendUnaryData<EnvResponse>,
@@ -83,6 +88,7 @@ export const updateEnv = async (
message: 'id parameter is required',
});
}
const envService = Container.get(EnvService);
const data = await envService.update(
pick(call.request.env, ['id', 'name', 'value', 'remarks']) as EnvItem,
@@ -92,6 +98,7 @@ export const updateEnv = async (
callback(e);
}
};
export const deleteEnvs = async (
call: ServerUnaryCall<DeleteEnvsRequest, Response>,
callback: sendUnaryData<Response>,
@@ -103,6 +110,7 @@ export const deleteEnvs = async (
message: 'ids parameter is required',
});
}
const envService = Container.get(EnvService);
await envService.remove(call.request.ids);
callback(null, { code: 200 });
@@ -110,6 +118,7 @@ export const deleteEnvs = async (
callback(e);
}
};
export const moveEnv = async (
call: ServerUnaryCall<MoveEnvRequest, EnvResponse>,
callback: sendUnaryData<EnvResponse>,
@@ -122,6 +131,7 @@ export const moveEnv = async (
message: 'id parameter is required',
});
}
const envService = Container.get(EnvService);
const data = await envService.move(call.request.id, {
fromIndex: call.request.fromIndex,
@@ -132,6 +142,7 @@ export const moveEnv = async (
callback(e);
}
};
export const disableEnvs = async (
call: ServerUnaryCall<DisableEnvsRequest, Response>,
callback: sendUnaryData<Response>,
@@ -143,6 +154,7 @@ export const disableEnvs = async (
message: 'ids parameter is required',
});
}
const envService = Container.get(EnvService);
await envService.disabled(call.request.ids);
callback(null, { code: 200 });
@@ -150,6 +162,7 @@ export const disableEnvs = async (
callback(e);
}
};
export const enableEnvs = async (
call: ServerUnaryCall<EnableEnvsRequest, Response>,
callback: sendUnaryData<Response>,
@@ -161,6 +174,7 @@ export const enableEnvs = async (
message: 'ids parameter is required',
});
}
const envService = Container.get(EnvService);
await envService.enabled(call.request.ids);
callback(null, { code: 200 });
@@ -168,6 +182,7 @@ export const enableEnvs = async (
callback(e);
}
};
export const updateEnvNames = async (
call: ServerUnaryCall<UpdateEnvNamesRequest, Response>,
callback: sendUnaryData<Response>,
@@ -179,6 +194,7 @@ export const updateEnvNames = async (
message: 'ids parameter is required',
});
}
const envService = Container.get(EnvService);
await envService.updateNames({
ids: call.request.ids,
@@ -189,6 +205,7 @@ export const updateEnvNames = async (
callback(e);
}
};
export const getEnvById = async (
call: ServerUnaryCall<GetEnvByIdRequest, EnvResponse>,
callback: sendUnaryData<EnvResponse>,
@@ -201,6 +218,7 @@ export const getEnvById = async (
message: 'id parameter is required',
});
}
const envService = Container.get(EnvService);
const data = await envService.getDb({ id: call.request.id });
callback(null, {
@@ -211,6 +229,7 @@ export const getEnvById = async (
callback(e);
}
};
export const systemNotify = async (
call: ServerUnaryCall<SystemNotifyRequest, Response>,
callback: sendUnaryData<Response>,
@@ -228,6 +247,7 @@ export const systemNotify = async (
callback(e);
}
};
export const recordNotifyLog = async (
call: ServerUnaryCall<RecordNotifyLogRequest, Response>,
callback: sendUnaryData<Response>,
@@ -247,9 +267,12 @@ export const recordNotifyLog = async (
callback(error);
}
};
const normalizeCronData = (data: CronItem | null): CronItem | undefined => {
if (!data) return undefined;
const cron = data;
// create() returns a Sequelize instance; spreading it omits attribute getters.
const cron =
data instanceof Model ? (data.get({ plain: true }) as CronItem) : data;
return {
...cron,
labels: cron.labels ?? [],
@@ -260,6 +283,7 @@ const normalizeCronData = (data: CronItem | null): CronItem | undefined => {
task_after: cron.task_after ?? undefined,
};
};
export const getCronDetail = async (
call: ServerUnaryCall<CronDetailRequest, CronDetailResponse>,
callback: sendUnaryData<CronDetailResponse>,
@@ -281,6 +305,7 @@ export const getCronDetail = async (
callback(e);
}
};
export const createCron = async (
call: ServerUnaryCall<CreateCronRequest, CronResponse>,
callback: sendUnaryData<CronResponse>,
@@ -293,6 +318,7 @@ export const createCron = async (
callback(e);
}
};
export const updateCron = async (
call: ServerUnaryCall<UpdateCronRequest, CronResponse>,
callback: sendUnaryData<CronResponse>,
@@ -300,6 +326,7 @@ export const updateCron = async (
try {
const cronService = Container.get(CronService);
const { id, ...fields } = call.request;
const updateRequest = {
id,
...Object.entries(fields).reduce((acc: any, [key, value]) => {
@@ -309,12 +336,14 @@ export const updateCron = async (
return acc;
}, {}),
} as UpdateCronRequest;
const data = (await cronService.update(updateRequest)) as CronItem;
callback(null, { code: 200, data: normalizeCronData(data) });
} catch (e: any) {
callback(e);
}
};
export const deleteCrons = async (
call: ServerUnaryCall<DeleteCronsRequest, Response>,
callback: sendUnaryData<Response>,
@@ -327,6 +356,7 @@ export const deleteCrons = async (
callback(e);
}
};
export const getCrons = async (
call: ServerUnaryCall<GetCronsRequest, CronsResponse>,
callback: sendUnaryData<CronsResponse>,
@@ -354,6 +384,7 @@ export const getCrons = async (
});
}
};
export const getCronById = async (
call: ServerUnaryCall<GetCronByIdRequest, CronResponse>,
callback: sendUnaryData<CronResponse>,
@@ -366,6 +397,7 @@ export const getCronById = async (
message: 'id parameter is required',
});
}
const cronService = Container.get(CronService);
const data = (await cronService.getDb({ id: call.request.id })) as CronItem;
callback(null, { code: 200, data: normalizeCronData(data) });
@@ -377,6 +409,7 @@ export const getCronById = async (
});
}
};
export const enableCrons = async (
call: ServerUnaryCall<EnableCronsRequest, Response>,
callback: sendUnaryData<Response>,
@@ -388,6 +421,7 @@ export const enableCrons = async (
message: 'ids parameter is required',
});
}
const cronService = Container.get(CronService);
await cronService.enabled(call.request.ids);
callback(null, { code: 200 });
@@ -395,6 +429,7 @@ export const enableCrons = async (
callback(e);
}
};
export const disableCrons = async (
call: ServerUnaryCall<DisableCronsRequest, Response>,
callback: sendUnaryData<Response>,
@@ -406,6 +441,7 @@ export const disableCrons = async (
message: 'ids parameter is required',
});
}
const cronService = Container.get(CronService);
await cronService.disabled(call.request.ids);
callback(null, { code: 200 });
@@ -413,6 +449,7 @@ export const disableCrons = async (
callback(e);
}
};
export const runCrons = async (
call: ServerUnaryCall<RunCronsRequest, Response>,
callback: sendUnaryData<Response>,
@@ -424,6 +461,7 @@ export const runCrons = async (
message: 'ids parameter is required',
});
}
const cronService = Container.get(CronService);
await cronService.run(call.request.ids);
callback(null, { code: 200 });
+1 -2
View File
@@ -4,7 +4,6 @@ import path from 'path';
import { request } from 'undici';
import { HealthCheckRequest, HealthCheckResponse } from '../protos/health';
import config from '../config';
import { database } from '../data';
function formatError(error: unknown): string {
if (!(error instanceof Error)) {
@@ -55,7 +54,7 @@ const check = async (
switch (call.request.service) {
// Local scheduler liveness only: never call HTTP from this probe.
case 'scheduler':
return callback(null, { status: database.healthy ? 1 : 2 });
return callback(null, { status: 1 });
case 'cron': {
const healthUrl = `http://localhost:${config.port}${
config.baseUrl || ''
+283 -233
View File
File diff suppressed because it is too large Load Diff
+31 -19
View File
@@ -1,4 +1,3 @@
import { Query, QuerySpec, orderBy } from '../data/queries';
import { Service, Inject } from 'typedi';
import winston from 'winston';
import { CrontabView, CrontabViewModel } from '../data/cronView';
@@ -8,12 +7,12 @@ import {
minPosition,
stepPosition,
} from '../data/env';
import { FindOptions } from 'sequelize';
@Service()
export default class CronViewService {
constructor(
@Inject('logger')
private logger: winston.Logger,
) {}
constructor(@Inject('logger') private logger: winston.Logger) {}
public async create(payload: CrontabView): Promise<CrontabView> {
let position = initPosition;
const views = await this.list();
@@ -23,51 +22,61 @@ export default class CronViewService {
position = position / 2;
const tab = new CrontabView({ ...payload, position });
const doc = await this.insert(tab);
await this.checkPosition(tab.position!);
return doc;
}
public async insert(payload: CrontabView): Promise<CrontabView> {
return await CrontabViewModel.insert(payload);
return await CrontabViewModel.create(payload, { returning: true });
}
public async update(payload: CrontabView): Promise<CrontabView> {
const doc = await this.getDb({ id: payload.id });
const tab = new CrontabView({ ...doc, ...payload });
const newDoc = await this.updateDb(tab);
return newDoc;
}
public async updateDb(payload: CrontabView): Promise<CrontabView> {
await CrontabViewModel.nativeUpdate({ id: payload.id }, payload);
await CrontabViewModel.update(payload, { where: { id: payload.id } });
return await this.getDb({ id: payload.id });
}
public async remove(ids: number[]) {
await CrontabViewModel.nativeDelete({ id: ids });
await CrontabViewModel.destroy({ where: { id: ids } });
}
public async list(): Promise<CrontabView[]> {
try {
const result = await CrontabViewModel.find(
{},
{
orderBy: orderBy([['position', 'DESC']]),
},
);
const result = await CrontabViewModel.findAll({
where: {},
order: [['position', 'DESC']],
});
return result;
} catch (error) {
throw error;
}
}
public async getDb(query: QuerySpec['where']): Promise<CrontabView> {
const doc: any = await CrontabViewModel.findOne({ ...query }, {});
public async getDb(
query: FindOptions<CrontabView>['where'],
): Promise<CrontabView> {
const doc: any = await CrontabViewModel.findOne({ where: { ...query } });
if (!doc) {
throw new Error(`CronView ${JSON.stringify(query)} not found`);
}
return doc;
return doc.get({ plain: true });
}
public async disabled(ids: number[]) {
await CrontabViewModel.nativeUpdate({ id: ids }, { isDisabled: 1 });
await CrontabViewModel.update({ isDisabled: 1 }, { where: { id: ids } });
}
public async enabled(ids: number[]) {
await CrontabViewModel.nativeUpdate({ id: ids }, { isDisabled: 0 });
await CrontabViewModel.update({ isDisabled: 0 }, { where: { id: ids } });
}
private async checkPosition(position: number) {
const precisionPosition = parseFloat(position.toPrecision(16));
if (precisionPosition < minPosition || precisionPosition > maxPosition) {
@@ -79,9 +88,11 @@ export default class CronViewService {
}
}
}
private getPrecisionPosition(position: number): number {
return parseFloat(position.toPrecision(16));
}
public async move({
id,
fromIndex,
@@ -107,6 +118,7 @@ export default class CronViewService {
id,
position: this.getPrecisionPosition(targetPosition),
});
await this.checkPosition(targetPosition);
return newDoc;
}
+93 -48
View File
@@ -1,5 +1,3 @@
import { textSearch } from '../data/queries';
import { Query, QuerySpec, orderBy } from '../data/queries';
import { Service, Inject } from 'typedi';
import winston from 'winston';
import config from '../config';
@@ -12,6 +10,7 @@ import {
} from '../data/dependence';
import { spawn } from 'cross-spawn';
import SockService from './sock';
import { FindOptions, Op } from 'sequelize';
import {
fileExist,
getPid,
@@ -28,13 +27,14 @@ import taskLimit from '../shared/pLimit';
import { detectOS } from '../config/util';
import { LINUX_DEPENDENCE_COMMAND } from '../config/const';
import { t, tf } from '../shared/i18n';
@Service()
export default class DependenceService {
constructor(
@Inject('logger')
private logger: winston.Logger,
@Inject('logger') private logger: winston.Logger,
private sockService: SockService,
) {}
public async create(payloads: Dependence[]): Promise<Dependence[]> {
const tabs = payloads.map((x) => {
getInstallCommand(x.type, x.name);
@@ -45,14 +45,14 @@ export default class DependenceService {
this.installDependenceOneByOne(docs);
return docs;
}
public async insert(payloads: Dependence[]): Promise<Dependence[]> {
const docs = await DependenceModel.insertMany(payloads);
const docs = await DependenceModel.bulkCreate(payloads);
return docs;
}
public async update(
payload: Dependence & {
id: string;
},
payload: Dependence & { id: string },
): Promise<Dependence> {
const { id, ...other } = payload;
const doc = await this.getDb({ id });
@@ -66,12 +66,14 @@ export default class DependenceService {
this.installDependenceOneByOne([newDoc]);
return newDoc;
}
private async updateDb(payload: Dependence): Promise<Dependence> {
await DependenceModel.nativeUpdate({ id: payload.id }, payload);
await DependenceModel.update(payload, { where: { id: payload.id } });
return await this.getDb({ id: payload.id });
}
public async remove(ids: number[], force = false): Promise<Dependence[]> {
const docs = await DependenceModel.find({ id: ids }, {});
const docs = await DependenceModel.findAll({ where: { id: ids } });
for (const doc of docs) {
taskLimit.removeQueuedDependency(doc);
}
@@ -86,18 +88,22 @@ export default class DependenceService {
);
const unInstalledDeps = docs.filter((x) => !installedDeps.includes(x));
await this.removeDb(unInstalledDeps.map((x) => x.id!));
if (installedDeps.length) {
await DependenceModel.nativeUpdate(
{ id: installedDeps.map((x) => x.id!) },
await DependenceModel.update(
{ status: DependenceStatus.queued, log: [] },
{ where: { id: installedDeps.map((x) => x.id!) } },
);
this.installDependenceOneByOne(installedDeps, false, force);
}
return docs;
}
public async removeDb(ids: number[]) {
await DependenceModel.nativeDelete({ id: ids });
await DependenceModel.destroy({ where: { id: ids } });
}
public async dependencies(
{
searchValue,
@@ -124,7 +130,13 @@ export default class DependenceService {
condition.status = status.split(',').map(Number);
}
if (searchValue) {
Object.assign(condition, textSearch(['name'], searchValue));
const encodeText = encodeURI(searchValue);
condition.name = {
[Op.or]: [
{ [Op.like]: `%${searchValue}%` },
{ [Op.like]: `%${encodeText}%` },
],
};
}
try {
return await this.find(condition, sort);
@@ -132,6 +144,7 @@ export default class DependenceService {
throw error;
}
}
private async refreshInstalledStatuses(type?: DependenceTypes) {
const cacheDependenceTypes = [
DependenceTypes.nodejs,
@@ -140,13 +153,13 @@ export default class DependenceService {
if (type !== undefined && !cacheDependenceTypes.includes(type)) {
return;
}
const docs = await DependenceModel.find(
{
const docs = await DependenceModel.findAll({
where: {
status: DependenceStatus.installed,
type: type === undefined ? { ['$in']: cacheDependenceTypes } : type,
type: type === undefined ? { [Op.in]: cacheDependenceTypes } : type,
},
{},
);
});
const checks = await concurrentRun(
docs
.filter(
@@ -162,13 +175,15 @@ export default class DependenceService {
const missingIds = (checks || []).filter(
(id): id is number => id !== undefined,
);
if (missingIds.length) {
await DependenceModel.nativeUpdate(
{ id: missingIds },
await DependenceModel.update(
{ status: DependenceStatus.installFailed },
{ where: { id: missingIds } },
);
}
}
private async isDependenceInstalled(dependency: Dependence) {
let depName = dependency.name.trim();
const depVersionStr = versionDependenceCommandTypes[dependency.type];
@@ -186,6 +201,7 @@ export default class DependenceService {
depVersion = parsedVersion;
}
}
const depInfo = (
await promiseExecSuccess(getGetCommand(dependency.type, depName))
)
@@ -195,10 +211,12 @@ export default class DependenceService {
(dependency.type === DependenceTypes.nodejs &&
depInfo.split(' ')?.[0] === depName) ||
dependency.type === DependenceTypes.python3;
return Boolean(
depInfo && nameMatches && (!depVersion || depInfo.includes(depVersion)),
);
}
public installDependenceOneByOne(
docs: Dependence[],
isInstall: boolean = true,
@@ -207,27 +225,32 @@ export default class DependenceService {
docs.forEach((dep) => {
this.installOrUninstallDependency(dep, isInstall, force);
});
return taskLimit.waitDependencyQueueDone();
}
public async reInstall(ids: number[]): Promise<Dependence[]> {
await DependenceModel.nativeUpdate(
{ id: ids },
await DependenceModel.update(
{ status: DependenceStatus.queued, log: [] },
{ where: { id: ids } },
);
const docs = await DependenceModel.find({ id: ids }, {});
const docs = await DependenceModel.findAll({ where: { id: ids } });
for (const doc of docs) {
taskLimit.removeQueuedDependency(doc);
}
this.installDependenceOneByOne(docs, true, true);
return docs;
}
public async cancel(ids: number[]) {
const docs = await DependenceModel.find({ id: ids }, {});
const docs = await DependenceModel.findAll({ where: { id: ids } });
for (const doc of docs) {
taskLimit.removeQueuedDependency(doc);
let depInstallCommand = getInstallCommand(doc.type, doc.name);
let depUnInstallCommand = getUninstallCommand(doc.type, doc.name);
const isLinuxDependence = doc.type === DependenceTypes.linux;
if (isLinuxDependence) {
const osType = await detectOS();
if (!osType) {
@@ -245,37 +268,44 @@ export default class DependenceService {
pid && (await killTask(pid));
}
}
await DependenceModel.nativeUpdate(
{ id: ids },
await DependenceModel.update(
{ status: DependenceStatus.cancelled },
{ where: { id: ids } },
);
}
private async find(query: any, sort: any = []): Promise<Dependence[]> {
const docs = await DependenceModel.find(
{ ...query },
{
orderBy: orderBy([...sort, ['createdAt', 'DESC']]),
},
);
const docs = await DependenceModel.findAll({
where: { ...query },
order: [...sort, ['createdAt', 'DESC']],
});
return docs;
}
public async getDb(query: QuerySpec['where']): Promise<Dependence> {
const doc: any = await DependenceModel.findOne({ ...query }, {});
public async getDb(
query: FindOptions<Dependence>['where'],
): Promise<Dependence> {
const doc: any = await DependenceModel.findOne({ where: { ...query } });
if (!doc) {
throw new Error(`Dependency ${JSON.stringify(query)} not found`);
}
return doc;
return doc.get({ plain: true });
}
private async updateLog(ids: number[], log: string): Promise<void> {
taskLimit.updateDepLog(async () => {
const docs = await DependenceModel.find({ id: ids }, {});
const docs = await DependenceModel.findAll({ where: { id: ids } });
for (const doc of docs) {
const newLog = doc?.log ? [...doc.log, log] : [log];
await DependenceModel.nativeUpdate({ id: doc.id }, { log: newLog });
await DependenceModel.update(
{ log: newLog },
{ where: { id: doc.id } },
);
}
return null;
});
}
public installOrUninstallDependency(
dependency: Dependence,
isInstall: boolean = true,
@@ -292,14 +322,14 @@ export default class DependenceService {
if (!isInstall && force) {
return this.removeDb([dependency.id!]).then(() => null);
}
return DependenceModel.nativeUpdate(
{ id: dependency.id },
return DependenceModel.update(
{
status: isInstall
? DependenceStatus.installFailed
: DependenceStatus.removeFailed,
log: [String(error)],
},
{ where: { id: dependency.id } },
).then(() => null);
}
return new Promise(async (resolve) => {
@@ -320,9 +350,9 @@ export default class DependenceService {
taskLimit.removeQueuedDependency(dependency);
if (isLinuxDependence) {
if (!osType) {
await DependenceModel.nativeUpdate(
{ id: depIds },
await DependenceModel.update(
{ status: DependenceStatus.installFailed },
{ where: { id: depIds } },
);
const startTime = dayjs();
const message = tf(
@@ -345,10 +375,12 @@ export default class DependenceService {
}
linuxCommand = LINUX_DEPENDENCE_COMMAND[osType];
}
const status = isInstall
? DependenceStatus.installing
: DependenceStatus.removing;
await DependenceModel.nativeUpdate({ id: depIds }, { status });
await DependenceModel.update({ status }, { where: { id: depIds } });
let command = validatedCommand;
if (isLinuxDependence) {
command = isInstall
@@ -356,6 +388,7 @@ export default class DependenceService {
: `${linuxCommand.uninstall} ${depName.trim()}`;
}
const startTime = dayjs();
const message = tf(
'开始%s依赖 %s,开始时间 %s\n\n',
actionText,
@@ -369,6 +402,7 @@ export default class DependenceService {
status,
});
this.updateLog(depIds, message);
// 判断是否已经安装过依赖
// pip must resolve optional dependencies even if the base distribution
// is already present; its version alone cannot prove extras are ready.
@@ -382,6 +416,7 @@ export default class DependenceService {
if (isLinuxDependence) {
getCommand = `${linuxCommand.info} ${depName}`;
}
let depVersion = '';
if (!isPythonDependence && depName.includes(depVersionStr)) {
const symbolRegx = new RegExp(
@@ -396,6 +431,7 @@ export default class DependenceService {
const depInfo = (await promiseExecSuccess(getCommand))
.replace(/\s{2,}/, ' ')
.replace(/\s+$/, '');
if (
depInfo &&
((isNodeDependence && depInfo.split(' ')?.[0] === depName) ||
@@ -420,9 +456,9 @@ export default class DependenceService {
status: DependenceStatus.installed,
});
this.updateLog(depIds, _message);
await DependenceModel.nativeUpdate(
{ id: depIds },
await DependenceModel.update(
{ status: DependenceStatus.installed },
{ where: { id: depIds } },
);
return resolve(null);
}
@@ -436,6 +472,7 @@ export default class DependenceService {
const cp = spawn(`${proxyStr} ${command}`, {
shell: '/bin/bash',
});
cp.stdout.on('data', async (data) => {
this.sockService.sendMessage({
type: socketMessageType,
@@ -445,6 +482,7 @@ export default class DependenceService {
});
this.updateLog(depIds, data.toString());
});
cp.stderr.on('data', async (data) => {
this.sockService.sendMessage({
type: socketMessageType,
@@ -454,6 +492,7 @@ export default class DependenceService {
});
this.updateLog(depIds, data.toString());
});
cp.on('error', async (err) => {
this.sockService.sendMessage({
type: socketMessageType,
@@ -463,10 +502,12 @@ export default class DependenceService {
});
this.updateLog(depIds, JSON.stringify(err));
});
cp.on('exit', async (code) => {
const endTime = dayjs();
const isSucceed = code === 0;
const resultText = isSucceed ? t('成功') : t('失败');
const message =
'\n' +
tf(
@@ -490,20 +531,24 @@ export default class DependenceService {
status: exitStatus,
});
this.updateLog(depIds, message);
const docs = await DependenceModel.find({ id: depIds }, {});
const docs = await DependenceModel.findAll({ where: { id: depIds } });
const _docIds = docs
.filter((x) => x.status !== DependenceStatus.cancelled)
.map((x) => x.id!);
if (_docIds.length > 0) {
await DependenceModel.nativeUpdate(
{ id: _docIds },
await DependenceModel.update(
{ status: exitStatus },
{ where: { id: _docIds } },
);
}
// 如果删除依赖成功或者强制删除
if ((isSucceed || force) && !isInstall) {
this.removeDb(depIds);
}
resolve(null);
});
});
+85 -41
View File
@@ -1,6 +1,5 @@
import { textSearch } from '../data/queries';
import { Query, QuerySpec, orderBy } from '../data/queries';
import groupBy from 'lodash/groupBy';
import { FindOptions, Op } from 'sequelize';
import { Inject, Service } from 'typedi';
import winston from 'winston';
import config from '../config';
@@ -14,13 +13,12 @@ import {
stepPosition,
} from '../data/env';
import { writeFileWithLock } from '../shared/utils';
import { database } from '../data';
import { sequelize } from '../data';
@Service()
export default class EnvService {
constructor(
@Inject('logger')
private logger: winston.Logger,
) {}
constructor(@Inject('logger') private logger: winston.Logger) { }
public async create(payloads: Env[]): Promise<Env[]> {
const envs = await this.envs();
let position = initPosition;
@@ -41,14 +39,16 @@ export default class EnvService {
await this.checkPosition(tabs[tabs.length - 1].position!);
return docs;
}
public async insert(payloads: Env[]): Promise<Env[]> {
const result: Env[] = [];
for (const env of payloads) {
const doc = await EnvModel.insert(env);
const doc = await EnvModel.create(env, { returning: true });
result.push(doc);
}
return result;
}
public async update(payload: Env): Promise<Env> {
const doc = await this.getDb({ id: payload.id });
const tab = new Env({ ...doc, ...payload });
@@ -56,14 +56,17 @@ export default class EnvService {
await this.set_envs();
return newDoc;
}
private async updateDb(payload: Env): Promise<Env> {
await EnvModel.nativeUpdate({ id: payload.id }, { ...payload });
await EnvModel.update({ ...payload }, { where: { id: payload.id } });
return await this.getDb({ id: payload.id });
}
public async remove(ids: number[]) {
await EnvModel.nativeDelete({ id: ids });
await EnvModel.destroy({ where: { id: ids } });
await this.set_envs();
}
public async move(
id: number,
{
@@ -86,13 +89,16 @@ export default class EnvService {
? (envs[toIndex].position! + envs[toIndex - 1].position!) / 2
: (envs[toIndex].position! + envs[toIndex + 1].position!) / 2;
}
const newDoc = await this.update({
id,
position: this.getPrecisionPosition(targetPosition),
});
await this.checkPosition(targetPosition, envs[toIndex].position!);
return newDoc;
}
private async checkPosition(position: number, edge: number = 0) {
const precisionPosition = this.getPrecisionPosition(position);
if (
@@ -108,20 +114,40 @@ export default class EnvService {
}
}
}
private getPrecisionPosition(position: number): number {
return Math.trunc(parseFloat(position.toPrecision(16)));
}
public async envs(searchText: string = '', query: any = {}): Promise<Env[]> {
let condition = { ...query };
if (searchText) {
const encodeText = encodeURI(searchText);
const reg = {
[Op.or]: [
{ [Op.like]: `%${searchText}%` },
{ [Op.like]: `%${encodeText}%` },
],
};
condition = {
...condition,
...textSearch(['name', 'value', 'remarks'], searchText),
[Op.or]: [
{
name: reg,
},
{
value: reg,
},
{
remarks: reg,
},
],
};
}
try {
const result = await this.find(condition, [
['isPinned', 'DESC'],
[sequelize.literal('COALESCE(`isPinned`, 0)'), 'DESC'],
['position', 'DESC'],
['createdAt', 'ASC'],
]);
@@ -130,73 +156,90 @@ export default class EnvService {
throw error;
}
}
private async find(query: any, sort: any = []): Promise<Env[]> {
const docs = await EnvModel.find(
{ ...query },
{
orderBy: orderBy([...sort]),
},
);
return docs.map((x) => x);
const docs = await EnvModel.findAll({
where: { ...query },
order: [...sort],
});
return docs.map((x) => x.get({ plain: true }));
}
public async getDb(query: QuerySpec['where']): Promise<Env> {
const doc: any = await EnvModel.findOne({ ...query }, {});
public async getDb(query: FindOptions<Env>['where']): Promise<Env> {
const doc: any = await EnvModel.findOne({ where: { ...query } });
if (!doc) {
throw new Error(`Env ${JSON.stringify(query)} not found`);
}
return doc;
return doc.get({ plain: true });
}
public async disabled(ids: number[]) {
await EnvModel.nativeUpdate({ id: ids }, { status: EnvStatus.disabled });
await EnvModel.update(
{ status: EnvStatus.disabled },
{ where: { id: ids } },
);
await this.set_envs();
}
public async enabled(ids: number[]) {
await EnvModel.nativeUpdate({ id: ids }, { status: EnvStatus.normal });
await EnvModel.update({ status: EnvStatus.normal }, { where: { id: ids } });
await this.set_envs();
}
public async updateNames({ ids, name }: { ids: number[]; name: string }) {
await EnvModel.nativeUpdate({ id: ids }, { name });
await EnvModel.update({ name }, { where: { id: ids } });
await this.set_envs();
}
public async pin(ids: number[]) {
await EnvModel.nativeUpdate({ id: ids }, { isPinned: 1 });
await EnvModel.update({ isPinned: 1 }, { where: { id: ids } });
}
public async unPin(ids: number[]) {
await EnvModel.nativeUpdate({ id: ids }, { isPinned: 0 });
await EnvModel.update({ isPinned: 0 }, { where: { id: ids } });
}
public async addLabels(ids: number[], labels: string[]) {
await database.transaction(async () => {
const docs = await EnvModel.find({ id: ids }, {});
await sequelize.transaction(async (transaction) => {
const docs = await EnvModel.findAll({
where: { id: ids },
transaction,
});
for (const doc of docs) {
const env = doc;
await EnvModel.nativeUpdate(
{ id: env.id },
const env = doc.get({ plain: true });
await EnvModel.update(
{ labels: Array.from(new Set([...(env.labels || []), ...labels])) },
{ where: { id: env.id }, transaction },
);
}
});
return await this.find({ id: ids });
}
public async removeLabels(ids: number[], labels: string[]) {
await database.transaction(async () => {
const docs = await EnvModel.find({ id: ids }, {});
await sequelize.transaction(async (transaction) => {
const docs = await EnvModel.findAll({
where: { id: ids },
transaction,
});
for (const doc of docs) {
const env = doc;
await EnvModel.nativeUpdate(
{ id: env.id },
const env = doc.get({ plain: true });
await EnvModel.update(
{
labels: (env.labels || []).filter(
(label: string) => !labels.includes(label),
),
},
{ where: { id: env.id }, transaction },
);
}
});
return await this.find({ id: ids });
}
public async set_envs() {
const envs = await this.envs('', {
name: { ['$ne']: null },
name: { [Op.not]: null },
status: EnvStatus.normal,
});
const groups = groupBy(envs, 'name');
@@ -206,6 +249,7 @@ export default class EnvService {
for (const key in groups) {
if (Object.prototype.hasOwnProperty.call(groups, key)) {
const group = groups[key];
// 忽略不符合bash要求的环境变量名称
if (/^[a-zA-Z_][0-9a-zA-Z_]*$/.test(key)) {
let value = group
@@ -218,9 +262,9 @@ export default class EnvService {
.map((x) => x.value)
.join('&')
.replace(/\\/g, '\\\\')}`;
js_env_string += `process.env[${JSON.stringify(
key,
)}]=${JSON.stringify(group.map((x) => x.value).join('&'))};\n`;
js_env_string += `process.env[${JSON.stringify(key)}]=${JSON.stringify(
group.map((x) => x.value).join('&'),
)};\n`;
py_env_string += `os.environ['${key}']='''${_env_value.replace(
/\'/g,
"\\'",
-4
View File
@@ -1,4 +1,3 @@
import { database } from '../data';
import { Service } from 'typedi';
import Logger from '../loaders/logger';
import cronClient from '../schedule/client';
@@ -11,7 +10,6 @@ interface HealthStatus {
services: {
http: boolean;
grpc: boolean;
database: boolean;
};
metrics: {
uptime: number;
@@ -35,7 +33,6 @@ export class HealthService {
services: {
http: true,
grpc: true,
database: database.healthy,
},
metrics: {
uptime: Math.floor((Date.now() - this.startTime) / 1000),
@@ -69,7 +66,6 @@ export class HealthService {
Logger.error('gRPC server check failed:', err);
}
if (!status.services.database) status.status = 'error';
return status;
}
}
+49 -21
View File
@@ -1,9 +1,9 @@
import { textSearch } from '../data/queries';
import { Service, Inject } from 'typedi';
import winston from 'winston';
import { createRandomString } from '../config/util';
import { App, AppModel } from '../data/open';
import { v4 as uuidV4 } from 'uuid';
import sequelize, { Op } from 'sequelize';
import { shareStore } from '../shared/store';
import { t } from '../shared/i18n';
import { lock } from 'proper-lockfile';
@@ -11,19 +11,18 @@ import path from 'path';
import os from 'os';
import { writeFile, rename, rm, stat, chmod } from 'fs/promises';
import config from '../config';
@Service()
export default class OpenService {
constructor(
@Inject('logger')
private logger: winston.Logger,
) {}
constructor(@Inject('logger') private logger: winston.Logger) {}
// Every database mutation and full authentication-cache publication shares
// this cross-process lock. Lock a separate marker, not token.json itself.
private async withAppsLock<T>(operation: () => Promise<T>): Promise<T> {
const target = path.join(config.configPath, '.apps-state');
const release = await lock(target, {
realpath: false,
stale: 10000,
stale: 10_000,
retries: { retries: 10, factor: 2, minTimeout: 100, maxTimeout: 3000 },
});
try {
@@ -32,10 +31,11 @@ export default class OpenService {
await release();
}
}
public async initializeSystemApp(): Promise<void> {
await this.withAppsLock(async () => {
const doc = await AppModel.findOne({ name: 'system' }, {});
const app = doc;
const doc = await AppModel.findOne({ where: { name: 'system' } });
const app = doc?.get({ plain: true });
if (!app) {
await this.insert({
name: 'system',
@@ -44,23 +44,26 @@ export default class OpenService {
client_secret: createRandomString(24, 24),
});
} else if (!app.scopes.includes('dashboard')) {
await AppModel.nativeUpdate(
{ id: app.id },
await AppModel.update(
{ scopes: [...app.scopes, 'dashboard'] },
{ where: { id: app.id } },
);
}
await shareStore.updateApps(await this.find({}));
});
}
public async refreshApps(): Promise<void> {
await this.withAppsLock(async () => {
await shareStore.updateApps(await this.find({}));
});
}
public async findApps(): Promise<App[] | null> {
const docs = await this.find({});
return docs;
}
public async create(payload: App): Promise<App> {
return this.withAppsLock(async () => {
const tab = { ...payload };
@@ -72,10 +75,12 @@ export default class OpenService {
return { ...doc, tokens: [] };
});
}
private async insert(payload: App): Promise<App> {
const doc = await AppModel.insert(payload);
return doc;
const doc = await AppModel.create(payload, { returning: true });
return doc.get({ plain: true });
}
public async update(payload: App): Promise<App> {
const newDoc = await this.updateDb({
name: payload.name,
@@ -84,28 +89,32 @@ export default class OpenService {
} as App);
return { ...newDoc, tokens: [] };
}
private async updateDb(payload: Partial<App>): Promise<App> {
return this.withAppsLock(async () => {
await AppModel.nativeUpdate({ id: payload.id }, payload);
await AppModel.update(payload, { where: { id: payload.id } });
const apps = await this.find({});
await shareStore.updateApps(apps);
return apps?.find((x) => x.id === payload.id) as App;
});
}
public async getDb(query: Record<string, any>): Promise<App> {
const doc = await AppModel.findOne(query, {});
const doc = await AppModel.findOne({ where: query });
if (!doc) {
throw new Error(`App ${JSON.stringify(query)} not found`);
}
return doc;
return doc.get({ plain: true });
}
public async remove(ids: number[]) {
return this.withAppsLock(async () => {
await AppModel.nativeDelete({ id: ids });
await AppModel.destroy({ where: { id: ids } });
const apps = await this.find({});
await shareStore.updateApps(apps);
});
}
public async resetSecret(id: number): Promise<App> {
const tab: Partial<App> = {
client_secret: createRandomString(24, 24),
@@ -121,6 +130,7 @@ export default class OpenService {
const newDoc = await this.updateDb(tab);
return newDoc;
}
public async list(
searchText: string = '',
sort: any = {},
@@ -128,7 +138,18 @@ export default class OpenService {
): Promise<App[]> {
let condition = { ...query };
if (searchText) {
condition = { ...condition, ...textSearch(['name'], searchText) };
const encodeText = encodeURI(searchText);
const reg = {
[Op.or]: [
{ [Op.like]: `%${searchText}%` },
{ [Op.like]: `%${encodeText}%` },
],
};
condition = {
...condition,
name: reg,
};
}
try {
const result = await this.find(condition);
@@ -139,16 +160,19 @@ export default class OpenService {
throw error;
}
}
private async find(query: Record<string, any>, sort?: any): Promise<App[]> {
const docs = await AppModel.find({ ...query }, {});
return docs.map((x) => x);
const docs = await AppModel.findAll({ where: { ...query } });
return docs.map((x) => x.get({ plain: true }));
}
public async authToken(credentials: {
client_id: string;
client_secret: string;
}): Promise<any> {
return this.withAppsLock(() => this.issueToken(credentials));
}
// Caller holds withAppsLock through database update and cache publication.
private async issueToken({
client_id,
@@ -159,7 +183,7 @@ export default class OpenService {
}): Promise<any> {
let token = uuidV4();
const expiration = Math.round(Date.now() / 1000) + 2592000; // 2592000 30天
const doc = await AppModel.findOne({ client_id, client_secret }, {});
const doc = await AppModel.findOne({ where: { client_id, client_secret } });
if (doc) {
const timestamp = Math.round(Date.now() / 1000);
const invalidTokens = (doc.tokens || []).filter(
@@ -175,7 +199,10 @@ export default class OpenService {
} else {
tokens = [...invalidTokens, { value: token, expiration }];
}
await AppModel.nativeUpdate({ client_id, client_secret }, { tokens });
await AppModel.update(
{ tokens },
{ where: { client_id, client_secret } },
);
const apps = await this.find({});
await shareStore.updateApps(apps);
return {
@@ -190,6 +217,7 @@ export default class OpenService {
return { code: 400, message: t('client_id 或 client_seret 有误') };
}
}
public async generateSystemToken(renew = false): Promise<{
value: string;
expiration: number;
+41 -25
View File
@@ -1,10 +1,11 @@
import fs from 'fs/promises';
import path from 'path';
import dayjs from 'dayjs';
import { Op } from 'sequelize';
import { Service } from 'typedi';
import config from '../config';
import { CrontabStatModel } from '../data/cronStats';
import { database } from '../data';
import { sequelize } from '../data';
import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
import { AuthDataType, SystemModel } from '../data/system';
import {
@@ -14,50 +15,54 @@ import {
normalizeRetentionPolicy,
RetentionPolicy,
} from '../shared/retention';
export interface StorageCleanupRequest extends RetentionPolicy {
dependenceCacheTypes?: DependenceCacheType[];
compactDatabase?: boolean;
}
function runningInstanceWhere(days: number) {
const cutoff = dayjs().subtract(days, 'day').unix();
return {
status: { ['$ne']: InstanceStatus.running },
['$or']: [
{ finished_at: { ['$lt']: cutoff } },
status: { [Op.ne]: InstanceStatus.running },
[Op.or]: [
{ finished_at: { [Op.lt]: cutoff } },
{
finished_at: { ['$eq']: null },
started_at: { ['$lt']: cutoff },
finished_at: { [Op.is]: null },
started_at: { [Op.lt]: cutoff },
},
],
};
}
function cronStatWhere(days: number) {
return {
date: { ['$lt']: dayjs().subtract(days, 'day').format('YYYY-MM-DD') },
date: { [Op.lt]: dayjs().subtract(days, 'day').format('YYYY-MM-DD') },
};
}
@Service()
export default class RetentionService {
public async updatePolicy(policy: Partial<RetentionPolicy>) {
const normalized = normalizeRetentionPolicy(policy);
const systemConfig = await SystemModel.findOne(
{ type: AuthDataType.systemConfig },
{},
);
const systemConfig = await SystemModel.findOne({
where: { type: AuthDataType.systemConfig },
});
if (!systemConfig) {
throw new Error('System config not found');
}
await SystemModel.nativeUpdate(
{ id: systemConfig.id },
await SystemModel.update(
{
info: {
...systemConfig.info,
...normalized,
},
},
{ where: { id: systemConfig.id } },
);
return normalized;
}
public async preview(request: StorageCleanupRequest) {
const policy = normalizeRetentionPolicy(request);
const dependenceCacheTypes = (request.dependenceCacheTypes || []).filter(
@@ -65,12 +70,14 @@ export default class RetentionService {
);
const [runningInstances, cronStats, dependenceCaches] = await Promise.all([
policy.runningInstanceRetentionDays > 0
? RunningInstanceModel.count(
runningInstanceWhere(policy.runningInstanceRetentionDays),
)
? RunningInstanceModel.count({
where: runningInstanceWhere(policy.runningInstanceRetentionDays),
})
: 0,
policy.cronStatRetentionDays > 0
? CrontabStatModel.count(cronStatWhere(policy.cronStatRetentionDays))
? CrontabStatModel.count({
where: cronStatWhere(policy.cronStatRetentionDays),
})
: 0,
Promise.all(
dependenceCacheTypes.map(async (type) => ({
@@ -81,6 +88,7 @@ export default class RetentionService {
})),
),
]);
return {
policy,
runningInstances,
@@ -93,23 +101,29 @@ export default class RetentionService {
compactDatabase: Boolean(request.compactDatabase),
};
}
public async cleanup(request: StorageCleanupRequest) {
const preview = await this.preview(request);
const deleted = await database.transaction(async () => {
const deleted = await sequelize.transaction(async (transaction) => {
const runningInstances =
preview.policy.runningInstanceRetentionDays > 0
? await RunningInstanceModel.nativeDelete(
runningInstanceWhere(preview.policy.runningInstanceRetentionDays),
)
? await RunningInstanceModel.destroy({
where: runningInstanceWhere(
preview.policy.runningInstanceRetentionDays,
),
transaction,
})
: 0;
const cronStats =
preview.policy.cronStatRetentionDays > 0
? await CrontabStatModel.nativeDelete(
cronStatWhere(preview.policy.cronStatRetentionDays),
)
? await CrontabStatModel.destroy({
where: cronStatWhere(preview.policy.cronStatRetentionDays),
transaction,
})
: 0;
return { runningInstances, cronStats };
});
const dependenceCaches = [];
for (const cache of preview.dependenceCaches) {
await fs.rm(path.join(config.dependenceCachePath, cache.type), {
@@ -118,12 +132,14 @@ export default class RetentionService {
});
dependenceCaches.push(cache);
}
if (
request.compactDatabase &&
(deleted.runningInstances || deleted.cronStats)
) {
await database.query('VACUUM');
await sequelize.query('VACUUM');
}
return {
deleted: {
...deleted,
+101 -70
View File
@@ -1,5 +1,3 @@
import { textSearch } from '../data/queries';
import { Query, QuerySpec, orderBy } from '../data/queries';
import { Service, Inject } from 'typedi';
import winston from 'winston';
import config from '../config';
@@ -21,6 +19,7 @@ import {
rmPath,
} from '../config/util';
import fs from 'fs/promises';
import { FindOptions, Op } from 'sequelize';
import path from 'path';
import ScheduleService, { TaskCallbacks } from './schedule';
import { SimpleIntervalSchedule } from 'toad-scheduler';
@@ -43,16 +42,17 @@ import CrontabService from './cron';
import taskLimit from '../shared/pLimit';
import { logStreamManager } from '../shared/logStreamManager';
import { LogReadOptions, readLogChunk } from '../shared/logReader';
@Service()
export default class SubscriptionService {
constructor(
@Inject('logger')
private logger: winston.Logger,
@Inject('logger') private logger: winston.Logger,
private scheduleService: ScheduleService,
private sockService: SockService,
private sshKeyService: SshKeyService,
private crontabService: CrontabService,
) {}
public async list(
searchText?: string,
ids?: string,
@@ -60,23 +60,37 @@ export default class SubscriptionService {
let query = {};
const subIds = JSON.parse(ids || '[]');
if (searchText) {
query = textSearch(['name', 'url'], searchText);
const reg = {
[Op.or]: [
{ [Op.like]: `%${searchText}%` },
{ [Op.like]: `%${encodeURI(searchText)}%` },
],
};
query = {
[Op.or]: [
{
name: reg,
},
{
url: reg,
},
],
};
}
try {
const result = await SubscriptionModel.find(
{ ...query, ...(ids ? { id: subIds } : undefined) },
{
orderBy: orderBy([
['is_disabled', 'ASC'],
['createdAt', 'DESC'],
]),
},
);
const result = await SubscriptionModel.findAll({
where: { ...query, ...(ids ? { id: subIds } : undefined) },
order: [
['is_disabled', 'ASC'],
['createdAt', 'DESC'],
],
});
return result;
} catch (error) {
throw error;
}
}
public async handleTask(
doc: Subscription,
needCreate = true,
@@ -87,6 +101,7 @@ export default class SubscriptionService {
const { url } = formatUrl(doc);
doc.command = formatCommand(doc, url as string);
}
if (doc.schedule_type === 'crontab') {
this.scheduleService.cancelCronTask(doc as any);
needCreate &&
@@ -107,34 +122,34 @@ export default class SubscriptionService {
));
}
}
public async setSshConfig() {
const docs = await SubscriptionModel.find({}, {});
const docs = await SubscriptionModel.findAll();
await this.sshKeyService.setSshConfig(docs);
}
private taskCallbacks(doc: Subscription): TaskCallbacks {
return {
onBefore: async (startTime) => {
const logTime = startTime.format('YYYY-MM-DD-HH-mm-ss');
const logPath = `${doc.alias}/${logTime}.log`;
await SubscriptionModel.nativeUpdate(
{ id: doc.id },
await SubscriptionModel.update(
{
status: SubscriptionStatus.running,
log_path: logPath,
},
{ where: { id: doc.id } },
);
const absolutePath = await handleLogPath(
logPath as string,
tf('## 开始执行... %s\n', startTime.format('YYYY-MM-DD HH:mm:ss')),
);
// 执行sub_before
let beforeStr = '';
try {
if (doc.sub_before) {
await logStreamManager.write(
absolutePath,
`\n## ${t('执行before命令...')}\n\n`,
);
await logStreamManager.write(absolutePath, `\n## ${t('执行before命令...')}\n\n`);
beforeStr = await promiseExec(doc.sub_before);
}
} catch (error: any) {
@@ -146,11 +161,11 @@ export default class SubscriptionService {
}
},
onStart: async (cp: ChildProcessWithoutNullStreams, startTime) => {
await SubscriptionModel.nativeUpdate(
{ id: doc.id },
await SubscriptionModel.update(
{
pid: cp.pid,
},
{ where: { id: doc.id } },
);
},
onEnd: async (cp, endTime, diff) => {
@@ -158,6 +173,7 @@ export default class SubscriptionService {
try {
const sub = await this.getDb({ id: doc.id });
absolutePath = await handleLogPath(sub.log_path as string);
// 执行 sub_after
let afterStr = '';
try {
@@ -170,12 +186,12 @@ export default class SubscriptionService {
}
} catch (error: any) {
afterStr =
(error.stderr && error.stderr.toString()) ||
JSON.stringify(error);
(error.stderr && error.stderr.toString()) || JSON.stringify(error);
}
if (afterStr) {
await logStreamManager.write(absolutePath, `${afterStr}\n`);
}
await logStreamManager.write(
absolutePath,
'\n' +
@@ -190,10 +206,10 @@ export default class SubscriptionService {
try {
if (absolutePath) await logStreamManager.closeStream(absolutePath);
} finally {
await SubscriptionModel.nativeUpdate({ id: doc.id }, {
status: SubscriptionStatus.idle,
pid: null,
} as any);
await SubscriptionModel.update(
{ status: SubscriptionStatus.idle, pid: null } as any,
{ where: { id: doc.id } },
);
this.sockService.sendMessage({
type: 'runSubscriptionEnd',
message: t('订阅执行完成'),
@@ -214,19 +230,22 @@ export default class SubscriptionService {
},
};
}
public async create(payload: Subscription): Promise<Subscription> {
assertSubscriptionAlias(payload.alias);
return withSubscriptionMutation(async () => {
const tab = new Subscription(payload);
const doc = await this.insert(tab);
await this.handleTask(doc);
await this.handleTask(doc.get({ plain: true }));
await this.setSshConfig();
return doc;
});
}
public async insert(payload: Subscription): Promise<SubscriptionInstance> {
return await SubscriptionModel.insert(payload);
return await SubscriptionModel.create(payload, { returning: true });
}
public async update(payload: Subscription): Promise<Subscription> {
assertSubscriptionAlias(payload.alias);
return withSubscriptionMutation(async () => {
@@ -239,6 +258,7 @@ export default class SubscriptionService {
return newDoc;
});
}
private async removeSshConfigForSubscription(doc: Subscription) {
if (doc.type !== 'private-repo' || doc.pull_type !== 'ssh-key') return;
// Invalid historical rows must remain deletable without deriving unsafe paths.
@@ -249,10 +269,12 @@ export default class SubscriptionService {
}
await this.sshKeyService.removeSSHKey(doc.alias, '');
}
public async updateDb(payload: Subscription): Promise<Subscription> {
await SubscriptionModel.nativeUpdate({ id: payload.id }, payload);
await SubscriptionModel.update(payload, { where: { id: payload.id } });
return await this.getDb({ id: payload.id });
}
public async status({
ids,
status,
@@ -278,16 +300,16 @@ export default class SubscriptionService {
if (last_running_time > 0) {
options.last_running_time = last_running_time;
}
return await SubscriptionModel.nativeUpdate({ id: ids }, { ...options });
return await SubscriptionModel.update(
{ ...options },
{ where: { id: ids } },
);
}
public async remove(
ids: number[],
query: {
force?: boolean;
},
) {
public async remove(ids: number[], query: { force?: boolean }) {
return withSubscriptionMutation(async () => {
const docs = await SubscriptionModel.find({ id: ids }, {});
const docs = await SubscriptionModel.findAll({ where: { id: ids } });
const storagePaths: string[] = [];
const retainedPaths: string[] = [];
const collectPaths = (
@@ -299,9 +321,7 @@ export default class SubscriptionService {
// A URL accepted by one engine can produce an invalid name in another.
// Validate each candidate independently, retaining the safe candidate.
if (!Object.values(names).every(isSubscriptionStorageName)) continue;
const resolve = deleting
? resolveSubscriptionStoragePath
: path.resolve;
const resolve = deleting ? resolveSubscriptionStoragePath : path.resolve;
const scriptPath = resolve(config.scriptPath, names.script);
if (scriptPath) targets.push(scriptPath);
if (names.repo) {
@@ -309,10 +329,7 @@ export default class SubscriptionService {
if (repoPath) targets.push(repoPath);
}
if (names.raw) {
const rawPath = resolve(
path.join(config.dataPath, 'raw'),
names.raw,
);
const rawPath = resolve(path.join(config.dataPath, 'raw'), names.raw);
if (rawPath) targets.push(rawPath);
}
}
@@ -325,65 +342,70 @@ export default class SubscriptionService {
resolveSubscriptionStoragePath(config.repoPath, doc.alias);
collectPaths(doc, storagePaths, true);
}
const remaining = await SubscriptionModel.find({}, {});
const remaining = await SubscriptionModel.findAll({ where: {} });
for (const doc of remaining) {
if (!ids.includes(doc.id!)) collectPaths(doc, retainedPaths);
}
}
for (const doc of docs) {
await this.handleTask(doc, false);
await this.handleTask(doc.get({ plain: true }), false);
}
await SubscriptionModel.nativeDelete({ id: ids });
await SubscriptionModel.destroy({ where: { id: ids } });
for (const doc of docs) {
await this.removeSshConfigForSubscription(doc);
}
await this.setSshConfig();
if (query?.force === true) {
const crons = await CrontabModel.find({ sub_id: ids }, {});
const crons = await CrontabModel.findAll({ where: { sub_id: ids } });
if (crons?.length) {
await this.crontabService.remove(crons.map((x) => x.id!));
}
for (const storagePath of new Set(storagePaths)) {
// rm is recursive: an ancestor must also be retained if a surviving
// subscription uses a nested branch, and vice versa.
const shared = retainedPaths.some(
(retained) =>
retained === storagePath ||
retained.startsWith(`${storagePath}${path.sep}`) ||
storagePath.startsWith(`${retained}${path.sep}`),
const shared = retainedPaths.some((retained) =>
retained === storagePath ||
retained.startsWith(`${storagePath}${path.sep}`) ||
storagePath.startsWith(`${retained}${path.sep}`),
);
if (!shared) await rmPath(storagePath);
}
}
});
}
public async getDb(query: QuerySpec['where']): Promise<Subscription> {
const doc = await SubscriptionModel.findOne({ ...query }, {});
public async getDb(
query: FindOptions<Subscription>['where'],
): Promise<Subscription> {
const doc = await SubscriptionModel.findOne({ where: { ...query } });
if (!doc) {
throw new Error(`Subscription ${JSON.stringify(query)} not found`);
}
return doc;
return doc.get({ plain: true });
}
public async run(ids: number[]) {
await SubscriptionModel.nativeUpdate(
{ id: ids },
await SubscriptionModel.update(
{ status: SubscriptionStatus.queued },
{ where: { id: ids } },
);
ids.forEach((id) => {
this.runSingle(id);
});
}
public async stop(ids: number[]) {
const docs = await SubscriptionModel.find({ id: ids }, {});
const docs = await SubscriptionModel.findAll({ where: { id: ids } });
let failure: unknown;
for (const doc of docs) {
try {
if (doc.pid) {
await killTask(doc.pid, true);
}
await SubscriptionModel.nativeUpdate(
{ id: doc.id, pid: doc.pid ?? null },
await SubscriptionModel.update(
{ status: SubscriptionStatus.idle, pid: null } as any,
{ where: { id: doc.id, pid: doc.pid ?? null } },
);
} catch (error) {
this.logger.error(error);
@@ -392,12 +414,15 @@ export default class SubscriptionService {
}
if (failure) throw failure;
}
private async runSingle(subscriptionId: number) {
const subscription = await this.getDb({ id: subscriptionId });
if (subscription.status !== SubscriptionStatus.queued) {
return;
}
const command = formatCommand(subscription);
this.scheduleService.runTask(command, this.taskCallbacks(subscription), {
name: subscription.name,
schedule: subscription.schedule,
@@ -406,22 +431,25 @@ export default class SubscriptionService {
runOrigin: 'subscription',
});
}
public async disabled(ids: number[]) {
await SubscriptionModel.nativeUpdate({ id: ids }, { is_disabled: 1 });
const docs = await SubscriptionModel.find({ id: ids }, {});
await SubscriptionModel.update({ is_disabled: 1 }, { where: { id: ids } });
const docs = await SubscriptionModel.findAll({ where: { id: ids } });
await this.setSshConfig();
for (const doc of docs) {
await this.handleTask(doc, false);
await this.handleTask(doc.get({ plain: true }), false);
}
}
public async enabled(ids: number[]) {
await SubscriptionModel.nativeUpdate({ id: ids }, { is_disabled: 0 });
const docs = await SubscriptionModel.find({ id: ids }, {});
await SubscriptionModel.update({ is_disabled: 0 }, { where: { id: ids } });
const docs = await SubscriptionModel.findAll({ where: { id: ids } });
await this.setSshConfig();
for (const doc of docs) {
await this.handleTask(doc);
await this.handleTask(doc.get({ plain: true }));
}
}
public async log(id: number, options: LogReadOptions = {}) {
const doc = await this.getDb({ id });
if (!doc || !doc.log_path) {
@@ -433,14 +461,17 @@ export default class SubscriptionService {
truncated: false,
};
}
const absolutePath = await handleLogPath(doc.log_path as string);
return await readLogChunk(absolutePath, options);
}
public async logs(id: number) {
const doc = await this.getDb({ id });
if (!doc) {
return [];
}
if (doc.log_path) {
const relativeDir = path.dirname(`${doc.log_path}`);
const dir = path.dirname(resolveLogPath(config.logPath, doc.log_path));
+68 -34
View File
@@ -1,4 +1,3 @@
import { Query, QuerySpec, orderBy } from '../data/queries';
import { persistNotificationAudit } from '../shared/notificationAudit';
import { spawn } from 'cross-spawn';
import { execFile } from 'child_process';
@@ -47,16 +46,18 @@ import os from 'os';
import dayjs from 'dayjs';
import { t, setLang } from '../shared/i18n';
import { updateLinuxMirrorFile } from '../config/util';
@Service()
export default class SystemService {
@Inject((type) => NotificationService)
private notificationService!: NotificationService;
constructor(
@Inject('logger')
private logger: winston.Logger,
@Inject('logger') private logger: winston.Logger,
private scheduleService: ScheduleService,
private sockService: SockService,
) {}
public async getSystemConfig() {
const doc = await this.getDb({ type: AuthDataType.systemConfig });
return {
@@ -64,19 +65,22 @@ export default class SystemService {
info: { ...doc.info, timezone: doc.info?.timezone || 'Asia/Shanghai' },
};
}
private async updateAuthDb(payload: SystemInfo): Promise<SystemInfo> {
const { id, ...others } = payload;
await SystemModel.nativeUpdate({ id }, others);
await SystemModel.update(others, { where: { id } });
const doc = await this.getDb({ id });
return doc;
}
public async getDb(query: any): Promise<SystemInfo> {
const doc = await SystemModel.findOne(query, {});
const doc = await SystemModel.findOne({ where: query });
if (!doc) {
throw new Error(`System ${JSON.stringify(query)} not found`);
}
return doc;
return doc.get({ plain: true });
}
public async updateNotificationMode(notificationInfo: NotificationInfo) {
const code = Math.random().toString().slice(-6);
const isSuccess = await this.notificationService.testNotify(
@@ -94,6 +98,7 @@ export default class SystemService {
return { code: 400, message: t('通知发送失败,请检查参数') };
}
}
public async updateLogRemoveFrequency(info: SystemModelInfo) {
const oDoc = await this.getSystemConfig();
const result = await this.updateAuthDb({
@@ -120,6 +125,7 @@ export default class SystemService {
}
return { code: 200, data: info };
}
public async updateCronConcurrency(info: SystemModelInfo) {
const limit = info.cronConcurrency ?? 0;
if (!Number.isInteger(limit) || limit < 0 || limit > 2147483647) {
@@ -136,6 +142,7 @@ export default class SystemService {
return { code: 200, data: info };
});
}
public async restoreCronConcurrency() {
await withSchedulerMutation(async () => {
const doc = await this.getSystemConfig();
@@ -144,6 +151,7 @@ export default class SystemService {
await cronClient.setConcurrency(limit);
});
}
public async updateDependenceProxy(info: SystemModelInfo) {
const oDoc = await this.getSystemConfig();
await this.updateAuthDb({
@@ -162,6 +170,7 @@ export default class SystemService {
}
return { code: 200, data: info };
}
public async updateNodeMirror(info: SystemModelInfo, res?: Response) {
const oDoc = await this.getSystemConfig();
await this.updateAuthDb({
@@ -173,13 +182,12 @@ export default class SystemService {
cmd = `pnpm config set registry ${shellQuote(info.nodeMirror)}`;
}
let command = `cd && ${cmd}`;
const docs = await DependenceModel.find(
{
const docs = await DependenceModel.findAll({
where: {
type: DependenceTypes.nodejs,
status: DependenceStatus.installed,
},
{},
);
});
if (docs.length > 0) {
command += ` && pnpm i -g`;
}
@@ -211,6 +219,7 @@ export default class SystemService {
},
);
}
public async updatePythonMirror(info: SystemModelInfo) {
const oDoc = await this.getSystemConfig();
await this.updateAuthDb({
@@ -224,6 +233,7 @@ export default class SystemService {
await promiseExec(cmd);
return { code: 200, data: info };
}
public async updateLinuxMirror(
info: SystemModelInfo,
res?: Response,
@@ -270,11 +280,13 @@ export default class SystemService {
},
);
}
public async checkUpdate() {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 15000);
try {
const currentVersionContent = await parseVersion(config.versionFile);
const { body, statusCode } = await request(
`${config.lastVersionFile}?t=${Date.now()}`,
{ signal: controller.signal },
@@ -289,6 +301,7 @@ export default class SystemService {
) {
throw new Error('Invalid version response');
}
return {
code: 200,
data: {
@@ -310,6 +323,7 @@ export default class SystemService {
clearTimeout(timeout);
}
}
private checkHasNewVersion(curVersion: string, lastVersion: string) {
const curArr = curVersion.split('.').map((x) => parseInt(x, 10));
const lastArr = lastVersion.split('.').map((x) => parseInt(x, 10));
@@ -328,21 +342,25 @@ export default class SystemService {
}
return false;
}
public async updateSystem() {
return this.startUpgradeJob('download');
}
private upgradeJobs() {
return require(path.join(
config.rootPath,
'cli/src/internal/maintenance/panelUpgrade.cjs',
));
}
public async getUpgradeStatus() {
return {
code: 200,
data: await this.upgradeJobs().getStatus(config.tmpPath),
};
}
private async startUpgradeJob(action: 'download' | 'apply') {
try {
const data = await this.upgradeJobs().startJob(
@@ -365,6 +383,7 @@ export default class SystemService {
return { code: 400, message: t('升级任务启动失败,请检查日志后重试') };
}
}
public async reloadSystem(target?: 'system' | 'data') {
if (target && !['system', 'data'].includes(target)) {
return { code: 400, message: t('参数错误') };
@@ -382,6 +401,7 @@ export default class SystemService {
});
return { code: 200 };
}
public async notify({
title,
content,
@@ -398,38 +418,35 @@ export default class SystemService {
if (notificationInfo && typeString) {
notificationInfo.type = typeString;
}
const isSuccess = await this.notificationService.notify(
title,
content,
notificationInfo,
);
if (isSuccess) {
return { code: 200, message: t('通知发送成功') };
} else {
return { code: 400, message: t('通知发送失败,请检查系统设置/通知配置') };
}
}
public async recordNotifyLog(info: NotifyLogInfo): Promise<void> {
await persistNotificationAudit(info);
}
public async getNotifyLog(): Promise<Array<NotifyLogInfo>> {
const docs = await SystemModel.find(
{ type: AuthDataType.notifyLog },
{
orderBy: orderBy([['id', 'DESC']]),
limit: 200,
},
);
const docs = await SystemModel.findAll({
where: { type: AuthDataType.notifyLog },
order: [['id', 'DESC']],
limit: 200,
});
return docs.map((x) => ({ ...x.info, id: x.id }));
}
public async run(
{
command,
logPath,
}: {
command: string;
logPath?: string;
},
{ command, logPath }: { command: string; logPath?: string },
callback: TaskCallbacks,
) {
if (!command.startsWith(TASK_COMMAND)) {
@@ -446,14 +463,17 @@ export default class SystemService {
},
);
}
public async stop({ command, pid }: { command: string; pid: number }) {
if (!pid && !command) {
return { code: 400, message: t('参数错误') };
}
if (pid) {
await killTask(pid);
return { code: 200 };
}
if (!command.startsWith(TASK_COMMAND)) {
command = `${TASK_COMMAND} ${command}`;
}
@@ -465,6 +485,7 @@ export default class SystemService {
return { code: 400, message: t('任务未找到') };
}
}
public async exportData(res: Response, type?: string[]) {
try {
let dataDirs = ['db', 'upload'];
@@ -506,6 +527,7 @@ export default class SystemService {
return res.send({ code: 400, message: error.message });
}
}
public async importData() {
try {
await promiseExec(`rm -rf ${path.join(config.tmpPath, 'data')}`);
@@ -517,6 +539,7 @@ export default class SystemService {
return { code: 400, message: error.message };
}
}
public async getSystemLog(
res: Response,
query: {
@@ -536,6 +559,7 @@ export default class SystemService {
.reverse()
.filter((x) => x.title.endsWith('.log'))
.filter((x) => x.createTime >= startTime && x.createTime <= endTime);
const limit = Math.min(
Math.max(Number(query.limit) || 1024 * 1024, 1),
1024 * 1024,
@@ -543,10 +567,7 @@ export default class SystemService {
const total = logs.reduce((size, log) => size + (log.size || 0), 0);
let remaining = limit;
const selected: Array<
(typeof logs)[number] & {
start: number;
length: number;
}
(typeof logs)[number] & { start: number; length: number }
> = [];
for (let index = logs.length - 1; index >= 0 && remaining > 0; index--) {
const log = logs[index];
@@ -557,6 +578,7 @@ export default class SystemService {
remaining -= length;
}
}
const contentLength = selected.reduce((size, log) => size + log.length, 0);
res.set({
'Content-Length': contentLength,
@@ -568,6 +590,7 @@ export default class SystemService {
res.end();
return;
}
const currentLog = fileNames.shift();
if (currentLog) {
const currentFileStream = fs.createReadStream(
@@ -584,6 +607,7 @@ export default class SystemService {
}
})(res, selected);
}
public async deleteSystemLog() {
const result = await readDirs(config.systemLogPath, config.systemLogPath);
const logs = result.reverse().filter((x) => x.title.endsWith('.log'));
@@ -601,6 +625,7 @@ export default class SystemService {
}
}
}
public async updateTimezone(info: SystemModelInfo) {
if (!info.timezone) {
info.timezone = 'Asia/Shanghai';
@@ -617,6 +642,7 @@ export default class SystemService {
return { code: 400, message: t('设置时区失败') };
}
}
public async updateLanguage(info: SystemModelInfo) {
const oDoc = await this.getSystemConfig();
const lang = info.lang || 'zh';
@@ -636,6 +662,7 @@ export default class SystemService {
setLang(lang);
return { code: 200, data: { lang } };
}
public async updatePanelTitle(info: SystemModelInfo) {
const oDoc = await this.getSystemConfig();
const panelTitle = info.panelTitle?.trim() || '';
@@ -645,36 +672,43 @@ export default class SystemService {
});
return { code: 200, data: { panelTitle } };
}
public async updateGlobalSshKey(info: SystemModelInfo) {
const oDoc = await this.getSystemConfig();
const result = await this.updateAuthDb({
...oDoc,
info: { ...oDoc.info, ...info },
});
// Apply the global SSH key
const SshKeyService = require('./sshKey').default;
const Container = require('typedi').Container;
const sshKeyService = Container.get(SshKeyService);
if (info.globalSshKey) {
await sshKeyService.addGlobalSSHKey(info.globalSshKey, 'global');
} else {
await sshKeyService.removeGlobalSSHKey('global');
}
return { code: 200, data: result };
}
public async cleanDependence(type: 'node' | 'python3') {
if (!type || !['node', 'python3'].includes(type)) {
return { code: 400, message: t('参数错误') };
}
const finalPath = path.join(config.dependenceCachePath, type);
await fs.promises.rm(finalPath, { recursive: true, force: true });
await DependenceModel.nativeUpdate(
{
type:
type === 'node' ? DependenceTypes.nodejs : DependenceTypes.python3,
status: DependenceStatus.installed,
},
await DependenceModel.update(
{ status: DependenceStatus.installFailed },
{
where: {
type:
type === 'node' ? DependenceTypes.nodejs : DependenceTypes.python3,
status: DependenceStatus.installed,
},
},
);
return { code: 200 };
}
+71 -20
View File
@@ -34,27 +34,27 @@ import {
} from '../shared/password';
import { serializeAuthMutation } from '../shared/authMutation';
import { LoginThrottle } from '../shared/loginThrottle';
@Service()
export default class UserService {
private loginThrottle = new LoginThrottle();
@Inject((type) => NotificationService)
private notificationService!: NotificationService;
constructor(
@Inject('logger')
private logger: winston.Logger,
@Inject('logger') private logger: winston.Logger,
private scheduleService: ScheduleService,
private sockService: SockService,
) {}
@serializeAuthMutation
public async login(
payloads: {
username: string;
password: string;
},
payloads: { username: string; password: string },
req: Request,
): Promise<any> {
return this.authenticate(payloads, req);
}
private async authenticate(
payloads: {
username: string;
@@ -89,6 +89,7 @@ export default class UserService {
platform,
blockedIps = [],
} = content;
if (
ip &&
blockedIps.some((blockedIp) => normalizeClientIp(blockedIp) === ip)
@@ -106,6 +107,7 @@ export default class UserService {
this.getLoginLog();
return { code: 403, message: t('该 IP 已被列入黑名单') };
}
const waitTime = this.loginThrottle.retryAfter(ip, timestamp);
if (waitTime > 0) {
return {
@@ -114,8 +116,10 @@ export default class UserService {
data: waitTime,
};
}
const passwordMatches =
username === cUsername && (await verifyPassword(password, cPassword));
if (passwordMatches && twoFactorActivated && needTwoFactor) {
await this.updateAuthInfo(content, {
isTwoFactorChecking: true,
@@ -126,6 +130,7 @@ export default class UserService {
message: '',
};
}
if (passwordMatches) {
this.loginThrottle.reset(ip);
const data = createRandomString(50, 100);
@@ -134,6 +139,7 @@ export default class UserService {
expiresIn: config.jwt.expiresIn || expiration,
algorithm: 'HS384',
});
const tokenInfo: TokenInfo = {
value: token,
timestamp,
@@ -141,11 +147,13 @@ export default class UserService {
address,
platform: req.platform,
};
const updatedTokens = this.addTokenToList(
tokens,
req.platform,
tokenInfo,
);
await this.updateAuthInfo(content, {
password: isPasswordHash(cPassword)
? cPassword
@@ -244,13 +252,16 @@ export default class UserService {
}
}
}
@serializeAuthMutation
public async logout(platform: string, tokenValue: string): Promise<any> {
if (!platform || !tokenValue) {
this.logger.warn('Invalid logout parameters - empty platform or token');
return;
}
const authInfo = await this.getAuthInfo();
// Verify the token exists before attempting to remove it
const tokenExists = this.findTokenInList(
authInfo.tokens,
@@ -264,36 +275,45 @@ export default class UserService {
);
return;
}
const updatedTokens = this.removeTokenFromList(
authInfo.tokens,
platform,
tokenValue,
);
await this.updateAuthInfo(authInfo, {
token: authInfo.token === tokenValue ? '' : authInfo.token,
tokens: updatedTokens,
});
}
public async getLoginLog(): Promise<Array<SystemModelInfo | undefined>> {
const docs = await SystemModel.find({ type: AuthDataType.loginLog }, {});
const docs = await SystemModel.findAll({
where: { type: AuthDataType.loginLog },
});
if (docs && docs.length > 0) {
const result = docs.sort(
(a, b) => b.info!.timestamp! - a.info!.timestamp!,
);
if (result.length > 100) {
const ids = result.slice(100).map((x) => x.id!);
await SystemModel.nativeDelete({ id: ids });
await SystemModel.destroy({
where: { id: ids },
});
}
return result.map((x) => x.info);
}
return [];
}
public async getIpBlacklist(): Promise<string[]> {
const authInfo = await this.getAuthInfo();
return uniq((authInfo.blockedIps || []).map(normalizeClientIp)).filter(
Boolean,
);
}
@serializeAuthMutation
public async blockIp(ip: string): Promise<string[]> {
const authInfo = await this.getAuthInfo();
@@ -304,6 +324,7 @@ export default class UserService {
await this.updateAuthInfo(authInfo, { blockedIps });
return blockedIps;
}
@serializeAuthMutation
public async unblockIp(ip: string): Promise<string[]> {
const authInfo = await this.getAuthInfo();
@@ -314,10 +335,12 @@ export default class UserService {
await this.updateAuthInfo(authInfo, { blockedIps });
return blockedIps;
}
private async insertDb(payload: SystemInfo): Promise<SystemInfo> {
const doc = await SystemModel.insert({ ...payload });
const doc = await SystemModel.create({ ...payload }, { returning: true });
return doc;
}
@serializeAuthMutation
public async initializeUser({
username,
@@ -343,6 +366,7 @@ export default class UserService {
});
return { code: 200, message: t('更新成功') };
}
@serializeAuthMutation
public async updateUsernameAndPassword({
username,
@@ -365,12 +389,14 @@ export default class UserService {
});
return { code: 200, message: t('更新成功') };
}
@serializeAuthMutation
public async updateAvatar(avatar: string) {
const authInfo = await this.getAuthInfo();
await this.updateAuthInfo(authInfo, { avatar });
return { code: 200, data: avatar, message: t('更新成功') };
}
@serializeAuthMutation
public async initTwoFactor() {
const secret = authenticator.generateSecret();
@@ -382,6 +408,7 @@ export default class UserService {
await this.updateAuthInfo(authInfo, { twoFactorSecret: secret });
return { secret, url: otpauth };
}
@serializeAuthMutation
public async activeTwoFactor(code: string) {
const authInfo = await this.getAuthInfo();
@@ -400,17 +427,14 @@ export default class UserService {
}
return isValid;
}
@serializeAuthMutation
public async twoFactorLogin(
{
username,
password,
code,
}: {
username: string;
password: string;
code: string;
},
}: { username: string; password: string; code: string },
req: any,
) {
const authInfo = await this.getAuthInfo();
@@ -458,6 +482,7 @@ export default class UserService {
return { code: 430, message: t('验证失败') };
}
}
@serializeAuthMutation
public async deactivateTwoFactor() {
const authInfo = await this.getAuthInfo();
@@ -471,6 +496,7 @@ export default class UserService {
});
return true;
}
public async getAuthInfo() {
const authInfo = await shareStore.getAuthInfo();
if (authInfo) {
@@ -479,6 +505,7 @@ export default class UserService {
const doc = await this.getDb({ type: AuthDataType.authConfig });
return (doc.info || {}) as AuthInfo;
}
private async updateAuthInfo(authInfo: AuthInfo, info: Partial<AuthInfo>) {
const result = { ...authInfo, ...info };
await shareStore.updateAuthInfo(result);
@@ -490,27 +517,34 @@ export default class UserService {
this.sockService.getClients().forEach((conn) => conn.close('401'));
}
}
public async getNotificationMode(): Promise<NotificationInfo> {
const doc = await this.getDb({ type: AuthDataType.notification });
return (doc.info || {}) as NotificationInfo;
}
private async updateAuthDb(payload: SystemInfo): Promise<any> {
let doc = await SystemModel.findOne({ type: payload.type }, {});
let doc = await SystemModel.findOne({ where: { type: payload.type } });
if (doc) {
await SystemModel.nativeUpdate({ id: doc.id }, payload);
doc = await SystemModel.findOne({ id: doc.id });
const updateResult = await SystemModel.update(payload, {
where: { id: doc.id },
returning: true,
});
doc = updateResult[1][0];
} else {
doc = await SystemModel.insert(payload);
doc = await SystemModel.create(payload, { returning: true });
}
return doc;
}
public async getDb(query: any): Promise<SystemInfo> {
const doc = await SystemModel.findOne({ ...query }, {});
const doc = await SystemModel.findOne({ where: { ...query } });
if (!doc) {
throw new Error(`${JSON.stringify(query)} not found`);
}
return doc;
return doc.get({ plain: true });
}
public async updateNotificationMode(notificationInfo: NotificationInfo) {
const code = Math.random().toString().slice(-6);
const isSuccess = await this.notificationService.testNotify(
@@ -528,10 +562,12 @@ export default class UserService {
return { code: 400, message: t('通知发送失败,请检查参数') };
}
}
private normalizeTokens(
tokens: Record<string, string | TokenInfo[]>,
): Record<string, TokenInfo[]> {
const normalized: Record<string, TokenInfo[]> = {};
for (const [platform, value] of Object.entries(tokens)) {
if (typeof value === 'string') {
// Legacy format: convert string token to TokenInfo array
@@ -553,8 +589,10 @@ export default class UserService {
normalized[platform] = value || [];
}
}
return normalized;
}
private addTokenToList(
tokens: Record<string, string | TokenInfo[]>,
platform: string,
@@ -568,12 +606,16 @@ export default class UserService {
);
maxTokensPerPlatform = config.maxTokensPerPlatform;
}
const normalized = this.normalizeTokens(tokens);
if (!normalized[platform]) {
normalized[platform] = [];
}
// Add new token
normalized[platform].unshift(tokenInfo);
// Limit the number of active tokens per platform
if (normalized[platform].length > maxTokensPerPlatform) {
normalized[platform] = normalized[platform].slice(
@@ -581,32 +623,40 @@ export default class UserService {
maxTokensPerPlatform,
);
}
return normalized;
}
private removeTokenFromList(
tokens: Record<string, string | TokenInfo[]>,
platform: string,
tokenValue: string,
): Record<string, TokenInfo[]> {
const normalized = this.normalizeTokens(tokens);
if (normalized[platform]) {
normalized[platform] = normalized[platform].filter(
(t) => t.value !== tokenValue,
);
}
return normalized;
}
private findTokenInList(
tokens: Record<string, string | TokenInfo[]>,
platform: string,
tokenValue: string,
): TokenInfo | undefined {
const normalized = this.normalizeTokens(tokens);
if (normalized[platform]) {
return normalized[platform].find((t) => t.value === tokenValue);
}
return undefined;
}
@serializeAuthMutation
public async resetAuthInfo(info: Partial<AuthInfo>) {
const { retries, twoFactorActivated, password, username } = info;
@@ -623,6 +673,7 @@ export default class UserService {
},
(x) => !isNil(x),
);
if (password !== undefined) {
payload.password = await hashPassword(password);
}
+2 -4
View File
@@ -135,10 +135,8 @@ export function parseCronSchedule(schedule: unknown): CronSchedule {
};
Object.freeze(patterns);
Object.freeze(parsed);
if (scheduleCache.size >= 512) {
const oldest = scheduleCache.keys().next().value;
if (oldest !== undefined) scheduleCache.delete(oldest);
}
if (scheduleCache.size >= 512)
scheduleCache.delete(scheduleCache.keys().next().value);
scheduleCache.set(cacheKey, parsed);
return parsed;
}
-2
View File
@@ -1,8 +1,6 @@
const messages: Record<string, Record<string, string>> = {
zh: {},
en: {
'认证服务暂时不可用,请稍后重试':
'Authentication service is temporarily unavailable; try again shortly.',
'升级任务启动失败,请检查日志后重试':
'Failed to start the update. Check the logs and try again.',
'[schedule][停止已运行任务] 任务ID: %s, PID: %s':
+15 -14
View File
@@ -1,16 +1,17 @@
import { Query, QuerySpec, orderBy } from '../data/queries';
import { Op } from 'sequelize';
import {
SystemModel,
AuthDataType,
NotifyStatus,
NotifyLogInfo,
} from '../data/system';
// Both backend sends and script reports use the same bounded, best-effort sink.
export async function persistNotificationAudit(
info: NotifyLogInfo,
): Promise<void> {
try {
await SystemModel.insert({
await SystemModel.create({
type: AuthDataType.notifyLog,
info: {
timestamp: Date.now(),
@@ -26,19 +27,19 @@ export async function persistNotificationAudit(
taskId: /^\d+$/.test(info.taskId || '') ? info.taskId : undefined,
},
});
const oldest = await SystemModel.find(
{ type: AuthDataType.notifyLog },
{
fields: ['id'],
orderBy: orderBy([['id', 'DESC']]),
offset: 200,
limit: 1,
},
);
const oldest = await SystemModel.findAll({
where: { type: AuthDataType.notifyLog },
attributes: ['id'],
order: [['id', 'DESC']],
offset: 200,
limit: 1,
});
if (oldest[0]?.id) {
await SystemModel.nativeDelete({
type: AuthDataType.notifyLog,
id: { ['$lte']: oldest[0].id },
await SystemModel.destroy({
where: {
type: AuthDataType.notifyLog,
id: { [Op.lte]: oldest[0].id },
},
});
}
} catch (error) {
+7 -14
View File
@@ -137,20 +137,13 @@ class TaskLimit {
this.manualCronoLimit.concurrency = limit;
return;
}
try {
const doc = await SystemModel.findOne({
type: AuthDataType.systemConfig,
});
if (doc?.info?.cronConcurrency) {
this.cronLimit.concurrency = doc.info.cronConcurrency;
this.manualCronoLimit.concurrency = doc.info.cronConcurrency;
}
} catch (error) {
// Imports can precede the isolated initializer on a fresh installation.
// Use defaults until that initializer creates Auths; never create schema here.
if ((error as { code?: string }).code !== 'DATABASE_SCHEMA_MISSING')
throw error;
await SystemModel.sync();
const doc = await SystemModel.findOne({
where: { type: AuthDataType.systemConfig },
});
if (doc?.info?.cronConcurrency) {
this.cronLimit.concurrency = doc.info.cronConcurrency;
this.manualCronoLimit.concurrency = doc.info.cronConcurrency;
}
}
+90 -31
View File
@@ -4,7 +4,6 @@ import taskLimit from './pLimit';
import Logger from '../loaders/logger';
import { ICron } from '../protos/cron';
import { CrontabModel, CrontabStatus } from '../data/cron';
import { killTask } from '../config/util';
import { RunningInstanceModel, InstanceStatus } from '../data/runningInstance';
import dayjs from 'dayjs';
import { observeChildProcess, asError } from './childProcess';
@@ -17,10 +16,10 @@ export function runCron(
const execute = async () => {
try {
if (!isCurrent()) return;
// Check if the cron is already running and stop it (only if multiple instances are not allowed)
// A single-instance trigger must not interrupt work already in progress.
try {
const existingCron = await CrontabModel.findOne({
id: Number(cron.id),
where: { id: Number(cron.id) },
});
// A mutation may commit while this queued run is reading the database.
@@ -30,33 +29,93 @@ export function runCron(
const allowSingleInstances =
existingCron?.allow_multiple_instances === 0;
if (
allowSingleInstances &&
existingCron &&
existingCron.pid &&
(existingCron.status === CrontabStatus.running ||
existingCron.status === CrontabStatus.queued)
) {
Logger.info(
t('[schedule][停止已运行任务] 任务ID: %s, PID: %s'),
cron.id,
existingCron.pid,
);
await killTask(existingCron.pid, true);
// Mark old running instances as stopped
const stoppedAt = dayjs().unix();
await RunningInstanceModel.nativeUpdate(
{
cron_id: Number(cron.id),
status: InstanceStatus.running,
},
{ status: InstanceStatus.stopped, finished_at: stoppedAt },
);
// Update the status to idle after killing
await CrontabModel.nativeUpdate(
{ id: Number(cron.id) },
{ status: CrontabStatus.idle, pid: undefined },
);
if (allowSingleInstances) {
// A queued manual run is an active reservation, even if its stored PID
// belongs to a previous execution.
if (
existingCron.status === CrontabStatus.queued &&
existingCron.pid
) {
Logger.info(
'[schedule][Skip queued single-instance task] Task ID: %s, PID: %s',
cron.id,
existingCron.pid,
);
return;
}
if (
existingCron.status === CrontabStatus.running &&
existingCron.pid
) {
let missing = false;
try {
process.kill(existingCron.pid, 0);
} catch (error) {
const code = (error as NodeJS.ErrnoException).code;
if (code === 'ESRCH') missing = true;
else if (code !== 'EPERM') throw error;
}
if (!missing) {
Logger.info(
'[schedule][Skip active single-instance task] Task ID: %s, PID: %s',
cron.id,
existingCron.pid,
);
return;
}
// Capture old instance IDs before releasing the stale cron state so
// a concurrent replacement cannot have its instance marked stopped.
const staleInstances = await RunningInstanceModel.findAll({
attributes: ['id'],
where: {
cron_id: Number(cron.id),
pid: existingCron.pid,
status: InstanceStatus.running,
},
});
if (!isCurrent()) return;
const [recovered] = await CrontabModel.update(
{
status: CrontabStatus.idle,
pid: null,
queued_token: null,
} as any,
{
where: {
id: Number(cron.id),
pid: existingCron.pid,
status: CrontabStatus.running,
...(existingCron.log_path !== undefined
? { log_path: existingCron.log_path }
: {}),
...(existingCron.queued_token !== undefined
? { queued_token: existingCron.queued_token }
: {}),
...(existingCron.last_execution_time !== undefined
? { last_execution_time: existingCron.last_execution_time }
: {}),
...(existingCron.isDisabled !== undefined
? { isDisabled: existingCron.isDisabled }
: {}),
allow_multiple_instances: 0,
},
},
);
if (recovered !== 1) return;
const staleIds = staleInstances.map((instance) => instance.id!);
if (staleIds.length) {
await RunningInstanceModel.update(
{ status: InstanceStatus.stopped, finished_at: dayjs().unix() },
{ where: { id: staleIds, status: InstanceStatus.running } },
);
}
Logger.info(
'[schedule][Recovered stale single-instance task] Task ID: %s, PID: %s',
cron.id,
existingCron.pid,
);
}
}
} catch (error) {
Logger.error(
@@ -67,7 +126,7 @@ export function runCron(
throw error;
}
// In particular, invalidation can happen while stopping a previous run.
// Invalidation can happen during the asynchronous database check.
if (!isCurrent()) return;
Logger.info(
t('[schedule][开始执行任务] 参数 %s'),
+1 -1
View File
@@ -28,7 +28,7 @@ const backend = read(path.resolve(__dirname, '../app.js'));
const root = process.env.QL_DIR || path.resolve(__dirname, '../../..');
const version = read(path.join(root, 'version.yaml'))
?.toString()
.match(/^\uFEFF?version:\s*["']?(\d+\.\d+\.\d+)/m)?.[1];
.match(/^\uFEFF?version:\s*["']?(\d+\.\d+\.\d+(?:-[\w.-]+)?(?:\+[\w.-]+)?)/m)?.[1];
export const runtimeBuild = Object.freeze({
instanceId: randomUUID(),
+15 -5
View File
@@ -1,4 +1,4 @@
import { DatabaseClient } from '../data/databaseClient';
import { QueryTypes, Sequelize } from 'sequelize';
// Append new entries; IDs are persisted and must not be renumbered or reused.
const columns = [
@@ -27,19 +27,25 @@ const columns = [
{ table: 'Crontabs', column: 'queued_token', type: 'VARCHAR(255)' },
];
export async function migrateSchema(database: DatabaseClient): Promise<void> {
await database.transaction(async () => {
export async function migrateSchema(database: Sequelize): Promise<void> {
await database.transaction(async (transaction) => {
await database.query(
'CREATE TABLE IF NOT EXISTS "SchemaMigrations" ("id" TEXT PRIMARY KEY, "applied_at" TEXT NOT NULL)',
{ transaction },
);
const applied = await database.query<{ id: string }>(
'SELECT "id" FROM "SchemaMigrations"',
{ type: QueryTypes.SELECT, transaction },
);
const appliedIds = new Set(applied.map(({ id }) => id));
for (const { table, column, type } of columns) {
const id = `add-${table}-${column}`;
const fields = await database.query<{ name: string }>(
`PRAGMA table_info("${table}")`,
{
type: QueryTypes.SELECT,
transaction,
},
);
if (fields.length === 0)
throw new Error(`Migration table is missing: ${table}`);
@@ -47,12 +53,16 @@ export async function migrateSchema(database: DatabaseClient): Promise<void> {
// table/column/type come only from the static migration manifest above.
await database.query(
`ALTER TABLE "${table}" ADD COLUMN "${column}" ${type}`,
{ transaction },
);
}
if (!appliedIds.has(id)) {
await database.query(
'INSERT INTO "SchemaMigrations" ("id", "applied_at") VALUES (?, ?)',
[id, new Date().toISOString()],
'INSERT INTO "SchemaMigrations" ("id", "applied_at") VALUES (:id, :appliedAt)',
{
replacements: { id, appliedAt: new Date().toISOString() },
transaction,
},
);
}
}
+5 -7
View File
@@ -1,7 +1,9 @@
import { AuthInfo } from '../data/system';
import { App } from '../data/open';
import Keyv from 'keyv';
import { CacheStore } from '../data/cacheStore';
import KeyvSqlite from '@keyv/sqlite';
import config from '../config';
import path from 'path';
export enum EKeyv {
'apps' = 'apps',
@@ -15,12 +17,8 @@ export interface IKeyvStore {
lang: string;
}
const keyvSqlite = new CacheStore();
// A stopped database is unavailable, not a cache miss or an expired session.
export const keyvStore = new Keyv<IKeyvStore>({
store: keyvSqlite,
throwOnErrors: true,
});
const keyvSqlite = new KeyvSqlite(path.join(config.dbPath, 'keyv.sqlite'));
export const keyvStore = new Keyv<IKeyvStore>({ store: keyvSqlite });
export const shareStore = {
getAuthInfo() {
+31 -12
View File
@@ -2,15 +2,21 @@ import express, { Application, Request } from 'express';
import { AuthDataType, SystemModel } from '../data/system';
import { normalizeClientIp } from './clientIp';
import { t } from './i18n';
const DEFAULT_TRUST_PROXY = 'loopback';
type TrustProxyValue = boolean | number | string;
let activeApp: Application | undefined;
function getEnvironmentSetting(): string {
return process.env.QL_TRUST_PROXY?.trim() || '';
}
function normalizeSetting(value?: string): string {
return value?.trim() || DEFAULT_TRUST_PROXY;
}
export function resolveTrustProxy(value?: string): TrustProxyValue {
const setting = normalizeSetting(value);
if (setting === 'true' || setting === 'false') {
@@ -21,6 +27,7 @@ export function resolveTrustProxy(value?: string): TrustProxyValue {
}
return setting;
}
function validateTrustProxy(value: string): string {
const setting = normalizeSetting(value);
if (setting.length > 500 || /[\r\n]/.test(setting)) {
@@ -29,22 +36,25 @@ function validateTrustProxy(value: string): string {
if (/^\d+$/.test(setting) && Number(setting) > 20) {
throw new Error(t('代理层数不能超过 20'));
}
const probe = express();
probe.set('trust proxy', resolveTrustProxy(setting));
return setting;
}
async function getStoredSetting(): Promise<string> {
const doc = await SystemModel.findOne(
{ type: AuthDataType.systemConfig },
{},
);
const info = (doc?.info || {}) as Record<string, unknown>;
const doc = await SystemModel.findOne({
where: { type: AuthDataType.systemConfig },
});
const info = (doc?.get('info') || {}) as Record<string, unknown>;
return typeof info.trustProxy === 'string' ? info.trustProxy : '';
}
export async function getTrustProxyConfig() {
const environmentSetting = getEnvironmentSetting();
const storedSetting = await getStoredSetting();
const trustProxy = normalizeSetting(environmentSetting || storedSetting);
return {
trustProxy,
source: environmentSetting
@@ -55,31 +65,36 @@ export async function getTrustProxyConfig() {
editable: !environmentSetting,
};
}
export async function initializeTrustProxy(app: Application) {
activeApp = app;
const { trustProxy } = await getTrustProxyConfig();
app.set('trust proxy', resolveTrustProxy(trustProxy));
}
export async function updateTrustProxy(value: string) {
if (getEnvironmentSetting()) {
throw new Error(t('环境变量 QL_TRUST_PROXY 已生效,系统设置不可覆盖'));
}
const trustProxy = validateTrustProxy(value);
const doc = await SystemModel.findOne(
{ type: AuthDataType.systemConfig },
{},
);
const doc = await SystemModel.findOne({
where: { type: AuthDataType.systemConfig },
});
if (!doc) {
throw new Error(t('系统配置不存在'));
}
const plain = doc;
await SystemModel.nativeUpdate(
{ id: plain.id },
const plain = doc.get({ plain: true });
await SystemModel.update(
{ info: { ...(plain.info || {}), trustProxy } as any },
{ where: { id: plain.id } },
);
activeApp?.set('trust proxy', resolveTrustProxy(trustProxy));
return getTrustProxyConfig();
}
function parseForwardedFor(value: string | string[] | undefined): string[] {
const values = Array.isArray(value) ? value : value ? [value] : [];
return values
@@ -87,6 +102,7 @@ function parseForwardedFor(value: string | string[] | undefined): string[] {
.map((item) => normalizeClientIp(item))
.filter(Boolean);
}
export async function diagnoseClientIp(req: Request) {
const remoteAddress = normalizeClientIp(req.socket.remoteAddress);
const forwardedFor = parseForwardedFor(req.headers['x-forwarded-for']);
@@ -96,6 +112,7 @@ export async function diagnoseClientIp(req: Request) {
const trust = req.app.get('trust proxy fn') as
| ((ip: string, hop: number) => boolean)
| undefined;
let selectedIndex = Math.max(hopsFromApp.length - 1, 0);
for (let index = 0; index < hopsFromApp.length - 1; index += 1) {
if (!trust?.(hopsFromApp[index], index)) {
@@ -103,6 +120,7 @@ export async function diagnoseClientIp(req: Request) {
break;
}
}
const hops = hopsFromApp.map((ip, index) => ({
ip,
hop: index,
@@ -113,6 +131,7 @@ export async function diagnoseClientIp(req: Request) {
? 'client'
: 'not_checked',
}));
return {
...(await getTrustProxyConfig()),
remoteAddress,
+1 -1
View File
@@ -23,5 +23,5 @@
"esModuleInterop": true
},
"include": ["./**/*"],
"exclude": ["node_modules", "./data/databaseWorker.ts"]
"exclude": ["node_modules"]
}
+5 -16
View File
@@ -61,7 +61,9 @@ async function withPanelLifecycle(root, tmp, env, operation, signal) {
async function targetVersion(source) {
const text = await fs.readFile(path.join(source, 'version.yaml'), 'utf8');
const version = text.match(/^\uFEFF?version:\s*["']?(\d+\.\d+\.\d+)/m)?.[1];
const version = text.match(
/^\uFEFF?version:\s*["']?(\d+\.\d+\.\d+(?:-[\w.-]+)?(?:\+[\w.-]+)?)/m,
)?.[1];
if (!version) throw new Error('Invalid update version');
return version;
}
@@ -164,10 +166,7 @@ async function readState(tmp) {
}
}
async function getStatus(
tmp,
{ root = path.resolve(__dirname, '../../../..') } = {},
) {
async function getStatus(tmp) {
const state = await readState(tmp);
if (active.has(state.status)) {
let alive = false;
@@ -201,17 +200,7 @@ async function getStatus(
await handle.close();
}
}
const pkg = JSON.parse(
await fs
.readFile(path.join(root, 'package.json'), 'utf8')
.catch((error) => {
if (error.code !== 'ENOENT') throw error;
return '{}';
}),
);
// Read the installed package on every poll: the worker's completion record
// survives replacement, while this marker disappears only with the new runtime.
return { ...state, migrationRequired: Boolean(pkg.qinglongMigrationBridge) };
return state;
}
// Manual restarts accept the currently prepared package, including one retained
+30 -179
View File
@@ -1,6 +1,5 @@
// One installed verifier handles release downloads and local offline archives.
const fs = require('node:fs/promises');
const { constants, createReadStream } = require('node:fs');
const path = require('node:path');
const crypto = require('node:crypto');
const os = require('node:os');
@@ -177,39 +176,23 @@ async function verifyPayload(selected, manifest, target) {
});
const names = Object.keys(actual).sort(),
expected = Object.keys(manifest.files).sort();
const differences = [...new Set([...expected, ...names])].filter(
(name) =>
!Object.hasOwn(actual, name) ||
!Object.hasOwn(manifest.files, name) ||
JSON.stringify(actual[name]) !== JSON.stringify(manifest.files[name]),
);
if (differences.length)
if (
JSON.stringify(names) !== JSON.stringify(expected) ||
names.some(
(name) =>
JSON.stringify(actual[name]) !== JSON.stringify(manifest.files[name]),
)
)
throw new Error(
'Platform bundle files or dependency checksums do not match: ' +
JSON.stringify({
count: differences.length,
files: differences.slice(0, 5).map((name) => ({
name,
expected: manifest.files[name] || null,
actual: actual[name] || null,
})),
}),
'Platform bundle files or dependency checksums do not match',
);
if (
sha256(await fs.readFile(path.join(selected.source, 'pnpm-lock.yaml'))) !==
manifest.lockfileSha256
)
throw new Error('Platform bundle lockfile mismatch');
const pkg = JSON.parse(
await fs.readFile(path.join(selected.source, 'package.json'), 'utf8'),
);
await fs.access(
path.join(
selected.source,
pkg.dependencies?.['@mikro-orm/sql']
? 'node_modules/@mikro-orm/sql/package.json'
: 'node_modules/sqlite3/package.json',
),
path.join(selected.source, 'node_modules/sqlite3/package.json'),
);
}
@@ -219,58 +202,34 @@ async function smokeDependencies(source, signal) {
const path = require('node:path'),fs = require('node:fs');
const root = fs.realpathSync(process.cwd());
const r = require('node:module').createRequire(path.join(root,'package.json'));
const pkg=r('./package.json'), modern=Boolean(pkg.dependencies?.['@mikro-orm/sql']);
if (modern) {
const [major,minor]=process.versions.node.split('.').map(Number);
if(major<22 || (major===22 && minor<17)) throw new Error('Bundle requires Node >=22.17; the existing Node is retained');
}
for (const name of [...(modern?['@mikro-orm/core','@mikro-orm/sql']:['sqlite3','sequelize']),'cron-parser','@grpc/grpc-js','undici']) {
for (const name of ['sqlite3','sequelize','cron-parser','@grpc/grpc-js','undici']) {
if (!fs.realpathSync(r.resolve(name)).startsWith(root + path.sep)) throw new Error('Dependency outside staged payload: '+name);
r(name);
}
if (modern) {
const {DatabaseSync}=require('node:sqlite'), db=new DatabaseSync(':memory:');
try {
db.exec('CREATE TABLE verify_bundle(value INTEGER);INSERT INTO verify_bundle VALUES(42)');
if(db.prepare('SELECT value FROM verify_bundle').get().value!==42) throw new Error('SQLite readback failed');
} finally {db.close();}
} else {
const sqlite=r('sqlite3'), db=new sqlite.Database(':memory:');
db.serialize(()=>{db.run('CREATE TABLE verify_bundle(value INTEGER)');db.run('INSERT INTO verify_bundle VALUES(42)');
db.get('SELECT value FROM verify_bundle',(err,row)=>{if(err||row.value!==42){console.error(err||'SQLite readback failed');process.exitCode=1;}db.close();});});
}
`;
// Match the server's normal runtime. Node 20.18's built-in HTTP parser needs
// WebAssembly, which --jitless disables; only archive workers need that flag.
await execute(
process.execPath,
[...serverRuntimeArgs(process), '-e', probe],
{
cwd: source,
env: { ...process.env, NODE_PATH: '', NODE_OPTIONS: '' },
// ARM1176 devices need the same startup allowance as the backend probe.
// Keep this bounded and cancellable without forcing an unnecessary repair.
timeout: nodeRuntimeArgs().length ? 300000 : 30000,
signal,
},
);
await execute(process.execPath, [...serverRuntimeArgs(process), '-e', probe], {
cwd: source,
env: { ...process.env, NODE_PATH: '', NODE_OPTIONS: '' },
// ARM1176 devices need the same startup allowance as the backend probe.
// Keep this bounded and cancellable without forcing an unnecessary repair.
timeout: nodeRuntimeArgs().length ? 300000 : 30000,
signal,
});
}
async function extractArchive(archive, destination) {
const entries = new Map(),
directories = new Map([['qinglong', 0o755]]);
const header = (entry) => ({
type: entry.type,
size: entry.size,
mode: entry.mode,
linkpath: entry.linkpath,
});
links = new Set();
let unpacked = 0,
invalid;
await tar.t({
file: archive,
strict: true,
maxReadSize: 256 * 1024,
onReadEntry(entry) {
try {
const name = entry.path.replace(/\/$/, '');
@@ -278,19 +237,16 @@ async function extractArchive(archive, destination) {
!name ||
name.includes('\\') ||
/[\x00-\x1f\x7f]/.test(name) ||
name.split('/').length > 1024 ||
name.split('/').some((p) => !p || p === '.' || p === '..') ||
!(name === 'qinglong' || name.startsWith('qinglong/')) ||
entries.has(name) ||
!['File', 'OldFile', 'Directory', 'SymbolicLink'].includes(entry.type)
)
throw new Error(`Unsafe bundle archive entry: ${name}`);
if (!Number.isSafeInteger(entry.size) || entry.size < 0)
throw new Error('Invalid bundle archive entry size');
unpacked += entry.size;
if (unpacked > 2 * 1024 ** 3 || entries.size > 200000)
throw new Error('Bundle archive exceeds extraction limits');
entries.set(name, header(entry));
entries.set(name, entry.type);
if (entry.type === 'SymbolicLink') {
if (
path.posix.isAbsolute(entry.linkpath) ||
@@ -301,6 +257,7 @@ async function extractArchive(archive, destination) {
)
)
throw new Error('Unsafe bundle symlink');
links.add(name);
}
} catch (error) {
invalid ||= error;
@@ -308,101 +265,22 @@ async function extractArchive(archive, destination) {
},
});
if (invalid) throw invalid;
for (const [name, entry] of entries) {
if (entry.type === 'Directory')
directories.set(name, ((entry.mode ?? 0o755) & 0o777) | 0o700);
for (const name of entries.keys()) {
let parent = path.posix.dirname(name);
while (parent !== '.') {
if (entries.has(parent) && entries.get(parent).type !== 'Directory')
throw new Error('Archive writes through a symlink or non-directory');
if (!directories.has(parent)) directories.set(parent, 0o755);
if (links.has(parent))
throw new Error('Archive writes through a symlink');
parent = path.posix.dirname(parent);
}
}
// The extraction root must be new and private. Create every parent before
// links, and keep directories private/searchable until all writes finish.
const directoryNames = [...directories.keys()].sort(
(a, b) => a.split('/').length - b.split('/').length,
);
for (const name of directoryNames) {
const filename = path.join(destination, name);
await fs.mkdir(filename, { mode: 0o700 });
await fs.chmod(filename, 0o700);
}
let writes = Promise.resolve(),
failure;
const active = new Set(),
seen = new Set();
const parser = tar.t({
await tar.x({
file: archive,
cwd: destination,
strict: true,
noResume: true,
onReadEntry(entry) {
active.add(entry);
// A parser failure can destroy queued entries before their turn starts.
entry.on('error', () => {});
writes = writes
.then(async () => {
if (failure) return entry.resume();
const name = entry.path.replace(/\/$/, '');
if (
seen.has(name) ||
JSON.stringify(entries.get(name)) !== JSON.stringify(header(entry))
)
throw new Error('Archive changed during extraction');
seen.add(name);
const filename = path.join(destination, name);
if (entry.type === 'Directory') return entry.resume();
if (entry.type === 'SymbolicLink') {
await fs.symlink(entry.linkpath, filename);
return entry.resume();
}
const mode = (entry.mode ?? 0o644) & 0o777;
const handle = await fs.open(
filename,
constants.O_WRONLY |
constants.O_CREAT |
constants.O_EXCL |
constants.O_NOFOLLOW,
0o600,
);
try {
// tar's fs-minipass writer discards successful partial write/writev
// results. Node's writeFile loops until each chunk is fully written.
// One entry/chunk at a time bounds memory and open FDs on ARMv6.
for await (const chunk of entry) await handle.writeFile(chunk);
await handle.chmod(mode);
} finally {
await handle.close();
}
})
.catch((error) => {
failure ||= error;
entry.resume();
})
.finally(() => active.delete(entry));
},
preservePaths: false,
noChown: true,
umask: 0,
});
const input = createReadStream(archive, { highWaterMark: 256 * 1024 });
try {
await new Promise((resolve, reject) => {
input.once('error', reject);
parser.once('error', reject);
parser.once('end', resolve);
input.pipe(parser);
});
} catch (error) {
failure ||= error;
for (const entry of active) entry.destroy(error);
} finally {
input.destroy();
}
await writes;
if (failure) throw failure;
if (seen.size !== entries.size)
throw new Error('Archive changed during extraction');
for (const name of directoryNames.reverse())
await fs.chmod(path.join(destination, name), directories.get(name));
}
async function stageBundle({
@@ -419,8 +297,6 @@ async function stageBundle({
repair = repairDependencies,
}) {
let local = env.QL_UPGRADE_FILE;
let pendingMigration = false,
requiresModern = false;
if (!local) {
const current = JSON.parse(
await fs
@@ -431,9 +307,6 @@ async function stageBundle({
}),
);
if (current.qinglongUpgradeFormat !== 2) return null; // One-time compatibility path for older releases.
pendingMigration = Boolean(current.qinglongMigrationBridge);
requiresModern =
pendingMigration || Boolean(current.dependencies?.['@mikro-orm/sql']);
}
const target = await runtimeTarget();
const branch = ['develop', 'debian-dev'].includes(env.QL_BRANCH)
@@ -488,16 +361,7 @@ async function stageBundle({
indexUrl,
]);
const index = JSON.parse(await fs.readFile(indexFile, 'utf8'));
if (pendingMigration && !Object.hasOwn(index, 'runtimePackages'))
throw new Error('更新索引尚未提供第二步运行包,请稍后重试');
requiresModern ||= Object.hasOwn(index, 'runtimePackages');
// Old format-2 clients read packages and receive an updater bridge. Once
// installed, this selector uses only the actual runtime packages. A broken
// runtime entry must never fall back to the bridge or a different platform.
const packages = Object.hasOwn(index, 'runtimePackages')
? index.runtimePackages
: index.packages;
const entry = packages?.[targetKey(target)];
const entry = index.packages?.[targetKey(target)];
if (
index.format !== 2 ||
!/^[a-f0-9]{40}$/.test(index.sourceCommit) ||
@@ -539,10 +403,6 @@ async function stageBundle({
const manifest = JSON.parse(
await fs.readFile(path.join(incoming, 'bundle.json'), 'utf8'),
);
if (manifest.migrationBridge)
throw new Error(
'This is a legacy upgrader bridge; download the runtime package instead',
);
if (expectedCommit && manifest.sourceCommit !== expectedCommit)
throw new Error('Update index and bundle commit differ');
const selected = {
@@ -553,15 +413,6 @@ async function stageBundle({
await fs.rename(path.join(incoming, 'static'), selected.static);
await report({ phase: 'verifying', percent: null });
await verifyPayload(selected, manifest, target);
if (
requiresModern &&
!JSON.parse(
await fs.readFile(path.join(selected.source, 'package.json'), 'utf8'),
).dependencies?.['@mikro-orm/sql']
)
throw new Error(
'第二步更新包仍是旧数据库后端,已保留当前服务,请稍后重试',
);
const build = await verifyStatic(selected.source, selected.static, env);
if (build.sourceCommit !== manifest.sourceCommit)
throw new Error('Bundle and build commit differ');
@@ -92,7 +92,9 @@ async function prestartBackend(
.digest('hex');
const version = (
await fs.readFile(path.join(selected.source, 'version.yaml'), 'utf8')
).match(/^\uFEFF?version:\s*["']?(\d+\.\d+\.\d+)/m)?.[1];
).match(
/^\uFEFF?version:\s*["']?(\d+\.\d+\.\d+(?:-[\w.-]+)?(?:\+[\w.-]+)?)/m,
)?.[1];
// An allowlist prevents NODE_OPTIONS, startup hooks, real data paths and
// package-manager settings from escaping into the disposable backend.
const env = {
@@ -210,28 +212,6 @@ async function prestartBackend(
}
}
async function cleanSqliteBuildArtifacts(directory) {
const entries = await fs.readdir(directory, { withFileTypes: true }).catch((error) => {
if (error.code !== 'ENOENT') throw error;
return [];
});
if (!entries.length) return;
const pkg = JSON.parse(await fs.readFile(path.join(directory, 'package.json'), 'utf8'));
const runtimePath = path.posix.normalize(pkg.binary?.module_path || '.');
const copiedRuntime = runtimePath.startsWith('lib/binding/');
// Our legacy fork copies its runtime to lib/binding. Other sqlite3 versions
// load build/Release directly: retain that addon and remove only the compiler
// helper directory. Never traverse a build-directory symlink while cleaning.
for (const entry of entries) {
if (entry.name !== 'build' && !/^build-tmp-napi-v\d+$/.test(entry.name)) continue;
const build = path.join(directory, entry.name);
await fs.rm(
copiedRuntime || entry.isSymbolicLink() ? build : path.join(build, 'node_gyp_bins'),
{ recursive: true, force: true },
);
}
}
async function repairDependencies(source, { target, signal, env }) {
const workspace = await fs.mkdtemp(
path.join(path.dirname(source), 'repair-'),
@@ -321,12 +301,7 @@ async function repairDependencies(source, { target, signal, env }) {
'--child-concurrency=1',
'--network-concurrency=2',
]);
if (pkg.dependencies?.sqlite3) {
await run(['rebuild', 'sqlite3', '@whyour/sqlite3']);
await cleanSqliteBuildArtifacts(
path.join(workspace, 'node_modules/sqlite3'),
);
}
await run(['rebuild', 'sqlite3', '@whyour/sqlite3']);
if (
!(await fs.readFile(path.join(source, 'pnpm-lock.yaml'))).equals(
await fs.readFile(path.join(workspace, 'pnpm-lock.yaml')),
@@ -417,6 +392,5 @@ module.exports = {
prestartBackend,
dependencyFailure,
repairDependencies,
cleanSqliteBuildArtifacts,
probeLimits,
};
+17 -10
View File
@@ -1,8 +1,9 @@
# Run Node package installation natively on the builder. Node/npm can spin at
# 100% CPU when Alpine s390x is emulated through QEMU.
FROM --platform=$BUILDPLATFORM node:24-alpine3.24 AS builder
FROM --platform=$BUILDPLATFORM node:24.18.1-alpine3.24 AS builder
ARG TARGETARCH
ARG TARGETVARIANT
ENV NPM_CONFIG_PREFIX=/opt/node-global
ENV PATH=/opt/node-global/bin:${PATH}
@@ -17,20 +18,24 @@ RUN set -x \
ppc64le) NODE_ARCH=ppc64 ;; \
*) NODE_ARCH="${TARGETARCH}" ;; \
esac \
&& case "${TARGETARCH}/${TARGETVARIANT:-}" in \
arm/v6) NODE_LIBC=musl-armv6 ;; \
arm/v7) NODE_LIBC=musl-armv7 ;; \
arm/*) echo "Unsupported ARM variant: ${TARGETVARIANT:-unknown}" >&2; exit 1 ;; \
*) NODE_LIBC=musl ;; \
esac \
&& npm_config_target_platform=linux \
npm_config_target_arch="${NODE_ARCH}" \
npm_config_target_libc=musl \
QL_BUILD_ONLY=1 pnpm install --prod --frozen-lockfile
npm_config_target_libc="${NODE_LIBC}" \
pnpm install --prod --frozen-lockfile --side-effects-cache=false
# The amd64 APK runtime has crashed in native URL parsing on dev. Keep its
# URL implementation bundled with the pinned upstream musl Node binary.
# Pin amd64 to upstream's musl runtime after native URL parser crashes with APK Node.
FROM --platform=linux/amd64 node:24.18.1-alpine3.24@sha256:f70403e87646dc51b45295f4b8b70cdad0b63d2297c4c9899119b03f7af7a6b3 AS node-runtime-amd64
FROM python:3.11-alpine
FROM python:3.11-alpine3.24
ARG TARGETARCH
ARG TARGETVARIANT
COPY --from=node-runtime-amd64 /usr/local/bin/node /tmp/node-runtime-amd64
ARG QL_MAINTAINER="whyour"
LABEL maintainer="${QL_MAINTAINER}"
@@ -52,7 +57,8 @@ EXPOSE 5700
COPY --from=builder /opt/node-global/lib/node_modules/. /usr/local/lib/node_modules/
COPY --from=builder /opt/node-global/bin/. /usr/local/bin/
RUN set -x \
RUN --mount=type=bind,from=node-runtime-amd64,source=/usr/local/bin/node,target=/tmp/node-runtime-amd64 \
set -x \
&& apk update -f \
&& apk upgrade \
&& apk --no-cache add -f bash \
@@ -71,9 +77,10 @@ RUN set -x \
unzip \
npm \
&& if [ "$TARGETARCH" = amd64 ]; then \
mv /tmp/node-runtime-amd64 /usr/local/bin/node; \
cp /tmp/node-runtime-amd64 /usr/local/bin/node; \
ln -sf /usr/local/bin/node /usr/bin/node; \
else rm /tmp/node-runtime-amd64; fi \
fi \
&& node --version | grep -E '^v24\.' \
&& rm -rf /var/cache/apk/* \
&& apk update \
&& ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
+17 -10
View File
@@ -1,8 +1,9 @@
# Run Node package installation natively on the builder. Node/npm can spin at
# 100% CPU when Alpine s390x is emulated through QEMU.
FROM --platform=$BUILDPLATFORM node:24-alpine3.24 AS builder
FROM --platform=$BUILDPLATFORM node:24.18.1-alpine3.24 AS builder
ARG TARGETARCH
ARG TARGETVARIANT
ENV NPM_CONFIG_PREFIX=/opt/node-global
ENV PATH=/opt/node-global/bin:${PATH}
@@ -17,20 +18,24 @@ RUN set -x \
ppc64le) NODE_ARCH=ppc64 ;; \
*) NODE_ARCH="${TARGETARCH}" ;; \
esac \
&& case "${TARGETARCH}/${TARGETVARIANT:-}" in \
arm/v6) NODE_LIBC=musl-armv6 ;; \
arm/v7) NODE_LIBC=musl-armv7 ;; \
arm/*) echo "Unsupported ARM variant: ${TARGETVARIANT:-unknown}" >&2; exit 1 ;; \
*) NODE_LIBC=musl ;; \
esac \
&& npm_config_target_platform=linux \
npm_config_target_arch="${NODE_ARCH}" \
npm_config_target_libc=musl \
QL_BUILD_ONLY=1 pnpm install --prod --frozen-lockfile
npm_config_target_libc="${NODE_LIBC}" \
pnpm install --prod --frozen-lockfile --side-effects-cache=false
# The amd64 APK runtime has crashed in native URL parsing on dev. Keep its
# URL implementation bundled with the pinned upstream musl Node binary.
# Pin amd64 to upstream's musl runtime after native URL parser crashes with APK Node.
FROM --platform=linux/amd64 node:24.18.1-alpine3.24@sha256:f70403e87646dc51b45295f4b8b70cdad0b63d2297c4c9899119b03f7af7a6b3 AS node-runtime-amd64
FROM python:3.10-alpine
FROM python:3.10-alpine3.24
ARG TARGETARCH
ARG TARGETVARIANT
COPY --from=node-runtime-amd64 /usr/local/bin/node /tmp/node-runtime-amd64
ARG QL_MAINTAINER="whyour"
LABEL maintainer="${QL_MAINTAINER}"
@@ -52,7 +57,8 @@ EXPOSE 5700
COPY --from=builder /opt/node-global/lib/node_modules/. /usr/local/lib/node_modules/
COPY --from=builder /opt/node-global/bin/. /usr/local/bin/
RUN set -x \
RUN --mount=type=bind,from=node-runtime-amd64,source=/usr/local/bin/node,target=/tmp/node-runtime-amd64 \
set -x \
&& apk update -f \
&& apk upgrade \
&& apk --no-cache add -f bash \
@@ -71,9 +77,10 @@ RUN set -x \
unzip \
npm \
&& if [ "$TARGETARCH" = amd64 ]; then \
mv /tmp/node-runtime-amd64 /usr/local/bin/node; \
cp /tmp/node-runtime-amd64 /usr/local/bin/node; \
ln -sf /usr/local/bin/node /usr/bin/node; \
else rm /tmp/node-runtime-amd64; fi \
fi \
&& node --version | grep -E '^v24\.' \
&& rm -rf /var/cache/apk/* \
&& apk update \
&& ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
+9 -6
View File
@@ -1,9 +1,10 @@
FROM python:3.11.14-slim-bookworm AS nodebuilder
# Download and verify the target Node archive without running under QEMU.
FROM --platform=$BUILDPLATFORM debian:bookworm-slim AS nodebuilder
ARG TARGETARCH
ARG TARGETVARIANT
COPY docker/install-node-runtime.sh /tmp/install-node-runtime.sh
RUN apt-get update && apt-get install --no-install-recommends -y ca-certificates curl xz-utils libatomic1 libstdc++6 && \
sh /tmp/install-node-runtime.sh /usr/local
COPY docker/install-node-debian.sh /tmp/install-node-debian.sh
RUN apt-get update && \
apt-get install --no-install-recommends -y ca-certificates curl xz-utils && \
sh /tmp/install-node-debian.sh "${TARGETARCH}"
FROM python:3.11.14-slim-bookworm AS builder
COPY package.json .npmrc pnpm-lock.yaml /tmp/build/
@@ -14,9 +15,10 @@ RUN set -x && \
apt-get update && \
apt-get install --no-install-recommends -y libatomic1 && \
npm i -g pnpm@8.3.1 && \
test "$(node --version)" = "v24.18.1" && \
test "$(pnpm --version)" = "8.3.1" && \
cd /tmp/build && \
QL_BUILD_ONLY=1 pnpm install --prod --frozen-lockfile
pnpm install --prod --frozen-lockfile
FROM python:3.11.14-slim-bookworm
@@ -74,6 +76,7 @@ RUN set -x && \
git config --global user.name "qinglong" && \
git config --global http.postBuffer 524288000 && \
npm install -g pnpm@8.3.1 pm2 ts-node typescript@5 && \
test "$(node --version)" = "v24.18.1" && \
test "$(pnpm --version)" = "8.3.1" && \
npm cache clean --force && \
rm -rf /root/.cache && \
+10 -7
View File
@@ -1,11 +1,12 @@
FROM python:3.10.19-slim-bookworm@sha256:23f63358922e79a794f71be8f3723c84e5ccca9638af3f74456dc73d6184499e AS nodebuilder
# Download and verify the target Node archive without running under QEMU.
FROM --platform=$BUILDPLATFORM debian:bookworm-slim AS nodebuilder
ARG TARGETARCH
ARG TARGETVARIANT
COPY docker/install-node-runtime.sh /tmp/install-node-runtime.sh
RUN apt-get update && apt-get install --no-install-recommends -y ca-certificates curl xz-utils libatomic1 libstdc++6 && \
sh /tmp/install-node-runtime.sh /usr/local
COPY docker/install-node-debian.sh /tmp/install-node-debian.sh
RUN apt-get update && \
apt-get install --no-install-recommends -y ca-certificates curl xz-utils && \
sh /tmp/install-node-debian.sh "${TARGETARCH}"
# Keep the full release matrix, including s390x, on the same immutable base.
# Keep s390x on the same immutable Python base as the other Debian targets.
# The floating 3.10-slim-bookworm tag no longer includes s390x.
FROM python:3.10.19-slim-bookworm@sha256:23f63358922e79a794f71be8f3723c84e5ccca9638af3f74456dc73d6184499e AS builder
COPY package.json .npmrc pnpm-lock.yaml /tmp/build/
@@ -16,8 +17,9 @@ RUN set -x && \
apt-get update && \
apt-get install --no-install-recommends -y libatomic1 && \
npm i -g pnpm@8.3.1 && \
test "$(node --version)" = "v24.18.1" && \
cd /tmp/build && \
QL_BUILD_ONLY=1 pnpm install --prod --frozen-lockfile
pnpm install --prod --frozen-lockfile
FROM python:3.10.19-slim-bookworm@sha256:23f63358922e79a794f71be8f3723c84e5ccca9638af3f74456dc73d6184499e
@@ -74,6 +76,7 @@ RUN set -x && \
git config --global user.name "qinglong" && \
git config --global http.postBuffer 524288000 && \
npm install -g pnpm@8.3.1 pm2 ts-node typescript@5 && \
test "$(node --version)" = "v24.18.1" && \
npm cache clean --force && \
rm -rf /root/.cache && \
rm -rf /root/.npm && \
-22
View File
@@ -1,22 +0,0 @@
ARG LEGACY_IMAGE
ARG BASE_IMAGE
FROM ${LEGACY_IMAGE} AS legacy
FROM ${BASE_IMAGE} AS bridge
ARG BASE_IMAGE
ENV QL_BUNDLE_BASE_IMAGE=${BASE_IMAGE}
ARG TARGETARCH
ARG TARGETVARIANT
USER root
COPY --from=legacy /ql/ /legacy/
RUN ulimit -c 0; if [ "$TARGETARCH/$TARGETVARIANT" = arm/v6 ]; then \
export QEMU_CPU=arm1176; \
apk add --no-cache build-base nodejs-dev && \
node --jitless /ql/scripts/build-migration-bridge.cjs /legacy /ql /out && \
for archive in /out/qinglong-*-linux-armv6-musl.tar.gz; do \
case "$archive" in */qinglong-migration-*) continue ;; esac; \
node /ql/shell/install-bundle.cjs --root /tmp/ql-bundle-offline-smoke --file "$archive" --install && \
rm -rf /tmp/ql-bundle-offline-smoke || exit 1; \
done; \
else node /ql/scripts/build-migration-bridge.cjs /legacy /ql /out; fi
FROM scratch
COPY --from=bridge /out/ /
+5 -8
View File
@@ -6,14 +6,11 @@ ENV QL_BUNDLE_BASE_IMAGE=${BASE_IMAGE}
ARG TARGETARCH
ARG TARGETVARIANT
USER root
# SQLite comes from the retained Node runtime; no architecture-specific addon.
RUN ulimit -c 0; if [ "$TARGETARCH/$TARGETVARIANT" = arm/v6 ]; then \
RUN ulimit -c 0; \
if [ "$TARGETARCH/$TARGETVARIANT" = arm/v6 ]; then \
export QEMU_CPU=arm1176; \
node --jitless /ql/scripts/build-upgrade-bundle.cjs /ql /out && \
for archive in /out/*.tar.gz; do \
node /ql/shell/install-bundle.cjs --root /tmp/ql-bundle-offline-smoke --file "$archive" --install && \
rm -rf /tmp/ql-bundle-offline-smoke || exit 1; \
done; \
else node /ql/scripts/build-upgrade-bundle.cjs /ql /out; fi
set -- --jitless; \
fi; \
node "$@" /ql/scripts/build-upgrade-bundle.cjs /ql /out
FROM scratch
COPY --from=bundle /out/ /
+22
View File
@@ -0,0 +1,22 @@
#!/bin/sh
set -eu
# The official Node 24 Bookworm image does not cover s390x. Use the same
# verified upstream release archive on every supported Debian architecture.
version=24.18.1
case "${1:?Missing Docker TARGETARCH}" in
amd64) arch=x64; checksum=d6c664df3f3f61458e8c277585571328522d705166723a7c7823a9253a4d15a0 ;;
arm64) arch=arm64; checksum=7201e3a09dc825bac57867c81913e2b8f0ef87d04cb9082af4cda82f6ff3d88c ;;
ppc64le) arch=ppc64le; checksum=c47812c13862be7f6e71194c434c4d78a99096e72efb9c31a2627437e056d669 ;;
s390x) arch=s390x; checksum=acf57bd726f35afc2ba8475194e5a8bd532c00e1a3b2436295b2fbdde199d0e9 ;;
*) echo "Node ${version} has no supported Debian binary for TARGETARCH=$1" >&2; exit 1 ;;
esac
archive="node-v${version}-linux-${arch}.tar.xz"
curl --fail --location --silent --show-error --retry 3 \
"https://nodejs.org/dist/v${version}/${archive}" -o "/tmp/${archive}"
printf '%s %s\n' "${checksum}" "/tmp/${archive}" | sha256sum --check --status
tar -xJf "/tmp/${archive}" -C /usr/local --strip-components=1 --no-same-owner
rm "/tmp/${archive}"
# The target binary is executed and version-checked in the target builder/runtime stages.
test -x /usr/local/bin/node
-30
View File
@@ -1,30 +0,0 @@
#!/bin/sh
set -eu
# Official binaries, pinned to their release checksums. Node 24 no longer
# publishes ARMv7 binaries, so that existing Debian target uses Node 22 LTS.
node_version=24.18.1
case "${TARGETARCH:?}/${TARGETVARIANT:-}" in
amd64/*) node_arch=x64; node_sha=d6c664df3f3f61458e8c277585571328522d705166723a7c7823a9253a4d15a0 ;;
arm64/*) node_arch=arm64; node_sha=7201e3a09dc825bac57867c81913e2b8f0ef87d04cb9082af4cda82f6ff3d88c ;;
ppc64le/*) node_arch=ppc64le; node_sha=c47812c13862be7f6e71194c434c4d78a99096e72efb9c31a2627437e056d669 ;;
s390x/*) node_arch=s390x; node_sha=acf57bd726f35afc2ba8475194e5a8bd532c00e1a3b2436295b2fbdde199d0e9 ;;
arm/v7) node_version=22.23.2; node_arch=armv7l; node_sha=ef8f26a3de19acd8c23548e6c3cfc2052610b0e67abb5fd64dbd92c8b1c1245b ;;
*) echo "Unsupported Debian Node architecture: ${TARGETARCH}/${TARGETVARIANT:-}" >&2; exit 1 ;;
esac
if [ "${1:-}" = --print ]; then
printf '%s %s %s\n' "$node_version" "$node_arch" "$node_sha"
exit 0
fi
node_prefix=${1:-/usr/local}
node_work_dir=$(mktemp -d)
trap 'rm -rf "$node_work_dir"' EXIT
node_archive="node-v${node_version}-linux-${node_arch}.tar.xz"
curl --fail --location --retry 3 --connect-timeout 20 --max-time 300 \
"https://nodejs.org/dist/v${node_version}/${node_archive}" -o "$node_work_dir/$node_archive"
printf '%s %s\n' "$node_sha" "$node_work_dir/$node_archive" | sha256sum -c -
mkdir -p "$node_prefix"
tar -xJf "$node_work_dir/$node_archive" -C "$node_prefix" --strip-components=1 --no-same-owner
"$node_prefix/bin/node" -e 'const {DatabaseSync}=require("node:sqlite"); const db=new DatabaseSync(":memory:"); try {db.exec("CREATE TABLE node_probe(value INTEGER);INSERT INTO node_probe VALUES(42)");if(db.prepare("SELECT value FROM node_probe").get().value!==42)throw Error("Node SQLite probe failed");}finally{db.close();}'
-5
View File
@@ -1,5 +0,0 @@
{
"sourceCommit": "99a4d0eff8431898d7638fefff305591fe326ed5",
"musl": "ghcr.io/whyour/qinglong@sha256:0847b0035083cd439c00ff3087392f67035b26842209c110ee976f51ec13fbcd",
"glibc": "ghcr.io/whyour/qinglong@sha256:d2775474223793a5dfe0060910acdcd9bab4c2b70ff6b2ef770ec536afb206f6"
}
+13 -27
View File
@@ -1,18 +1,12 @@
# 平台完整升级包
CI 从同次构建的不可变镜像摘要导出完整 tar.gz,包含源码中的运行脚本和模板、前后端编译产物、锁文件和已经安装的生产 `node_modules`。不包含 Node 可执行文件、用户数据、`.env`、Git 历史或 npm 缓存。保留现有 Node 和用户脚本依赖。面板最低需要 Node 22.17,推荐 Node 24;Node 20 安装先升级运行时,应用更新包不会替换 Node。
CI 从同次构建的不可变镜像摘要导出完整 tar.gz,包含源码中的运行脚本和模板、前后端编译产物、锁文件和已经安装的生产 `node_modules`。不包含 Node 可执行文件、用户数据、`.env`、Git 历史或 npm 缓存。保留现有 Node 和用户脚本依赖。
## 在线升级
`package.json` 中 `qinglongUpgradeFormat: 2` 表示使用完整包。满足 Node 22.17 的旧格式 2 面板可在面板内分两步迁移,不需要手动运行安装器或更换镜像:第一次下载独立的兼容升级器包,预启动通过后手动重启;此时仍运行旧数据库后端,界面提示“升级器已更新,请继续升级新版”。点击“继续升级新版”完成第二次下载和新版后端预启动,再手动重启,健康与版本检查通过后才显示“升级成功”并隐藏侧栏状态。关闭页面、刷新或换浏览器不会丢失第二步提示;Node 不满足最低要求会在第一步准备阶段被旧校验器拒绝,旧服务继续运行。
`package.json` 中 `qinglongUpgradeFormat: 2` 表示使用完整包。旧版升级器仍可通过原归档路径完成首次迁移;完整包版本不会在平台包缺失、损坏或不兼容时退回设备端 npm 安装。仅当完整包已通过校验、依赖加载或后端预启动明确报告依赖错误时,在线升级会按同一冻结锁文件修复一次生产依赖,然后再次预启动;第二次失败即停止,保留正在运行的旧服务。
兼容包单独命名为 `qinglong-migration-<commit>-<platform>.tar.gz`,来自固定摘要、固定源码提交的旧镜像,保留真实旧 SQLite/Sequelize 依赖和旧数据库后端,只替换升级器、前端和构建身份;它不伪造 SQLite 模块,也不执行用户数据迁移。正常运行包仍为 `qinglong-<commit>-<platform>.tar.gz`,只使用 node:sqlite。索引的 `packages` 为旧客户端保留兼容包,`runtimePackages` 指向正式运行包;新版升级器优先且只使用完整的 `runtimePackages`,不因损坏或缺失而退回兼容包。新版在线/离线入口会拒绝误用兼容包。生产索引发布前,每个平台都必须使用未经修改的旧升级器真实下载、校验、预启动第一步,再使用包内新版升级器下载、校验、预启动第二步。两步失败均沿用原有回滚流程。
固定旧镜像中的 generic ARM SQLite addon 实际使用 ARMv7 指令。ARMv6 musl 兼容包会在 CI 构建时从同一旧依赖源码重新编译该 addon,使用匹配构建 Node 的本地头文件、单并发及 ARMv6/VFP 硬浮点参数,并检查 ELF 指令集,拒绝 ARMv7、Thumb-2 或 VFPv3 产物。旧锁文件及后端保持不变,其他架构不执行此步骤;正常运行包仍不包含旧 SQLite,目标设备无需为第一步迁移联网编译。
旧源码归档升级器继续保留原兼容路径;完整包版本不会在平台包缺失、损坏或不兼容时退回设备端 npm 安装。仅当完整包已通过校验、依赖加载或后端预启动明确报告依赖错误时,在线升级会按同一冻结锁文件修复一次生产依赖,然后再次预启动;第二次失败即停止,保留正在运行的旧服务。第一步旧后端的检查仍使用真实旧依赖,第二步才切换到 MikroORM 和 Node 内置 SQLite。
面板下载任务与本地离线入口共用 `platformBundle.cjs`:平台匹配 → 归档安全检查 → 全文件/链接/权限和锁文件校验 → MikroORM/cron-parser/gRPC/undici 加载及 node:sqlite 内存库读写(旧包继续检查 SQLite/Sequelize) → 隔离后端预启动 → 发布就绪指针。预启动使用现有 Node、临时数据目录和 loopback 随机端口,运行真实数据库建表/迁移和事务读写、gRPC 服务,以及与正常启动共用的 HTTP 中间件、路由和 WebSocket 初始化,再通过 `/api/health` 确认双服务与目标版本、提交和后端摘要;不运行任务、订阅、通知、用户启动钩子和系统依赖安装。临时库不是用户数据库副本,因此实际应用后的健康检查和回退仍不可省略。点击重启时,接口只检查就绪记录并提交后台任务;完整文件与依赖复验由独立工作进程执行,复验通过后才停止旧服务,避免小设备的预检耗时超过 HTTP 请求期限。离线升级同步面板任务状态,并与在线任务互斥;中断准备时保留之前的可用就绪包。原有服务停止、文件备份、启动健康检查及失败恢复仍由共享升级引擎处理。
面板下载任务与本地离线入口共用 `platformBundle.cjs`:平台匹配 → 归档安全检查 → 全文件/链接/权限和锁文件校验 → SQLite/Sequelize/cron-parser/gRPC/undici 加载及 SQLite 内存库读写 → 隔离后端预启动 → 发布就绪指针。预启动使用现有 Node、临时数据目录和 loopback 随机端口,运行真实数据库建表/迁移和事务读写、gRPC 服务,以及与正常启动共用的 HTTP 中间件、路由和 WebSocket 初始化,再通过 `/api/health` 确认双服务与目标版本、提交和后端摘要;不运行任务、订阅、通知、用户启动钩子和系统依赖安装。临时库不是用户数据库副本,因此实际应用后的健康检查和回退仍不可省略。点击重启时,接口只检查就绪记录并提交后台任务;完整文件与依赖复验由独立工作进程执行,复验通过后才停止旧服务,避免小设备的预检耗时超过 HTTP 请求期限。离线升级同步面板任务状态,并与在线任务互斥;中断准备时保留之前的可用就绪包。原有服务停止、文件备份、启动健康检查及失败恢复仍由共享升级引擎处理。
正式升级包跟随 master 同一次版本镜像构建;develop 的普通提交也自动构建开发包。两者都直接使用该次 Alpine/Debian 镜像的不可变 digest。Git tag 工作流和 `static_sync_only` 不生成完整包。需要重建当前开发包时,可手动运行同一工作流:
@@ -24,13 +18,13 @@ gh workflow run build-docker-image.yml --ref develop
开发包只保留最新一个已验证批次:所有平台资产和渠道索引的大小、SHA256 校验成功后,删除其他名称匹配的开发构建 prerelease;不删除正式包、渠道索引或无法分类的旧 release。构建或上传失败不会删除上一套可用包。旧批次被删除后,已选中旧索引但尚未下载完的客户端可能需要重新发起下载;准备失败仍保留旧服务和数据。
开发和正式包都必须在 12 个平台完成归档、解包后复验及依赖试运行,并且全部上传资产的大小和 SHA256 复验通过后,才更新各自索引。平台范围与现有 Alpine/Debian 镜像一致:ARMv6 仅提供 musl 包,不提供 ARMv6 glibc 包。
开发和正式包都必须在 11 个平台完成归档、解包后复验及依赖试运行,并且全部上传资产的大小和 SHA256 复验通过后,才更新各自索引。平台范围与现有 Alpine/Debian 镜像一致:ARMv6 和 ARMv7 仅提供 musl 包,Debian 不再发布 ARMv7 镜像或 glibc 平台包。
版本入口为 GitHub prerelease `upgrade-v<version>` 下的 `latest.json`;在线正式渠道保留 `upgrade-master/latest.json`,兼容现有客户端。二者指向同一套完整包,指定版本与 latest 不重复生成产物。索引记录渠道、版本、源码提交、构建批次、准确字节数、SHA256,以及 `upgrade-build-v<version>-<run-id>-<run-attempt>` 下的固定下载 URL。相同版本的镜像 tag 重新发布时,完整包也重新生成,版本和正式渠道索引同步覆盖到新批次;同一提交重跑也允许产生新内容。已选中旧索引的下载继续使用旧批次资产,避免覆盖文件导致哈希不匹配。master 的镜像构建和完整包发布整条流水线串行执行,避免镜像 tag 与完整包被并发重发交错覆盖;旧工作流明确重跑时也允许同版本、同提交重新发布。过期源码提交不能推进对应渠道或版本索引;平台包上传失败时不切换索引。
## 手动离线安装和升级
从 `upgrade-v<version>` 的说明或 `latest.json` 的 `runtimePackages` 中取得当前批次的下载链接,下载匹配设备的正常运行 `.tar.gz` 和独立 `install-bundle.cjs`;两者都来自同一个 `upgrade-build-v<version>-<run-id>-<run-attempt>` release。不要下载 `qinglong-migration-` 包作为离线安装包。独立安装器已内置解析依赖,在目标设备正常情况无需 npm、pnpm 或重新安装 SQLite。离线准备和预启动默认不联网;依赖不符合要求时直接报错。可对照同批次 release 的 `latest.json` 核对归档 SHA256。
从 `upgrade-v<version>` 的说明或 `latest.json` 中取得当前批次的下载链接,下载匹配设备的 `.tar.gz` 和独立 `install-bundle.cjs`;两者都来自同一个 `upgrade-build-v<version>-<run-id>-<run-attempt>` release。独立安装器已内置解析依赖,在目标设备正常情况无需 npm、pnpm 或重新安装 SQLite。离线准备和预启动默认不联网;依赖不符合要求时直接报错。可对照同批次 release 的 `latest.json` 核对归档 SHA256。
已有安装(默认应用并重启):
@@ -38,7 +32,7 @@ gh workflow run build-docker-image.yml --ref develop
node install-bundle.cjs --root /ql --file /absolute/path/qinglong-<commit>-linux-arm64-musl.tar.gz
```
已迁移的安装只准备,稍后手动重启(首次从旧格式 2 迁移应直接执行上一条新版独立安装器命令,不能调用旧版 `ql reload system`):
只准备,稍后手动重启:
```sh
node install-bundle.cjs --root /ql --file /absolute/path/package.tar.gz --download-only
@@ -51,7 +45,7 @@ ql reload system
node install-bundle.cjs --root /ql --file /absolute/path/package.tar.gz --download-only --repair-deps
```
修复需要已有 pnpm 8.3.1。新包在私有目录按 `--prod --frozen-lockfile` 重装生产 JavaScript 依赖,不编译 SQLite。只有旧包显式依赖 sqlite3 时才需要编译工具并重建 SQLite;旧 ARMv6 包强制源码编译和 ARMv6 硬浮点指令,禁止下载 generic ARM 的 ARMv7 二进制。修复仅执行一次;不安装或替换 Node、不改锁文件、不改已校验源码和静态产物。应用前以原包校验清单和独立记录的修复依赖摘要复验。
修复需要已有 pnpm 8.3.1 及必要编译工具。依赖在私有目录按 `--prod --frozen-lockfile` 重装,仅重建 SQLite;ARMv6 强制源码编译和 ARMv6 硬浮点指令,禁止下载 generic ARM 的 ARMv7 二进制。修复仅执行一次;不安装或替换 Node、不改锁文件、不改已校验源码和静态产物。应用前以原包校验清单和独立记录的修复依赖摘要复验。
修复继承调用环境中的标准 HTTP/HTTPS/ALL/NO_PROXY(含小写形式),其余 Node 预加载和编译参数仍隔离。修复安装或 SQLite 重建失败时,清理私有目录,保留当前服务、当前依赖及上一次就绪包;不会标记本次更新就绪。预启动或修复主进程退出时立即终止其进程组,防止持有输出管道的子进程继续写入已清理目录。
@@ -70,7 +64,7 @@ QL_DIR=/opt/qinglong QL_DATA_DIR=/opt/qinglong/data bash /opt/qinglong/shell/sta
`--install` 拒绝非空目录,不覆盖既有配置。该包安装青龙及其 Node 生产依赖;首次启动仍需已有 Python、Bash、OpenSSL、PM2、nginx 等相应运行环境,不会在线安装或替换它们。手动启动脚本会在启动工作进程前完成首次 gRPC 证书准备,避免慢设备生成密钥耗尽工作进程的就绪期限;已有证书继续复用。服务启动和系统自启按原手动安装流程配置。
ARMv6 小设备使用 `linux-armv6-musl.tar.gz`,要求已有兼容的 musl 系统及硬浮点 Node。ARMv6 glibc(例如 Raspbian/Raspberry Pi OS 32 位系统)不在平台包支持范围内,不能使用 ARMv7 或 musl 包替代;安装器仍会检查实际架构和 libc,拒绝不匹配的包。
ARMv6 小设备使用 `linux-armv6-musl.tar.gz`,要求已有兼容的 musl 系统及硬浮点 Node。ARMv6 glibc(例如 Raspbian/Raspberry Pi OS 32 位系统)不在平台包支持范围内,不能使用其他架构或 musl 包替代;安装器仍会检查实际架构和 libc,拒绝不匹配的包。
## npm 面板包
@@ -80,9 +74,9 @@ npm 会排除根目录的 `pnpm-lock.yaml`。主包的原生 `npm pack/publish`
`package.json#files` 是 npm 包内容的唯一项目规则,不再维护根 `.npmignore`:npm 在根包使用 `files` 时会停用该目录的 `.npmignore` 和 `.gitignore` 规则。打包测试在没有 `.npmignore` 的独立源码目录中注入私有配置、密钥占位文件、缓存及生成的预加载配置,执行真实 `npm pack` 和安装,检查运行文件完整、私有文件及其内容不进入归档。此规则只用于 npm 包,完整平台 tar.gz 仍由平台包构建器处理。
干净源码完成 `pnpm build:front`、`pnpm build:back`、`pnpm build:info` 后,运行 `node scripts/verify-panel-package.cjs pack /tmp/qinglong-npm` 生成并检查真实 npm 归档。CI 在一次性 Linux 容器中,以 Node 22.17.0 和 Node 24 安装同一个归档,使用真实生产依赖和已安装的 `qinglong`、`ql` 入口,验证启动、损坏下载保留旧服务、在线准备与重启、离线准备与 Bash 重启及账号、配置、环境变量、任务和脚本保留。测试还安装完整性摘要固定的已发布 `@whyour/qinglong@2.23.0`,在旧服务中建立真实数据,再用新 `.tgz` 执行 npm 覆盖安装,验证账号、配置、环境变量、任务和脚本保留。之后升级到同次 Debian 镜像不可变 digest 通过正式 `Dockerfile.upgrade-bundle` 导出的原始平台包,确认运行目录变为 pnpm 布局。该单平台验收包仅上传短期 CI artifact,不发布 Release,也不推进开发或正式更新索引。
干净源码完成 `pnpm build:front`、`pnpm build:back`、`pnpm build:info` 后,运行 `node scripts/verify-panel-package.cjs pack /tmp/qinglong-npm` 生成并检查真实 npm 归档。CI 在一次性 Linux 容器中,以 Node 20.18.1 和 Node 24 安装同一个归档,使用真实生产依赖和已安装的 `qinglong`、`ql` 入口,验证启动、损坏下载保留旧服务、在线准备与重启、离线准备与 Bash 重启及账号、配置、环境变量、任务和脚本保留。测试还安装完整性摘要固定的已发布 `@whyour/qinglong@2.23.0`,在旧服务中建立真实数据,再用新 `.tgz` 执行 npm 覆盖安装,验证账号、配置、环境变量、任务和脚本保留。之后升级到同次 Debian 镜像不可变 digest 通过正式 `Dockerfile.upgrade-bundle` 导出的原始平台包,确认运行目录变为 pnpm 布局。该单平台验收包仅上传短期 CI artifact,不发布 Release,也不推进开发或正式更新索引。
Umi 4.4.4 的构建依赖仍使用 Node 24 已移除的 `http_parser`。前端源码构建使用 `.nvmrc` 中的 Node 22,CI 从该文件读取版本,Vercel 前后端项目均配置为 Node 22.x。`package.json` 不声明 `engines.node`,避免 Vercel 用开放的最低运行版本范围覆盖项目的构建版本;`pnpm build:front` 直接执行构建,不下载或切换 Node。面板运行时最低为 Node 22.17,平台包清单、npm preinstall 和数据库进程均检查最低 Node 要求。CI 的 Node 20.18.1 项使用实际新 npm 归档验证拒绝安装,以及本地/全局旧包、`.env`、外置数据和旧服务保留;它不启动新后端。构建专用环境可跳过安装阶段的运行时探测,后端启动不跳过版本检查。
Umi 4.4.4 的构建依赖仍使用 Node 24 已移除的 `http_parser`。前端源码构建使用 `.nvmrc` 中的 Node 22,CI 从该文件读取版本,Vercel 前后端项目均配置为 Node 22.x。`package.json` 不声明 `engines.node`,避免 Vercel 用开放的最低运行版本范围覆盖项目的构建版本;`pnpm build:front` 直接执行构建,不下载或切换 Node。面板运行时仍支持 Node 20.18.1 及以上,平台包的清单与安装器独立校验最低 Node 要求,npm 实装验收继续覆盖 Node 20/24。
服务集成测试只能在专用容器中运行:设置 `QL_PANEL_PACKAGE_TEST_ISOLATED=1`、指向隔离 PM2 的 `QL_PANEL_TEST_PM2` 和指向正式构建脚本输出目录的 `QL_PANEL_TEST_BUNDLE_DIR`,然后执行 `node scripts/verify-panel-package.cjs verify /tmp/qinglong-npm /tmp/qinglong-npm-results`。容器需已有 Bash、curl、OpenSSL、Python、jq、nginx 和 procps。旧正式包会重置 PATH,因此验收容器将 PM2 6.0.14 安装到 `/usr/local/bin/pm2`,并以此设置 `QL_PANEL_TEST_PM2`;隔离通过专属容器和独立 `PM2_HOME` 保证。测试创建独立的 npm prefix、HOME、PM2_HOME 和数据目录,结束后清理。
@@ -90,21 +84,13 @@ Umi 4.4.4 的构建依赖仍使用 Node 24 已移除的 `http_parser`。前端
master 发布任务等待上述测试通过,直接发布已验证的 `.tgz`,不重新构建或打包。develop 只生成测试归档与验收记录,不发布 npm。
## 数据库兼容与回滚
正常运行包使用 MikroORM 7 的官方 NodeSqliteDialect。Sequelize 和 sqlite3 只作为开发测试依赖,用于验证旧库读写,不进入正常运行包的生产依赖;独立的第一步兼容包仍携带固定旧镜像的真实旧依赖和后端,以满足已安装旧升级器的检查。九张业务表及既有迁移记录保留原名和结构,不运行 ORM 自动改表;日期保持旧版 UTC 文本格式,JSON、ID、排序精度及 Keyv 缓存格式保持兼容,旧版可读新版写入的数据。
每个后端进程延迟启动一个有界数据库子进程,避免同步 node:sqlite 阻塞 HTTP 和调度事件循环。事务独占请求队列,嵌套事务使用 SAVEPOINT;事务异常回滚。数据库进程退出或超时后拒绝后续请求、健康检查失败,不自动重放写入;恢复由服务重启完成。后端正常退出会终止其数据库子进程,预启动监督器还会终止整个临时进程组。数据库进程在打开 SQLite 前启动一个仅监视父进程的轻量线程;即使主 SQL 线程阻塞,父进程被强杀后也会终止整个数据库进程并释放事务锁,避免阻止旧服务重启。
## 平台兼容
- 最低 Node 为 22.17,Node-API 至少 6;安装器只检查,不替换 Node。Debian amd64、arm64、ppc64le、s390x 使用 Node 24;官方 Node 24 不提供 Linux ARMv7 二进制,Debian ARMv7 使用 Node 22 LTS。
- 最低 Node 为 20.18.1,Node-API 至少 6;安装器只检查,不替换 Node。
- 以 Node 实际运行架构、ARM 指令集和 libc 类型/版本选择包;不依据 Docker 架构别名猜测。
- glibc、musl 分包,并记录构建系统 libc 下限。系统库仍以目标设备加载测试作为最后检查。
- ARMv6 与 ARMv7 使用不同包名。当前生产依赖没有 SQLite 原生 addon,使用设备 Node 自带的 node:sqlite。ARMv6 Node 本身仍须兼容 ARM1176 和硬浮点;构建需通过内置 SQLite 的加载和内存库读写,失败则不发布渠道。
- ARMv6 的归档构建、在线更新工作进程及独立离线安装器使用 `--jitless`,避开 ARM1176 上 V8 的优化编译崩溃;保留现有 Node 二进制,Node 24 及以上的 ARMv6 面板进程通过共享 `nodeRuntime` 使用 `--no-opt --no-sparkplug --no-maglev`,其他架构和较低 Node 版本保持原启动参数。ARMv6 的 gRPC 工作进程首次启动及重建最多等待 2 分钟;升级后的整体健康检查最多等待 5 分钟,覆盖慢设备按顺序启动数据库、调度与 HTTP 工作进程的耗时;其他平台仍为 60 秒,超时或版本校验失败仍执行回退。依赖加载与后端预启动保留 HTTP/WebAssembly 所需能力。预启动使用一个后端和一个数据库子进程,按顺序初始化数据库、gRPC、HTTP,不启动 PM2/cluster;后端的 ARMv6 V8 堆上限 128 MiB、期限 5 分钟,其他平台为 192 MiB/60 秒。数据库执行主线程的 V8 堆上限 128 MiB,ARMv6 使用 `--jitless`;监视线程不加载 ORM、HTTP 或 ts-node,并保留进程的 V8 模式。主线程和监视线程的 V8 限制不是 RSS 总内存上限,系统仍需为现有服务和 OpenSSL 留出内存。依赖修复的包管理器及编译使用单并发、低网络并发和有界时长;ARMv6 包管理器采用 `--jitless`、256 MiB 堆上限,并优先复用版本匹配的现有 Node 头文件与实际运行时编译配置;超时、中断会终止整个进程组并清理临时目录。
- ARMv6 保留独立包名。ARMv6 在目标镜像中从源码编译 SQLite,禁止复用 generic `arm` 的 ARMv7 预编译件;构建需通过加载和内存库读写,失败则不发布渠道。
- ARMv6 的归档构建、在线更新工作进程及独立离线安装器使用 `--jitless`,避开 ARM1176 上 V8 的优化编译崩溃;保留现有 Node 二进制,Node 24 及以上的 ARMv6 面板进程通过共享 `nodeRuntime` 使用 `--no-opt --no-sparkplug --no-maglev`,其他架构和较低 Node 版本保持原启动参数。ARMv6 的 gRPC 工作进程首次启动及重建最多等待 2 分钟;升级后的整体健康检查最多等待 5 分钟,覆盖慢设备按顺序启动数据库、调度与 HTTP 工作进程的耗时;其他平台仍为 60 秒,超时或版本校验失败仍执行回退。依赖加载与后端预启动使用现有 Node 的正常模式,使 Node 20.18.1 的内置 HTTP/WebAssembly 组件保持可用。预启动只有一个后端进程,按顺序初始化数据库、gRPC、HTTP,不启动 PM2/cluster;ARMv6 V8 堆上限 128 MiB、期限 5 分钟,其他平台为 192 MiB/60 秒。堆上限不是 RSS 总内存上限,系统仍需为现有服务和 OpenSSL 留出内存。依赖修复的包管理器及编译使用单并发、低网络并发和有界时长;ARMv6 包管理器采用 `--jitless`、256 MiB 堆上限,并优先复用版本匹配的现有 Node 头文件与实际运行时编译配置;超时、中断会终止整个进程组并清理临时目录。
- tar 中允许 pnpm 的包内符号链接;拒绝绝对路径、越界路径、包外链接、硬链接、设备条目和经符号链接写入的条目。重启前再次校验依赖,暂存失败保留上一次可用的就绪包。
归档先完整预检路径、类型、大小与链接,再向新建的私有目录逐个流式写入文件。文件写入使用 Node 的完整写入接口,处理短写后才关闭文件;目录权限在写入结束后恢复,避免受调用进程的 umask 影响。读取、解析或写入失败时,等候待写文件关闭后再清理,不发布就绪状态。校验失败会列出最多五个文件的预期和实际权限、哈希或链接信息,不输出文件内容。
自托管镜像可设置 `QL_UPGRADE_INDEX_URL` 指向兼容索引;要求 HTTPS,测试时仅放行 loopback HTTP。索引和所有平台归档应部署到同一可访问源,避免依赖额外的 raw 域名。
+11
View File
@@ -0,0 +1,11 @@
# 青龙 2.24.0-dev.0 开发候选
这是 `develop` 渠道的开发构建,上一正式版本为 2.23.0。此版本尚未正式发布。
## 本轮变化
- Alpine 与 Debian 镜像统一使用 Node.js 24;SQLite 依赖升级至 1.1.4。
- Alpine 镜像保留 arm/v6 和 arm/v7,Debian 镜像暂不提供 arm/v7。
- ARM64 升级包改用原生 runner 构建,避免在 x64 QEMU 下归档时发生异常。
升级前请备份完整持久化数据,并在目标设备验证脚本、依赖和通知。正式发布说明将在候选验收后整理。
+13 -16
View File
@@ -1,7 +1,7 @@
{
"name": "@whyour/qinglong",
"packageManager": "pnpm@8.3.1",
"version": "2.23.0",
"version": "2.24.0-dev.0",
"description": "Timed task management platform supporting Python3, JavaScript, Shell, Typescript",
"repository": {
"type": "git",
@@ -49,7 +49,6 @@
"version.yaml"
],
"scripts": {
"preinstall": "node -e \"if (process.env.QL_BUILD_ONLY !== '1' && process.env.VERCEL !== '1') require('./shell/runtime-guard.cjs').assertNodeRuntime()\"",
"build:cli": "node cli/scripts/build.cjs",
"check:cli": "npm --prefix cli run check",
"test:cli": "npm run build:cli && node cli/scripts/test.cjs",
@@ -58,7 +57,7 @@
"start:back": "nodemon ./back/app.ts",
"start:front": "max dev",
"build:front": "max build",
"build:back": "tsc -p back/tsconfig.json && tsc -p back/data/tsconfig.worker.json",
"build:back": "tsc -p back/tsconfig.json",
"test": "TS_NODE_PROJECT=back/tsconfig.json node -r ts-node/register/transpile-only --test test/back/*.test.cjs test/front/*.test.cjs",
"benchmark:execution": "TS_NODE_PROJECT=back/tsconfig.json node -r ts-node/register/transpile-only scripts/benchmark-execution.cjs",
"build:info": "node scripts/write-build-info.cjs",
@@ -112,7 +111,7 @@
}
},
"overrides": {
"sqlite3": "npm:@whyour/sqlite3@1.1.2",
"sqlite3": "npm:@whyour/sqlite3@1.1.4",
"@codemirror/state": "6.5.4",
"@codemirror/view": "6.39.16",
"proxy-addr": "2.0.8",
@@ -122,11 +121,11 @@
}
},
"dependencies": {
"@ant-design/plots": "^2.6.8",
"@bufbuild/protobuf": "^2.10.0",
"@grpc/grpc-js": "1.14.5",
"@grpc/proto-loader": "^0.8.0",
"@mikro-orm/core": "7.2.5",
"@mikro-orm/sql": "7.2.5",
"@keyv/sqlite": "^4.0.1",
"@otplib/preset-default": "^12.0.1",
"body-parser": "^1.20.3",
"celebrate": "^15.0.3",
@@ -148,7 +147,7 @@
"ip2region": "2.3.0",
"js-yaml": "^4.1.0",
"jsonwebtoken": "^9.0.2",
"keyv": "5.6.0",
"keyv": "^5.2.3",
"lodash": "^4.17.21",
"multer": "2.4.0",
"node-cron": "4.6.0",
@@ -158,18 +157,19 @@
"ps-tree": "^1.2.0",
"reflect-metadata": "^0.2.2",
"request-ip": "3.3.0",
"sequelize": "6.37.8",
"sockjs": "^0.3.24",
"tar": "7.5.20",
"sqlite3": "npm:@whyour/sqlite3@1.1.4",
"toad-scheduler": "^3.0.1",
"typedi": "^0.10.0",
"undici": "^7.9.0",
"uuid": "^11.0.3",
"winston": "^3.17.0",
"winston-daily-rotate-file": "^5.0.0"
"winston-daily-rotate-file": "^5.0.0",
"tar": "7.5.20"
},
"devDependencies": {
"@ant-design/icons": "^5.0.1",
"@ant-design/plots": "^2.6.8",
"@ant-design/pro-layout": "6.38.22",
"@codemirror/state": "6.5.4",
"@codemirror/view": "6.39.16",
@@ -187,7 +187,7 @@
"@types/jsonwebtoken": "^8.5.8",
"@types/lodash": "^4.14.185",
"@types/multer": "^1.4.7",
"@types/node": "^22.20.5",
"@types/node": "^17.0.21",
"@types/nodemailer": "^6.4.4",
"@types/proper-lockfile": "^4.1.4",
"@types/ps-tree": "^1.1.6",
@@ -233,16 +233,13 @@
"react-intl-universal": "^2.12.0",
"react-router-dom": "6.26.1",
"react-split-pane": "^0.1.92",
"sequelize": "6.37.8",
"sockjs-client": "^1.6.0",
"sqlite3": "npm:@whyour/sqlite3@1.1.2",
"ts-node": "^10.9.2",
"ts-proto": "^2.6.1",
"tslib": "^2.4.0",
"typescript": "5.9.3",
"typescript": "5.2.2",
"vh-check": "^2.0.5",
"virtualizedtableforantd4": "1.3.0"
},
"qinglongUpgradeFormat": 2,
"qinglongRuntimeMinimum": "22.17.0"
"qinglongUpgradeFormat": 2
}
+652 -545
View File
File diff suppressed because it is too large Load Diff
-272
View File
@@ -1,272 +0,0 @@
// A separate first-hop package for installed format-2 verifiers. Its database
// backend and dependencies remain the pinned legacy implementation. Only the
// updater and frontend are replaced; the next download uses runtimePackages.
const fs = require('node:fs/promises');
const { createReadStream } = require('node:fs');
const path = require('node:path');
const http = require('node:http');
const assert = require('node:assert/strict');
const { pipeline } = require('node:stream/promises');
const { once } = require('node:events');
const { collectBuildFiles } = require('../docker/build-manifest.cjs');
const { buildBundle } = require('./build-upgrade-bundle.cjs');
const artifacts = require('../cli/src/internal/maintenance/upgradeArtifacts.cjs');
const bundle = require('../cli/src/internal/maintenance/platformBundle.cjs');
const baseline = require('../docker/migration-bridge-images.json');
const { rebuildMigrationSqlite } = require('./rebuild-migration-sqlite.cjs');
const read = async (file) => JSON.parse(await fs.readFile(file, 'utf8'));
async function prepareBridge(legacy, current, destination) {
const oldBuild = await artifacts.verifyUpgrade(
legacy,
path.join(legacy, 'static'),
);
const build = await artifacts.verifyUpgrade(
current,
path.join(current, 'static'),
);
const pkg = await read(path.join(legacy, 'package.json'));
const next = await read(path.join(current, 'package.json'));
if (
oldBuild.sourceCommit !== baseline.sourceCommit ||
pkg.qinglongUpgradeFormat !== 2 ||
!pkg.dependencies?.sqlite3 ||
!pkg.dependencies?.sequelize ||
!next.dependencies?.['@mikro-orm/sql']
)
throw new Error(
'Migration bridge requires the pinned legacy backend and a modern runtime',
);
// Never copy user state, even if this builder is used outside Docker.
const excluded = new Set(['.git', '.tmp', '.env', 'data', 'output']);
await fs.cp(legacy, destination, {
recursive: true,
verbatimSymlinks: true,
filter: (file) =>
!excluded.has(path.relative(legacy, file).split(path.sep)[0]),
});
const maintenance = 'cli/src/internal/maintenance';
await fs.rm(path.join(destination, maintenance), {
recursive: true,
force: true,
});
await fs.cp(
path.join(current, maintenance),
path.join(destination, maintenance),
{ recursive: true },
);
await fs.rm(path.join(destination, 'static/dist'), {
recursive: true,
force: true,
});
await fs.cp(
path.join(current, 'static/dist'),
path.join(destination, 'static/dist'),
{ recursive: true },
);
const migrationBridge = {
format: 1,
legacySourceCommit: oldBuild.sourceCommit,
sourceCommit: build.sourceCommit,
targetVersion: next.version,
};
await fs.writeFile(
path.join(destination, 'package.json'),
JSON.stringify(
{
...pkg,
qinglongRuntimeMinimum: '22.17.0',
qinglongMigrationBridge: migrationBridge,
},
null,
2,
),
);
await fs.writeFile(
path.join(destination, 'static/build-info.json'),
JSON.stringify({
...oldBuild,
sourceCommit: build.sourceCommit,
files: collectBuildFiles(path.join(destination, 'static')),
}),
);
// No backend bytes or dependency lock may change in the updater hop.
assert.equal(
bundle.sha256(
await fs.readFile(path.join(destination, 'static/build/app.js')),
),
oldBuild.files['build/app.js'],
);
assert.equal(
bundle.sha256(await fs.readFile(path.join(destination, 'pnpm-lock.yaml'))),
oldBuild.lockfileSha256,
);
return migrationBridge;
}
async function verifyBothDownloads(legacy, current, directory, records) {
let index, workspace;
const files = new Map(
records.map((r) => ['/' + r.filename, path.join(directory, r.filename)]),
);
const server = http.createServer((req, res) => {
if (!index) {
res.writeHead(503);
res.end();
return;
}
if (req.url === '/latest.json') {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(index));
} else if (files.has(req.url)) {
const record = records.find((r) => '/' + r.filename === req.url);
res.setHeader('Content-Length', record.size);
pipeline(createReadStream(files.get(req.url)), res).catch(() =>
res.destroy(),
);
} else {
res.writeHead(404);
res.end();
}
});
try {
server.listen(0, '127.0.0.1');
await once(server, 'listening');
const base = `http://127.0.0.1:${server.address().port}`;
const entry = (r) => ({
size: r.size,
sha256: r.sha256,
url: base + '/' + r.filename,
});
const [bridge, runtime] = records;
index = {
format: 2,
sourceCommit: runtime.sourceCommit,
packages: { [bridge.target]: entry(bridge) },
runtimePackages: { [runtime.target]: entry(runtime) },
};
workspace = await fs.mkdtemp(path.join(directory, '.legacy-client-'));
const env = {
...process.env,
QL_BRANCH: 'develop',
QL_UPGRADE_INDEX_URL: base + '/latest.json',
QL_UPGRADE_FILE: '',
NO_PROXY: '127.0.0.1',
no_proxy: '127.0.0.1',
};
const old = require(path.join(
legacy,
'cli/src/internal/maintenance/upgradeArtifacts.cjs',
));
const firstTmp = path.join(workspace, 'first');
await fs.mkdir(firstTmp);
// These are the unchanged, genuinely installed old download, dependency
// smoke, prestart and readiness verifiers. No injected probe or verifier.
const selected = await old.stageUpgrade({
root: legacy,
tmp: firstTmp,
env,
run: (program, args, capture) =>
old.runProcess(env, program, args, capture),
});
await old.selectedUpgrade(firstTmp, 'develop', env);
assert(
(await read(path.join(selected.source, 'package.json')))
.qinglongMigrationBridge,
);
const upgraded = require(path.join(
selected.source,
'cli/src/internal/maintenance/upgradeArtifacts.cjs',
));
const secondTmp = path.join(workspace, 'second');
await fs.mkdir(secondTmp);
const next = await upgraded.stageUpgrade({
root: selected.source,
tmp: secondTmp,
env,
run: (program, args, capture) =>
upgraded.runProcess(env, program, args, capture),
});
await upgraded.selectedUpgrade(secondTmp, 'develop', env);
const pkg = await read(path.join(next.source, 'package.json'));
assert(pkg.dependencies['@mikro-orm/sql']);
assert.equal(pkg.qinglongMigrationBridge, undefined);
await assert.rejects(
fs.access(path.join(next.source, 'node_modules/sqlite3')),
{ code: 'ENOENT' },
);
await assert.rejects(
fs.access(path.join(next.source, 'node_modules/sequelize')),
{ code: 'ENOENT' },
);
const oldBundle = require(path.join(
legacy,
'cli/src/internal/maintenance/platformBundle.cjs',
));
const manifest = await read(
path.join(path.dirname(selected.source), 'bundle.json'),
);
assert.throws(
() =>
oldBundle.assertCompatible(
manifest,
{ ...manifest.target, libcVersion: manifest.target.libcMinimum },
'20.18.1',
),
/requires Node/,
);
return {
legacyOnlineVerified: true,
modernOnlineVerified: true,
legacyNode20Rejected: true,
};
} finally {
server.closeAllConnections();
if (server.listening) await new Promise((resolve) => server.close(resolve));
if (workspace) await fs.rm(workspace, { recursive: true, force: true });
}
}
async function buildMigrationBridge(legacy, current, output) {
legacy = path.resolve(legacy);
current = path.resolve(current);
output = path.resolve(output);
await fs.mkdir(output, { recursive: true });
const workspace = await fs.mkdtemp(path.join(output, '.migration-'));
let bridge;
try {
const root = path.join(workspace, 'root');
await prepareBridge(legacy, current, root);
await rebuildMigrationSqlite(root);
const runtime = await buildBundle(current, output);
bridge = await buildBundle(root, output);
const verification = await verifyBothDownloads(legacy, current, output, [
bridge,
runtime,
]);
bridge = { ...bridge, migrationVerified: true, ...verification };
await fs.writeFile(
path.join(output, bridge.filename + '.json'),
JSON.stringify(bridge) + '\n',
);
console.log(JSON.stringify(bridge));
return bridge;
} catch (error) {
if (bridge)
await fs.rm(path.join(output, bridge.filename + '.json'), {
force: true,
});
throw error;
} finally {
await fs.rm(workspace, { recursive: true, force: true });
}
}
module.exports = { prepareBridge, verifyBothDownloads, buildMigrationBridge };
if (require.main === module)
buildMigrationBridge(process.argv[2], process.argv[3], process.argv[4]).catch(
(error) => {
console.error(error);
process.exitCode = 1;
},
);
+18 -31
View File
@@ -1,15 +1,10 @@
const fs = require('node:fs/promises');
const { createWriteStream } = require('node:fs');
const path = require('node:path');
const { pipeline } = require('node:stream/promises');
const tar = require('tar');
const bundle = require('../cli/src/internal/maintenance/platformBundle.cjs');
const {
verifyUpgrade,
} = require('../cli/src/internal/maintenance/upgradeArtifacts.cjs');
const {
prestartBackend,
} = require('../cli/src/internal/maintenance/upgradePrestart.cjs');
async function normalizeModes(directory) {
for (const entry of await fs.readdir(directory, { withFileTypes: true })) {
@@ -24,17 +19,6 @@ async function normalizeModes(directory) {
}
}
async function writeBundleArchive(workspace, filename) {
// Core streams complete partial writes; tar's file option uses fs-minipass,
// whose writer can silently discard a successful short write/writev result.
await pipeline(
tar.c({ gzip: true, portable: true, noMtime: true, cwd: workspace }, [
'qinglong',
]),
createWriteStream(filename, { highWaterMark: 256 * 1024 }),
);
}
async function buildBundle(root, output) {
root = path.resolve(root);
output = path.resolve(output);
@@ -42,10 +26,6 @@ async function buildBundle(root, output) {
if (process.platform !== 'linux')
throw new Error('Release bundles must be built and probed on Linux');
const build = await verifyUpgrade(root, path.join(root, 'static'));
const pkg = JSON.parse(
await fs.readFile(path.join(root, 'package.json'), 'utf8'),
);
const migrationBridge = pkg.qinglongMigrationBridge;
await bundle.smokeDependencies(root);
await fs.mkdir(output, { recursive: true });
const workspace = await fs.mkdtemp(path.join(output, '.bundle-'));
@@ -84,8 +64,9 @@ async function buildBundle(root, output) {
const manifest = {
format: 2,
sourceCommit: build.sourceCommit,
version: pkg.version,
...(migrationBridge ? { migrationBridge } : {}),
version: JSON.parse(
await fs.readFile(path.join(root, 'package.json'), 'utf8'),
).version,
lockfileSha256: build.lockfileSha256,
target: {
platform: target.platform,
@@ -95,7 +76,7 @@ async function buildBundle(root, output) {
endianness: target.endianness,
armFloatAbi: target.armFloatAbi,
},
nodeMinimum: '22.17.0',
nodeMinimum: '20.18.1',
nodeApi: 6,
files: await bundle.inventory({ source, static: staticRoot }),
};
@@ -104,10 +85,19 @@ async function buildBundle(root, output) {
JSON.stringify(manifest),
);
console.log('Bundle inventory verified; writing archive');
const filename = `qinglong-${migrationBridge ? 'migration-' : ''}${
build.sourceCommit
}-${bundle.targetKey(target)}.tar.gz`;
await writeBundleArchive(workspace, path.join(output, filename));
const filename = `qinglong-${build.sourceCommit}-${bundle.targetKey(
target,
)}.tar.gz`;
await tar.c(
{
gzip: true,
portable: true,
noMtime: true,
cwd: workspace,
file: path.join(output, filename),
},
['qinglong'],
);
// Verify the bytes users receive, not just the pre-archive build directory.
const roundtrip = path.join(workspace, 'roundtrip');
await fs.mkdir(roundtrip);
@@ -121,8 +111,6 @@ async function buildBundle(root, output) {
await verifyUpgrade(extracted.source, extracted.static);
await bundle.smokeDependencies(extracted.source);
console.log('Archived production dependencies and SQLite verified');
await prestartBackend(extracted, { target });
console.log('Archived backend database, gRPC and HTTP startup verified');
const bytes = await fs.readFile(path.join(output, filename));
const record = {
format: 2,
@@ -131,7 +119,6 @@ async function buildBundle(root, output) {
target: bundle.targetKey(target),
filename,
baseImage: process.env.QL_BUNDLE_BASE_IMAGE,
...(migrationBridge ? { migrationBridge } : {}),
size: bytes.length,
sha256: bundle.sha256(bytes),
};
@@ -145,7 +132,7 @@ async function buildBundle(root, output) {
await fs.rm(workspace, { recursive: true, force: true });
}
}
module.exports = { buildBundle, normalizeModes, writeBundleArchive };
module.exports = { buildBundle, normalizeModes };
if (require.main === module)
buildBundle(process.argv[2] || '/ql', process.argv[3] || '/out').catch(
(e) => {
+41 -68
View File
@@ -13,69 +13,44 @@ const platforms = [
'linux-s390x-musl',
'linux-ia32-musl',
'linux-x64-glibc',
'linux-armv7-glibc',
'linux-arm64-glibc',
'linux-ppc64-glibc',
'linux-s390x-glibc',
];
function releaseIndex(directory, commit, repository, options = {}) {
const {
version,
releaseTag = `upgrade-${commit}`,
migration = false,
} = options;
const { version, releaseTag = `upgrade-${commit}` } = options;
if (
!/^[a-f0-9]{40}$/.test(commit) ||
!/^[\w.-]+\/[\w.-]+$/.test(repository) ||
!/^[\w.+-]+$/.test(releaseTag)
)
throw new Error('Invalid release source');
const readPackages = (bridge) => {
const packages = {};
for (const target of platforms) {
const filename = `qinglong-${
bridge ? 'migration-' : ''
}${commit}-${target}.tar.gz`;
const record = JSON.parse(
fs.readFileSync(path.join(directory, filename + '.json')),
);
const bytes = fs.readFileSync(path.join(directory, filename));
if (
record.format !== 2 ||
record.sourceCommit !== commit ||
(version !== undefined &&
(bridge ? record.migrationBridge?.targetVersion : record.version) !==
version) ||
(bridge &&
(record.migrationVerified !== true ||
record.migrationBridge?.sourceCommit !== commit ||
record.migrationBridge?.legacySourceCommit !==
require('../docker/migration-bridge-images.json')
.sourceCommit)) ||
(!bridge && record.migrationBridge) ||
record.target !== target ||
record.filename !== filename ||
record.size !== bytes.length ||
record.sha256 !== sha256(bytes)
)
throw new Error(`Unverified platform bundle: ${target}`);
packages[target] = {
size: record.size,
sha256: record.sha256,
url: `https://github.com/${repository}/releases/download/${encodeURIComponent(
releaseTag,
)}/${filename}`,
};
}
return packages;
};
const runtimePackages = readPackages(false);
return {
format: 2,
sourceCommit: commit,
packages: migration ? readPackages(true) : runtimePackages,
...(migration ? { runtimePackages } : {}),
};
const packages = {};
for (const target of platforms) {
const filename = `qinglong-${commit}-${target}.tar.gz`;
const record = JSON.parse(
fs.readFileSync(path.join(directory, filename + '.json')),
);
const bytes = fs.readFileSync(path.join(directory, filename));
if (
record.format !== 2 ||
record.sourceCommit !== commit ||
(version !== undefined && record.version !== version) ||
record.target !== target ||
record.filename !== filename ||
record.size !== bytes.length ||
record.sha256 !== sha256(bytes)
)
throw new Error(`Unverified platform bundle: ${target}`);
packages[target] = {
size: record.size,
sha256: record.sha256,
url: `https://github.com/${repository}/releases/download/${encodeURIComponent(
releaseTag,
)}/${filename}`,
};
}
return { format: 2, sourceCommit: commit, packages };
}
function publishBundles({
directory,
@@ -107,7 +82,6 @@ function publishBundles({
...releaseIndex(directory, commit, repository, {
version,
releaseTag: tag,
migration: true,
}),
version,
buildId,
@@ -139,13 +113,11 @@ function publishBundles({
}
};
const notes =
`Verified platform bundles for v${version}, source ${commit}, build ${buildId}. Existing Node runtime is retained.\n\n` +
Object.entries(index.runtimePackages)
`${development ? 'Development' : 'Verified'} platform bundles for v${version}, source ${commit}, build ${buildId}. Existing Node runtime is retained.\n\n` +
Object.entries(index.packages)
.map(([target, entry]) => `- [${target}](${entry.url})`)
.join('\n') +
`\n- [Standalone offline installer](${index.installer.url})\n\n` +
'Legacy format-2 panels update the upgrader first, then download the runtime. ' +
'The packages index contains the separate legacy bridge; runtimePackages contains the normal online/offline packages.';
`\n- [Standalone offline installer](${index.installer.url})`;
const ensureRelease = (ref, title) => {
if (!lookup(ref))
gh(
@@ -175,17 +147,18 @@ function publishBundles({
console.log('Newer source exists; leaving version and channel unchanged');
return false;
}
ensureRelease(tag, `Platform upgrade v${version} (${buildId})`);
ensureRelease(
tag,
`${development ? 'Development platform upgrade' : 'Platform upgrade'} v${version} (${buildId})`,
);
const assets = platforms
.flatMap((target) =>
[false, true].flatMap((bridge) => {
const archive = path.join(
directory,
`qinglong-${bridge ? 'migration-' : ''}${commit}-${target}.tar.gz`,
);
return [archive, archive + '.json'];
}),
)
.flatMap((target) => {
const archive = path.join(
directory,
`qinglong-${commit}-${target}.tar.gz`,
);
return [archive, archive + '.json'];
})
.concat(
path.join(directory, 'install-bundle.cjs'),
path.join(directory, 'latest.json'),
-64
View File
@@ -1,64 +0,0 @@
// The legacy image's generic ARM prebuild contains ARMv7 instructions. Rebuild
// only the first-hop bridge on the builder; never add sqlite3 to normal bundles
// or ask an ARMv6 device to compile it during the migration.
const fs = require('node:fs/promises');
const path = require('node:path');
const { createRequire } = require('node:module');
const { execFile } = require('node:child_process');
const { promisify } = require('node:util');
const bundle = require('../cli/src/internal/maintenance/platformBundle.cjs');
const { repairDependencies, cleanSqliteBuildArtifacts } = require('../cli/src/internal/maintenance/upgradePrestart.cjs');
function assertArmv6Addon(attributes) {
// Alpine's ARM CRT objects are ARMv6KZ (the ARM1176 architecture). Linking
// them raises a base-v6 object's ELF tag without introducing ARMv7/Thumb-2.
if (
!/Tag_CPU_arch:\s+v6(?:K|KZ|ZK)?\s*$/m.test(attributes) ||
/Tag_THUMB_ISA_use:\s+Thumb-2|Tag_FP_arch:\s+VFPv[3-9]|Tag_Advanced_SIMD_arch:\s+\S/.test(attributes) ||
!/Tag_ABI_VFP_args:\s+VFP registers/.test(attributes)
)
throw new Error('Legacy migration SQLite must use ARMv6 hard-float instructions: ' +
String(attributes).split('\n').filter((line) =>
/Tag_(CPU_arch|THUMB_ISA_use|FP_arch|ABI_VFP_args|Advanced_SIMD_arch):/.test(line),
).slice(0, 5).map((line) => line.trim()).join('; '));
}
async function rebuildMigrationSqlite(root, {
target,
run = promisify(execFile),
runtime = process,
repair = repairDependencies,
} = {}) {
target ||= await bundle.runtimeTarget();
if (target.arch !== 'armv6' || target.libc !== 'musl') return false;
const pkg = JSON.parse(await fs.readFile(path.join(root, 'package.json')));
if (!pkg.qinglongMigrationBridge || !pkg.dependencies?.sqlite3)
throw new Error('Only a legacy migration bridge may rebuild SQLite');
const prefix = path.resolve(path.dirname(runtime.execPath), '..');
const header = await fs.readFile(path.join(prefix, 'include/node/node_version.h'), 'utf8');
const version = ['MAJOR', 'MINOR', 'PATCH'].map((part) =>
header.match(new RegExp(`#define\\s+NODE_${part}_VERSION\\s+(\\d+)`))?.[1],
).join('.');
if (version !== runtime.versions.node)
throw new Error('Migration SQLite requires headers matching the existing Node');
const lock = await fs.readFile(path.join(root, 'pnpm-lock.yaml'));
// The Node 18 legacy builder skipped node-gyp 12's optional dependency.
// The existing frozen repair installs that locked graph using the current
// Node, then forces an ARMv6 source build with local headers and one job.
await repair(root, { target, env: runtime.env });
// Repair replaces node_modules, so resolve its new layout after completion.
const rebuilt = path.dirname(createRequire(path.join(root, 'package.json')).resolve('sqlite3/package.json'));
const addon = path.join(rebuilt, 'lib/binding/napi-v6-linux-arm/node_sqlite3.node');
const { stdout } = await run('readelf', ['-A', addon], { timeout: 30000 });
assertArmv6Addon(stdout);
if (!lock.equals(await fs.readFile(path.join(root, 'pnpm-lock.yaml'))))
throw new Error('Migration SQLite rebuild changed the frozen lockfile');
// node-pre-gyp copies the addon to lib/binding and then renames build to
// build-tmp-napi-v<N>. Both layouts contain compiler objects and an absolute
// node_gyp_bins/python3 link; none belongs in a portable runtime archive.
await cleanSqliteBuildArtifacts(rebuilt);
console.log('Legacy bridge SQLite rebuilt and verified for ARMv6 hard-float');
return true;
}
module.exports = { rebuildMigrationSqlite, assertArmv6Addon };
+44
View File
@@ -0,0 +1,44 @@
"""Check serialized Docker layers, including files hidden by later deletions."""
import hashlib
import json
import sys
import tarfile
archive, architecture = sys.argv[1:]
assert architecture in {"amd64", "arm"}
nodes = []
with tarfile.open(archive) as image:
manifests = json.load(image.extractfile("manifest.json"))
assert len(manifests) == 1, "Expected one platform image"
manifest = manifests[0]
config = json.load(image.extractfile(manifest["Config"]))
assert config["architecture"] == architecture
for name in manifest["Layers"]:
with tarfile.open(fileobj=image.extractfile(name), mode="r|*") as layer:
for member in layer:
filename = member.name.removeprefix("./")
assert not (
filename == "tmp/node-runtime-amd64" and member.size > 0
), "Temporary amd64 Node remains in an immutable layer"
if filename not in {"usr/local/bin/node", "usr/bin/node"}:
continue
if not member.isfile():
continue
with layer.extractfile(member) as binary:
header = binary.read(20)
assert header[:4] == b"\x7fELF", "Expected a native Node binary"
digest = hashlib.sha256(header)
while chunk := binary.read(1024 * 1024):
digest.update(chunk)
endian = "little" if header[5] == 1 else "big"
machine = int.from_bytes(header[18:20], endian)
assert machine == {"amd64": 62, "arm": 40}[architecture], (
"Wrong Node architecture in an image layer"
)
nodes.append({"path": filename, "size": member.size, "sha256": digest.hexdigest()})
assert len(nodes) == 1, "Expected exactly one retained Node binary across all layers"
assert nodes[0]["path"] == (
"usr/local/bin/node" if architecture == "amd64" else "usr/bin/node"
)
print(json.dumps({"pass": True, "architecture": architecture, "layers": len(manifest["Layers"]), "nodes": nodes}))
+114 -519
View File
@@ -16,134 +16,61 @@ const execute = promisify(execFile);
const repository = path.resolve(__dirname, '..');
const hash = (bytes) => createHash('sha256').update(bytes).digest('hex');
const read = async (file) => JSON.parse(await fs.readFile(file, 'utf8'));
const run = (program, args, options = {}) =>
execute(program, args, {
encoding: 'utf8',
maxBuffer: 8 * 1024 * 1024,
timeout: 120000,
...options,
});
const run = (program, args, options = {}) => execute(program, args, {
encoding: 'utf8', maxBuffer: 8 * 1024 * 1024, timeout: 120000,
...options,
});
const requiredFiles = [
'.env.example',
'back/protos/api.proto',
'ecosystem.config.js',
'version.yaml',
'shell/upgrade.cjs',
'shell/install-bundle.cjs',
'shell/install-package.cjs',
'shell/upgrade-lock.yaml',
'shell/runtime-guard.cjs',
'shell/node-runtime-probe.cjs',
'shell/start.sh',
'shell/start-simplify.sh',
'shell/task.sh',
'shell/preload/client.js',
'shell/preload/client.py',
'shell/preload/sitecustomize.js',
'shell/preload/sitecustomize.py',
'shell/preload/esm-loader.mjs',
'shell/preload/notificationAudit.js',
'sample/config.sample.sh',
'sample/auth.sample.json',
'static/build-info.json',
'static/build/app.js',
'static/build/token.js',
'.env.example', 'back/protos/api.proto', 'ecosystem.config.js', 'version.yaml',
'shell/upgrade.cjs', 'shell/install-bundle.cjs', 'shell/install-package.cjs',
'shell/upgrade-lock.yaml', 'shell/node-runtime-probe.cjs', 'shell/start.sh', 'shell/start-simplify.sh',
'shell/task.sh', 'shell/preload/client.js', 'shell/preload/client.py',
'shell/preload/sitecustomize.js', 'shell/preload/sitecustomize.py',
'shell/preload/esm-loader.mjs', 'shell/preload/notificationAudit.js',
'sample/config.sample.sh', 'sample/auth.sample.json',
'static/build-info.json', 'static/build/app.js', 'static/build/token.js',
'static/build/bootstrap/upgradeProbe.js',
...[
'databaseClient',
'databaseErrors',
'databaseProcess',
'databaseWorker',
'repository',
'schema',
'queries',
'cacheStore',
'statistics',
].map((name) => `static/build/data/${name}.js`),
'static/dist/index.html',
...[
'upgradeArtifacts',
'platformBundle',
'upgradePrestart',
'panelUpgrade',
'nodeRuntime',
].map((name) => `cli/src/internal/maintenance/${name}.cjs`),
...['upgradeArtifacts', 'platformBundle', 'upgradePrestart', 'panelUpgrade', 'nodeRuntime']
.map((name) => `cli/src/internal/maintenance/${name}.cjs`),
];
function checkPackedFiles(files) {
const names = new Set(files.map((file) => file.path));
for (const name of requiredFiles)
assert.ok(names.has(name), `Missing npm runtime: ${name}`);
for (const name of requiredFiles) assert.ok(names.has(name), `Missing npm runtime: ${name}`);
for (const name of names) {
assert.ok(
/^(?:package\.json$|LICENSE$|README[^/]*\.md$|\.env\.example$|ecosystem\.config\.js$|version\.yaml$|back\/protos\/api\.proto$|cli\/(?:README[^/]*\.md$|src\/internal\/maintenance\/[^/]+\.cjs$)|sample\/|shell\/|static\/)/.test(
name,
),
`Unexpected npm file: ${name}`,
);
assert.ok(
!/^shell\/(?:pub\.sh$|preload\/(?:env\.|notify\.|__ql_notify__\.|lang_env\.sh|.*-notify\.json))/.test(
name,
),
`Generated or release-only runtime file: ${name}`,
);
assert.ok(
!/(?:^|\/)__pycache__\/|\.pyc$|(?:^|\/)\.DS_Store$/.test(name),
name,
);
assert.ok(/^(?:package\.json$|LICENSE$|README[^/]*\.md$|\.env\.example$|ecosystem\.config\.js$|version\.yaml$|back\/protos\/api\.proto$|cli\/(?:README[^/]*\.md$|src\/internal\/maintenance\/[^/]+\.cjs$)|sample\/|shell\/|static\/)/.test(name), `Unexpected npm file: ${name}`);
assert.ok(!/^shell\/(?:pub\.sh$|preload\/(?:env\.|notify\.|__ql_notify__\.|lang_env\.sh|.*-notify\.json))/.test(name), `Generated or release-only runtime file: ${name}`);
assert.ok(!/(?:^|\/)__pycache__\/|\.pyc$|(?:^|\/)\.DS_Store$/.test(name), name);
assert.notEqual(name, 'sample/tool.ts');
}
}
async function packPanel(output) {
const git = (...args) => run('git', args, { cwd: repository });
assert.equal(
(await git('status', '--porcelain', '--untracked-files=all')).stdout.trim(),
'',
'Pack only from a clean source checkout',
);
await run(
process.execPath,
[path.join(repository, 'docker/verify-build.cjs')],
{ cwd: repository },
);
assert.equal((await git('status', '--porcelain', '--untracked-files=all')).stdout.trim(), '', 'Pack only from a clean source checkout');
await run(process.execPath, [path.join(repository, 'docker/verify-build.cjs')], { cwd: repository });
await fs.mkdir(output, { recursive: true });
const cache = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-npm-pack-cache-'));
try {
const result = await run(
'npm',
['pack', '--json', '--cache', cache, '--pack-destination', output],
{ cwd: repository },
);
const result = await run('npm', ['pack', '--json', '--cache', cache, '--pack-destination', output], { cwd: repository });
const [manifest] = JSON.parse(result.stdout);
checkPackedFiles(manifest.files);
assert.equal(manifest.name, '@whyour/qinglong');
const build = await read(path.join(repository, 'static/build-info.json'));
const names = new Set(manifest.files.map((file) => file.path));
for (const name of Object.keys(build.files))
assert.ok(
names.has(`static/${name}`),
`npm filtered a verified build artifact: ${name}`,
);
for (const name of Object.keys(build.files)) assert.ok(names.has(`static/${name}`), `npm filtered a verified build artifact: ${name}`);
const receipt = {
filename: manifest.filename,
sha256: hash(await fs.readFile(path.join(output, manifest.filename))),
sourceCommit: build.sourceCommit,
size: manifest.size,
fileCount: manifest.files.length,
filename: manifest.filename, sha256: hash(await fs.readFile(path.join(output, manifest.filename))),
sourceCommit: build.sourceCommit, size: manifest.size, fileCount: manifest.files.length,
};
await fs.writeFile(
path.join(output, 'package-receipt.json'),
JSON.stringify(receipt, null, 2),
);
await fs.writeFile(path.join(output, 'package-receipt.json'), JSON.stringify(receipt, null, 2));
console.log(JSON.stringify(receipt));
return receipt;
} finally {
await fs.rm(cache, { recursive: true, force: true });
// Also recover an interrupted npm pack whose postpack hook did not execute.
await fs.rm(path.join(repository, 'shell/upgrade-lock.yaml'), {
force: true,
});
await fs.rm(path.join(repository, 'shell/upgrade-lock.yaml'), { force: true });
}
}
@@ -157,13 +84,8 @@ async function unusedPort() {
// Consume the unmodified output of Dockerfile.upgrade-bundle / buildBundle.
async function platformFixture(root, directory, expectedCommit) {
assert.ok(
directory,
'Set QL_PANEL_TEST_BUNDLE_DIR to the official platform bundle output',
);
const names = (await fs.readdir(directory)).filter((name) =>
name.endsWith('.tar.gz'),
);
assert.ok(directory, 'Set QL_PANEL_TEST_BUNDLE_DIR to the official platform bundle output');
const names = (await fs.readdir(directory)).filter((name) => name.endsWith('.tar.gz'));
assert.equal(names.length, 1, 'Expected one official platform package');
const archive = path.resolve(directory, names[0]);
const record = await read(archive + '.json');
@@ -183,140 +105,64 @@ async function platformFixture(root, directory, expectedCommit) {
async function legacyArchive(temp, env) {
const directory = path.join(temp, 'legacy');
await fs.mkdir(directory);
const result = await run(
'npm',
[
'pack',
'@whyour/qinglong@2.23.0',
'--ignore-scripts',
'--json',
'--registry=https://registry.npmjs.org',
'--pack-destination',
directory,
'--cache',
path.join(temp, 'cache'),
],
{ cwd: temp, env, timeout: 300000 },
);
const result = await run('npm', ['pack', '@whyour/qinglong@2.23.0', '--ignore-scripts', '--json', '--registry=https://registry.npmjs.org', '--pack-destination', directory, '--cache', path.join(temp, 'cache')], { cwd: temp, env, timeout: 300000 });
const [record] = JSON.parse(result.stdout);
assert.equal(record.name, '@whyour/qinglong');
assert.equal(record.version, '2.23.0');
assert.equal(record.filename, 'whyour-qinglong-2.23.0.tgz');
const archive = path.join(directory, record.filename);
const integrity =
'sha512-' +
createHash('sha512')
.update(await fs.readFile(archive))
.digest('base64');
assert.equal(
integrity,
'sha512-x6B9s1/SRKkHe7jRQtb3LzMluJ3O3PAAssrr/5jaFMMl7DZvABSEiqkkZMu0r1BNtiCF72uj8OqXN9pCnF2WVw==',
'Published legacy package integrity changed',
);
const integrity = 'sha512-' + createHash('sha512').update(await fs.readFile(archive)).digest('base64');
assert.equal(integrity, 'sha512-x6B9s1/SRKkHe7jRQtb3LzMluJ3O3PAAssrr/5jaFMMl7DZvABSEiqkkZMu0r1BNtiCF72uj8OqXN9pCnF2WVw==', 'Published legacy package integrity changed');
return { archive, integrity, version: record.version };
}
async function verifyPanel(input, output) {
assert.equal(process.platform, 'linux', 'Service tests require Linux');
assert.equal(
process.env.QL_PANEL_PACKAGE_TEST_ISOLATED,
'1',
'Run only inside a disposable Linux container: QL_PANEL_PACKAGE_TEST_ISOLATED=1',
);
assert.equal(process.env.QL_PANEL_PACKAGE_TEST_ISOLATED, '1', 'Run only inside a disposable Linux container: QL_PANEL_PACKAGE_TEST_ISOLATED=1');
const stat = await fs.stat(input);
if (stat.isDirectory()) {
const names = (await fs.readdir(input)).filter((name) =>
name.endsWith('.tgz'),
);
const names = (await fs.readdir(input)).filter((name) => name.endsWith('.tgz'));
assert.equal(names.length, 1, 'Expected one exact npm archive');
input = path.join(input, names[0]);
}
const expected = await read(
path.join(path.dirname(input), 'package-receipt.json'),
);
const expected = await read(path.join(path.dirname(input), 'package-receipt.json'));
assert.equal(path.basename(input), expected.filename);
assert.equal(
hash(await fs.readFile(input)),
expected.sha256,
'npm archive differs from its build receipt',
);
assert.equal(hash(await fs.readFile(input)), expected.sha256, 'npm archive differs from its build receipt');
assert.equal((await fs.stat(input)).size, expected.size);
assert.match(expected.sourceCommit, /^[a-f0-9]{40}$/);
if (process.env.GITHUB_SHA)
assert.equal(expected.sourceCommit, process.env.GITHUB_SHA);
if (process.env.GITHUB_SHA) assert.equal(expected.sourceCommit, process.env.GITHUB_SHA);
const nodeSha = hash(await fs.readFile(process.execPath));
const pm2 = process.env.QL_PANEL_TEST_PM2;
assert.ok(
pm2 && path.isAbsolute(pm2),
'Set QL_PANEL_TEST_PM2 to an isolated PM2 executable',
);
const temp = await fs.realpath(
await fs.mkdtemp(path.join(os.tmpdir(), 'ql-npm-install-')),
);
const home = path.join(temp, 'home'),
prefix = path.join(temp, 'prefix'),
data = path.join(temp, 'data');
assert.ok(pm2 && path.isAbsolute(pm2), 'Set QL_PANEL_TEST_PM2 to an isolated PM2 executable');
const temp = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'ql-npm-install-')));
const home = path.join(temp, 'home'), prefix = path.join(temp, 'prefix'), data = path.join(temp, 'data');
await fs.mkdir(home);
await fs.mkdir(data);
const port = await unusedPort();
let grpcPort = await unusedPort();
while (grpcPort === port) grpcPort = await unusedPort();
const env = {
...process.env,
HOME: home,
PM2_HOME: path.join(temp, '.pm2'),
QL_DATA_DIR: data,
QL_BRANCH: 'develop',
NODE_ENV: 'production',
QL_CONTAINER: 'false',
BACK_PORT: String(port),
GRPC_PORT: String(grpcPort),
QlPort: String(port),
QlGrpcPort: String(grpcPort),
BIND_HOST: '127.0.0.1',
BIND_HOST_GRPC: '127.0.0.1',
NODE_PATH: '',
NODE_OPTIONS: '',
PATH: [path.join(prefix, 'bin'), path.dirname(pm2), process.env.PATH].join(
path.delimiter,
),
const env = { ...process.env, HOME: home, PM2_HOME: path.join(temp, '.pm2'),
QL_DATA_DIR: data, QL_BRANCH: 'develop', NODE_ENV: 'production', QL_CONTAINER: 'false',
BACK_PORT: String(port), GRPC_PORT: String(grpcPort), QlPort: String(port), QlGrpcPort: String(grpcPort),
BIND_HOST: '127.0.0.1', BIND_HOST_GRPC: '127.0.0.1',
NODE_PATH: '', NODE_OPTIONS: '',
PATH: [path.join(prefix, 'bin'), path.dirname(pm2), process.env.PATH].join(path.delimiter),
};
for (const key of [
'QL_CLI_ROOT',
'QL_DIR',
'QL_UPGRADE_FILE',
'QL_UPGRADE_REPAIR_DEPS',
'QL_UPGRADE_LAUNCHER_PID',
'QL_UPGRADE_INDEX_URL',
])
delete env[key];
for (const key of ['QL_CLI_ROOT', 'QL_DIR', 'QL_UPGRADE_FILE', 'QL_UPGRADE_REPAIR_DEPS', 'QL_UPGRADE_LAUNCHER_PID', 'QL_UPGRADE_INDEX_URL']) delete env[key];
let root, fixtureServer, token;
const api = async (route, method = 'GET', body, anonymous = false) => {
const response = await fetch(`http://127.0.0.1:${port}/api/${route}`, {
method,
headers: {
'Content-Type': 'application/json',
...(!anonymous && token ? { Authorization: `Bearer ${token}` } : {}),
},
body: body === undefined ? undefined : JSON.stringify(body),
signal: AbortSignal.timeout(5000),
method, headers: { 'Content-Type': 'application/json', ...(!anonymous && token ? { Authorization: `Bearer ${token}` } : {}) },
body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(5000),
});
const result = await response.json();
try {
assert.equal(result.code, 200, `${method} ${route}: ${result.message}`);
} catch (error) {
error.status = response.status;
throw error;
}
assert.equal(result.code, 200, `${method} ${route}: ${result.message}`);
return result.data;
};
const healthy = async () => {
const deadline = Date.now() + 120000;
while (Date.now() < deadline) {
try {
const value = await api('health', 'GET', undefined, true);
if (value.status === 'ok') return value;
} catch {}
try { const value = await api('health', 'GET', undefined, true); if (value.status === 'ok') return value; } catch {}
await delay(300);
}
throw new Error('Installed npm backend did not become healthy');
@@ -328,81 +174,32 @@ async function verifyPanel(input, output) {
const value = await api('system/update-status');
assert.equal(value.id, id);
if (value.status === status) return value;
if (value.status === 'failed' && status !== 'failed')
throw new Error(value.error);
if (value.status === 'failed' && status !== 'failed') throw new Error(value.error);
} catch (error) {
if (
error.status !== 503 &&
!/fetch failed|ECONNRESET|ECONNREFUSED|aborted|timeout/i.test(
error.message,
)
)
throw error;
if (!/fetch failed|ECONNRESET|ECONNREFUSED|aborted|timeout/i.test(error.message)) throw error;
}
await delay(250);
}
throw new Error(`Installed npm update never reached ${status}`);
};
try {
const install = (archive) =>
run(
'npm',
[
'install',
'--global',
'--prefix',
prefix,
'--force',
'--omit=dev',
'--no-audit',
'--no-fund',
'--cache',
path.join(temp, 'cache'),
archive,
],
{ cwd: temp, env, timeout: 600000 },
);
const start = () =>
run(path.join(prefix, 'bin/qinglong'), ['reload'], {
cwd: temp,
env,
timeout: 180000,
});
const stop = () =>
run(pm2, ['delete', 'ecosystem.config.js'], { cwd: root, env });
console.log(
`Installing actual npm archive with production lifecycle scripts on ${process.version}`,
);
const install = (archive) => run('npm', ['install', '--global', '--prefix', prefix, '--force', '--omit=dev', '--no-audit', '--no-fund', '--cache', path.join(temp, 'cache'), archive], { cwd: temp, env, timeout: 600000 });
const start = () => run(path.join(prefix, 'bin/qinglong'), ['reload'], { cwd: temp, env, timeout: 180000 });
const stop = () => run(pm2, ['delete', 'ecosystem.config.js'], { cwd: root, env });
console.log(`Installing actual npm archive with production lifecycle scripts on ${process.version}`);
await install(input);
const globalRoot = (
await run('npm', ['root', '--global', '--prefix', prefix], {
cwd: temp,
env,
})
).stdout.trim();
const globalRoot = (await run('npm', ['root', '--global', '--prefix', prefix], { cwd: temp, env })).stdout.trim();
root = path.join(globalRoot, '@whyour/qinglong');
env.QL_DIR = root;
const runtimeProbes = [];
const checkRuntime = async () => {
const result = await run(
process.execPath,
[path.join(root, 'shell/node-runtime-probe.cjs')],
{ cwd: temp, env },
);
const result = await run(process.execPath, [path.join(root, 'shell/node-runtime-probe.cjs')], { cwd: temp, env, timeout: 540000 });
runtimeProbes.push(JSON.parse(result.stdout.trim()));
};
await checkRuntime();
// Use a subprocess: cached candidate modules must not mask missing files
// after old-package installation and the real npm overwrite below.
await run(
process.execPath,
[
'-e',
`const root=process.argv[1],p=require('node:path');const m=require(root+'/package.json');for(const target of Object.values(m.bin))require('node:fs').accessSync(p.join(root,target));for(const name of ['platformBundle','upgradePrestart','panelUpgrade'])require(root+'/cli/src/internal/maintenance/'+name+'.cjs');require(root+'/shell/upgrade.cjs').verifyUpgrade(root,p.join(root,'static')).catch(e=>{console.error(e);process.exitCode=1});`,
root,
],
{ env, cwd: temp },
);
await run(process.execPath, ['-e', `const root=process.argv[1],p=require('node:path');const m=require(root+'/package.json');for(const target of Object.values(m.bin))require('node:fs').accessSync(p.join(root,target));for(const name of ['platformBundle','upgradePrestart','panelUpgrade'])require(root+'/cli/src/internal/maintenance/'+name+'.cjs');require(root+'/shell/upgrade.cjs').verifyUpgrade(root,p.join(root,'static')).catch(e=>{console.error(e);process.exitCode=1});`, root], { env, cwd: temp });
env.QL_DATA_DIR = path.join(temp, 'fresh/data');
await fs.mkdir(env.QL_DATA_DIR, { recursive: true });
console.log('Starting the freshly installed npm package');
@@ -410,75 +207,40 @@ async function verifyPanel(input, output) {
const fresh = await healthy();
assert.equal(fresh.build.sourceCommit, expected.sourceCommit);
assert.equal(fresh.services.grpc, true);
assert.match(
await fetch(`http://127.0.0.1:${port}/`).then((r) => r.text()),
/<html/i,
);
assert.match(await fetch(`http://127.0.0.1:${port}/`).then((r) => r.text()), /<html/i);
await stop();
await fs.rm(path.join(temp, 'fresh'), { recursive: true, force: true });
env.QL_DATA_DIR = data;
console.log('Installing the integrity-pinned published legacy npm package');
const legacy = await legacyArchive(temp, env);
await install(legacy.archive);
assert.equal(
(await read(path.join(root, 'package.json'))).version,
legacy.version,
);
await assert.rejects(
fs.access(
path.join(root, 'cli/src/internal/maintenance/platformBundle.cjs'),
),
{ code: 'ENOENT' },
);
const legacyBackendSha = hash(
await fs.readFile(path.join(root, 'static/build/app.js')),
);
assert.equal((await read(path.join(root, 'package.json'))).version, legacy.version);
await assert.rejects(fs.access(path.join(root, 'cli/src/internal/maintenance/platformBundle.cjs')), { code: 'ENOENT' });
const legacyBackendSha = hash(await fs.readFile(path.join(root, 'static/build/app.js')));
await start();
await healthy();
const credentials = {
username: 'npm_package_verification',
password: `Test_${randomUUID()}!`,
};
const credentials = { username: 'npm_package_verification', password: `Test_${randomUUID()}!` };
await api('user/init', 'PUT', credentials, true);
token = (await api('user/login', 'POST', credentials, true)).token;
const nonce = randomUUID();
await api('envs', 'POST', [{ name: 'NPM_PACKAGE_PRESERVE', value: nonce }]);
const script = path.join(data, 'scripts/npm-package-check.js');
await fs.writeFile(script, `console.log(${JSON.stringify(nonce)});\n`);
const cron = await api('crons', 'POST', {
name: 'npm package retained task',
command: `node ${script}`,
schedule: '0 0 1 1 *',
});
const cron = await api('crons', 'POST', { name: 'npm package retained task', command: `node ${script}`, schedule: '0 0 1 1 *' });
const userConfig = path.join(data, 'config/config.sh');
await fs.appendFile(
userConfig,
`\n# retained npm package setting ${nonce}\n`,
);
await fs.appendFile(userConfig, `\n# retained npm package setting ${nonce}\n`);
const userConfigSha = hash(await fs.readFile(userConfig));
const preservedData = async () => {
assert.equal((await api('user')).username, credentials.username);
assert.ok(
(await api('envs')).some(
(x) => x.name === 'NPM_PACKAGE_PRESERVE' && x.value === nonce,
),
);
assert.ok((await api('envs')).some((x) => x.name === 'NPM_PACKAGE_PRESERVE' && x.value === nonce));
assert.equal((await api(`crons/${cron.id}`)).command, cron.command);
assert.equal(hash(await fs.readFile(userConfig)), userConfigSha);
assert.equal(
await fs.readFile(script, 'utf8'),
`console.log(${JSON.stringify(nonce)});\n`,
);
assert.equal(await fs.readFile(script, 'utf8'), `console.log(${JSON.stringify(nonce)});\n`);
};
await preservedData();
console.log(
'Stopping legacy service and verifying npm automatically preserves its .env during overwrite',
);
console.log('Stopping legacy service and verifying npm automatically preserves its .env during overwrite');
await stop();
await fs.appendFile(
path.join(root, '.env'),
`\n# retained custom npm environment ${nonce}\n`,
);
await fs.appendFile(path.join(root, '.env'), `\n# retained custom npm environment ${nonce}\n`);
const savedEnv = await fs.readFile(path.join(root, '.env'));
await install(input);
assert.deepEqual(await fs.readFile(path.join(root, '.env')), savedEnv);
@@ -486,62 +248,25 @@ async function verifyPanel(input, output) {
const configSha = hash(savedEnv);
const local = createRequire(path.join(root, 'package.json'));
const artifacts = local('./shell/upgrade.cjs');
const build = await artifacts.verifyUpgrade(
root,
path.join(root, 'static'),
);
const build = await artifacts.verifyUpgrade(root, path.join(root, 'static'));
assert.equal(build.sourceCommit, expected.sourceCommit);
assert.notEqual(
hash(await fs.readFile(path.join(root, 'static/build/app.js'))),
legacyBackendSha,
);
assert.equal(
hash(await fs.readFile(path.join(root, 'pnpm-lock.yaml'))),
build.lockfileSha256,
);
const fixture = await platformFixture(
root,
process.env.QL_PANEL_TEST_BUNDLE_DIR,
expected.sourceCommit,
);
let badChecksum = true,
requests = 0;
assert.notEqual(hash(await fs.readFile(path.join(root, 'static/build/app.js'))), legacyBackendSha);
assert.equal(hash(await fs.readFile(path.join(root, 'pnpm-lock.yaml'))), build.lockfileSha256);
const fixture = await platformFixture(root, process.env.QL_PANEL_TEST_BUNDLE_DIR, expected.sourceCommit);
let badChecksum = true, requests = 0;
fixtureServer = http.createServer((request, response) => {
requests++;
if (request.url === '/latest.json') {
response.setHeader('Content-Type', 'application/json');
response.end(
JSON.stringify({
format: 2,
sourceCommit: fixture.sourceCommit,
packages: {
[fixture.target]: {
size: fixture.size,
sha256: badChecksum ? '0'.repeat(64) : fixture.sha256,
url: `http://127.0.0.1:${
fixtureServer.address().port
}/bundle.tar.gz`,
},
},
}),
);
response.end(JSON.stringify({ format: 2, sourceCommit: fixture.sourceCommit, packages: { [fixture.target]: { size: fixture.size, sha256: badChecksum ? '0'.repeat(64) : fixture.sha256, url: `http://127.0.0.1:${fixtureServer.address().port}/bundle.tar.gz` } } }));
} else if (request.url === '/bundle.tar.gz') {
response.setHeader('Content-Length', fixture.size);
sync.createReadStream(fixture.archive).pipe(response);
} else {
response.statusCode = 404;
response.end();
}
} else { response.statusCode = 404; response.end(); }
});
await new Promise((resolve) =>
fixtureServer.listen(0, '127.0.0.1', resolve),
);
env.QL_UPGRADE_INDEX_URL = `http://127.0.0.1:${
fixtureServer.address().port
}/latest.json`;
console.log(
'Starting the overwritten npm package against legacy user data',
);
await new Promise((resolve) => fixtureServer.listen(0, '127.0.0.1', resolve));
env.QL_UPGRADE_INDEX_URL = `http://127.0.0.1:${fixtureServer.address().port}/latest.json`;
console.log('Starting the overwritten npm package against legacy user data');
await start();
const original = await healthy();
assert.equal(original.build.sourceCommit, build.sourceCommit);
@@ -558,22 +283,13 @@ async function verifyPanel(input, output) {
assert.equal((await healthy()).build.instanceId, original.build.instanceId);
await preserved();
badChecksum = false;
console.log(
'Testing installed panel download, prestart and explicit application',
);
console.log('Testing installed panel download, prestart and explicit application');
const accepted = await api('system/update', 'PUT');
const ready = await job(accepted.id, 'ready');
assert.equal((await healthy()).build.instanceId, original.build.instanceId);
const pointer = await read(
path.join(root, '.tmp/upgrade-ready-develop.json'),
);
const receipt = await read(
path.join(root, '.tmp', pointer.directory, 'prestart.json'),
);
assert.ok(
receipt.completedAt <= ready.updatedAt,
'Readiness was recorded before backend prestart',
);
const pointer = await read(path.join(root, '.tmp/upgrade-ready-develop.json'));
const receipt = await read(path.join(root, '.tmp', pointer.directory, 'prestart.json'));
assert.ok(receipt.completedAt <= ready.updatedAt, 'Readiness was recorded before backend prestart');
const applying = await api('update/system', 'PUT');
await job(applying.id, 'completed');
const afterOnline = await healthy();
@@ -581,59 +297,20 @@ async function verifyPanel(input, output) {
assert.notEqual(afterOnline.build.instanceId, original.build.instanceId);
assert.equal(afterOnline.build.sourceCommit, fixture.sourceCommit);
await fs.access(path.join(root, 'node_modules/.pnpm'));
assert.match(
await fs.realpath(path.join(root, 'node_modules/@mikro-orm/sql')),
/[/\\]node_modules[/\\]\.pnpm[/\\]/,
);
await assert.rejects(fs.access(path.join(root, 'node_modules/sqlite3')), {
code: 'ENOENT',
});
await assert.rejects(fs.access(path.join(root, 'node_modules/sequelize')), {
code: 'ENOENT',
});
assert.match(await fs.realpath(path.join(root, 'node_modules/sqlite3')), /[/\\]node_modules[/\\]\.pnpm[/\\]/);
await fs.access(path.join(root, 'scripts/build-upgrade-bundle.cjs'));
await artifacts.verifyUpgrade(root, path.join(root, 'static'));
assert.match(
await fetch(`http://127.0.0.1:${port}/`).then((r) => r.text()),
/<html/i,
);
assert.match(await fetch(`http://127.0.0.1:${port}/`).then((r) => r.text()), /<html/i);
await preserved();
console.log(
'Testing offline installer and Bash reload from the installed npm package',
);
console.log('Testing offline installer and Bash reload from the installed npm package');
const requestsBeforeOffline = requests;
await run(
process.execPath,
[
path.join(root, 'shell/install-bundle.cjs'),
'--root',
root,
'--file',
fixture.archive,
'--download-only',
],
{ cwd: temp, env, timeout: 240000 },
);
assert.equal(
requests,
requestsBeforeOffline,
'Offline preparation contacted the index',
);
assert.equal(
(await healthy()).build.instanceId,
afterOnline.build.instanceId,
);
await run(path.join(prefix, 'bin/ql'), ['reload', 'system'], {
cwd: temp,
env,
timeout: 240000,
});
await run(process.execPath, [path.join(root, 'shell/install-bundle.cjs'), '--root', root, '--file', fixture.archive, '--download-only'], { cwd: temp, env, timeout: 240000 });
assert.equal(requests, requestsBeforeOffline, 'Offline preparation contacted the index');
assert.equal((await healthy()).build.instanceId, afterOnline.build.instanceId);
await run(path.join(prefix, 'bin/ql'), ['reload', 'system'], { cwd: temp, env, timeout: 240000 });
const afterOffline = await healthy();
await checkRuntime();
assert.notEqual(
afterOffline.build.instanceId,
afterOnline.build.instanceId,
);
assert.notEqual(afterOffline.build.instanceId, afterOnline.build.instanceId);
assert.equal(afterOffline.build.sourceCommit, fixture.sourceCommit);
await fs.access(path.join(root, 'node_modules/.pnpm'));
assert.equal(hash(await fs.readFile(process.execPath)), nodeSha);
@@ -643,66 +320,36 @@ async function verifyPanel(input, output) {
let taskExecuted = false;
for (let i = 0; i < 100; i++) {
const log = String(await api(`crons/${cron.id}/log`));
if (
log.includes(nonce) &&
/完成|执行结束|finished|completed/i.test(log)
) {
taskExecuted = true;
break;
}
if (log.includes(nonce) && /完成|执行结束|finished|completed/i.test(log)) { taskExecuted = true; break; }
await delay(300);
}
assert.ok(
taskExecuted,
'Retained task did not execute after npm package upgrades',
);
assert.ok(taskExecuted, 'Retained task did not execute after npm package upgrades');
const result = {
node: process.version,
platform: fixture.target,
archiveSha256: hash(await fs.readFile(input)),
sourceCommit: build.sourceCommit,
installedProductionDependencies: true,
restoredLockVerified: true,
legacyNpmVersion: legacy.version,
legacyNpmIntegrity: legacy.integrity,
legacyNpmOverwriteInstalled: true,
legacyExternalDataPreserved: true,
node: process.version, platform: fixture.target,
archiveSha256: hash(await fs.readFile(input)), sourceCommit: build.sourceCommit,
installedProductionDependencies: true, restoredLockVerified: true,
legacyNpmVersion: legacy.version, legacyNpmIntegrity: legacy.integrity,
legacyNpmOverwriteInstalled: true, legacyExternalDataPreserved: true,
customRootEnvironmentPreservedAutomatically: true,
officialPlatformBundle: {
baseImage: fixture.baseImage,
sourceCommit: fixture.sourceCommit,
sha256: fixture.sha256,
},
officialPlatformBundle: { baseImage: fixture.baseImage, sourceCommit: fixture.sourceCommit, sha256: fixture.sha256 },
migratedFromNpmToOfficialPnpmLayout: true,
existingNodePreserved: true,
nativeUrlRuntimeProbes: runtimeProbes,
frontendAndBackendStarted: true,
checksumFailureKeptOldService: true,
prestartBeforeReady: true,
onlineActualRestart: true,
offlinePreparationDidNotContactIndex: true,
offlineBashActualRestart: true,
accountEnvironmentConfigurationAndScriptsPreserved: true,
retainedTaskExecuted: true,
initialBuild: original.build,
afterOnline: afterOnline.build,
afterOffline: afterOffline.build,
existingNodePreserved: true, nativeUrlRuntimeProbes: runtimeProbes,
frontendAndBackendStarted: true, checksumFailureKeptOldService: true,
prestartBeforeReady: true, onlineActualRestart: true,
offlinePreparationDidNotContactIndex: true, offlineBashActualRestart: true,
accountEnvironmentConfigurationAndScriptsPreserved: true, retainedTaskExecuted: true,
initialBuild: original.build, afterOnline: afterOnline.build, afterOffline: afterOffline.build,
};
if (output) {
await fs.mkdir(output, { recursive: true });
await fs.writeFile(
path.join(output, `verification-${process.version}.json`),
JSON.stringify(result, null, 2),
);
await fs.writeFile(path.join(output, `verification-${process.version}.json`), JSON.stringify(result, null, 2));
}
console.log(JSON.stringify(result, null, 2));
return result;
} catch (error) {
if (root) {
for (const file of ['qinglong-error.log', 'qinglong-out.log']) {
const text = await fs
.readFile(path.join(env.PM2_HOME, 'logs', file), 'utf8')
.catch(() => '');
const text = await fs.readFile(path.join(env.PM2_HOME, 'logs', file), 'utf8').catch(() => '');
if (text) console.error(text.slice(-6000));
}
}
@@ -712,71 +359,19 @@ async function verifyPanel(input, output) {
fixtureServer.closeAllConnections();
await new Promise((resolve) => fixtureServer.close(resolve));
}
// A failed assertion must not remove a prefix that its detached updater is
// still extracting into. Identify only this private installation's job.
if (root) {
const state = await read(
path.join(root, '.tmp/panel-upgrade.json'),
).catch(() => undefined);
if (
Number.isSafeInteger(state?.pid) &&
state.pid > 1 &&
typeof state.id === 'string'
) {
const command = (
await fs
.readFile(`/proc/${state.pid}/cmdline`, 'utf8')
.catch(() => '')
).split('\0');
if (
command.includes(
path.join(root, 'cli/src/internal/maintenance/panelUpgrade.cjs'),
) &&
command.includes('worker') &&
command.includes(state.id)
) {
const signal = (name) => {
try {
process.kill(-state.pid, name);
return true;
} catch (error) {
if (error.code !== 'ESRCH') throw error;
return false;
}
};
signal('SIGTERM');
const deadline = Date.now() + 30000;
while (signal(0) && Date.now() < deadline) await delay(100);
if (signal(0)) signal('SIGKILL');
}
}
}
await run(pm2, ['kill'], { env, cwd: temp }).catch(() => {});
await run('nginx', ['-s', 'quit'], { env, cwd: temp }).catch(() => {});
await fs.rm(temp, {
recursive: true,
force: true,
maxRetries: 3,
retryDelay: 100,
});
await fs.rm(temp, { recursive: true, force: true });
}
}
module.exports = { checkPackedFiles, packPanel, verifyPanel };
if (require.main === module) {
const [action, input, output] = process.argv.slice(2);
const operation =
action === 'pack' && input
? packPanel(path.resolve(input))
: action === 'verify' && input
const operation = action === 'pack' && input
? packPanel(path.resolve(input))
: action === 'verify' && input
? verifyPanel(path.resolve(input), output && path.resolve(output))
: Promise.reject(
new Error(
'Usage: node scripts/verify-panel-package.cjs pack OUTPUT | verify ARCHIVE_OR_DIRECTORY [RESULT_DIRECTORY]',
),
);
operation.catch((error) => {
console.error(error);
process.exitCode = 1;
});
: Promise.reject(new Error('Usage: node scripts/verify-panel-package.cjs pack OUTPUT | verify ARCHIVE_OR_DIRECTORY [RESULT_DIRECTORY]'));
operation.catch((error) => { console.error(error); process.exitCode = 1; });
}
-180
View File
@@ -1,180 +0,0 @@
// Verify the actual archive under an unsupported Node without touching any
// existing installation. This is also the Node 20 negative CI matrix entry.
const fs = require('node:fs/promises');
const path = require('node:path');
const os = require('node:os');
const assert = require('node:assert/strict');
const { execFile, spawn } = require('node:child_process');
const { promisify } = require('node:util');
const execute = promisify(execFile);
async function verifyRuntimeFloor(input, output) {
assert.equal(
Number(process.versions.node.split('.')[0]),
20,
'Run this rejection test with Node 20',
);
const names = (await fs.readdir(input)).filter((name) =>
name.endsWith('.tgz'),
);
assert.equal(names.length, 1, 'Expected one actual npm archive');
const archive = path.resolve(input, names[0]);
const temp = await fs.realpath(
await fs.mkdtemp(path.join(os.tmpdir(), 'ql-node-floor-')),
);
const env = {
PATH: process.env.PATH,
HOME: temp,
TMPDIR: temp,
npm_config_cache: path.join(temp, 'cache'),
npm_config_userconfig: path.join(temp, 'npmrc'),
npm_config_globalconfig: path.join(temp, 'global-npmrc'),
};
const npm = (args) =>
execute('npm', args, {
cwd: temp,
env,
encoding: 'utf8',
timeout: 600000,
maxBuffer: 4 * 1024 * 1024,
});
try {
await fs.writeFile(env.npm_config_userconfig, '');
await fs.writeFile(env.npm_config_globalconfig, '');
const legacy = path.join(temp, 'legacy');
await fs.mkdir(legacy);
await fs.writeFile(
path.join(legacy, 'package.json'),
JSON.stringify({ name: '@whyour/qinglong', version: '2.22.0' }),
);
await fs.writeFile(path.join(legacy, 'legacy-marker'), 'old application');
const [old] = JSON.parse(
(
await execute('npm', ['pack', '--json', '--ignore-scripts'], {
cwd: legacy,
env,
})
).stdout,
);
for (const global of [false, true]) {
const prefix = path.join(temp, global ? 'global' : 'local');
const installed = path.join(
prefix,
global
? 'lib/node_modules/@whyour/qinglong'
: 'node_modules/@whyour/qinglong',
);
const args = [
'install',
...(global ? ['--global'] : []),
'--prefix',
prefix,
'--omit=dev',
'--no-audit',
'--no-fund',
];
await npm([...args, path.join(legacy, old.filename)]);
const environment = Buffer.from(
'QL_DATA_DIR=/external/data\r\nSYNTHETIC_SECRET=keep\n',
);
const data = path.join(temp, global ? 'global-data' : 'local-data');
await fs.writeFile(path.join(installed, '.env'), environment);
await fs.writeFile(data, 'synthetic old data');
const child = spawn(
process.execPath,
[
'-e',
`
const fs=require('node:fs'),http=require('node:http');
const value=fs.readFileSync(process.argv[1],'utf8')+fs.readFileSync(process.argv[2],'utf8');
const server=http.createServer((q,r)=>r.end(value));
server.listen(0,'127.0.0.1',()=>process.send(server.address().port));
`,
path.join(installed, '.env'),
data,
],
{ env, stdio: ['ignore', 'ignore', 'pipe', 'ipc'] },
);
const stopped = new Promise((resolve) => child.once('exit', resolve));
try {
const port = await new Promise((resolve, reject) => {
const timer = setTimeout(
() => reject(new Error('Old service startup timed out')),
10000,
);
child.once('message', (value) => {
clearTimeout(timer);
resolve(value);
});
child.once('error', (error) => {
clearTimeout(timer);
reject(error);
});
child.once('exit', () => {
clearTimeout(timer);
reject(new Error('Old service exited'));
});
});
const url = `http://127.0.0.1:${port}`;
const before = await (await fetch(url)).text();
await assert.rejects(npm([...args, archive]), (error) => {
assert.equal(
error.killed,
false,
`npm did not reach the runtime guard: ${error.message}`,
);
assert.match(error.stderr + error.stdout, /requires Node >=22\.17/);
return true;
});
assert.equal(
JSON.parse(await fs.readFile(path.join(installed, 'package.json')))
.version,
'2.22.0',
);
assert.deepEqual(
await fs.readFile(path.join(installed, '.env')),
environment,
);
assert.equal(await fs.readFile(data, 'utf8'), 'synthetic old data');
assert.equal(child.exitCode, null);
assert.equal(await (await fetch(url)).text(), before);
} finally {
child.kill('SIGKILL');
await stopped;
}
}
const result = {
node: process.version,
unsupportedRuntimeRejected: true,
localAndGlobalOldPackagePreserved: true,
environmentAndDataPreserved: true,
oldServiceStayedHealthy: true,
};
if (output) {
await fs.mkdir(output, { recursive: true });
await fs.writeFile(
path.join(output, `verification-${process.version}.json`),
JSON.stringify(result, null, 2),
);
}
console.log(JSON.stringify(result, null, 2));
return result;
} finally {
await fs.rm(temp, { recursive: true, force: true });
}
}
module.exports = { verifyRuntimeFloor };
if (require.main === module) {
const [input, output] = process.argv.slice(2);
if (!input)
throw new Error(
'Usage: node scripts/verify-panel-runtime-floor.cjs ARCHIVE_DIRECTORY [RESULT_DIRECTORY]',
);
verifyRuntimeFloor(path.resolve(input), output && path.resolve(output)).catch(
(error) => {
console.error(error);
process.exitCode = 1;
},
);
}
-17
View File
@@ -1,17 +0,0 @@
const minimum = '22.17.0';
function assertNodeRuntime(version = process.versions.node) {
const [major, minor] = version.split('.').map(Number);
if (!Number.isInteger(major) || !Number.isInteger(minor) || major < 22 || (major === 22 && minor < 17)) {
throw Object.assign(new Error(`Qinglong requires Node >=${minimum}; upgrade Node before installing. Application updates retain the existing Node.`), { code: 'UNSUPPORTED_NODE' });
}
if (version === process.versions.node) {
try {
if (typeof require('node:sqlite').DatabaseSync !== 'function') throw new Error();
} catch {
throw Object.assign(new Error('This Node build does not provide node:sqlite; use a Node build with SQLite support.'), { code: 'UNSUPPORTED_NODE_SQLITE' });
}
}
}
module.exports = { minimum, assertNodeRuntime };
+2 -4
View File
@@ -20,16 +20,14 @@ export default function UpgradeNotice({
const showNotice =
activeStatuses.includes(job.status) ||
job.status === 'ready' ||
job.status === 'failed' ||
Boolean(job.migrationRequired);
job.status === 'failed';
if (!showNotice && !open) return null;
const text = !connected
? intl.get('连接已断开,正在重新连接')
: intl.get(upgradeStatusLabel(job));
const label =
activeStatuses.includes(job.status) &&
job.phase === 'downloading' &&
job.percent != null
job.phase === 'downloading' && job.percent != null
? `${text} ${job.percent}%`
: text;
return (
+2 -11
View File
@@ -15,7 +15,6 @@ export interface UpgradeJob {
version?: string;
error?: string;
log?: string;
migrationRequired?: boolean;
}
export const activeStatuses = ['starting', 'running', 'restarting'];
@@ -35,8 +34,6 @@ export const phaseLabels: Record<string, string> = {
// Status is authoritative; old progress fields must not claim completion.
export function upgradeStatusLabel(job: UpgradeJob): string {
if (job.migrationRequired && ['idle', 'completed'].includes(job.status))
return '升级器已更新,请继续升级新版';
if (job.status === 'ready') return '下载完成,等待重启';
if (job.status === 'completed') return '升级成功';
if (job.status === 'failed') return '更新失败';
@@ -97,14 +94,8 @@ export function createUpgradeStore() {
) {
sessionStorage.removeItem(completionKey);
notification.success({
message: intl.get(
result.data.migrationRequired ? '升级器已更新' : '升级成功',
),
description: intl.get(
result.data.migrationRequired
? '当前仍运行旧版,请继续下载并重启以完成新版升级。'
: '已重新启动并运行新版本,无需再次重启。',
),
message: intl.get('升级成功'),
description: intl.get('已重新启动并运行新版本,无需再次重启。'),
duration: 8,
});
}
-8
View File
@@ -758,14 +758,6 @@
"暂存更新已变化,请重新下载更新。": "The staged update changed. Download the update again.",
"启动版本与更新包不一致": "The running build does not match the update package.",
"下载完成,等待重启": "Download complete, restart to update",
"升级器已更新,请继续升级新版": "Upgrader updated; continue to the new version",
"升级器已更新": "Upgrader updated",
"当前仍运行旧版,请继续下载并重启以完成新版升级。": "The previous version is still running. Download the new version and restart to finish upgrading.",
"继续升级新版": "Continue upgrading",
"升级器已更新。第二步将下载并预启动新版,准备成功后再重启应用。": "The upgrader is updated. Step 2 downloads and tests the new backend before you restart to apply it.",
"下载新版": "Download new version",
"更新索引尚未提供第二步运行包,请稍后重试": "The update index does not yet provide the second-step runtime. Please try again later.",
"第二步更新包仍是旧数据库后端,已保留当前服务,请稍后重试": "The second-step package still uses the legacy database backend. Your current service is unchanged. Please try again later.",
"升级成功": "Update successful",
"已重新启动并运行新版本,无需再次重启。": "Restarted and running the new version. No further restart is needed."
}
-8
View File
@@ -758,14 +758,6 @@
"暂存更新已变化,请重新下载更新。": "暂存更新已变化,请重新下载更新。",
"启动版本与更新包不一致": "启动版本与更新包不一致",
"下载完成,等待重启": "下载完成,等待重启",
"升级器已更新,请继续升级新版": "升级器已更新,请继续升级新版",
"升级器已更新": "升级器已更新",
"当前仍运行旧版,请继续下载并重启以完成新版升级。": "当前仍运行旧版,请继续下载并重启以完成新版升级。",
"继续升级新版": "继续升级新版",
"升级器已更新。第二步将下载并预启动新版,准备成功后再重启应用。": "升级器已更新。第二步将下载并预启动新版,准备成功后再重启应用。",
"下载新版": "下载新版",
"更新索引尚未提供第二步运行包,请稍后重试": "更新索引尚未提供第二步运行包,请稍后重试",
"第二步更新包仍是旧数据库后端,已保留当前服务,请稍后重试": "第二步更新包仍是旧数据库后端,已保留当前服务,请稍后重试",
"升级成功": "升级成功",
"已重新启动并运行新版本,无需再次重启。": "已重新启动并运行新版本,无需再次重启。"
}
+4 -26
View File
@@ -35,18 +35,6 @@ const CheckUpdate = ({ systemInfo }: any) => {
};
const checkUpgrade = async () => {
if (job.migrationRequired) {
Modal.confirm({
title: intl.get('继续升级新版'),
content: intl.get(
'升级器已更新。第二步将下载并预启动新版,准备成功后再重启应用。',
),
okText: intl.get('下载新版'),
cancelText: intl.get('以后再说'),
onOk: startUpdate,
});
return;
}
if (!upgradeStore.beginMutation()) return;
try {
const { code, data } = await request.put(
@@ -113,9 +101,7 @@ const CheckUpdate = ({ systemInfo }: any) => {
});
const label = intl.get(upgradeStatusLabel(job));
const hasJob =
Boolean(job.migrationRequired) ||
(job.status !== 'idle' && job.status !== 'loading');
const hasJob = job.status !== 'idle' && job.status !== 'loading';
const actions = !active && (
<Space wrap>
<Button
@@ -124,15 +110,9 @@ const CheckUpdate = ({ systemInfo }: any) => {
disabled={!connected || job.status === 'loading'}
onClick={ready ? showReloadModal : checkUpgrade}
>
{intl.get(
ready
? '重启并更新'
: job.migrationRequired
? '继续升级新版'
: '检查更新',
)}
{intl.get(ready ? '重启并更新' : '检查更新')}
</Button>
{!ready && !completed && !job.migrationRequired && (
{!ready && !completed && (
<Button
disabled={busy || !connected || job.status === 'loading'}
onClick={() => reloadSystem(false)}
@@ -168,9 +148,7 @@ const CheckUpdate = ({ systemInfo }: any) => {
? intl.get(job.error || '更新失败,请检查网络及日志或稍后再试') ||
job.error
: intl.get(
job.migrationRequired && !ready && !active
? '当前仍运行旧版,请继续下载并重启以完成新版升级。'
: completed
completed
? '已重新启动并运行新版本,无需再次重启。'
: ready
? '更新已就绪,重启后应用新版本'
+15 -16
View File
@@ -14,6 +14,7 @@ function temporary(t) {
t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
return dir;
}
test('environment values remain literal when the generated JavaScript loads', async () => {
const output = {};
const Env = load(path.resolve('back/services/env.ts'), {
@@ -41,6 +42,7 @@ test('environment values remain literal when the generated JavaScript loads', as
assert.equal(context.process.env[`VALUE_${i}`], value),
);
});
test('subscription shell parameters round trip without expansion or command substitution', () => {
const { formatCommand } = load(path.resolve('back/config/subscription.ts'));
const payload =
@@ -81,6 +83,7 @@ test('subscription shell parameters round trip without expansion or command subs
}
}
});
test('SSH alias traversal and symlinks cannot write or remove outside files', async (t) => {
const base = temporary(t);
const root = path.join(base, 'ssh');
@@ -138,17 +141,14 @@ test('SSH alias traversal and symlinks cannot write or remove outside files', as
service.generateSingleSshConfig('safe', 'host', '$(printf BAD)'),
);
});
function cronService(config, updates = []) {
const Cron = load(path.resolve('back/services/cron.ts'), {
typedi: decorators,
'../config': config,
'../data/cron': {
CrontabStatus: { running: 0, idle: 1, queued: 3 },
CrontabModel: {
nativeUpdate: async (where, v) => {
return updates.push(v);
},
},
CrontabModel: { update: async (v) => updates.push(v) },
},
'../data/runningInstance': { RunningInstanceModel: {}, InstanceStatus: {} },
'../config/util': { fileExist: async (p) => fs.existsSync(p) },
@@ -183,6 +183,7 @@ function subscriptionService(config, model = {}) {
}).default;
return new Subscription(logger, {}, {}, {}, {});
}
test('stored cron/subscription paths cannot read or enumerate outside the log root', async (t) => {
const base = temporary(t);
const root = path.join(base, 'log');
@@ -223,6 +224,7 @@ test('stored cron/subscription paths cannot read or enumerate outside the log ro
(await subscription.logs(1)).some((x) => x.filename === 'normal.log'),
);
});
test('subscription aliases are checked before persistence and forced deletion', async (t) => {
const base = temporary(t);
for (const dir of ['scripts', 'repo']) fs.mkdirSync(path.join(base, dir));
@@ -233,12 +235,8 @@ test('subscription aliases are checked before persistence and forced deletion',
repoPath: path.join(base, 'repo'),
},
{
find: async (where, read = {}) => {
return [{ alias: '../victim' }];
},
nativeDelete: async (where, read = {}) => {
return writes++;
},
findAll: async () => [{ alias: '../victim' }],
destroy: async () => writes++,
},
);
service.insert = service.updateDb = async () => writes++;
@@ -256,17 +254,14 @@ test('subscription aliases are checked before persistence and forced deletion',
);
assert.equal(writes, 0);
});
function systemService(config, overrides = {}) {
const System = load(path.resolve('back/services/system.ts'), {
typedi: decorators,
'../config': config,
'../config/const': {},
'../data/dependence': {
DependenceModel: {
find: async (where, read = {}) => {
return [];
},
},
DependenceModel: { findAll: async () => [] },
DependenceTypes: {},
DependenceStatus: {},
},
@@ -282,6 +277,7 @@ function systemService(config, overrides = {}) {
}).default;
return new System(logger, {}, {});
}
test('backup rejects unrecognized paths before spawning; valid export uses argv without a shell', async (t) => {
const base = temporary(t);
const calls = [];
@@ -337,6 +333,7 @@ test('backup rejects unrecognized paths before spawning; valid export uses argv
assert.equal(calls[0][2].shell, undefined);
assert.equal(downloaded, true);
});
test('proxy, language and mirror settings stay literal at their shell boundaries', async (t) => {
const base = temporary(t);
const commands = [];
@@ -376,6 +373,7 @@ test('proxy, language and mirror settings stay literal at their shell boundaries
}
assert.equal((await service.reloadSystem('$(printf BAD)')).code, 400);
});
test('the ignored-log sentinel remains supported without exposing device files', async (t) => {
const root = temporary(t);
const cron = cronService({ logPath: root });
@@ -387,6 +385,7 @@ test('the ignored-log sentinel remains supported without exposing device files',
cron.status({ ids: [], status: 0, log_path: '/dev/zero' }),
);
});
test('normal backup creates an archive containing the selected data directories', async (t) => {
const base = temporary(t);
const dataPath = path.join(base, 'data');
-129
View File
@@ -1,129 +0,0 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const path = require('node:path');
const os = require('node:os');
const fs = require('node:fs/promises');
const express = require('express');
const jwt = require('jsonwebtoken');
const load = require('../helpers/load-security-module.cjs');
test('authentication cache propagates database read and write failures', async () => {
const failure = new Error('synthetic unavailable database');
const CacheStore = load(path.resolve('back/data/cacheStore.ts'), {
'.': {
database: {
request: async () => {
throw failure;
},
},
},
}).CacheStore;
const { shareStore } = load(path.resolve('back/shared/store.ts'), {
'../data/cacheStore': { CacheStore },
});
await assert.rejects(shareStore.getAuthInfo(), (error) => error === failure);
await assert.rejects(
shareStore.updateAuthInfo({ username: 'qa' }),
(error) => error === failure,
);
});
test('HTTP authentication distinguishes unavailable storage from an invalid session', async (t) => {
const root = await fs.mkdtemp(
path.join(os.tmpdir(), 'ql-auth-availability-'),
);
const secret = 'synthetic-auth-availability';
const token = jwt.sign({}, secret, { algorithm: 'HS384', expiresIn: '1h' });
const auth = { username: 'qa', password: 'configured', token };
let unavailable = false;
let initialized = 0;
const read = async (value) => {
if (unavailable) throw new Error(`SQL failure must not expose ${token}`);
return value;
};
const router = express.Router();
router.get('/system/update-status', (_req, res) => res.json({ code: 200 }));
router.get('/envs', (_req, res) => res.json({ code: 200 }));
router.get('/health', (_req, res) => res.json({ code: 200 }));
router.put('/user/init', (_req, res) => {
initialized++;
res.json({ code: 200 });
});
const app = express();
load(path.resolve('back/loaders/express.ts'), {
'../config': {
rootPath: root,
uploadPath: path.join(root, 'upload'),
configPath: path.join(root, 'config'),
api: { prefix: '/api' },
apiWhiteList: ['/api/health', '/api/user/init'],
jwt: { secret },
},
'../api': () => router,
'../config/util': {
getToken: (req) =>
(req.headers.authorization || '').replace(/^Bearer /, ''),
getPlatform: () => 'desktop',
},
'../shared/store': {
shareStore: {
getAuthInfo: () => read(auth),
getApps: () =>
read([
{
scopes: ['envs'],
tokens: [
{ value: 'app-token', expiration: Date.now() / 1000 + 3600 },
],
},
]),
},
},
'../shared/i18n': { t: (value) => value },
'../config/serverEnv': { serveEnv: (_req, res) => res.end() },
'../data/open': {},
'../data/system': {},
}).default({ app });
const server = app.listen(0, '127.0.0.1');
t.after(async () => {
await new Promise((resolve) => server.close(resolve));
await fs.rm(root, { recursive: true, force: true });
});
await new Promise((resolve) => server.once('listening', resolve));
const request = async (route, value, method = 'GET') => {
const response = await fetch(
`http://127.0.0.1:${server.address().port}${route}`,
{
method,
headers: value ? { Authorization: `Bearer ${value}` } : {},
signal: AbortSignal.timeout(3000),
},
);
return { status: response.status, body: await response.text() };
};
assert.equal((await request('/api/system/update-status', token)).status, 200);
unavailable = true;
for (const [route, value, method] of [
['/api/system/update-status', token],
['/open/envs', 'app-token'],
['/api/user/init', undefined, 'PUT'],
]) {
const response = await request(route, value, method);
assert.equal(response.status, 503);
assert.equal(JSON.parse(response.body).code, 503);
assert.ok(!response.body.includes(token));
assert.ok(!response.body.includes('SQL failure'));
}
assert.equal(initialized, 0);
assert.equal((await request('/api/health')).status, 200);
unavailable = false;
assert.equal((await request('/api/system/update-status', token)).status, 200);
const invalid = jwt.sign({ unrecognized: true }, secret, {
algorithm: 'HS384',
expiresIn: '1h',
});
assert.equal(
(await request('/api/system/update-status', invalid)).status,
401,
);
});
-69
View File
@@ -1,69 +0,0 @@
const assert = require('node:assert/strict');
const test = require('node:test');
const express = require('express');
const { DatabaseClient } = require('../../back/data/databaseClient');
const { Repository } = require('../../back/data/repository');
const { initialize } = require('../helpers/repository-fixture.cjs');
function mockModule(modulePath, exports) {
const filename = require.resolve(modulePath);
require.cache[filename] = {
id: filename,
filename,
loaded: true,
exports,
children: [],
paths: [],
};
}
test('instance API retains newest-first ordering independently of insertion order', async (t) => {
const db = new DatabaseClient(':memory:');
t.after(() => db.close());
await initialize(db);
const instances = new Repository('RunningInstance', () => db);
for (const [cron_id, started_at] of [
[7, 200],
[7, 300],
[8, 900],
[7, 100],
]) {
await instances.insert({ cron_id, started_at, status: 2 });
}
for (const service of ['cron', 'cronView']) {
mockModule(`../../back/services/${service}`, {
__esModule: true,
default: class {},
});
}
mockModule('../../back/schedule/client', { __esModule: true, default: {} });
mockModule('../../back/shared/i18n', { t: (message) => message });
mockModule('../../back/data/runningInstance', {
RunningInstanceModel: instances,
InstanceStatus: { running: 0 },
});
const app = express.Router();
require('../../back/api/cron').default(app);
const router = app.stack.find((layer) => layer.name === 'router').handle;
const handler = router.stack
.find((layer) => layer.route?.path === '/:id/instances')
.route.stack.at(-1).handle;
let response;
await handler(
{ params: { id: 7 } },
{
send: (body) => {
response = body;
},
},
(error) => {
throw error;
},
);
assert.equal(response.code, 200);
assert.deepEqual(
response.data.map((row) => row.started_at),
[300, 200, 100],
);
assert(response.data.every((row) => row.cron_id === 7));
});
+12 -69
View File
@@ -10,40 +10,15 @@ test('system and node modes route extended cron once and keep portable rules in
const end = source.indexOf(' private async getLogName', begin);
const saveBegin = source.indexOf(' private async setCrontab(');
const saveEnd = source.indexOf(' public importCrontab(', saveBegin);
const js = ts.transpileModule(
`class Fixture {${source.slice(begin, end)}${source.slice(
saveBegin,
saveEnd,
)}}; module.exports = Fixture;`,
{
compilerOptions: { target: ts.ScriptTarget.ES2020 },
},
).outputText;
const js = ts.transpileModule(`class Fixture {${source.slice(begin, end)}${source.slice(saveBegin, saveEnd)}}; module.exports = Fixture;`, {
compilerOptions: { target: ts.ScriptTarget.ES2020 },
}).outputText;
for (const mode of ['system', 'node']) {
const module = { exports: {} };
let file = '',
installs = 0;
new Function(
'module',
'process',
'execSync',
'ScheduleType',
'config',
'writeFileWithLock',
'CrontabModel',
js,
)(
module,
{ env: { QL_SCHEDULER: mode } },
() => {
installs++;
},
ScheduleType,
{ crontabFile: '/unused' },
async (_, data) => {
file = data;
},
{ nativeUpdate: async () => {} },
let file = '', installs = 0;
new Function('module', 'process', 'execSync', 'ScheduleType', 'config', 'writeFileWithLock', 'CrontabModel', js)(
module, { env: { QL_SCHEDULER: mode } }, () => { installs++; }, ScheduleType,
{ crontabFile: '/unused' }, async (_, data) => { file = data; }, { update: async () => {} },
);
const service = new module.exports();
service.makeCommand = (row) => `task ${row.id}.js`;
@@ -65,52 +40,20 @@ test('system and node modes route extended cron once and keep portable rules in
{ id: 'named', schedule: '0 0 * JAN MON' },
{ id: 'macro', schedule: '@daily' },
{ id: 'short', schedule: '*' },
{
id: 'extra',
schedule: '* * * * *',
extra_schedules: [{ schedule: '0 * * * *' }],
},
{ id: 'extra', schedule: '* * * * *', extra_schedules: [{ schedule: '0 * * * *' }] },
{ id: 'once', schedule: '@once' },
{ id: 'boot', schedule: '@boot' },
];
for (const row of rows) {
const special = ['once', 'boot'].includes(row.id);
const portable = ['plain', 'spaces'].includes(row.id);
assert.equal(
service.shouldUseCronClient(row),
!special && (mode === 'node' || !portable),
`${mode}: ${row.id}`,
);
assert.equal(service.shouldUseCronClient(row), !special && (mode === 'node' || !portable), `${mode}: ${row.id}`);
}
await service.setCrontab({ data: rows, total: rows.length });
assert.equal(installs, mode === 'system' ? 1 : 0);
for (const id of [
'numeric-step',
'sunday-seven',
'day-step',
'restricted-days',
'restricted-week',
'annually',
'midnight',
'minutely',
'seconds',
'question',
'last',
'nth',
'named',
'macro',
'short',
'extra',
'once',
'boot',
]) {
const line = file
.split('\n')
.find((line) => line.endsWith(`task ${id}.js`));
assert.ok(
line.startsWith('# '),
`${mode}: ${id} cannot also run from system crontab`,
);
for (const id of ['numeric-step', 'sunday-seven', 'day-step', 'restricted-days', 'restricted-week', 'annually', 'midnight', 'minutely', 'seconds', 'question', 'last', 'nth', 'named', 'macro', 'short', 'extra', 'once', 'boot']) {
const line = file.split('\n').find((line) => line.endsWith(`task ${id}.js`));
assert.ok(line.startsWith('# '), `${mode}: ${id} cannot also run from system crontab`);
}
}
});
+21 -12
View File
@@ -1,19 +1,28 @@
const { DatabaseClient } = require('../../back/data/databaseClient');
const { repository, initialize } = require('../helpers/repository-fixture.cjs');
const { orderBy } = require('../../back/data/queries');
require('ts-node/register/transpile-only');
const assert = require('node:assert/strict');
const { test } = require('node:test');
const { Sequelize, DataTypes } = require('sequelize');
const dayjs = require('dayjs');
const express = require('express');
test('today result lists include mixed and deleted tasks and exclude other dates', async (t) => {
const db = new DatabaseClient(':memory:');
const db = new Sequelize({
dialect: 'sqlite',
storage: ':memory:',
logging: false,
});
t.after(() => db.close());
const crons = repository(db, 'Crontab');
const stats = repository(db, 'CrontabStat');
const crons = db.define('Cron', {
name: DataTypes.STRING,
command: DataTypes.STRING,
});
const stats = db.define('Stat', {
ref_id: DataTypes.INTEGER,
date: DataTypes.STRING,
fail_count: DataTypes.INTEGER,
success_count: DataTypes.INTEGER,
});
const replacements = {
'../../back/data': { database: db },
'../../back/data/cron': { CrontabModel: crons },
'../../back/data/cronStats': { CrontabStatModel: stats },
'../../back/data/runningInstance': {},
@@ -33,14 +42,14 @@ test('today result lists include mixed and deleted tasks and exclude other dates
else delete require.cache[resolved];
});
}
await initialize(db);
await crons.insertMany([
await db.sync();
await crons.bulkCreate([
{ id: 1, name: 'Recovered task', command: 'task recovered.js' },
{ id: 2, name: '', command: 'task unnamed.js' },
{ id: 3, name: 'Successful task', command: 'task success.js' },
]);
const today = dayjs().format('YYYY-MM-DD');
await stats.insertMany([
await stats.bulkCreate([
{ ref_id: 1, date: today, fail_count: 2, success_count: 1 },
{ ref_id: 2, date: today, fail_count: 1, success_count: 0 },
{ ref_id: 3, date: today, fail_count: 0, success_count: 4 },
@@ -126,7 +135,7 @@ test('today result lists include mixed and deleted tasks and exclude other dates
},
],
});
await stats.nativeDelete({});
await stats.destroy({ where: {} });
await successHandler(
{},
{
@@ -151,7 +160,7 @@ test('today result lists include mixed and deleted tasks and exclude other dates
},
);
assert.deepEqual(response, { code: 200, data: [] });
await db.query('DROP TABLE CrontabStats');
await stats.drop();
let forwarded;
await handler(
{},
-222
View File
@@ -1,222 +0,0 @@
const test = require('node:test'),
assert = require('node:assert/strict');
const fs = require('node:fs/promises'),
path = require('node:path'),
os = require('node:os');
const {
Sequelize,
DataTypes,
Op,
QueryTypes,
where,
col,
} = require('sequelize');
const { DatabaseClient } = require('../../back/data/databaseClient');
const { Repository } = require('../../back/data/repository');
const { initialize } = require('../helpers/repository-fixture.cjs');
const names = {
App: 'Apps',
Auth: 'Auths',
Crontab: 'Crontabs',
CrontabStat: 'CrontabStats',
CrontabView: 'CrontabViews',
Dependence: 'Dependences',
Env: 'Envs',
RunningInstance: 'RunningInstances',
Subscription: 'Subscriptions',
};
async function legacy(file) {
const database = new Sequelize({
dialect: 'sqlite',
storage: file,
logging: false,
});
const models = {};
for (const [name, tableName] of Object.entries(names)) {
const columns = await database.query(`PRAGMA table_info("${tableName}")`, {
type: QueryTypes.SELECT,
});
const attrs = Object.fromEntries(
columns.map((c) => [
c.name,
{
type:
c.type === 'JSON'
? DataTypes.JSON
: c.type === 'DATETIME'
? DataTypes.DATE
: c.type === 'TINYINT(1)'
? DataTypes.BOOLEAN
: /INT|NUMBER/.test(c.type)
? DataTypes.NUMBER
: DataTypes.STRING,
...(c.pk ? { primaryKey: true, autoIncrement: true } : {}),
},
]),
);
models[name] = database.define(name, attrs, {
tableName,
freezeTableName: true,
});
}
return { database, models };
}
test('all nine legacy tables remain readable in both versions without changing schema, IDs, dates or .env', async (t) => {
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-orm-legacy-'));
t.after(() => fs.rm(dir, { recursive: true, force: true }));
const file = path.join(dir, 'database.sqlite'),
envFile = path.join(dir, '.env');
const environment = 'QL_DATA_DIR=/synthetic/old-data\nSYNTHETIC_ENV=keep\n';
await fs.writeFile(envFile, environment);
const seed = new Sequelize({
dialect: 'sqlite',
storage: file,
logging: false,
});
try {
const sql = await fs.readFile(
path.resolve('test/fixtures/sequelize-2.23-schema.sql'),
'utf8',
);
for (const statement of sql
.replace(/^--.*$/gm, '')
.split(';')
.map((x) => x.trim())
.filter(Boolean))
await seed.query(statement);
} finally {
await seed.close();
}
let old = await legacy(file);
const payloads = {
App: { name: 'legacy', scopes: ['envs'], client_id: 'synthetic-client' },
Auth: {
type: 'authConfig',
info: {
username: 'synthetic-user',
password: 'synthetic-hash',
tokens: { desktop: ['keep'] },
},
},
Crontab: {
name: 'legacy',
command: 'echo synthetic',
schedule: '0 0 * * *',
saved: true,
labels: ['中文'],
},
CrontabStat: { ref_id: 7, date: '2026-10-21', run_count: 2 },
CrontabView: {
name: 'legacy',
filters: [{ property: 'status', operation: 'In', value: [1] }],
},
Dependence: { name: 'legacy', type: 0, log: ['旧日志'] },
Env: {
name: 'legacy',
value: '旧值',
labels: ['中文'],
position: 4500000000000000,
},
RunningInstance: { cron_id: 7, started_at: 100, status: 2, exit_code: 143 },
Subscription: {
name: 'legacy',
alias: 'legacy',
pull_option: { username: 'synthetic', password: 'synthetic-only' },
},
};
const date = new Date('2020-01-02T03:04:05.006Z');
for (const [name, payload] of Object.entries(payloads))
await old.models[name].create({
id: 7,
...payload,
createdAt: date,
updatedAt: date,
});
const schema = await old.database.query(
"SELECT type,name,sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' ORDER BY type,name",
{ type: QueryTypes.SELECT },
);
await old.database.close();
const client = new DatabaseClient(file);
t.after(() => client.close());
await initialize(client);
for (const [name, payload] of Object.entries(payloads)) {
const repository = new Repository(name, () => client),
row = await repository.findOne({ id: 7 });
for (const [field, value] of Object.entries(payload))
assert.deepEqual(row[field], value, `${name}.${field}`);
assert.equal(row.createdAt.toISOString(), date.toISOString());
const added = await repository.insert({
...payload,
id: undefined,
...(payload.name ? { name: 'new' } : {}),
...(payload.alias ? { alias: 'new' } : {}),
...(payload.date ? { date: '2026-10-22' } : {}),
});
assert(added.id > 7, name);
assert(added.createdAt instanceof Date);
}
assert.deepEqual(
await client.query(
"SELECT type,name,sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' ORDER BY type,name",
),
schema,
);
await client.close();
old = await legacy(file);
try {
for (const name of Object.keys(payloads)) {
const rows = await old.models[name].findAll({ order: [['id', 'ASC']] });
assert.equal(rows.length, 2);
assert.equal(rows[0].createdAt.toISOString(), date.toISOString());
assert(rows[1].createdAt instanceof Date);
assert(!Number.isNaN(+rows[1].createdAt));
}
assert.equal(await fs.readFile(envFile, 'utf8'), environment);
} finally {
await old.database.close();
}
});
test('native filters retain legacy NULL, IN, nested groups and JSON text matching', async (t) => {
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-orm-filters-'));
t.after(() => fs.rm(dir, { recursive: true, force: true }));
const file = path.join(dir, 'database.sqlite');
const client = new DatabaseClient(file);
t.after(() => client.close());
await initialize(client);
const repository = new Repository('Env', () => client);
for (const row of [
{ name: 'one', value: 'a', status: 0, labels: ['中文', 'keep'] },
{ name: 'two', value: 'b', status: 1, labels: [] },
{ name: null, value: 'c', status: 0, labels: null },
])
await repository.insert(row);
const old = await legacy(file);
t.after(() => old.database.close());
const cases = [
[{ status: [0, 1] }, { status: [0, 1] }],
[{ name: { [Op.not]: null } }, { name: { $ne: null } }],
[
{ [Op.or]: [{ name: 'missing' }, {}] },
{ $or: [{ name: 'missing' }, {}] },
],
[{ [Op.and]: [{ status: 0 }, {}] }, { $and: [{ status: 0 }, {}] }],
[{ [Op.or]: [] }, { $or: [] }],
[
where(col('labels'), { [Op.like]: '%keep%' }),
{ labels: { $like: '%keep%' } },
],
[{ name: { [Op.notIn]: ['one'] } }, { name: { $nin: ['one'] } }],
];
for (const [before, after] of cases) {
const expected = (
await old.models.Env.findAll({ where: before, order: [['id', 'ASC']] })
).map((x) => x.id);
const actual = (
await repository.find(after, { orderBy: [{ id: 'asc' }] })
).map((x) => x.id);
assert.deepEqual(actual, expected, JSON.stringify(after));
}
});
@@ -1,77 +0,0 @@
const test = require('node:test'),
assert = require('node:assert/strict');
const fs = require('node:fs/promises'),
path = require('node:path'),
os = require('node:os'),
{ spawnSync } = require('node:child_process');
const { minimum, assertNodeRuntime } = require('../../shell/runtime-guard.cjs');
const { databaseStartupFailure } = require('../../back/data/databaseErrors');
const {
dependencyFailure,
} = require('../../cli/src/internal/maintenance/upgradePrestart.cjs');
test('runtime policy matches package metadata and accepts the exact Node minimum', () => {
assert.equal(minimum, require('../../package.json').qinglongRuntimeMinimum);
for (const version of ['22.17.0', '22.23.2', '24.18.1', '26.0.0'])
assert.doesNotThrow(() => assertNodeRuntime(version));
for (const version of ['20.20.2', '22.16.9', 'invalid'])
assert.throws(() => assertNodeRuntime(version), {
code: 'UNSUPPORTED_NODE',
});
});
test('database startup repairs only third-party dependency import failures', () => {
for (const message of [
"Cannot find module '@mikro-orm/sql'",
"Cannot find module '/candidate/node_modules/@mikro-orm/core/missing.js'",
"Cannot find module './missing.js'\nRequire stack:\n- /candidate/node_modules/@mikro-orm/sql/index.js",
]) {
const result = databaseStartupFailure(
Object.assign(new Error(message), { code: 'MODULE_NOT_FOUND' }),
);
assert.equal(dependencyFailure(result), true);
}
for (const value of [
new Error('heap out of memory'),
undefined,
Object.assign(
new Error(
"Cannot find module './schema'\nRequire stack:\n- /candidate/static/build/data/databaseWorker.js",
),
{ code: 'MODULE_NOT_FOUND' },
),
Object.assign(new Error("Cannot find module 'node:sqlite'"), {
code: 'MODULE_NOT_FOUND',
}),
])
assert.equal(dependencyFailure(databaseStartupFailure(value)), false);
});
test('Node 20 rejects backend initialization before changing its old database or .env', async (t) => {
const node20 = process.env.QL_TEST_NODE20;
if (!node20) {
t.skip('Set QL_TEST_NODE20 to verify an existing Node 20 runtime');
return;
}
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-runtime-floor-'));
t.after(() => fs.rm(dir, { recursive: true, force: true }));
const data = path.join(dir, 'data'),
db = path.join(data, 'db/database.sqlite'),
env = path.join(dir, '.env');
await fs.mkdir(path.dirname(db), { recursive: true });
await fs.writeFile(db, 'old synthetic database marker');
await fs.writeFile(env, 'SYNTHETIC_ENV=keep\n');
const result = spawnSync(
node20,
[path.resolve('static/build/bootstrap/database.js')],
{
env: { ...process.env, QL_DIR: dir, QL_DATA_DIR: data },
encoding: 'utf8',
timeout: 10000,
},
);
assert.equal(result.status, 1, result.stderr);
assert.match(result.stderr, /Node >=22\.17/);
assert.equal(await fs.readFile(db, 'utf8'), 'old synthetic database marker');
assert.equal(await fs.readFile(env, 'utf8'), 'SYNTHETIC_ENV=keep\n');
});
-515
View File
@@ -1,515 +0,0 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs/promises');
const path = require('node:path');
const os = require('node:os');
const { DatabaseClient } = require('../../back/data/databaseClient');
const { Repository } = require('../../back/data/repository');
const { initialSchema } = require('../../back/data/schema');
const { migrateSchema } = require('../../back/shared/schemaMigrations');
test('requests buffered before bootstrap and during owner-monitor startup retain their order', async (t) => {
const { fork } = require('node:child_process');
const directory = await fs.mkdtemp(
path.join(os.tmpdir(), 'ql-db-bootstrap-queue-'),
);
const script = path.join(directory, 'delayed.ts');
await fs.writeFile(
script,
`setTimeout(()=>require(${JSON.stringify(
require.resolve('../../back/data/databaseProcess'),
)}),100)`,
);
const child = fork(script, [], {
execArgv: [
'--require',
require.resolve('ts-node/register/transpile-only'),
'--max-old-space-size=128',
],
env: {
...process.env,
TS_NODE_PROJECT: path.resolve('back/data/tsconfig.worker.json'),
},
stdio: ['ignore', 'ignore', 'pipe', 'ipc'],
});
const stopped = new Promise((resolve) => child.once('exit', resolve));
let stderr = '';
child.stderr.on('data', (chunk) => {
stderr += chunk;
});
t.after(async () => {
child.kill('SIGKILL');
await stopped;
await fs.rm(directory, { recursive: true, force: true });
});
const responses = new Promise((resolve, reject) => {
const replies = [];
const timer = setTimeout(
() => reject(new Error('Buffered startup requests were lost')),
30000,
);
child.on('message', (reply) => {
if (reply.error) {
clearTimeout(timer);
reject(new Error(reply.error.message));
return;
}
replies.push(reply);
if (replies.length === 2) {
clearTimeout(timer);
resolve(replies);
}
});
child.once('exit', () => {
clearTimeout(timer);
reject(new Error('Database exited during bootstrap: ' + stderr));
});
child.once('error', reject);
});
child.send({ storage: ':memory:', ownerPid: process.pid });
child.send({ id: 1, operation: 'query', args: ['SELECT 42 AS value'] });
child.send({ id: 2, operation: 'query', args: ['SELECT 43 AS value'] });
assert.deepEqual(await responses, [
{ id: 1, value: [{ value: 42 }] },
{ id: 2, value: [{ value: 43 }] },
]);
});
test('killing the owner during synchronous SQL releases the write lock and rolls back before restart', async (t) => {
const { fork } = require('node:child_process');
const { DatabaseSync } = require('node:sqlite');
const directory = await fs.mkdtemp(
path.join(os.tmpdir(), 'ql-db-owner-exit-'),
);
const file = path.join(directory, 'data.sqlite');
const script = path.join(directory, 'owner.cjs');
await fs.writeFile(
script,
`
const {DatabaseClient}=require(${JSON.stringify(
require.resolve('../../back/data/databaseClient'),
)});
const db=new DatabaseClient(process.argv[2]);
(async()=>{
await db.query('CREATE TABLE probe(value INTEGER)');
await db.query('INSERT INTO probe VALUES(1)');
await db.transaction(async()=>{
await db.query('UPDATE probe SET value=2');
const running=db.query('WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<200000000000) SELECT SUM(x) FROM n');
setTimeout(()=>process.send({databasePid:db.worker.pid}),300);
await running;
});
})().catch(error=>{console.error(error);process.exitCode=1});
`,
);
const owner = fork(script, [file], {
execArgv: ['--require', require.resolve('ts-node/register/transpile-only')],
env: {
...process.env,
TS_NODE_PROJECT: path.resolve('back/tsconfig.json'),
},
stdio: ['ignore', 'ignore', 'pipe', 'ipc'],
});
const exited = new Promise((resolve) => owner.once('exit', resolve));
let databasePid,
probe,
stderr = '',
unlocked = false;
owner.stderr.on('data', (chunk) => {
stderr += chunk;
});
t.after(async () => {
probe?.close();
owner.kill('SIGKILL');
await exited;
if (databasePid && !unlocked) {
try {
process.kill(databasePid, 'SIGKILL');
} catch (error) {
if (error.code !== 'ESRCH') throw error;
}
}
await fs.rm(directory, { recursive: true, force: true });
});
databasePid = await new Promise((resolve, reject) => {
const timer = setTimeout(
() => reject(new Error('Owner startup timed out: ' + stderr)),
30000,
);
owner.once('message', (message) => {
clearTimeout(timer);
resolve(message.databasePid);
});
owner.once('exit', () => {
clearTimeout(timer);
reject(new Error('Owner exited before SQL: ' + stderr));
});
owner.once('error', reject);
});
probe = new DatabaseSync(file);
probe.exec('PRAGMA busy_timeout=50');
assert.throws(() => probe.exec('BEGIN IMMEDIATE'), /locked/);
owner.kill('SIGKILL');
await exited;
const deadline = Date.now() + 5000;
do {
try {
probe.exec('BEGIN IMMEDIATE;ROLLBACK');
unlocked = true;
break;
} catch (error) {
if (!/locked/.test(error.message)) throw error;
}
await new Promise((resolve) => setTimeout(resolve, 50));
} while (Date.now() < deadline);
assert.equal(
unlocked,
true,
'The orphan database process still blocks the old service',
);
assert.equal(probe.prepare('SELECT value FROM probe').get().value, 1);
});
async function fixture(t, file = ':memory:') {
const db = new DatabaseClient(file);
t.after(() => db.close());
await db.transaction(async () => {
for (const sql of initialSchema) await db.query(sql);
await migrateSchema(db);
});
return {
db,
env: new Repository('Env', () => db),
auth: new Repository('Auth', () => db),
};
}
test('direct ts-node CLI startup registers the isolated database worker compiler', async () => {
const execute = require('node:util').promisify(
require('node:child_process').execFile,
);
const result = await execute(
process.execPath,
[
path.resolve('node_modules/ts-node/dist/bin.js'),
'--transpile-only',
'--project',
'back/tsconfig.json',
'-e',
`const {DatabaseClient}=require('./back/data/databaseClient');const db=new DatabaseClient(':memory:');db.query('SELECT 42 AS value').then(async rows=>{console.log(JSON.stringify(rows));await db.close()}).catch(async error=>{console.error(error);process.exitCode=1;await db.close().catch(()=>{})})`,
],
{ cwd: path.resolve('.'), timeout: 30000 },
);
assert.deepEqual(JSON.parse(result.stdout), [{ value: 42 }]);
});
test('worker round-trips legacy field names, JSON, dates, null and precise positions', async (t) => {
const { db, env } = await fixture(t);
const row = await env.insert({
name: '中文',
value: 'keep',
position: 4500000000000000,
labels: ['old', '中文'],
remarks: null,
});
assert.deepEqual((await env.findOne({ id: row.id })).labels, row.labels);
assert.equal(row.position, 4500000000000000);
assert.equal(row.remarks, null);
assert(row.createdAt instanceof Date);
const stored = (
await db.query('SELECT typeof(createdAt) AS type,createdAt FROM Envs')
)[0];
assert.equal(stored.type, 'text');
assert.match(
stored.createdAt,
/^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\.\d{3} \+00:00$/,
);
assert.equal(
(await db.query('SELECT count(*) AS n FROM SchemaMigrations'))[0].n,
15,
);
});
test('JSON LIKE uses serialized text without quoting the pattern', async (t) => {
const { env } = await fixture(t);
await env.insert({ name: 'a', value: 'b', labels: ['中文', 'old'] });
assert.equal((await env.find({ labels: { $like: '%old%' } })).length, 1);
assert.equal(
(await env.find({ $not: { labels: { $like: '%old%' } } })).length,
0,
);
});
test('transaction commit, failure and nested savepoints preserve ownership', async (t) => {
const { db, env } = await fixture(t);
await env.insert({ name: 'a', value: 'old' });
await db.transaction(async () => {
await env.nativeUpdate({ name: 'a' }, { value: 'committed' });
await assert.rejects(
db.transaction(async () => {
await env.nativeUpdate({ name: 'a' }, { value: 'rollback' });
throw Error('injected');
}),
/injected/,
);
assert.equal((await env.findOne({ name: 'a' })).value, 'committed');
});
await assert.rejects(
db.transaction(async () => {
await env.nativeUpdate({ name: 'a' }, { value: 'rollback all' });
throw Error('outer failure');
}),
/outer failure/,
);
assert.equal((await env.findOne({ name: 'a' })).value, 'committed');
});
test('unrelated reads wait for rollback and cannot see uncommitted data', async (t) => {
const { db, env } = await fixture(t);
await env.insert({ name: 'a', value: 'old' });
let release, writing;
const started = new Promise((r) => {
writing = r;
});
const gate = new Promise((r) => {
release = r;
});
const transaction = db.transaction(async () => {
await env.nativeUpdate({ name: 'a' }, { value: 'uncommitted' });
writing();
await gate;
throw Error('rollback');
});
const rejected = assert.rejects(transaction, /rollback/);
await started;
const read = env.findOne({ name: 'a' });
release();
await rejected;
assert.equal((await read).value, 'old');
});
test('expired async transaction scopes cannot write after commit', async (t) => {
const { db, env } = await fixture(t);
// Capture an asynchronous resource inside the scope, then release it afterwards.
let release, result;
const gate = new Promise((r) => {
release = r;
});
await db.transaction(async () => {
result = gate.then(() => env.insert({ name: 'late' }));
});
const rejected = assert.rejects(result, { code: 'TRANSACTION_SCOPE' });
release();
await rejected;
assert.equal(await env.count(), 0);
});
test('conditional updates keep compare-and-swap counts and do not clear undefined data', async (t) => {
const { env } = await fixture(t);
const row = await env.insert({ name: 'a', value: 'old' });
assert.equal(
await env.nativeUpdate({ id: row.id, value: 'newer' }, { value: 'bad' }),
0,
);
assert.equal(
await env.nativeUpdate(
{ id: row.id, value: 'old' },
{ value: 'new', remarks: undefined },
),
1,
);
assert.equal((await env.findOne({ id: row.id })).value, 'new');
await assert.rejects(env.nativeUpdate({ id: undefined }, { value: 'bad' }));
});
test('missing predicates cannot become unrestricted writes or disappear inside an IN filter', async (t) => {
const { env } = await fixture(t);
const row = await env.insert({ name: 'retained', value: 'old' });
for (const where of [undefined, null, { id: { $in: [row.id, undefined] } }]) {
await assert.rejects(env.nativeUpdate(where, { value: 'must not write' }));
await assert.rejects(env.nativeDelete(where));
}
assert.equal(await env.count(), 1);
assert.equal((await env.findOne({ id: row.id })).value, 'old');
});
test('worker failures reject writes, never replay them, and roll back SQLite', async (t) => {
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-db-worker-'));
t.after(() => fs.rm(dir, { recursive: true, force: true }));
const file = path.join(dir, 'database.sqlite');
const { db, env } = await fixture(t, file);
await env.insert({ name: 'a', value: 'old' });
await assert.rejects(
db.transaction(async () => {
await env.nativeUpdate({ name: 'a' }, { value: 'uncommitted' });
db.worker.kill('SIGKILL');
await db.stopped;
await env.count();
}),
);
assert.equal(db.healthy, false);
await assert.rejects(env.insert({ name: 'replayed' }));
const reopened = new DatabaseClient(file);
t.after(() => reopened.close());
const records = new Repository('Env', () => reopened);
assert.equal((await records.findOne({ name: 'a' })).value, 'old');
assert.equal(await records.count(), 1);
});
test('a long database query leaves main-thread timers responsive', async (t) => {
const { db } = await fixture(t);
const started = performance.now();
const expensive = db.query(
'WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<2000000) SELECT SUM(x) AS n FROM n',
);
const timer = await new Promise((r) =>
setTimeout(() => r(performance.now() - started), 10),
);
assert(timer < 150, `main-thread timer delayed ${timer}ms`);
assert.equal((await expensive)[0].n, 2000001000000);
});
test('existing Keyv authentication cache stays byte-compatible with legacy readers', async (t) => {
const { DatabaseSync } = require('node:sqlite');
const Keyv = require('keyv').default;
const { CacheStore } = require('../../back/data/cacheStore');
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-cache-compat-'));
t.after(() => fs.rm(dir, { recursive: true, force: true }));
const cacheFile = path.join(dir, 'keyv.sqlite');
const old = new DatabaseSync(cacheFile);
old.exec('CREATE TABLE keyv(key VARCHAR(255) PRIMARY KEY,value TEXT)');
const value = {
username: 'synthetic-user',
password: 'synthetic-hash',
tokens: { desktop: ['keep'] },
};
old
.prepare('INSERT INTO keyv VALUES(?,?)')
.run('keyv:authInfo', JSON.stringify({ value, expires: null }));
old.close();
const db = new DatabaseClient(':memory:', cacheFile);
t.after(() => db.close());
const cache = new Keyv({ store: new CacheStore(() => db) });
assert.deepEqual(await cache.get('authInfo'), value);
const next = { ...value, username: 'new-synthetic-user' };
await cache.set('authInfo', next);
const legacy = new DatabaseSync(cacheFile, { readOnly: true });
try {
assert.deepEqual(
JSON.parse(
legacy
.prepare('SELECT value FROM keyv WHERE key=?')
.get('keyv:authInfo').value,
).value,
next,
);
} finally {
legacy.close();
}
});
test('optional DTO fields and nested JSON values may be undefined on insert', async (t) => {
const { env, auth } = await fixture(t);
const created = await env.insert({
name: 'optional',
value: 'keep',
id: undefined,
remarks: undefined,
});
assert.equal(created.value, 'keep');
const row = await auth.insert({
type: 'notifyLog',
info: { title: 'keep', taskId: undefined },
});
assert.deepEqual(row.info, { title: 'keep' });
});
test('legacy null pin flags sort like zero flags before comparing positions', async (t) => {
const { env } = await fixture(t);
await env.insert({ name: 'null', position: 20, isPinned: null });
await env.insert({ name: 'zero', position: 10, isPinned: 0 });
await env.insert({ name: 'pinned', position: 1, isPinned: 1 });
assert.deepEqual(
(
await env.find(
{},
{ orderBy: [{ isPinned: 'desc' }, { position: 'desc' }] },
)
).map((x) => x.name),
['pinned', 'null', 'zero'],
);
});
test('a stalled transaction callback times out and rolls back before unrelated reads', async (t) => {
const { db, env } = await fixture(t);
db.timeoutMs = 100;
await env.insert({ name: 'a', value: 'old' });
let release;
const gate = new Promise((r) => {
release = r;
});
await assert.rejects(
db.transaction(async () => {
await env.nativeUpdate({ name: 'a' }, { value: 'uncommitted' });
await gate;
await env.insert({ name: 'too-late' });
}),
{ code: 'DATABASE_TRANSACTION_TIMEOUT' },
);
release();
assert.equal((await env.findOne({ name: 'a' })).value, 'old');
assert.equal(await env.count(), 1);
assert.equal(db.healthy, true);
});
test('a stuck database worker is terminated, rejects its queue and never replays writes', async (t) => {
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-db-timeout-'));
t.after(() => fs.rm(dir, { recursive: true, force: true }));
const file = path.join(dir, 'database.sqlite');
const { db, env } = await fixture(t, file);
await env.insert({ name: 'a', value: 'old' });
db.timeoutMs = 100;
await assert.rejects(
db.transaction(async () => {
await env.nativeUpdate({ name: 'a' }, { value: 'uncommitted' });
await db.query(
'WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<200000000) SELECT SUM(x) FROM n',
);
}),
(e) =>
['DATABASE_TRANSACTION_TIMEOUT', 'DATABASE_WORKER_TIMEOUT'].includes(
e.code,
),
);
assert.equal(db.healthy, false);
await assert.rejects(env.insert({ name: 'replayed' }));
await db.close();
const next = new DatabaseClient(file);
t.after(() => next.close());
const records = new Repository('Env', () => next);
assert.equal((await records.findOne({ name: 'a' })).value, 'old');
assert.equal(await records.count(), 1);
});
test('the database queue is bounded while a transaction owns its connection', async (t) => {
const { db, env } = await fixture(t);
db.queueLimit = 2;
db.timeoutMs = 3000;
let entered, release;
const started = new Promise((r) => {
entered = r;
});
const gate = new Promise((r) => {
release = r;
});
const transaction = db.transaction(async () => {
entered();
await gate;
});
await started;
const waiting = env.count();
await assert.rejects(env.count(), { code: 'DATABASE_BUSY' });
release();
await transaction;
assert.equal(await waiting, 0);
});
+9 -2
View File
@@ -1,5 +1,6 @@
const assert = require('node:assert/strict');
const test = require('node:test');
test('dependency listing marks cache entries missing from disk for reinstall', async (t) => {
const moduleStubs = new Map();
const stubModule = (modulePath, exports) => {
@@ -14,6 +15,7 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
paths: [],
};
};
const DependenceStatus = {
installing: 0,
installed: 1,
@@ -26,7 +28,7 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
];
const updates = [];
const DependenceModel = {
find: async (where, read = {}) => {
findAll: async ({ where }) => {
if (where.status === DependenceStatus.installed) {
return docs;
}
@@ -37,10 +39,11 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
: doc.status,
}));
},
nativeUpdate: async (where, { status }) => {
update: async ({ status }, { where }) => {
updates.push({ status, ids: where.id });
},
};
stubModule('../../back/config', {
__esModule: true,
default: {},
@@ -80,6 +83,7 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
__esModule: true,
default: class SockService {},
});
const servicePath = require.resolve('../../back/services/dependence');
const originalService = require.cache[servicePath];
delete require.cache[servicePath];
@@ -97,6 +101,7 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
}
}
});
const DependenceService = require('../../back/services/dependence').default;
const service = new DependenceService({}, {});
const result = await service.dependencies({
@@ -104,6 +109,7 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
type: 'nodejs',
status: '',
});
assert.deepEqual(updates, [
{ status: DependenceStatus.installFailed, ids: [1] },
]);
@@ -114,6 +120,7 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
{ id: 2, status: DependenceStatus.installed },
],
);
await service.dependencies({ searchValue: '', type: 'linux', status: '' });
assert.equal(updates.length, 1);
});
+7 -5
View File
@@ -1,6 +1,7 @@
const assert = require('node:assert/strict');
const test = require('node:test');
const express = require('express');
function mockModule(modulePath, exports) {
const filename = require.resolve(modulePath);
require.cache[filename] = {
@@ -12,6 +13,7 @@ function mockModule(modulePath, exports) {
paths: [],
};
}
mockModule('../../back/config', {
__esModule: true,
default: {
@@ -47,17 +49,15 @@ for (const service of ['config', 'script', 'log']) {
}
mockModule('../../back/data/runningInstance', {
InstanceStatus: { running: 'running' },
RunningInstanceModel: {
findOne: async (where, read = {}) => {
return null;
},
},
RunningInstanceModel: { findOne: async () => null },
});
const deprecatedRoutes = [
['config', '/configs/detail'],
['script', '/scripts/detail'],
['log', '/logs/detail'],
];
for (const [moduleName, replacement] of deprecatedRoutes) {
test(`${moduleName} filename route points callers to its detail API`, () => {
const app = express.Router();
@@ -68,10 +68,12 @@ for (const [moduleName, replacement] of deprecatedRoutes) {
);
const handler = deprecatedRoute.route.stack.at(-1).handle;
let responseBody;
handler(
{ params: { file: 'Example.js' }, query: {} },
{ send: (body) => (responseBody = body) },
);
assert.deepEqual(responseBody, {
code: 410,
message: `接口已下线,请使用 ${replacement} 接口`,
+16 -20
View File
@@ -5,6 +5,7 @@ const os = require('node:os');
const path = require('node:path');
const { execFileSync } = require('node:child_process');
const load = require('../helpers/load-security-module.cjs');
const types = { nodejs: 0, python3: 1, linux: 2 };
const statuses = { installed: 1, installFailed: 2, queued: 6, cancelled: 7 };
const util = load('back/config/util.ts', {
@@ -14,6 +15,7 @@ const util = load('back/config/util.ts', {
'../shared/utils': {},
'../data/dependence': { DependenceTypes: types },
});
function serviceFor(docs) {
const probes = [],
removed = [],
@@ -26,21 +28,18 @@ function serviceFor(docs) {
DependenceStatus: statuses,
versionDependenceCommandTypes: { 0: '@', 1: '==', 2: '=' },
DependenceModel: {
find: async (where, read = {}) => {
return docs.filter(
findAll: async ({ where }) =>
docs.filter(
(d) =>
(where.status === undefined || d.status === where.status) &&
(!where.id || where.id.includes(d.id)),
);
},
nativeUpdate: async (where, values) => {
),
update: async (values, { where }) => {
docs
.filter((d) => where.id.includes(d.id))
.forEach((d) => Object.assign(d, values));
},
nativeDelete: async (where, read = {}) => {
return removed.push(...where.id);
},
destroy: async ({ where }) => removed.push(...where.id),
},
},
'../config/util': {
@@ -63,6 +62,7 @@ function serviceFor(docs) {
};
return { service, probes, removed, uninstalls };
}
test('listing preserves Python source results and still detects missing named packages', async () => {
const docs = [
' requirements with spaces/requirements.txt ',
@@ -88,6 +88,7 @@ test('listing preserves Python source results and still detects missing named pa
// Source installs must continue to execute pip rather than treating the source as a package name.
assert.equal(util.getGetCommand(1, docs[0].name), 'echo ""');
});
test('deletion cleans previously misclassified Python sources and only queues intended records', async () => {
const docs = [
{ id: 1, name: 'requirements.txt', type: 1, status: 2 },
@@ -105,6 +106,7 @@ test('deletion cleans previously misclassified Python sources and only queues in
]);
assert.equal(docs[2].status, 2);
});
test('Alpine container mirror command runs without sudo and writes valid repositories', async (t) => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-alpine-mirror-'));
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
@@ -149,6 +151,7 @@ test('Alpine container mirror command runs without sudo and writes valid reposit
),
);
});
test('force deletion removes a source record even when its requirements file is gone', async () => {
const removed = [],
updates = [];
@@ -164,12 +167,8 @@ test('force deletion removes a source record even when its requirements file is
DependenceTypes: types,
DependenceStatus: { ...statuses, removeFailed: 5 },
DependenceModel: {
nativeDelete: async (where, read = {}) => {
return removed.push(...where.id);
},
nativeUpdate: async (where, values) => {
return updates.push(values);
},
destroy: async ({ where }) => removed.push(...where.id),
update: async (values) => updates.push(values),
},
},
'../shared/pLimit': {
@@ -187,6 +186,7 @@ test('force deletion removes a source record even when its requirements file is
await service.installOrUninstallDependency(dep, false, true);
assert.deepEqual(removed, [1]);
});
test('Python extras run pip even when the base package version is already installed', async (t) => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-extra-install-'));
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
@@ -217,12 +217,8 @@ test('Python extras run pip even when the base package version is already instal
DependenceStatus: { ...statuses, installing: 0, cancelled: 7 },
versionDependenceCommandTypes: { 0: '@', 1: '==', 2: '=' },
DependenceModel: {
find: async (where, read = {}) => {
return [dep];
},
nativeUpdate: async (where, values) => {
return Object.assign(dep, values);
},
findAll: async () => [dep],
update: async (values) => Object.assign(dep, values),
},
},
'../shared/pLimit': {
+10 -5
View File
@@ -9,8 +9,10 @@ const path = require('node:path');
const load = require('../helpers/load-security-module.cjs');
const { observeChildProcess } = require('../../back/shared/childProcess');
const { LogStreamManager } = require('../../back/shared/logStreamManager');
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
const logger = { info() {}, error() {} };
test(
'spawn failure settles without exit, and the next scheduled run can proceed',
{ timeout: 3000 },
@@ -34,11 +36,7 @@ test(
},
'../loaders/logger': logger,
'../data/cron': {
CrontabModel: {
findOne: async (where, read = {}) => {
return { isDisabled: 0 };
},
},
CrontabModel: { findOne: async () => ({ isDisabled: 0 }) },
CrontabStatus: {},
},
'../data/runningInstance': {
@@ -53,6 +51,7 @@ test(
assert.equal(releases, 2);
},
);
test(
'completion waits for slow log consumers and preserves the final output',
{ timeout: 5000 },
@@ -80,6 +79,7 @@ test(
assert.match(chunks.join(''), /错误/);
},
);
test('exit is not completion, and UTF-8 split across writes remains intact', async () => {
const child = new EventEmitter();
child.stdout = new PassThrough();
@@ -105,6 +105,7 @@ test('exit is not completion, and UTF-8 split across writes remains intact', asy
await observed.completed;
assert.equal(chunks.join(''), '末尾中文');
});
test(
'failed log sink drains large output instead of blocking the child',
{ timeout: 5000 },
@@ -123,6 +124,7 @@ test(
assert.equal(result.error.message, 'ENOSPC');
},
);
function scheduleFixture() {
let active = 0;
const limit = async (_params, fn) => {
@@ -148,6 +150,7 @@ function scheduleFixture() {
}).default;
return { service: new Schedule(logger), active: () => active };
}
test(
'PID response keeps queue capacity occupied until cleanup completes',
{ timeout: 4000 },
@@ -175,6 +178,7 @@ test(
assert.equal(fixture.active(), 0);
},
);
test(
'before/error/end callback failures settle and release capacity',
{ timeout: 3000 },
@@ -203,6 +207,7 @@ test(
assert.equal(fixture.active(), 0);
},
);
test(
'concurrent writes and closes preserve all bytes, and failed files can be closed',
{ timeout: 5000 },
+5 -5
View File
@@ -5,6 +5,7 @@ const os = require('node:os');
const path = require('node:path');
const express = require('express');
const load = require('../helpers/load-security-module.cjs');
function fixture(t, customData = false) {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-access-scope-'));
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
@@ -45,15 +46,12 @@ function fixture(t, customData = false) {
'../config/const': {},
'../data/runningInstance': {
InstanceStatus: { running: 'running' },
RunningInstanceModel: {
findOne: async (where, read = {}) => {
return null;
},
},
RunningInstanceModel: { findOne: async () => null },
},
};
return { config, mocks, secret };
}
function api(name, service, mocks) {
mocks.typedi = { Container: { get: () => service } };
const app = express.Router();
@@ -81,6 +79,7 @@ function api(name, service, mocks) {
return result;
};
}
for (const customData of [false, true]) {
test(`config APIs reject script compatibility paths (custom=${customData})`, async (t) => {
const { config, mocks, secret } = fixture(t, customData);
@@ -145,6 +144,7 @@ for (const customData of [false, true]) {
assert.equal(fs.readFileSync(secret, 'utf8'), 'PANEL-SECRET');
});
}
test('log APIs allow configuration-like names while retaining directory boundaries', async (t) => {
const { config, mocks, secret } = fixture(t);
const Log = load(path.resolve('back/services/log.ts'), mocks).default;
+20 -5
View File
@@ -2,7 +2,7 @@ const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const ts = require('typescript');
const { Sequelize, DataTypes, Model } = require('sequelize');
const load = require('../helpers/load-security-module.cjs');
const { CronResponse } = load('back/protos/api.ts');
const source = fs.readFileSync('back/schedule/api.ts', 'utf8');
@@ -11,16 +11,31 @@ const end = source.indexOf('export const getCronDetail', start);
const code = ts.transpileModule(source.slice(start, end), {
compilerOptions: { target: ts.ScriptTarget.ES2020 },
}).outputText;
const normalize = new Function(`${code}\nreturn normalizeCronData;`)();
const normalize = new Function('Model', `${code}\nreturn normalizeCronData;`)(
Model,
);
test('gRPC cron response preserves plain repository fields through protobuf serialization', async (t) => {
const model = {
test('gRPC cron response preserves Sequelize model fields through protobuf serialization', async (t) => {
const db = new Sequelize({
dialect: 'sqlite',
storage: ':memory:',
logging: false,
});
t.after(() => db.close());
const Cron = db.define('Cron', {
name: DataTypes.STRING,
command: DataTypes.STRING,
schedule: DataTypes.STRING,
labels: DataTypes.JSON,
extra_schedules: DataTypes.JSON,
});
const model = Cron.build({
id: 7,
name: 'smoke',
command: 'true',
schedule: '* * * * *',
labels: ['test'],
};
});
const decoded = CronResponse.decode(
CronResponse.encode({ code: 200, data: normalize(model) }).finish(),
);

Some files were not shown because too many files have changed in this diff Show More