Compare commits

..
Author SHA1 Message Date
whyour 7603702eca fix: reduce task startup overhead and lazy-load notifications 2026-09-28 09:42:54 +08:00
338 changed files with 4346 additions and 34940 deletions
-3
View File
@@ -21,6 +21,3 @@ JWT_EXPIRES_IN=
QINIU_AK=
QINIU_SK=
QINIU_SCOPE=
# Python 虚拟环境路径,留空则使用系统 Python
# PYTHON_VENV_DIR=./.venv
+89 -312
View File
@@ -2,56 +2,23 @@ name: Build And Push Docker Image
on:
push:
paths:
- "**"
- "!*.md"
# CLI is verified separately; its shared Bash upgrade engine ships in images.
- "!cli/**"
- "cli/src/internal/maintenance/upgradeArtifacts.cjs"
- "cli/src/internal/maintenance/platformBundle.cjs"
- "cli/src/internal/maintenance/upgradePrestart.cjs"
- "cli/src/internal/maintenance/panelUpgrade.cjs"
- "cli/src/internal/maintenance/nodeRuntime.cjs"
- "!.github/workflows/cli-package.yml"
paths-ignore:
- "*.md"
branches:
- "master"
- "develop"
tags:
- "v*"
workflow_dispatch:
inputs:
static_sync_only:
description: Synchronize an existing static snapshot to GitLab and verify Gitee
type: boolean
default: false
static_ref:
description: Static branch or version name, for example v2.22.0
type: string
default: ''
static_ref_type:
description: Synchronize a branch, or a version branch and Git tag
type: choice
options: [branch, tag]
default: branch
permissions:
contents: read
# Keep master image tags and their bundle indexes in one serialized release.
# A rerun may intentionally replace a version, including from the same commit.
# Develop image builds and their shared bundle index serialize separately.
# Git-tag builds and static synchronization remain independent.
concurrency:
group: version-images-${{ github.ref == 'refs/heads/master' && 'master' || (github.ref == 'refs/heads/develop' && !inputs.static_sync_only && 'develop') || format('{0}-{1}', github.run_id, github.run_attempt) }}
cancel-in-progress: false
jobs:
validate:
if: ${{ !inputs.static_sync_only }}
uses: ./.github/workflows/validate.yml
code_gitlab:
if: ${{ !inputs.static_sync_only }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
@@ -68,8 +35,37 @@ jobs:
git push --force --all gitlab 2>&1 || echo "::warning::GitLab push failed"
git push --force --tags gitlab 2>&1 || echo "::warning::GitLab tags push failed"
code_gitee:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Push to Gitee
env:
GITEE_TOKEN: ${{ secrets.GITEE_TOKEN }}
run: |
set +e
mkdir -p ~/.ssh
printf '%s\n' "${{ secrets.GITLAB_SSH_PK }}" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -T 10 gitee.com >> ~/.ssh/known_hosts 2>/dev/null
git remote add gitee git@gitee.com:whyour/qinglong.git 2>/dev/null
if git push --force --all gitee 2>&1; then
echo "::notice::Gitee push --all succeeded"
else
echo "::warning::Push failed, trying to create repo via API..."
curl -sS --connect-timeout 30 --max-time 60 \
-X POST "https://gitee.com/api/v5/user/repos" \
-H "Content-Type: application/json" \
-d '{"name":"qinglong","private":"false"}' \
"?access_token=$GITEE_TOKEN" 2>/dev/null
git push --force --all gitee 2>&1 && echo "::notice::Gitee push succeeded after repo creation" || echo "::warning::Gitee push failed after retry"
fi
git push --force --tags gitee 2>&1 || echo "::warning::Gitee tags push failed"
build-static:
if: ${{ !inputs.static_sync_only }}
needs: validate
runs-on: ubuntu-latest
steps:
@@ -80,7 +76,7 @@ jobs:
- uses: actions/setup-node@v6
with:
node-version-file: ".nvmrc"
node-version: "20"
cache: "pnpm"
cache-dependency-path: pnpm-lock.yaml
@@ -91,9 +87,6 @@ jobs:
pnpm build:back
pnpm build:info
- name: Pack the npm panel archive from the verified build
run: node scripts/verify-panel-package.cjs pack "$RUNNER_TEMP/npm-panel"
- uses: actions/upload-artifact@v6
with:
name: qinglong-static-${{ github.sha }}
@@ -101,137 +94,70 @@ jobs:
if-no-files-found: error
include-hidden-files: true
- uses: actions/upload-artifact@v6
with:
name: qinglong-npm-${{ github.sha }}
path: ${{ runner.temp }}/npm-panel/
if-no-files-found: error
retention-days: 14
- name: copy to static repo
env:
GITHUB_REPO: github.com/${{ github.repository_owner }}/qinglong-static
GITHUB_BRANCH: ${{ github.ref_name }}
GITHUB_REF_TYPE: ${{ github.ref_type }}
STATIC_TOKEN: ${{ secrets.API_TOKEN }}
run: |
: "${STATIC_TOKEN:?Missing static repository token}"
bash scripts/publish-static.sh "https://x-access-token:${STATIC_TOKEN}@${GITHUB_REPO}.git" "$GITHUB_BRANCH" "$GITHUB_REF_TYPE"
- name: publish verified upgrade download sizes
env:
GITHUB_REPO: github.com/${{ github.repository_owner }}/qinglong-static
GITHUB_BRANCH: ${{ github.ref_name }}
STATIC_TOKEN: ${{ secrets.API_TOKEN }}
run: |
: "${STATIC_TOKEN:?Missing static repository token}"
node scripts/publish-upgrade-info.cjs "https://x-access-token:${STATIC_TOKEN}@${GITHUB_REPO}.git" "$GITHUB_BRANCH" "$GITHUB_SHA"
mkdir -p tmp
cd ./tmp
cp -rf ../static/* ./
git init -b ${GITHUB_BRANCH} && git add .
git config --local user.name 'github-actions[bot]'
git config --local user.email 'github-actions[bot]@users.noreply.github.com'
git commit --allow-empty -m "copy static at $(date +'%Y-%m-%d %H:%M:%S')"
git push --force --quiet "https://${{ secrets.API_TOKEN }}@${GITHUB_REPO}.git" ${GITHUB_BRANCH}:${GITHUB_BRANCH}
static_gitlab:
needs: build-static
runs-on: ubuntu-latest
concurrency:
group: static-gitlab-${{ github.ref_name }}
cancel-in-progress: false
steps:
- uses: actions/checkout@v6
- name: Push qinglong-static to GitLab
env:
STATIC_BRANCH: ${{ github.ref_name }}
STATIC_REF_TYPE: ${{ github.ref_type }}
MIRROR_SSH_KEY: ${{ secrets.GITLAB_SSH_PK }}
run: |
set -euo pipefail
set +e
mkdir -p ~/.ssh
printf '%s\n' "$MIRROR_SSH_KEY" > ~/.ssh/id_rsa
printf '%s\n' "${{ secrets.GITLAB_SSH_PK }}" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -T 10 gitlab.com >> ~/.ssh/known_hosts 2>/dev/null
bash scripts/sync-static.sh git@gitlab.com:whyour/qinglong-static.git "$STATIC_BRANCH" "$STATIC_REF_TYPE"
git clone --depth=1 --single-branch https://github.com/whyour/qinglong-static.git static-mirror
cd static-mirror
git remote add gitlab git@gitlab.com:whyour/qinglong-static.git 2>/dev/null
git push --force --all gitlab 2>&1 || echo "::warning::GitLab static push failed"
git push --force --tags gitlab 2>&1 || echo "::warning::GitLab static tags push failed"
verify-gitee-mirrors:
static_gitee:
needs: build-static
runs-on: ubuntu-latest
timeout-minutes: 14
permissions:
contents: write
strategy:
fail-fast: false
matrix:
repository: [qinglong, qinglong-static]
steps:
- uses: actions/checkout@v6
# Gitee Pull mirrors and GitHub push webhooks own synchronization.
# Verification runs alongside Docker builds and needs no Gitee credentials.
- name: Wait for Gitee Pull mirror
if: ${{ matrix.repository == 'qinglong' }}
- name: Push qinglong-static to Gitee
env:
GH_TOKEN: ${{ github.token }}
QL_MIRROR_NOTIFY_REPO: whyour/qinglong
MIRROR_REPOSITORY: ${{ matrix.repository }}
MIRROR_REF: ${{ github.ref_name }}
MIRROR_REF_TYPE: ${{ github.ref_type }}
QL_MIRROR_INCLUDE_VERSION_BRANCH: ${{ matrix.repository == 'qinglong-static' }}
QL_MIRROR_NOTIFY_ON_START: 'false'
GITEE_TOKEN: ${{ secrets.GITEE_TOKEN }}
run: |
bash scripts/wait-for-mirror.sh "https://github.com/whyour/$MIRROR_REPOSITORY.git" "https://gitee.com/whyour/$MIRROR_REPOSITORY.git" "$MIRROR_REF" "$MIRROR_REF_TYPE"
- name: Notify and wait for Gitee static mirror
if: ${{ matrix.repository == 'qinglong-static' }}
env:
GH_TOKEN: ${{ secrets.API_TOKEN }}
QL_MIRROR_NOTIFY_REPO: whyour/qinglong-static
MIRROR_REF: ${{ github.ref_name }}
MIRROR_REF_TYPE: ${{ github.ref_type }}
QL_MIRROR_INCLUDE_VERSION_BRANCH: 'true'
run: |
bash scripts/wait-for-mirror.sh https://github.com/whyour/qinglong-static.git https://gitee.com/whyour/qinglong-static.git "$MIRROR_REF" "$MIRROR_REF_TYPE"
sync-existing-static:
if: ${{ github.event_name == 'workflow_dispatch' && inputs.static_sync_only }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
mirror: [gitlab]
concurrency:
group: static-${{ matrix.mirror }}-${{ inputs.static_ref }}
cancel-in-progress: false
steps:
- uses: actions/checkout@v6
- name: Synchronize existing static snapshot
env:
STATIC_REF: ${{ inputs.static_ref }}
STATIC_REF_TYPE: ${{ inputs.static_ref_type }}
MIRROR_HOST: ${{ matrix.mirror }}.com
MIRROR_SSH_KEY: ${{ secrets.GITLAB_SSH_PK }}
run: |
set -euo pipefail
set +e
mkdir -p ~/.ssh
printf '%s\n' "$MIRROR_SSH_KEY" > ~/.ssh/id_rsa
printf '%s\n' "${{ secrets.GITLAB_SSH_PK }}" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -T 10 "$MIRROR_HOST" >> ~/.ssh/known_hosts 2>/dev/null
bash scripts/sync-static.sh "git@$MIRROR_HOST:whyour/qinglong-static.git" "$STATIC_REF" "$STATIC_REF_TYPE"
ssh-keyscan -T 10 gitee.com >> ~/.ssh/known_hosts 2>/dev/null
git clone --depth=1 --single-branch https://github.com/whyour/qinglong-static.git static-mirror
cd static-mirror
git remote add gitee git@gitee.com:whyour/qinglong-static.git 2>/dev/null
verify-existing-static-gitee:
if: ${{ github.event_name == 'workflow_dispatch' && inputs.static_sync_only }}
runs-on: ubuntu-latest
timeout-minutes: 14
steps:
- uses: actions/checkout@v6
- name: Wait for existing Gitee static snapshot
env:
GH_TOKEN: ${{ secrets.API_TOKEN }}
QL_MIRROR_NOTIFY_REPO: whyour/qinglong-static
MIRROR_REF: ${{ inputs.static_ref }}
MIRROR_REF_TYPE: ${{ inputs.static_ref_type }}
QL_MIRROR_INCLUDE_VERSION_BRANCH: 'true'
run: |
bash scripts/wait-for-mirror.sh https://github.com/whyour/qinglong-static.git https://gitee.com/whyour/qinglong-static.git "$MIRROR_REF" "$MIRROR_REF_TYPE"
if git push --force --all gitee 2>&1; then
echo "::notice::Gitee static push succeeded"
else
echo "::warning::Push failed, trying to create repo via API..."
curl -sS --connect-timeout 30 --max-time 60 \
-X POST "https://gitee.com/api/v5/user/repos" \
-H "Content-Type: application/json" \
-d '{"name":"qinglong-static","private":"false"}' \
"?access_token=$GITEE_TOKEN" 2>/dev/null
git push --force --all gitee 2>&1 && echo "::notice::Gitee static push succeeded after repo creation" || echo "::warning::Gitee static push failed after retry"
fi
git push --force --tags gitee 2>&1 || echo "::warning::Gitee static tags push failed"
build-alpine:
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
needs: build-static
outputs:
image-digest: ${{ steps.image.outputs.digest }}
runs-on: ubuntu-22.04
permissions:
packages: write
@@ -297,7 +223,6 @@ jobs:
uses: docker/setup-buildx-action@v4
- name: Build and push (Alpine)
id: image
uses: docker/build-push-action@v7
with:
build-args: |
@@ -319,8 +244,6 @@ jobs:
build-debian:
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
needs: build-static
outputs:
image-digest: ${{ steps.image.outputs.digest }}
runs-on: ubuntu-22.04
permissions:
packages: write
@@ -385,7 +308,6 @@ jobs:
uses: docker/setup-buildx-action@v4
- name: Build and push (Debian)
id: image
uses: docker/build-push-action@v7
with:
build-args: |
@@ -550,82 +472,30 @@ jobs:
cache-from: type=registry,ref=whyour/qinglong:cache-debian-python3.10
cache-to: type=registry,ref=whyour/qinglong:cache-debian-python3.10,mode=max
# One unmodified amd64 package for migration acceptance only. This does not
# publish bundles or advance development/formal channel indexes.
build-npm-migration-bundle:
needs: build-debian
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v6
- uses: docker/setup-buildx-action@v4
- name: Export the official platform package from this run's image digest
env:
IMAGE_DIGEST: ${{ needs.build-debian.outputs.image-digest }}
run: |
set -euo pipefail
[[ "$IMAGE_DIGEST" =~ ^sha256:[a-f0-9]{64}$ ]]
image="ghcr.io/${GITHUB_REPOSITORY}@${IMAGE_DIGEST}"
docker buildx build --platform linux/amd64 --build-arg "BASE_IMAGE=$image" --file docker/Dockerfile.upgrade-bundle --output "type=local,dest=$RUNNER_TEMP/npm-migration-bundle" .
- uses: actions/upload-artifact@v6
with:
name: npm-migration-bundle-${{ github.sha }}
path: ${{ runner.temp }}/npm-migration-bundle/*
if-no-files-found: error
retention-days: 3
verify-npm-package:
needs: [build-static, build-npm-migration-bundle]
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
node: ['20.18.1', '22.17.0', '24']
container: node:${{ matrix.node }}-bookworm
steps:
- uses: actions/checkout@v6
- uses: actions/download-artifact@v6
with:
name: qinglong-npm-${{ github.sha }}
path: ${{ runner.temp }}/npm-panel/
- uses: actions/download-artifact@v6
with:
name: npm-migration-bundle-${{ github.sha }}
path: ${{ runner.temp }}/npm-migration-bundle/
- name: Install isolated service test tools
run: |
apt-get update
apt-get install -y --no-install-recommends python3 openssl curl jq nginx procps
# The published legacy launcher resets PATH to standard directories.
# /usr/local is confined to this disposable job container; PM2_HOME
# remains private to each verification run.
npm install --global --prefix /usr/local --no-audit --no-fund pm2@6.0.14
- name: Install and upgrade the exact npm archive
if: matrix.node != '20.18.1'
env:
QL_PANEL_PACKAGE_TEST_ISOLATED: '1'
QL_PANEL_TEST_PM2: /usr/local/bin/pm2
QL_PANEL_TEST_BUNDLE_DIR: ${{ runner.temp }}/npm-migration-bundle
run: node scripts/verify-panel-package.cjs verify "$RUNNER_TEMP/npm-panel" "$RUNNER_TEMP/npm-panel-results"
- name: Reject an unsupported runtime while preserving the old installation
if: matrix.node == '20.18.1'
run: node scripts/verify-panel-runtime-floor.cjs "$RUNNER_TEMP/npm-panel" "$RUNNER_TEMP/npm-panel-results"
- uses: actions/upload-artifact@v6
with:
name: qinglong-npm-verification-${{ matrix.node }}-${{ github.sha }}
path: ${{ runner.temp }}/npm-panel-results/
if-no-files-found: error
retention-days: 14
publish:
if: ${{ github.repository == 'whyour/qinglong' && github.ref == 'refs/heads/master' }}
needs: [build-alpine, build-debian, build-alpine310, build-debian310, verify-npm-package]
needs: [build-alpine, build-debian, build-alpine310, build-debian310]
runs-on: ubuntu-latest
permissions:
contents: read
id-token: write
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v3
with:
version: "8.3.1"
- uses: actions/setup-node@v3
with:
node-version: "20"
cache: "pnpm"
- name: build front and back
run: |
pnpm install --frozen-lockfile
pnpm build:front
pnpm build:back
- name: Set up Node for trusted publishing
uses: actions/setup-node@v6
with:
@@ -633,111 +503,18 @@ jobs:
registry-url: https://registry.npmjs.org
package-manager-cache: false
- uses: actions/download-artifact@v6
with:
name: qinglong-npm-${{ github.sha }}
path: npm-panel/
- name: Publish npm package with OIDC
shell: bash
run: |
set -euo pipefail
shopt -s nullglob
archives=(npm-panel/*.tgz)
if [ "${#archives[@]}" -ne 1 ]; then
echo '::error::Expected exactly one verified panel npm archive.'
exit 1
fi
archive="${archives[0]}"
metadata=$(tar -xOf "$archive" package/package.json)
name=$(jq -er '.name | select(. == "@whyour/qinglong")' <<< "$metadata")
version=$(jq -er '.version | select(type == "string")' <<< "$metadata")
name=$(node -p 'require("./package.json").name')
version=$(node -p 'require("./package.json").version')
if npm view "$name@$version" version --json --registry=https://registry.npmjs.org > "$RUNNER_TEMP/npm-version.json" 2> "$RUNNER_TEMP/npm-version-error.log"; then
jq -e --arg version "$version" '. == $version' "$RUNNER_TEMP/npm-version.json" > /dev/null
echo "::notice::$name@$version is already published; skipping npm publication for this image rebuild."
elif jq -e '.error.code == "E404"' "$RUNNER_TEMP/npm-version.json" > /dev/null; then
npm publish "$archive" --ignore-scripts --access public --registry=https://registry.npmjs.org
npm publish --access public --registry=https://registry.npmjs.org
else
echo '::error::Could not check the published npm version; refusing to publish.'
exit 1
fi
build-upgrade-bundles:
# Both channels publish packages from this run's immutable image digests.
if: ${{ !inputs.static_sync_only && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop') }}
needs: [build-alpine, build-debian]
runs-on: ubuntu-22.04
timeout-minutes: 90
permissions:
contents: read
packages: write
strategy:
fail-fast: false
matrix:
include:
- { platform: linux/amd64, libc: musl, name: x64-musl }
- { platform: linux/arm/v6, libc: musl, name: armv6-musl }
- { platform: linux/arm/v7, libc: musl, name: armv7-musl }
- { platform: linux/arm64, libc: musl, name: arm64-musl }
- { platform: linux/ppc64le, libc: musl, name: ppc64-musl }
- { platform: linux/s390x, libc: musl, name: s390x-musl }
- { platform: linux/386, libc: musl, name: ia32-musl }
- { platform: linux/amd64, libc: glibc, name: x64-glibc }
- { platform: linux/arm/v7, libc: glibc, name: armv7-glibc }
- { platform: linux/arm64, libc: glibc, name: arm64-glibc }
- { platform: linux/ppc64le, libc: glibc, name: ppc64-glibc }
- { platform: linux/s390x, libc: glibc, name: s390x-glibc }
steps:
- uses: actions/checkout@v6
- uses: docker/setup-qemu-action@v4
with:
cache-image: false
- uses: docker/setup-buildx-action@v4
- name: Build and probe the immutable platform package
env:
ALPINE_DIGEST: ${{ needs.build-alpine.outputs.image-digest }}
DEBIAN_DIGEST: ${{ needs.build-debian.outputs.image-digest }}
BUNDLE_LIBC: ${{ matrix.libc }}
BUNDLE_PLATFORM: ${{ matrix.platform }}
run: |
set -euo pipefail
digest="$ALPINE_DIGEST"
if [ "$BUNDLE_LIBC" = glibc ]; then digest="$DEBIAN_DIGEST"; fi
test -n "$digest"
image="ghcr.io/${GITHUB_REPOSITORY}@${digest}"
docker buildx build --platform "$BUNDLE_PLATFORM" --build-arg "BASE_IMAGE=$image" --file docker/Dockerfile.upgrade-bundle --output type=local,dest=output/upgrade .
- uses: actions/upload-artifact@v6
with:
name: upgrade-${{ matrix.name }}-${{ github.sha }}
path: output/upgrade/*
if-no-files-found: error
publish-upgrade-bundles:
if: ${{ !inputs.static_sync_only && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop') }}
needs: build-upgrade-bundles
runs-on: ubuntu-latest
permissions:
contents: write
concurrency:
group: upgrade-channel-${{ github.ref_name }}
cancel-in-progress: false
steps:
- uses: actions/checkout@v6
- uses: actions/setup-node@v6
with:
node-version: '24'
- uses: pnpm/action-setup@v6
with:
version: '8.3.1'
- run: pnpm install --prod --frozen-lockfile
- run: npm --prefix cli ci
- uses: actions/download-artifact@v7
with:
pattern: upgrade-*-${{ github.sha }}
path: output/upgrade
merge-multiple: true
- run: node scripts/build-bundle-installer.cjs output/upgrade/install-bundle.cjs
- name: Publish complete packages before advancing the channel
env:
GH_TOKEN: ${{ github.token }}
run: node scripts/publish-upgrade-bundles.cjs output/upgrade "$GITHUB_SHA" "$GITHUB_REPOSITORY" "$GITHUB_REF_NAME"
+8 -54
View File
@@ -21,27 +21,16 @@ on:
workflow_dispatch:
inputs:
publish:
description: Publish the verified CLI package (master stable / develop prerelease)
description: Publish the verified CLI package (master only)
type: boolean
default: false
prerelease_version:
description: Develop only, e.g. 0.1.2-beta.0 (alpha/beta/rc)
type: string
default: ''
permissions:
contents: read
concurrency:
# Keep validation runs separate from releases; never interrupt a running release.
# Master push releases share the same group as manually requested releases.
group: >-
cli-package-${{ github.workflow }}-${{ github.ref }}-${{
((github.event_name == 'workflow_dispatch' && inputs.publish) ||
(github.event_name == 'push' && github.ref == 'refs/heads/master')) && 'publish' || 'check' }}
cancel-in-progress: >-
${{ github.ref != 'refs/heads/master' &&
!(github.event_name == 'workflow_dispatch' && inputs.publish) }}
group: cli-package-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
jobs:
package:
@@ -63,20 +52,6 @@ jobs:
sudo apt-get update
sudo apt-get install -y --no-install-recommends bash ca-certificates curl git jq perl procps python3 unzip zip
npm install --global --ignore-scripts --no-audit --no-fund ts-node@10.9.2 typescript@5.2.2
- name: Set requested prerelease version before verification
if: github.event_name == 'workflow_dispatch' && inputs.prerelease_version != ''
working-directory: cli
env:
PRERELEASE_VERSION: ${{ inputs.prerelease_version }}
shell: bash
run: |
set -euo pipefail
if [[ "$GITHUB_REF" != refs/heads/develop ||
! "$PRERELEASE_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+-(alpha|beta|rc)\.[0-9]+$ ]]; then
echo '::error::Prereleases require develop and an explicit alpha/beta/rc version.'
exit 1
fi
npm version "$PRERELEASE_VERSION" --no-git-tag-version --ignore-scripts
- run: npm ci --prefix cli --no-audit --no-fund
- run: npm run check:cli
- run: npm run test:cli
@@ -97,11 +72,9 @@ jobs:
needs: package
if: >-
github.repository == 'whyour/qinglong' &&
((github.ref == 'refs/heads/master' &&
(github.event_name == 'push' ||
(github.event_name == 'workflow_dispatch' && inputs.publish))) ||
(github.ref == 'refs/heads/develop' && github.event_name == 'workflow_dispatch' &&
inputs.publish && inputs.prerelease_version != ''))
github.ref == 'refs/heads/master' &&
(github.event_name == 'push' ||
(github.event_name == 'workflow_dispatch' && inputs.publish))
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
@@ -118,8 +91,6 @@ jobs:
name: qinglong-cli-${{ github.sha }}
path: cli-package
- name: Publish verified npm archive
env:
PRERELEASE_VERSION: ${{ inputs.prerelease_version }}
shell: bash
run: |
set -euo pipefail
@@ -132,30 +103,13 @@ jobs:
archive="${archives[0]}"
metadata=$(tar -xOf "$archive" package/package.json)
name=$(jq -er '.name | select(. == "@whyour/qinglong-cli")' <<< "$metadata")
version=$(jq -er '.version | select(type == "string")' <<< "$metadata")
if [[ "$GITHUB_REF" == refs/heads/develop ]]; then
if [[ "$version" != "$PRERELEASE_VERSION" ||
! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+-(alpha|beta|rc)\.[0-9]+$ ]]; then
echo '::error::Archive version does not match the requested prerelease.'
exit 1
fi
tag="${BASH_REMATCH[1]}"
elif [[ "$GITHUB_REF" == refs/heads/master && "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
tag=latest
else
echo '::error::Refusing to publish this branch/version combination.'
exit 1
fi
version=$(jq -er '.version | select(type == "string" and test("^[0-9]+\\.[0-9]+\\.[0-9]+$"))' <<< "$metadata")
if npm view "$name@$version" version --json --registry=https://registry.npmjs.org > version.json 2> version-error.log; then
jq -e --arg version "$version" '. == $version' version.json > /dev/null
if [[ "$tag" != latest ]]; then
echo '::error::Prerelease version already exists; choose a new version to test this build.'
exit 1
fi
echo "::notice::$name@$version is already published; bump cli/package.json and its lockfile to release changes."
exit 0
elif ! jq -e '.error.code == "E404"' version.json > /dev/null; then
echo '::error::Could not check the published CLI version; refusing to publish.'
exit 1
fi
npm publish "./$archive" --access public --tag "$tag" --ignore-scripts --registry=https://registry.npmjs.org
npm publish "./$archive" --access public --tag latest --ignore-scripts --registry=https://registry.npmjs.org
-136
View File
@@ -1,136 +0,0 @@
name: Release readiness
on:
pull_request:
paths:
- '.github/workflows/release-readiness.yml'
- 'package.json'
- 'pnpm-lock.yaml'
workflow_dispatch:
permissions:
contents: read
concurrency:
group: release-readiness-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
regression:
uses: ./.github/workflows/validate.yml
static:
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v6
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
- uses: pnpm/action-setup@v6
with:
version: '8.3.1'
- uses: actions/setup-node@v6
with:
node-version: '22'
cache: pnpm
- run: pnpm install --frozen-lockfile
- run: pnpm build:front && pnpm build:back && pnpm build:info
- uses: actions/upload-artifact@v6
with:
name: release-readiness-static
path: static/
include-hidden-files: true
if-no-files-found: error
images:
needs: [regression, static]
runs-on: ubuntu-22.04
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
include:
- image: alpine
file: docker/Dockerfile
cache: alpine
platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x,linux/386
- image: debian
file: docker/Dockerfile.debian
cache: debian
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
- image: alpine-python3.10
file: docker/Dockerfile.310
cache: alpine-python3.10
platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x,linux/386
- image: debian-python3.10
file: docker/Dockerfile.debian310
cache: debian-python3.10
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
steps:
- uses: actions/checkout@v6
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
- name: Record source
id: source
env:
SOURCE_URL: ${{ github.event.pull_request.head.repo.clone_url || format('https://github.com/{0}.git', github.repository) }}
SOURCE_BRANCH: ${{ github.head_ref || github.ref_name }}
run: |
echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
echo "url=$SOURCE_URL" >> "$GITHUB_OUTPUT"
echo "branch=$SOURCE_BRANCH" >> "$GITHUB_OUTPUT"
- uses: actions/download-artifact@v7
with:
name: release-readiness-static
path: static/
- uses: docker/setup-qemu-action@v4
with:
cache-image: false
- uses: docker/setup-buildx-action@v4
- name: Build every release architecture
uses: docker/build-push-action@v7
with:
context: .
file: ${{ matrix.file }}
platforms: ${{ matrix.platforms }}
build-args: |
QL_URL=${{ steps.source.outputs.url }}
QL_BRANCH=${{ steps.source.outputs.branch }}
SOURCE_COMMIT=${{ steps.source.outputs.sha }}
push: false
provenance: false
sbom: false
cache-from: type=registry,ref=whyour/qinglong:cache-${{ matrix.cache }}
- name: Load native image for runtime verification
uses: docker/build-push-action@v7
with:
context: .
file: ${{ matrix.file }}
platforms: linux/amd64
build-args: |
QL_URL=${{ steps.source.outputs.url }}
QL_BRANCH=${{ steps.source.outputs.branch }}
SOURCE_COMMIT=${{ steps.source.outputs.sha }}
tags: qinglong-readiness:${{ matrix.image }}
load: true
push: false
provenance: false
sbom: false
- name: Verify native production runtime without external networking
run: |
docker run --rm --network none --entrypoint node \
qinglong-readiness:${{ matrix.image }} \
-e 'const assert=require("node:assert/strict"); const {createRequire}=require("node:module"); for(const [name,want] of Object.entries({"@grpc/grpc-js":"1.14.5",compression:"1.8.2",multer:"2.4.0"}))assert.equal(require(name+"/package.json").version,want); const expressRequire=createRequire(require.resolve("express")); assert.equal(expressRequire("proxy-addr/package.json").version,"2.0.8"); const sockRequire=createRequire(require.resolve("sockjs")); assert.equal(sockRequire("websocket-driver/package.json").version,"0.7.5"); assert.equal(typeof require("@mikro-orm/sql").NodeSqliteDialect,"function"); const db=new (require("node:sqlite").DatabaseSync)(":memory:"); assert.equal(db.prepare("SELECT 42 AS value").get().value,42); db.close(); require("nodemailer"); require("undici"); console.log("Minimal production patches and runtime imports verified")'
- name: Export native Debian image for isolated dev validation
if: matrix.image == 'debian'
run: |
docker save qinglong-readiness:debian | gzip -1 > release-readiness-debian-image.tar.gz
sha256sum release-readiness-debian-image.tar.gz > release-readiness-debian-image.sha256
docker image inspect qinglong-readiness:debian > release-readiness-debian-image.json
- uses: actions/upload-artifact@v6
if: matrix.image == 'debian'
with:
name: release-readiness-debian-image
path: release-readiness-debian-image.*
compression-level: 0
if-no-files-found: error
+1 -1
View File
@@ -18,7 +18,7 @@ jobs:
version: '8.3.1'
- uses: actions/setup-node@v6
with:
node-version: '24'
node-version: '20'
cache: pnpm
- run: pnpm install --frozen-lockfile
- run: cp .env.example .env
-10
View File
@@ -29,7 +29,6 @@ __pycache__
/shell/preload/*-notify.json
/shell/preload/__ql_notify__.*
/shell/preload/lang_env.sh
/shell/upgrade-lock.yaml
.deepseek/
.claude/
@@ -39,12 +38,3 @@ __pycache__
# Local security audit artifacts
/audit/
# Local browser tests, screenshots, reports, and temporary test output
/.playwright-cli/
/output/
/playwright-report/
/test-results/
/blob-report/
/coverage/
/.nyc_output/
+22
View File
@@ -0,0 +1,22 @@
/.tmp/
/.github/
/.vscode/
/.history/
/back/**/*.ts
/back/**/*.json
/cli/
/data/
/src/
/static/**/*.js.map
/static/**/*.gz
/.editorconfig
/.gitignore
/.prettierignore
/.prettierrc
/.umirc.ts
/nodemon.json
/pnpm-lock.yaml
/tsconfig.back.json
/tsconfig.json
/typings.d.ts
/.env
-1
View File
@@ -1 +0,0 @@
22
-12
View File
@@ -1,12 +0,0 @@
# Node.js 原生部署与 Python venv
面板要求 Node.js >=22.17,推荐 Node.js 24,并要求该 Node 构建提供 `node:sqlite`。应用升级保留现有 Node;旧 Node 20 环境需先升级运行时。远程管理 CLI 的最低版本要求独立于面板。
设置 `QL_DIR` 为安装目录,`QL_DATA_DIR` 为以 `/data` 结尾的绝对持久化数据路径。
如需隔离 Python,请在启动进程的环境中显式设置 `PYTHON_VENV_DIR`(绝对路径或相对 `QL_DIR` 的路径),例如 `export PYTHON_VENV_DIR=/srv/qinglong/.venv`。未设置时保留现有 Python prefix 行为。
- `qinglong`(`shell/start.sh`)安装系统依赖和 Node 工具,并安装 Python requests,然后调用公共启动脚本。
- `npm run start:simple`(`shell/start-simplify.sh`)使用已准备的 Node、pm2、nginx、jq 等依赖,不安装系统包、全局 npm 包或 requests;显式指定的 venv 不存在时仍会创建。请先在 venv 中安装 requests 和任务所需依赖。
- 两种启动方式均保留已有 `.env`;使用 venv 时清除继承的 `PYTHONPATH`/`PYTHONUSERBASE`,依赖查询、安装和卸载使用该 venv 的 Python。
该模式需具备现有启动流程所要求的 nginx 配置和 pm2 服务权限。生产环境设置变量应通过 systemd 的 `Environment` 或相应进程配置完成。已有系统 Python 依赖不自动迁移至 venv,需在 venv 中重新安装。
+1 -2
View File
@@ -1,6 +1,5 @@
import { NextFunction, Request, Response, Router } from 'express';
import { celebrate, Joi } from 'celebrate';
import { t } from '../shared/i18n';
import {
diagnoseClientIp,
getTrustProxyConfig,
@@ -37,7 +36,7 @@ export default (app: Router) => {
} catch (error) {
res.send({
code: 400,
message: error instanceof Error ? error.message : t('配置更新失败'),
message: error instanceof Error ? error.message : '配置更新失败',
});
}
},
+7 -3
View File
@@ -4,6 +4,7 @@ import { Logger } from 'winston';
import config from '../config';
import * as fs from 'fs/promises';
import { celebrate, Joi } from 'celebrate';
import { join, basename } from 'path';
import { SAMPLE_FILES } from '../config/const';
import { t } from '../shared/i18n';
import ConfigService from '../services/config';
@@ -72,12 +73,15 @@ export default (app: Router) => {
const logger: Logger = Container.get('logger');
try {
const { name, content } = req.body;
// Resolve path first to prevent traversal attacks
let basePath = config.configPath;
if (name.startsWith('data/scripts/')) {
return res.send({ code: 403, message: t('文件无法访问') });
basePath = join(config.rootPath, 'data/scripts');
}
const cleanName = name.replace(/^data\/scripts\//, '');
const normalized = resolveFileAccess(
config.configPath,
[name],
basePath,
[cleanName],
config.blackFileList,
);
if (!normalized) {
+13 -19
View File
@@ -5,7 +5,10 @@ import CronService from '../services/cron';
import CronViewService from '../services/cronView';
import { celebrate, Joi } from 'celebrate';
import { commonCronSchema } from '../validation/schedule';
import { RunningInstanceModel, InstanceStatus } from '../data/runningInstance';
import {
RunningInstanceModel,
InstanceStatus,
} from '../data/runningInstance';
import { t } from '../shared/i18n';
import cronClient from '../schedule/client';
@@ -16,17 +19,8 @@ export default (app: Router) => {
route.use(async (req, res, next) => {
// Keep stop/status callbacks available even when the scheduler is down.
if (
['POST', 'PUT', 'DELETE'].includes(req.method) &&
[
'/',
'/run',
'/enable',
'/disable',
'/views/enable',
'/views/disable',
].includes(req.path)
) {
if (['POST', 'PUT', 'DELETE'].includes(req.method) &&
['/', '/run', '/enable', '/disable', '/views/enable', '/views/disable'].includes(req.path)) {
try {
await cronClient.readiness.ensureReady();
} catch (error) {
@@ -503,8 +497,12 @@ export default (app: Router) => {
}),
async (req: Request<{ id: number }>, res: Response, next: NextFunction) => {
try {
const instances = await RunningInstanceModel.find({
cron_id: req.params.id,
const instances = await RunningInstanceModel.findAll({
where: {
cron_id: req.params.id,
},
order: [['started_at', 'DESC']],
raw: true,
});
return res.send({ code: 200, data: instances });
} catch (e) {
@@ -521,11 +519,7 @@ export default (app: Router) => {
instanceId: Joi.number().required(),
}),
}),
async (
req: Request<{ id: number; instanceId: number }>,
res: Response,
next: NextFunction,
) => {
async (req: Request<{ id: number; instanceId: number }>, res: Response, next: NextFunction) => {
try {
const cronService = Container.get(CronService);
const data = await cronService.stopInstance(req.params.instanceId);
+157 -93
View File
@@ -1,32 +1,38 @@
import { statistics } from '../data/statistics';
import { Query, QuerySpec, orderBy } from '../data/queries';
import { Router, Request, Response, NextFunction } from 'express';
import { Container } from 'typedi';
import { fn, col, where, Op } from 'sequelize';
import { CrontabModel } from '../data/cron';
import { CrontabStatModel } from '../data/cronStats';
import { RunningInstanceModel, InstanceStatus } from '../data/runningInstance';
import {
RunningInstanceModel,
InstanceStatus,
} from '../data/runningInstance';
import dayjs from 'dayjs';
import os from 'os';
import { isEmpty } from 'lodash';
import { t, tf } from '../shared/i18n';
const route = Router();
export default (app: Router) => {
app.use('/dashboard', route);
route.post('/record', async (req: Request, res: Response) => {
try {
const { ref_id, code, elapsed } = req.body;
if (!ref_id) return res.send({ code: 400, message: 'ref_id required' });
const today = dayjs().format('YYYY-MM-DD');
const isSuccess = code === 0 ? 1 : 0;
const isFail = code !== 0 ? 1 : 0;
const elapsedMs = (Number(elapsed) || 0) * 1000;
const existing = await CrontabStatModel.findOne(
{ ref_id: Number(ref_id), date: today },
{},
);
const existing = await CrontabStatModel.findOne({
where: { ref_id: Number(ref_id), date: today },
});
if (existing) {
await CrontabStatModel.nativeUpdate(
{ id: existing.id },
await CrontabStatModel.update(
{
run_count: (existing.run_count || 0) + 1,
success_count: (existing.success_count || 0) + isSuccess,
@@ -34,9 +40,10 @@ export default (app: Router) => {
total_time: (existing.total_time || 0) + elapsedMs,
max_time: Math.max(existing.max_time || 0, elapsedMs),
},
{ where: { id: existing.id } },
);
} else {
await CrontabStatModel.insert({
await CrontabStatModel.create({
ref_id: Number(ref_id),
date: today,
run_count: 1,
@@ -46,27 +53,41 @@ export default (app: Router) => {
max_time: elapsedMs,
});
}
res.send({ code: 200 });
} catch (e) {
res.send({ code: 500 });
}
});
route.get(
'/overview',
async (req: Request, res: Response, next: NextFunction) => {
try {
const today = dayjs().format('YYYY-MM-DD');
const [total, enabled, disabled, stats] = await Promise.all([
CrontabModel.count({}),
CrontabModel.count({ isDisabled: 0 }),
CrontabModel.count({ isDisabled: 1 }),
statistics.totals(today),
CrontabModel.count(),
CrontabModel.count({ where: { isDisabled: 0 } }),
CrontabModel.count({ where: { isDisabled: 1 } }),
CrontabStatModel.findOne({
attributes: [
[fn('SUM', col('run_count')), 'total_runs'],
[fn('SUM', col('success_count')), 'total_success'],
[fn('SUM', col('fail_count')), 'total_fail'],
[fn('SUM', col('total_time')), 'total_time'],
],
where: { date: today },
raw: true,
}),
]);
const row = stats as any;
const totalRuns = Number(row?.total_runs) || 0;
const totalSuccess = Number(row?.total_success) || 0;
const totalFail = Number(row?.total_fail) || 0;
const totalTime = Number(row?.total_time) || 0;
res.send({
code: 200,
data: {
@@ -76,10 +97,7 @@ export default (app: Router) => {
todayRuns: totalRuns,
todaySuccess: totalSuccess,
todayFail: totalFail,
successRate:
totalRuns > 0
? ((totalSuccess / totalRuns) * 100).toFixed(1)
: '0',
successRate: totalRuns > 0 ? ((totalSuccess / totalRuns) * 100).toFixed(1) : '0',
avgTime: totalRuns > 0 ? Math.round(totalTime / totalRuns) : 0,
},
});
@@ -88,25 +106,25 @@ export default (app: Router) => {
}
},
);
for (const isSuccess of [false, true]) {
route.get(
isSuccess ? '/successes' : '/failures',
async (req: Request, res: Response, next: NextFunction) => {
try {
const countColumn = isSuccess ? 'success_count' : 'fail_count';
const rows = (await statistics.results(
dayjs().format('YYYY-MM-DD'),
countColumn,
)) as any[];
const crons =
rows.length > 0
? await CrontabModel.find(
{ id: { ['$in']: rows.map((row) => Number(row.ref_id)) } },
{
fields: ['id', 'name', 'command'],
},
)
: [];
const rows = (await CrontabStatModel.findAll({
attributes: ['ref_id', [fn('SUM', col(countColumn)), 'result_count']],
where: { date: dayjs().format('YYYY-MM-DD'), [countColumn]: { [Op.gt]: 0 } },
group: ['ref_id'],
order: [[fn('SUM', col(countColumn)), 'DESC'], ['ref_id', 'ASC']],
raw: true,
})) as any[];
const crons = rows.length > 0 ? await CrontabModel.findAll({
attributes: ['id', 'name', 'command'],
where: { id: { [Op.in]: rows.map((row) => Number(row.ref_id)) } },
raw: true,
}) : [];
const cronMap = new Map(crons.map((cron) => [cron.id, cron]));
res.send({
code: 200,
@@ -117,9 +135,7 @@ export default (app: Router) => {
id,
name: cron?.name || cron?.command || tf('任务#%s', id),
command: cron?.command || '',
[isSuccess ? 'successCount' : 'failCount']: Number(
row.result_count,
),
[isSuccess ? 'successCount' : 'failCount']: Number(row.result_count),
deleted: !cron,
};
}),
@@ -130,6 +146,7 @@ export default (app: Router) => {
},
);
}
route.get(
'/trend',
async (req: Request, res: Response, next: NextFunction) => {
@@ -139,7 +156,22 @@ export default (app: Router) => {
for (let i = days - 1; i >= 0; i--) {
dates.push(dayjs().subtract(i, 'day').format('YYYY-MM-DD'));
}
const rows = (await statistics.trend(dates)) as any[];
const rows = (await CrontabStatModel.findAll({
attributes: [
'date',
[fn('SUM', col('run_count')), 'total_runs'],
[fn('SUM', col('success_count')), 'total_success'],
[fn('SUM', col('fail_count')), 'total_fail'],
],
where: {
date: { [Op.in]: dates },
},
group: ['date'],
order: [['date', 'ASC']],
raw: true,
})) as any[];
const dataMap: Record<string, any> = {};
rows.forEach((r: any) => {
dataMap[r.date] = {
@@ -148,52 +180,89 @@ export default (app: Router) => {
fail: Number(r.total_fail) || 0,
};
});
const data = dates.map((d) => ({
date: dayjs(d).format('MM-DD'),
...(dataMap[d] || { total: 0, success: 0, fail: 0 }),
}));
res.send({ code: 200, data });
} catch (e) {
next(e);
}
},
);
route.get(
'/top-time',
async (req: Request, res: Response, next: NextFunction) => {
try {
const today = dayjs().format('YYYY-MM-DD');
const rows = (await statistics.ranking(today, 'total_time')) as any[];
const rows = (await CrontabStatModel.findAll({
attributes: [
'ref_id',
[fn('SUM', col('total_time')), 'total_time'],
[fn('SUM', col('run_count')), 'run_count'],
[fn('MAX', col('max_time')), 'max_time'],
],
where: { date: today, run_count: { [Op.gt]: 0 } },
group: ['ref_id'],
order: [[fn('SUM', col('total_time')), 'DESC']],
limit: 5,
raw: true,
})) as any[];
const ids = rows.map((r) => Number(r.ref_id));
const crons = await CrontabModel.find({ id: { ['$in']: ids } }, {});
const nameMap: Record<number, string> = {};
crons.forEach((c: any) => {
nameMap[c.id] = c.name || c.command;
const crons = await CrontabModel.findAll({
where: { id: { [Op.in]: ids } },
raw: true,
});
const nameMap: Record<number, string> = {};
crons.forEach((c: any) => { nameMap[c.id] = c.name || c.command; });
const data = rows.map((r: any, i) => ({
rank: i + 1,
name: nameMap[Number(r.ref_id)] || tf('任务#%s', r.ref_id),
avgTime: Math.round(Number(r.total_time) / Number(r.run_count)),
maxTime: Number(r.max_time),
}));
res.send({ code: 200, data });
} catch (e) {
next(e);
}
},
);
route.get(
'/top-count',
async (req: Request, res: Response, next: NextFunction) => {
try {
const today = dayjs().format('YYYY-MM-DD');
const rows = (await statistics.ranking(today, 'run_count')) as any[];
const rows = (await CrontabStatModel.findAll({
attributes: [
'ref_id',
[fn('SUM', col('run_count')), 'run_count'],
[fn('SUM', col('total_time')), 'total_time'],
[fn('SUM', col('success_count')), 'success_count'],
],
where: { date: today, run_count: { [Op.gt]: 0 } },
group: ['ref_id'],
order: [[fn('SUM', col('run_count')), 'DESC']],
limit: 5,
raw: true,
})) as any[];
const ids = rows.map((r) => Number(r.ref_id));
const crons = await CrontabModel.find({ id: { ['$in']: ids } }, {});
const nameMap: Record<number, string> = {};
crons.forEach((c: any) => {
nameMap[c.id] = c.name || c.command;
const crons = await CrontabModel.findAll({
where: { id: { [Op.in]: ids } },
raw: true,
});
const nameMap: Record<number, string> = {};
crons.forEach((c: any) => { nameMap[c.id] = c.name || c.command; });
const data = rows.map((r: any, i) => ({
rank: i + 1,
name: nameMap[Number(r.ref_id)] || tf('任务#%s', r.ref_id),
@@ -201,42 +270,48 @@ export default (app: Router) => {
avgTime: Math.round(Number(r.total_time) / Number(r.run_count)),
successRate:
Number(r.run_count) > 0
? ((Number(r.success_count) / Number(r.run_count)) * 100).toFixed(
1,
)
? ((Number(r.success_count) / Number(r.run_count)) * 100).toFixed(1)
: '0',
}));
res.send({ code: 200, data });
} catch (e) {
next(e);
}
},
);
route.get(
'/runtime',
async (req: Request, res: Response, next: NextFunction) => {
try {
const runningInstances = await RunningInstanceModel.find(
{
const runningInstances = await RunningInstanceModel.findAll({
where: {
status: InstanceStatus.running,
},
{},
);
const queuedCrons = await CrontabModel.find(
{
raw: true,
});
const queuedCrons = await CrontabModel.findAll({
where: {
status: 3, // queued
},
{},
);
raw: true,
});
// Fetch cron names for running instances
const cronIds = [
...new Set(runningInstances.map((i: any) => i.cron_id)),
];
const crons =
cronIds.length > 0
? await CrontabModel.find({ id: cronIds }, {})
? await CrontabModel.findAll({
where: { id: cronIds },
raw: true,
})
: [];
const cronMap = new Map(crons.map((c: any) => [c.id, c]));
const now = dayjs().unix();
const running = runningInstances.map((inst: any) => {
const cron = cronMap.get(inst.cron_id);
@@ -249,18 +324,19 @@ export default (app: Router) => {
logPath: inst.log_path,
};
});
const dayAgo = dayjs().subtract(24, 'hour').unix();
const idleTasks = await CrontabModel.find(
{
const idleTasks = await CrontabModel.findAll({
where: {
isDisabled: 0,
status: 1,
last_execution_time: { ['$lt']: dayAgo },
last_execution_time: { [Op.lt]: dayAgo },
},
{
orderBy: orderBy([['last_execution_time', 'ASC']]),
limit: 5,
},
);
order: [['last_execution_time', 'ASC']],
limit: 5,
raw: true,
});
res.send({
code: 200,
data: {
@@ -281,45 +357,33 @@ export default (app: Router) => {
}
},
);
route.get(
'/labels',
async (req: Request, res: Response, next: NextFunction) => {
try {
const today = dayjs().format('YYYY-MM-DD');
const [crons, stats] = await Promise.all([
CrontabModel.find({ isDisabled: 0 }, {}),
CrontabStatModel.find({ date: today }, {}),
]);
const [crons, stats] = (await Promise.all([
CrontabModel.findAll({ where: { isDisabled: 0 }, raw: true }),
CrontabStatModel.findAll({ where: { date: today }, raw: true }),
]));
const statMap: Record<number, any> = {};
stats.forEach((s: any) => {
statMap[s.ref_id] = s;
});
const labelMap: Record<
string,
{
count: number;
runs: number;
success: number;
totalTime: number;
}
> = {};
stats.forEach((s: any) => { statMap[s.ref_id] = s; });
const labelMap: Record<string, { count: number; runs: number; success: number; totalTime: number }> = {};
crons.forEach((c) => {
let rawLabels = c.labels;
if (typeof rawLabels === 'string') rawLabels = JSON.parse(rawLabels);
const labels: string[] = Array.isArray(rawLabels)
? [
...new Set(
(rawLabels as string[]).filter((l: string) => !isEmpty(l)),
),
]
? [...new Set((rawLabels as string[]).filter((l: string) => !isEmpty(l)))]
: [];
if (labels.length === 0) {
labels.push(t('未分类'));
}
const st = statMap[c.id!];
labels.forEach((label: string) => {
if (!labelMap[label])
labelMap[label] = { count: 0, runs: 0, success: 0, totalTime: 0 };
if (!labelMap[label]) labelMap[label] = { count: 0, runs: 0, success: 0, totalTime: 0 };
labelMap[label].count += 1;
if (st) {
labelMap[label].runs += Number(st.run_count) || 0;
@@ -328,22 +392,24 @@ export default (app: Router) => {
}
});
});
const data = Object.entries(labelMap)
.map(([label, v]) => ({
label,
count: v.count,
todayRuns: v.runs,
successRate:
v.runs > 0 ? ((v.success / v.runs) * 100).toFixed(1) : '0',
successRate: v.runs > 0 ? ((v.success / v.runs) * 100).toFixed(1) : '0',
avgTime: v.runs > 0 ? Math.round(v.totalTime / v.runs) : 0,
}))
.sort((a, b) => b.todayRuns - a.todayRuns);
res.send({ code: 200, data });
} catch (e) {
next(e);
}
},
);
route.get(
'/system',
async (req: Request, res: Response, next: NextFunction) => {
@@ -356,9 +422,7 @@ export default (app: Router) => {
uptime: Math.floor(process.uptime()),
memTotal: os.totalmem(),
memFree: os.freemem(),
memUsagePercent: ((1 - os.freemem() / os.totalmem()) * 100).toFixed(
1,
),
memUsagePercent: ((1 - os.freemem() / os.totalmem()) * 100).toFixed(1),
heapUsed: Math.round(memUsage.heapUsed / 1024 / 1024),
heapTotal: Math.round(memUsage.heapTotal / 1024 / 1024),
loadAvg: os.loadavg().map((v) => Number(v.toFixed(2))),
+14 -17
View File
@@ -1,16 +1,24 @@
import { Joi, celebrate } from 'celebrate';
import { NextFunction, Request, Response, Router } from 'express';
import fs from 'fs';
import multer from 'multer';
import { Container } from 'typedi';
import { Logger } from 'winston';
import config from '../config';
import { safeJSONParse } from '../config/util';
import { t } from '../shared/i18n';
import EnvService from '../services/env';
const route = Router();
const upload = multer({
storage: multer.memoryStorage(),
limits: { fileSize: 5 * 1024 * 1024, files: 1 },
const storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, config.scriptPath);
},
filename: function (req, file, cb) {
cb(null, file.originalname);
},
});
const upload = multer({ storage: storage });
const labelSchema = Joi.array()
.items(Joi.string().trim().required())
.min(1)
@@ -274,23 +282,12 @@ export default (app: Router) => {
const logger: Logger = Container.get('logger');
try {
const envService = Container.get(EnvService);
let parseContent: unknown;
try {
parseContent = JSON.parse(req.file?.buffer.toString('utf8') || '');
} catch {
return res.status(400).send({ code: 400, message: t('参数错误') });
}
const fileContent = await fs.promises.readFile(req!.file!.path, 'utf8');
const parseContent = safeJSONParse(fileContent);
const data = Array.isArray(parseContent)
? parseContent
: [parseContent];
if (
data.length > 0 &&
data.every((x) =>
x && typeof x.name === 'string' &&
/^[a-zA-Z_][0-9a-zA-Z_]*$/.test(x.name) &&
typeof x.value === 'string' && x.value.length > 0,
)
) {
if (data.every((x) => x.name && x.value)) {
const result = await envService.create(
data.map((x) => ({
name: x.name,
-2
View File
@@ -3,7 +3,6 @@ import { Container } from 'typedi';
import OpenService from '../services/open';
import { Logger } from 'winston';
import { celebrate, Joi } from 'celebrate';
import { rateLimit } from 'express-rate-limit';
const route = Router();
export default (app: Router) => {
@@ -101,7 +100,6 @@ export default (app: Router) => {
route.get(
'/auth/token',
rateLimit({ windowMs: 15 * 60 * 1000, limit: 100 }),
celebrate({
query: {
client_id: Joi.string().required(),
+14 -199
View File
@@ -7,62 +7,20 @@ import { Logger } from 'winston';
import config from '../config';
import * as fs from 'fs/promises';
import { celebrate, Joi } from 'celebrate';
import path, { dirname, join, parse, resolve, sep } from 'path';
import path, { join, parse } from 'path';
import ScriptService from '../services/script';
import ScriptHistoryService from '../services/scriptHistory';
import { t } from '../shared/i18n';
import multer from 'multer';
import { writeFileWithLock } from '../shared/utils';
import {
ScriptHistoryError,
HistoryUnavailableError,
} from '../shared/scriptHistory';
const route = Router();
function isPathAllowed(targetPath: string): boolean {
const resolved = path.resolve(targetPath);
return config.writePathList.some((x) =>
Boolean(
resolveFileAccess(
x,
[resolved],
// Panel configuration secrets must not restrict user script filenames.
path.resolve(x) === path.resolve(config.configPath)
? config.blackFileList
: [],
),
),
Boolean(resolveFileAccess(x, [resolved], config.blackFileList)),
);
}
async function existingWritableRealPath(targetPath: string): Promise<string> {
if (!isPathAllowed(targetPath)) {
throw new ScriptHistoryError(403, '暂无权限');
}
const resolved = resolve(targetPath);
for (const writableRoot of config.writePathList) {
const root = resolve(writableRoot);
const rootPrefix = root.endsWith(sep) ? root : root + sep;
// Keep an explicit normalized boundary check at the filesystem operation.
if (!resolved.startsWith(rootPrefix)) continue;
const realPath = await fs.realpath(resolved);
const realRoot = await fs.realpath(root);
const realRootPrefix = realRoot.endsWith(sep) ? realRoot : realRoot + sep;
if (
!realPath.startsWith(realRootPrefix) ||
!resolveFileAccess(
realRoot,
[realPath],
root === resolve(config.configPath) ? config.blackFileList : [],
)
) {
throw new ScriptHistoryError(403, '暂无权限');
}
return realPath;
}
throw new ScriptHistoryError(403, '暂无权限');
}
const storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, config.tmpPath);
@@ -97,7 +55,11 @@ export default (app: Router) => {
];
if (req.query.path) {
if (
!resolveFileAccess(config.scriptPath, [req.query.path as string])
!resolveFileAccess(
config.scriptPath,
[req.query.path as string],
config.blackFileList,
)
) {
return res.send({ code: 403, message: t('暂无权限') });
}
@@ -153,82 +115,6 @@ export default (app: Router) => {
},
);
const historyError = (error: unknown, res: Response, next: NextFunction) => {
if (error instanceof ScriptHistoryError) {
return res.status(error.status).send({
code: error.status,
message: t(error.message),
...(error instanceof HistoryUnavailableError
? { historyUnavailable: true, currentHash: error.currentHash }
: {}),
});
}
return next(error);
};
const historyQuery = {
filename: Joi.string().required(),
path: Joi.string().optional().allow(''),
};
route.get(
'/history',
celebrate({ query: Joi.object(historyQuery).unknown(true) }),
async (req: Request, res: Response, next: NextFunction) => {
try {
const service = Container.get(ScriptHistoryService);
const data = await service.list(
(req.query.path as string) || '',
req.query.filename as string,
);
res.send({ code: 200, data });
} catch (error) {
historyError(error, res, next);
}
},
);
route.get(
'/history/detail',
celebrate({
query: Joi.object({
...historyQuery,
id: Joi.string().uuid().required(),
}).unknown(true),
}),
async (req: Request, res: Response, next: NextFunction) => {
try {
const service = Container.get(ScriptHistoryService);
const data = await service.detail(
(req.query.path as string) || '',
req.query.filename as string,
req.query.id as string,
);
res.send({ code: 200, data });
} catch (error) {
historyError(error, res, next);
}
},
);
route.put(
'/history/restore',
celebrate({
body: Joi.object({
...historyQuery,
id: Joi.string().uuid().required(),
expectedHash: Joi.string().hex().length(64).required(),
}),
}),
async (req: Request, res: Response, next: NextFunction) => {
try {
const { path = '', filename, id, expectedHash } = req.body;
const service = Container.get(ScriptHistoryService);
const data = await service.restore(path, filename, id, expectedHash);
res.send({ code: 200, data });
} catch (error) {
historyError(error, res, next);
}
},
);
route.get('/:file', (req: Request, res: Response) => {
return res.send({
code: 410,
@@ -250,11 +136,6 @@ export default (app: Router) => {
filename: Joi.string().required(),
path: Joi.string().optional().allow(''),
content: Joi.string().optional().allow(''),
skipHistory: Joi.boolean().optional(),
expectedHash: Joi.string()
.hex()
.length(64)
.when('skipHistory', { is: true, then: Joi.required() }),
originFilename: Joi.string().optional().allow(''),
directory: Joi.string().optional().allow(''),
file: Joi.string().optional().allow(''),
@@ -314,56 +195,8 @@ export default (app: Router) => {
if (!isPathAllowed(filePath) || !isPathAllowed(originFilePath)) {
return res.send({ code: 403, message: t('暂无权限') });
}
// Check the canonical parent directly at the directory creation boundary.
const parentPath = resolve(dirname(filePath));
let parentCreated = false;
for (const writableRoot of config.writePathList) {
const root = resolve(writableRoot);
const rootPrefix = root.endsWith(sep) ? root : root + sep;
if (parentPath === root) {
await fs.mkdir(root, { recursive: true });
} else if (parentPath.startsWith(rootPrefix)) {
await fs.mkdir(parentPath, { recursive: true });
} else {
continue;
}
parentCreated = true;
break;
}
if (!parentCreated) {
return res.send({ code: 403, message: t('暂无权限') });
}
await fs.mkdir(path, { recursive: true });
const fileExists = await fileExist(filePath);
if (fileExists && resolveFileAccess(config.scriptPath, [filePath])) {
let removeSource = filename !== originFilename;
let backupSource = '';
if (removeSource) {
const [originRealPath, targetRealPath] = await Promise.all([
existingWritableRealPath(originFilePath),
existingWritableRealPath(filePath),
]);
// Aliases of one script are an in-place save. Removing the source
// would also remove the destination behind a target symlink.
removeSource = originRealPath !== targetRealPath;
backupSource = originRealPath;
}
// Save-as removes the source after committing the destination. Keep
// its original content too: destination history only protects the
// file being overwritten, not the source being deleted.
if (removeSource) {
await fs.copyFile(
backupSource,
join(config.bakPath, originFilename.replace(/\//g, '')),
);
}
const service = Container.get(ScriptHistoryService);
const data = await service.save(path, filename, content, {
skipHistory: req.body.skipHistory,
expectedHash: req.body.expectedHash,
});
if (removeSource) await rmPath(originFilePath);
return res.send({ code: 200, data });
}
if (fileExists) {
await fs.copyFile(
originFilePath,
@@ -376,7 +209,7 @@ export default (app: Router) => {
await writeFileWithLock(filePath, content);
return res.send({ code: 200 });
} catch (e) {
return historyError(e, res, next);
return next(e);
}
},
);
@@ -388,11 +221,6 @@ export default (app: Router) => {
filename: Joi.string().required(),
path: Joi.string().optional().allow(''),
content: Joi.string().required().allow(''),
skipHistory: Joi.boolean().optional(),
expectedHash: Joi.string()
.hex()
.length(64)
.when('skipHistory', { is: true, then: Joi.required() }),
}),
}),
async (req: Request, res: Response, next: NextFunction) => {
@@ -410,14 +238,10 @@ export default (app: Router) => {
message: t('暂无权限'),
});
}
const service = Container.get(ScriptHistoryService);
const data = await service.save(path || '', filename, content, {
skipHistory: req.body.skipHistory,
expectedHash: req.body.expectedHash,
});
return res.send({ code: 200, data });
await writeFileWithLock(filePath, content);
return res.send({ code: 200 });
} catch (e) {
return historyError(e, res, next);
return next(e);
}
},
);
@@ -544,21 +368,12 @@ export default (app: Router) => {
if (!isPathAllowed(filePath)) {
return res.send({ code: 403, message: t('暂无权限') });
}
const logPath = resolveFileAccess(config.logPath, [
path,
`${name}.swap`,
]);
if (!logPath) {
return res.send({ code: 403, message: t('暂无权限') });
}
const logPath = join(config.logPath, path, `${name}.swap`);
const scriptService = Container.get(ScriptService);
const result = await scriptService.stopScript(filePath, pid);
setTimeout(() => {
const cleanupPath = resolveFileAccess(config.logPath, [logPath]);
if (cleanupPath) {
void rmPath(cleanupPath);
}
rmPath(logPath);
}, 3000);
res.send(result);
} catch (e) {
+9 -38
View File
@@ -8,12 +8,15 @@ import { celebrate, Joi } from 'celebrate';
import UserService from '../services/user';
import { t } from '../shared/i18n';
import { isDefaultAuthInfo } from '../shared/auth';
import { getUniqPath, handleLogPath, parseVersion } from '../config/util';
import { findCronId } from '../shared/cronCommand';
import {
getUniqPath,
handleLogPath,
parseVersion,
promiseExec,
} from '../config/util';
import dayjs from 'dayjs';
import multer from 'multer';
import { logStreamManager } from '../shared/logStreamManager';
import { rateLimit } from 'express-rate-limit';
const route = Router();
const storage = multer.diskStorage({
@@ -205,17 +208,6 @@ export default (app: Router) => {
},
);
route.get(
'/update-status',
async (req: Request, res: Response, next: NextFunction) => {
try {
res.send(await Container.get(SystemService).getUpgradeStatus());
} catch (e) {
return next(e);
}
},
);
route.put(
'/reload',
celebrate({
@@ -257,7 +249,6 @@ export default (app: Router) => {
route.put(
'/command-run',
rateLimit({ windowMs: 60 * 1000, limit: 60 }),
celebrate({
body: Joi.object({
command: Joi.string().required(),
@@ -267,13 +258,8 @@ export default (app: Router) => {
try {
const systemService = Container.get(SystemService);
const command = req.body.command;
const crontab = await fs
.readFile(config.crontabFile, 'utf8')
.catch((e) => {
if (e.code === 'ENOENT') return '';
throw e;
});
const id = findCronId(crontab, command);
const idStr = `cat ${config.crontabFile} | grep -E "${command}" | perl -pe "s|.*ID=(.*) ${command}.*|\\1|" | head -1 | awk -F " " '{print $1}' | xargs echo -n`;
let id = await promiseExec(idStr);
const uniqPath = await getUniqPath(command, id);
const logTime = dayjs().format('YYYY-MM-DD-HH-mm-ss-SSS');
const logPath = `${uniqPath}/${logTime}.log`;
@@ -288,9 +274,7 @@ export default (app: Router) => {
onEnd: async (cp, endTime, diff) => {
// Close the stream after task completion
try {
await logStreamManager.closeStream(
await handleLogPath(logPath),
);
await logStreamManager.closeStream(await handleLogPath(logPath));
} finally {
res.end();
}
@@ -394,19 +378,6 @@ export default (app: Router) => {
},
);
route.get(
'/notify-log',
async (req: Request, res: Response, next: NextFunction) => {
try {
const systemService = Container.get(SystemService);
const data = await systemService.getNotifyLog();
res.send({ code: 200, data });
} catch (e) {
return next(e);
}
},
);
route.delete(
'/log',
async (req: Request, res: Response, next: NextFunction) => {
+28 -13
View File
@@ -5,7 +5,6 @@ import { Container } from 'typedi';
import config from './config';
import Logger from './loaders/logger';
import { errStack } from './shared/errors';
import { initializeLanguageSync } from './shared/languageSync';
import { type GrpcServerService } from './services/grpc';
import { type HttpServerService } from './services/http';
@@ -28,7 +27,6 @@ class Application {
if (cluster.isPrimary) {
await this.initializeDatabase();
}
await initializeLanguageSync();
if (cluster.isPrimary) {
this.startMasterProcess();
} else {
@@ -41,18 +39,11 @@ class Application {
}
private startMasterProcess() {
const readyTimeout =
process.arch === 'arm' &&
Number(
(process.config.variables as { arm_version?: string | number }).arm_version,
) === 6
? 120000
: 30000;
// Fork gRPC worker first and wait for it to be ready
const grpcWorker = this.forkWorker('grpc');
// Wait for gRPC worker to signal it's ready before starting HTTP worker
this.waitForWorkerReady(grpcWorker, readyTimeout)
this.waitForWorkerReady(grpcWorker, 30000)
.then(() => {
Logger.info('[boot] gRPC worker is ready, starting HTTP worker');
this.httpWorker = this.forkWorker('http');
@@ -78,7 +69,7 @@ class Application {
Logger.warn('Unable to notify HTTP worker of scheduler exit');
}
const newGrpcWorker = this.forkWorker('grpc');
this.waitForWorkerReady(newGrpcWorker, readyTimeout)
this.waitForWorkerReady(newGrpcWorker, 30000)
.then(() => {
Logger.info('gRPC worker restarted and ready');
// Re-register cron jobs by notifying the HTTP worker
@@ -155,8 +146,32 @@ class Application {
}
private async setupMiddlewares(): Promise<express.Application> {
const { createHttpApplication } = await import('./loaders/httpApplication');
return createHttpApplication();
// Only the HTTP worker needs an Express application and its middleware.
const [
{ default: createExpress },
{ default: helmet },
{ default: cors },
{ default: compression },
{ monitoringMiddleware },
] = await Promise.all([
import('express'),
import('helmet'),
import('cors'),
import('compression'),
import('./middlewares/monitoring'),
]);
const app = createExpress();
app.use((req, res, next) => {
if (req.query.t) delete req.query.t;
next();
});
app.use(helmet({
contentSecurityPolicy: false,
}));
app.use(cors(config.cors));
app.use(compression());
app.use(monitoringMiddleware);
return app;
}
private setupMasterShutdown() {
+5 -2
View File
@@ -1,16 +1,19 @@
import { errStack } from '../shared/errors';
// Also terminate if the primary disappears without forwarding a signal.
const onParentDisconnect = () => process.exit(1);
process.once('disconnect', onParentDisconnect);
async function initializeDatabase() {
const { database } = await import('../data');
const { sequelize } = await import('../data');
try {
const { default: loadDatabase } = await import('../loaders/db');
await loadDatabase();
} finally {
await database.close();
await sequelize.close();
}
}
initializeDatabase()
.catch((error) => {
console.error(`[boot] Database initialization failed:\n${errStack(error)}`);
-141
View File
@@ -1,141 +0,0 @@
// A single, disposable backend process. Resolve startup dependencies, but never
// run appLoader: its initData/initTask hooks install dependencies and start user work.
import 'reflect-metadata';
import dns from 'dns';
import http from 'http';
// The first scheduler connection can outlive its short RPC deadline. Let the
// normal single-flight recovery finish within the supervised startup budget.
async function waitForUpgradeScheduler(
readiness: {
recover(): Promise<boolean>;
},
timeoutMs: number,
retryMs = 250,
) {
const deadline = Date.now() + timeoutMs;
do {
if (await readiness.recover()) return;
const remaining = deadline - Date.now();
if (remaining <= 0) break;
await new Promise((resolve) =>
setTimeout(resolve, Math.min(retryMs, remaining)),
);
} while (Date.now() < deadline);
throw new Error('Upgrade scheduler readiness failed');
}
async function probeUpgrade() {
if (!process.send || process.env.QL_UPGRADE_PROBE !== '1')
throw new Error('Upgrade probe requires its isolated supervisor');
dns.setDefaultResultOrder('ipv4first');
const { default: config } = await import('../config');
const { database } = await import('../data');
const { default: loadDatabase } = await import('../loaders/db');
const { Container } = await import('typedi');
const { GrpcServerService } = await import('../services/grpc');
const { HttpServerService } = await import('../services/http');
const { createHttpApplication } = await import('../loaders/httpApplication');
const { default: loadServer } = await import('../loaders/server');
const { default: loadExpress } = await import('../loaders/express');
await import('../loaders/app'); // Resolve normal startup hooks without running them.
const { default: inject } = await import('../loaders/depInjector');
const grpc = Container.get(GrpcServerService);
const server = Container.get(HttpServerService);
try {
await loadDatabase();
// Exercise an actual on-disk transaction as well as schema initialization.
await database.transaction(async () => {
await database.query('CREATE TABLE upgrade_probe (value INTEGER)');
await database.query('INSERT INTO upgrade_probe VALUES (42)');
const rows = await database.query('SELECT value FROM upgrade_probe');
if (
(
rows[0] as {
value: number;
}
)?.value !== 42
)
throw new Error('Upgrade database readback failed');
});
inject();
config.grpcPort = await grpc.initialize();
const { default: cronClient } = await import('../schedule/client');
cronClient.readiness.configure(async () => {});
await waitForUpgradeScheduler(
cronClient.readiness,
process.arch === 'arm' &&
Number(
(
process.config.variables as {
arm_version?: string | number;
}
).arm_version,
) === 6
? 60000
: 10000,
);
const app = await createHttpApplication();
loadExpress({ app });
const listener = await server.initialize(app, 0);
await loadServer({ server: listener });
const address = listener.address();
if (!address || typeof address === 'string')
throw new Error('No probe HTTP port');
const health = await new Promise<any>((resolve, reject) => {
const request = http.get(
`http://127.0.0.1:${address.port}/api/health`,
(response) => {
let body = '';
response.on('data', (chunk) => {
body += chunk;
});
response.on('error', reject);
response.on('end', () => {
try {
const result = JSON.parse(body);
if (
response.statusCode !== 200 ||
result.code !== 200 ||
result.data?.status !== 'ok' ||
!result.data.services?.grpc ||
!result.data.services?.http
)
throw new Error('Upgrade HTTP/gRPC health check failed');
resolve(result.data);
} catch (error) {
reject(error);
}
});
},
);
request.setTimeout(10000, () =>
request.destroy(new Error('Upgrade health request timed out')),
);
request.on('error', reject);
});
process.send!({
type: 'upgrade-probe',
ok: true,
build: health.build,
memory: process.memoryUsage(),
pid: process.pid,
});
} finally {
await server.shutdown();
grpc.getServer().forceShutdown();
await database.close();
}
}
process.once('disconnect', () => process.exit(1));
probeUpgrade()
.then(() => process.exit(0))
.catch((error) => {
process.send?.({
type: 'upgrade-probe',
ok: false,
code: error.code,
message: error.message,
stack: error.stack,
});
console.error(error);
process.exit(1);
});
-1
View File
@@ -27,7 +27,6 @@ export const SAMPLE_FILES = [
];
export const PYTHON_INSTALL_DIR = process.env.PYTHON_HOME;
export const PYTHON_VENV_DIR = process.env.PYTHON_VENV_DIR;
export const NotificationModeStringMap = {
0: 'gotify',
+47 -95
View File
@@ -1,5 +1,6 @@
import { execFileSync } from 'child_process';
import { execSync } from 'child_process';
import * as fs from 'fs/promises';
import * as os from 'os';
import path from 'path';
import config from './index';
import { fileExist } from './util';
@@ -23,120 +24,71 @@ const clientCertPath = path.join(certDir, 'client.crt');
let cachedConfig: GrpcTlsConfig | null = null;
function run(args: string[]): string {
return execFileSync('openssl', args, {
stdio: 'pipe',
timeout: 30000,
encoding: 'utf-8',
}).trim();
function run(cmd: string, execOpts?: Record<string, unknown>): string {
const opts = { stdio: 'pipe', timeout: 30000, encoding: 'utf-8', ...execOpts } as any;
return (execSync(cmd, opts) as string).trim();
}
async function tmpFile(prefix: string): Promise<string> {
const dir = (await fileExist(certDir)) ? certDir : os.tmpdir();
await fs.mkdir(dir, { recursive: true });
return path.join(dir, `.${prefix}_${Date.now()}_${Math.random().toString(36).slice(2)}.pem`);
}
async function generateAllCerts(): Promise<GrpcTlsConfig> {
Logger.info('[boot] Generating gRPC mTLS certificates...');
await fs.mkdir(certDir, { recursive: true, mode: 0o700 });
// Atomically reserve a private workspace for keys, CSRs and OpenSSL side files.
const workspace = await fs.mkdtemp(path.join(certDir, '.generate-'));
const caKeyTmp = path.join(workspace, 'ca.key');
const caCertTmp = path.join(workspace, 'ca.crt');
const serverKeyTmp = path.join(workspace, 'server.key');
const serverCsrTmp = path.join(workspace, 'server.csr');
const serverExtTmp = path.join(workspace, 'server.ext');
const clientKeyTmp = path.join(workspace, 'client.key');
const clientCsrTmp = path.join(workspace, 'client.csr');
const clientExtTmp = path.join(workspace, 'client.ext');
const caKeyTmp = await tmpFile('ca_key');
const caCertTmp = await tmpFile('ca_cert');
const serverKeyTmp = await tmpFile('server_key');
const serverCsrTmp = await tmpFile('server_csr');
const serverExtTmp = await tmpFile('server_ext');
const clientKeyTmp = await tmpFile('client_key');
const clientCsrTmp = await tmpFile('client_csr');
const clientExtTmp = await tmpFile('client_ext');
const srlTmp = path.join(path.dirname(caKeyTmp), '.grpc_ca.srl');
const cleanup = async () => {
await fs.rm(workspace, { recursive: true, force: true });
for (const f of [caKeyTmp, caCertTmp, serverKeyTmp, serverCsrTmp, serverExtTmp,
clientKeyTmp, clientCsrTmp, clientExtTmp, srlTmp]) {
try { await fs.unlink(f); } catch {}
}
};
try {
run(['genrsa', '-out', caKeyTmp, '2048']);
run([
'req',
'-new',
'-x509',
'-days',
'3650',
'-key',
caKeyTmp,
'-out',
caCertTmp,
'-subj',
'/CN=qinglong-ca/O=qinglong/C=CN',
]);
// 1. CA(私钥直接存盘,证书写入临时文件供签发使用)
run(`openssl genrsa -out '${caKeyTmp}' 2048 2>/dev/null`);
run(`openssl req -new -x509 -days 3650 -key '${caKeyTmp}' -out '${caCertTmp}' -subj '/CN=qinglong-ca/O=qinglong/C=CN' 2>/dev/null`);
const caKey = await fs.readFile(caKeyTmp, 'utf-8');
const caCert = await fs.readFile(caCertTmp, 'utf-8');
await fs.mkdir(certDir, { recursive: true });
await fs.writeFile(caKeyPath, caKey, { mode: 0o600 });
run(['genrsa', '-out', serverKeyTmp, '2048']);
run([
'req',
'-new',
'-key',
serverKeyTmp,
'-out',
serverCsrTmp,
'-subj',
'/CN=grpc-server',
]);
await fs.writeFile(
serverExtTmp,
'subjectAltName=DNS:localhost,IP:127.0.0.1,IP:::1\n',
// 2. 服务端
run(`openssl genrsa -out '${serverKeyTmp}' 2048 2>/dev/null`);
run(`openssl req -new -key '${serverKeyTmp}' -out '${serverCsrTmp}' -subj '/CN=grpc-server' 2>/dev/null`);
await fs.writeFile(serverExtTmp, 'subjectAltName=DNS:localhost,IP:127.0.0.1,IP:::1\n');
const serverCert = run(
`openssl x509 -req -days 3650 -in '${serverCsrTmp}' -CA '${caCertTmp}' -CAkey '${caKeyTmp}' -CAcreateserial -extfile '${serverExtTmp}' 2>/dev/null`,
);
const serverCert = run([
'x509',
'-req',
'-days',
'3650',
'-in',
serverCsrTmp,
'-CA',
caCertTmp,
'-CAkey',
caKeyTmp,
'-CAserial',
path.join(workspace, 'ca.srl'),
'-CAcreateserial',
'-extfile',
serverExtTmp,
]);
const serverKey = await fs.readFile(serverKeyTmp, 'utf-8');
run(['genrsa', '-out', clientKeyTmp, '2048']);
run([
'req',
'-new',
'-key',
clientKeyTmp,
'-out',
clientCsrTmp,
'-subj',
'/CN=grpc-client',
]);
// 3. 客户端
run(`openssl genrsa -out '${clientKeyTmp}' 2048 2>/dev/null`);
run(`openssl req -new -key '${clientKeyTmp}' -out '${clientCsrTmp}' -subj '/CN=grpc-client' 2>/dev/null`);
await fs.writeFile(clientExtTmp, 'extendedKeyUsage=clientAuth\n');
const clientCert = run([
'x509',
'-req',
'-days',
'3650',
'-in',
clientCsrTmp,
'-CA',
caCertTmp,
'-CAkey',
caKeyTmp,
'-CAserial',
path.join(workspace, 'ca.srl'),
'-CAcreateserial',
'-extfile',
clientExtTmp,
]);
const clientCert = run(
`openssl x509 -req -days 3650 -in '${clientCsrTmp}' -CA '${caCertTmp}' -CAkey '${caKeyTmp}' -CAcreateserial -extfile '${clientExtTmp}' 2>/dev/null`,
);
const clientKey = await fs.readFile(clientKeyTmp, 'utf-8');
Logger.info('[boot] gRPC mTLS certificates generated successfully');
return { caCert, serverCert, serverKey, clientCert, clientKey };
} finally {
await cleanup();
Logger.info('[boot] gRPC mTLS certificates generated successfully');
return { caCert, serverCert, serverKey, clientCert, clientKey };
} catch (e) {
await cleanup();
throw e;
}
}
+26 -23
View File
@@ -1,7 +1,5 @@
import { Subscription } from '../data/subscription';
import isNil from 'lodash/isNil';
import { shellQuote } from '../shared/shellQuote';
import { getSubscriptionSshAlias } from '../shared/subscriptionPath';
export function formatUrl(doc: Subscription) {
let url = doc.url;
@@ -9,7 +7,7 @@ export function formatUrl(doc: Subscription) {
if (doc.type === 'private-repo') {
if (doc.pull_type === 'ssh-key') {
host = doc.url!.replace(/.*\@([^\:]+)\:.*/, '$1');
url = doc.url!.replace(host, getSubscriptionSshAlias(doc.alias));
url = doc.url!.replace(host, doc.alias);
} else {
host = doc.url!.replace(/.*\:\/\/([^\/]+)\/.*/, '$1');
const { username, password } = doc.pull_option as any;
@@ -20,24 +18,29 @@ export function formatUrl(doc: Subscription) {
}
export function formatCommand(doc: Subscription, url?: string) {
const args =
doc.type === 'file'
? ['raw', url || formatUrl(doc).url, doc.proxy || '']
: [
'repo',
url || formatUrl(doc).url,
doc.whitelist || '',
doc.blacklist || '',
doc.dependences || '',
doc.branch || '',
doc.extensions || '',
doc.proxy || '',
];
args.push(
String(isNil(doc.autoAddCron) ? true : Boolean(doc.autoAddCron)),
String(isNil(doc.autoDelCron) ? true : Boolean(doc.autoDelCron)),
);
return `SUB_ID=${shellQuote(String(doc.id))} ql ${args
.map((arg) => shellQuote(arg!))
.join(' ')}`;
let command = `SUB_ID=${doc.id} ql `;
let _url = url || formatUrl(doc).url;
const {
type,
whitelist,
blacklist,
dependences,
branch,
extensions,
proxy,
autoAddCron,
autoDelCron,
} = doc;
if (type === 'file') {
command += `raw "${_url}" "${proxy || ''}" "${
isNil(autoAddCron) ? true : Boolean(autoAddCron)
}" "${isNil(autoDelCron) ? true : Boolean(autoDelCron)}"`;
} else {
command += `repo "${_url}" "${whitelist || ''}" "${blacklist || ''}" "${
dependences || ''
}" "${branch || ''}" "${extensions || ''}" "${proxy || ''}" "${
isNil(autoAddCron) ? true : Boolean(autoAddCron)
}" "${isNil(autoDelCron) ? true : Boolean(autoDelCron)}"`;
}
return command;
}
+100 -223
View File
@@ -1,25 +1,18 @@
import * as fs from 'fs/promises';
import { statSync } from 'fs';
import * as path from 'path';
import { exec, execSync } from 'child_process';
import psTreeFun from 'ps-tree';
import { promisify } from 'util';
import { load } from 'js-yaml';
import config from './index';
import { PYTHON_INSTALL_DIR, PYTHON_VENV_DIR, TASK_COMMAND } from './const';
import { PYTHON_INSTALL_DIR, TASK_COMMAND } from './const';
import Logger from '../loaders/logger';
import { writeFileWithLock } from '../shared/utils';
import { DependenceTypes } from '../data/dependence';
import { FormData } from 'undici';
import os from 'os';
import { maybeSudo, isInContainer } from './container';
import { shellQuote } from '../shared/shellQuote';
import { resolveFileAccess } from '../shared/fileAccess';
import {
mirrorWriteCommand,
normalizeMirror,
rewriteMirrorSources,
} from '../shared/linuxMirror';
export * from './share';
@@ -677,9 +670,7 @@ export async function setSystemTimezone(timezone: string): Promise<boolean> {
throw new Error('Invalid timezone');
}
await promiseExec(
maybeSudo(`ln -sf /usr/share/zoneinfo/${timezone} /etc/localtime`),
);
await promiseExec(maybeSudo(`ln -sf /usr/share/zoneinfo/${timezone} /etc/localtime`));
await promiseExec(`echo "${timezone}" | ${maybeSudo('tee /etc/timezone')}`);
return true;
@@ -689,177 +680,61 @@ export async function setSystemTimezone(timezone: string): Promise<boolean> {
}
}
// Helper function to check if a name is a GitHub URL
function getPythonExecutable(): string {
return PYTHON_VENV_DIR
? shellQuote(path.resolve(config.rootPath, PYTHON_VENV_DIR, 'bin/python3'))
: 'python3';
}
function getPythonPip(): string {
return PYTHON_VENV_DIR ? `${getPythonExecutable()} -m pip` : 'pip3';
}
function isGitHubUrl(name: string): boolean {
// Support git+https://, git+http://, https://, and http:// URLs
// This covers GitHub URLs and other git-compatible repositories
return !!name.match(/^(git\+https?:\/\/|https?:\/\/)/i);
}
// Helper function to check if a name is a requirements file
function isRequirementsFile(name: string): boolean {
return !!name.match(/requirements.*\.(txt|in)$/i);
}
// Helper function to check if a name is a pyproject.toml file
function isPyprojectToml(name: string): boolean {
return name.endsWith('pyproject.toml');
}
export function isPythonSourceDependency(name: string): boolean {
const source = name.trim();
return (
isGitHubUrl(source) || isRequirementsFile(source) || isPyprojectToml(source)
);
}
export function getGetCommand(type: DependenceTypes, name: string): string {
const trimmedName = name.trim();
// For Python dependencies installed from GitHub or requirements files,
// we can't reliably check if they're installed, so skip the check
if (type === DependenceTypes.python3) {
if (isPythonSourceDependency(trimmedName)) {
// Return a command that will always indicate not installed
// This ensures GitHub URLs and requirements files are always installed
return 'echo ""';
}
}
const baseCommands = {
[DependenceTypes.nodejs]: `pnpm ls -g | grep "${trimmedName}" | head -1`,
[DependenceTypes.python3]: `${getPythonExecutable()} -c ${shellQuote(
`from importlib.metadata import version, PackageNotFoundError
import importlib.util
import sys
[DependenceTypes.nodejs]: `pnpm ls -g | grep "${name}" | head -1`,
[DependenceTypes.python3]: `
python3 -c "exec('''
name='${name}'
try:
try:
from packaging.requirements import Requirement
except ImportError:
from pip._vendor.packaging.requirements import Requirement
requirement = Requirement(sys.argv[1])
if requirement.marker and not requirement.marker.evaluate():
print('not applicable')
else:
try:
installed = version(requirement.name)
except PackageNotFoundError:
if not requirement.specifier and not requirement.extras and not requirement.url:
if importlib.util.find_spec(requirement.name):
print(requirement.name)
else:
if requirement.specifier.contains(installed, prereleases=True):
print(installed)
except Exception:
pass`,
)} ${shellQuote(trimmedName)}`,
[DependenceTypes.linux]:
getOsTypeSync() === 'Alpine'
? `apk info -es ${trimmedName}`
: maybeSudo(`dpkg-query -s ${trimmedName}`),
from importlib.metadata import version
print(version(name))
except:
import importlib.util as u
import importlib.metadata as m
spec=u.find_spec(name)
print(name if spec else '')
''')"`,
[DependenceTypes.linux]: getOsTypeSync() === 'Alpine'
? `apk info -es ${name}`
: maybeSudo(`dpkg-query -s ${name}`),
};
return baseCommands[type];
}
export function getInstallCommand(type: DependenceTypes, name: string): string {
if (type === DependenceTypes.python3) {
const prefix =
PYTHON_INSTALL_DIR && !PYTHON_VENV_DIR
? ` --prefix=${shellQuote(PYTHON_INSTALL_DIR)}`
: '';
const pip = `${getPythonPip()} install --disable-pip-version-check${
PYTHON_VENV_DIR ? '' : ' --root-user-action=ignore'
}${prefix}`;
name = name.trim();
if (isRequirementsFile(name) || isPyprojectToml(name)) {
const file = [config.scriptPath, config.repoPath]
.filter(Boolean)
.map((root) => resolveFileAccess(root, [name]))
.find((file) => {
if (!file) return false;
try {
return statSync(file).isFile();
} catch {
return false;
}
});
if (!file)
throw new Error(
'Python dependency file must be inside scripts or repo',
);
return isRequirementsFile(name)
? `${pip} -r ${shellQuote(file)}`
: `${pip} ${shellQuote(path.dirname(file))}`;
}
return `${pip} -- ${shellQuote(name)}`;
}
const baseCommands = {
[DependenceTypes.nodejs]: 'pnpm add -g',
[DependenceTypes.linux]:
getOsTypeSync() === 'Alpine'
? 'apk add --no-check-certificate'
: maybeSudo('apt-get install -y'),
[DependenceTypes.python3]:
'pip3 install --disable-pip-version-check --root-user-action=ignore',
[DependenceTypes.linux]: getOsTypeSync() === 'Alpine'
? 'apk add --no-check-certificate'
: maybeSudo('apt-get install -y'),
};
return `${baseCommands[type]} ${name.trim()}`;
let command = baseCommands[type];
if (type === DependenceTypes.python3 && PYTHON_INSTALL_DIR) {
command = `${command} --prefix=${PYTHON_INSTALL_DIR}`;
}
return `${command} ${name.trim()}`;
}
export function getUninstallCommand(
type: DependenceTypes,
name: string,
): string {
if (type === DependenceTypes.python3) {
name = name.trim();
if (isRequirementsFile(name)) {
const file = [config.scriptPath, config.repoPath]
.filter(Boolean)
.map((root) => resolveFileAccess(root, [name]))
.find((file) => {
if (!file) return false;
try {
return statSync(file).isFile();
} catch {
return false;
}
});
if (!file)
throw new Error(
'Python dependency file must be inside scripts or repo',
);
return `${getPythonPip()} uninstall --disable-pip-version-check${
PYTHON_VENV_DIR ? '' : ' --root-user-action=ignore'
} -y -r ${shellQuote(file)}`;
}
if (isPyprojectToml(name) || isGitHubUrl(name)) {
// A source URL/project can install several distributions and has no safe
// inverse. Removing its panel entry leaves installed packages intact.
return `printf '%s\\n' ${shellQuote(
'Remove installed Python distributions separately by package name.',
)}`;
}
const packageName =
name.match(
/^([A-Za-z0-9][A-Za-z0-9._-]*)(?:\[[^\]]+\])?(?=\s*(?:[<>=!~;@]|$))/,
)?.[1] || name;
return `${getPythonPip()} uninstall --disable-pip-version-check${
PYTHON_VENV_DIR ? '' : ' --root-user-action=ignore'
} -y -- ${shellQuote(packageName)}`;
}
const baseCommands = {
[DependenceTypes.nodejs]: 'pnpm remove -g',
[DependenceTypes.linux]:
getOsTypeSync() === 'Alpine' ? 'apk del' : maybeSudo('apt-get remove -y'),
[DependenceTypes.python3]:
'pip3 uninstall --disable-pip-version-check --root-user-action=ignore -y',
[DependenceTypes.linux]: getOsTypeSync() === 'Alpine'
? 'apk del'
: maybeSudo('apt-get remove -y'),
};
return `${baseCommands[type]} ${name.trim()}`;
}
@@ -931,73 +806,75 @@ export async function detectOS(): Promise<
return osType;
}
async function getCurrentMirrorDomain(
filePath: string,
): Promise<string | null> {
const fileContent = await fs.readFile(filePath, 'utf8');
const lines = fileContent.split('\n');
for (const line of lines) {
if (line.trim().startsWith('#')) {
continue;
}
const match = line.match(/https?:\/\/[^\/]+/);
if (match) {
return match[0];
}
}
return null;
}
async function replaceDomainInFile(
filePath: string,
oldDomainWithScheme: string,
newDomainWithScheme: string,
): Promise<void> {
let fileContent = await fs.readFile(filePath, 'utf8');
let updatedContent = fileContent.replace(
new RegExp(oldDomainWithScheme, 'g'),
newDomainWithScheme,
);
if (!newDomainWithScheme.endsWith('/')) {
newDomainWithScheme += '/';
}
await writeFileWithLock(filePath, updatedContent);
}
async function _updateLinuxMirror(
osType: string,
mirror: string,
mirrorDomainWithScheme: string,
): Promise<string> {
const defaults: Record<string, string> = {
Debian: 'http://deb.debian.org/debian',
Ubuntu: 'http://archive.ubuntu.com/ubuntu',
Alpine: 'http://dl-cdn.alpinelinux.org/alpine',
};
if (!defaults[osType])
throw new Error('Unsupported OS type for updating mirrors.');
const target = mirror || defaults[osType];
normalizeMirror(target);
const S = osType !== 'Alpine' && isInContainer() ? 'sudo ' : '';
const update =
osType === 'Alpine' ? 'apk update' : `${S}apt-get update --error-on=any`;
let file =
osType === 'Alpine'
? '/etc/apk/repositories'
: `/etc/apt/sources.list.d/${osType.toLowerCase()}.sources`;
let content = '';
try {
content = await fs.readFile(file, 'utf8');
} catch (error: any) {
if (error.code !== 'ENOENT') throw error;
if (osType !== 'Alpine') {
try {
file = '/etc/apt/sources.list';
content = await fs.readFile(file, 'utf8');
} catch (legacyError: any) {
if (legacyError.code !== 'ENOENT') throw legacyError;
file = `/etc/apt/sources.list.d/${osType.toLowerCase()}.sources`;
const S = isInContainer() ? 'sudo ' : '';
let filePath: string, currentDomainWithScheme: string | null;
switch (osType) {
case 'Debian':
filePath = '/etc/apt/sources.list.d/debian.sources';
currentDomainWithScheme = await getCurrentMirrorDomain(filePath);
if (currentDomainWithScheme) {
return `${S}sed -i 's|${currentDomainWithScheme}|${mirrorDomainWithScheme || 'http://deb.debian.org'}|g' ${filePath} || (${S}mkdir -p /etc/apt/sources.list.d && echo -e "Types: deb\\nURIs: ${mirrorDomainWithScheme || 'http://deb.debian.org'}\\nSuites: \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-updates\\nComponents: main\\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg" | ${S}tee ${filePath}) && ${S}apt-get update`;
} else {
return `${S}mkdir -p /etc/apt/sources.list.d && echo -e "Types: deb\\nURIs: ${mirrorDomainWithScheme || 'http://deb.debian.org'}\\nSuites: \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-updates\\nComponents: main\\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg" | ${S}tee ${filePath} && ${S}apt-get update`;
}
}
case 'Ubuntu':
filePath = '/etc/apt/sources.list.d/ubuntu.sources';
currentDomainWithScheme = await getCurrentMirrorDomain(filePath);
if (currentDomainWithScheme) {
return `${S}sed -i 's|${currentDomainWithScheme}|${mirrorDomainWithScheme || 'http://archive.ubuntu.com'}|g' ${filePath} || (${S}mkdir -p /etc/apt/sources.list.d && echo -e "Types: deb\\nURIs: ${mirrorDomainWithScheme || 'http://archive.ubuntu.com'}\\nSuites: \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-updates \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-backports\\nComponents: main restricted universe multiverse\\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg" | ${S}tee ${filePath}) && ${S}apt-get update`;
} else {
return `${S}mkdir -p /etc/apt/sources.list.d && echo -e "Types: deb\\nURIs: ${mirrorDomainWithScheme || 'http://archive.ubuntu.com'}\\nSuites: \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-updates \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-backports\\nComponents: main restricted universe multiverse\\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg" | ${S}tee ${filePath} && ${S}apt-get update`;
}
case 'Alpine':
filePath = '/etc/apk/repositories';
currentDomainWithScheme = await getCurrentMirrorDomain(filePath);
if (currentDomainWithScheme) {
return `sed -i 's|${currentDomainWithScheme}|${mirrorDomainWithScheme || 'http://dl-cdn.alpinelinux.org'}|g' ${filePath} || (mkdir -p /etc/apk && echo -e "\\$(grep VERSION_ID /etc/os-release | cut -d= -f2 | cut -d. -f1,2)/main\\n\\$(grep VERSION_ID /etc/os-release | cut -d= -f2 | cut -d. -f1,2)/community" | sed "s|^|${mirrorDomainWithScheme || 'http://dl-cdn.alpinelinux.org'}/alpine/v|" | tee ${filePath}) && apk update`;
} else {
return `mkdir -p /etc/apk && echo -e "\\$(grep VERSION_ID /etc/os-release | cut -d= -f2 | cut -d. -f1,2)/main\\n\\$(grep VERSION_ID /etc/os-release | cut -d= -f2 | cut -d. -f1,2)/community" | sed "s|^|${mirrorDomainWithScheme || 'http://dl-cdn.alpinelinux.org'}/alpine/v|" | tee ${filePath} && apk update`;
}
default:
throw Error('Unsupported OS type for updating mirrors.');
}
if (!content.trim()) {
const release = await fs.readFile('/etc/os-release', 'utf8');
if (osType === 'Alpine') {
const version = release.match(/^VERSION_ID=["']?(\d+\.\d+)/m)?.[1];
if (!version)
throw new Error('Cannot determine Alpine repository version');
content = `${defaults.Alpine}/v${version}/main\n${defaults.Alpine}/v${version}/community\n`;
} else {
// The fallback may have read an empty legacy .list. Deb822 stanzas must
// be written to a .sources file, leaving the legacy file untouched.
file = `/etc/apt/sources.list.d/${osType.toLowerCase()}.sources`;
const codename = release.match(/^VERSION_CODENAME=["']?([\w-]+)/m)?.[1];
if (!codename)
throw new Error('Cannot determine APT repository codename');
const ubuntu = osType === 'Ubuntu';
content = `Types: deb\nURIs: ${
defaults[osType]
}\nSuites: ${codename} ${codename}-updates${
ubuntu ? ` ${codename}-backports` : ''
}\nComponents: ${
ubuntu ? 'main restricted universe multiverse' : 'main'
}\nSigned-By: /usr/share/keyrings/${osType.toLowerCase()}-archive-keyring.gpg\n`;
if (!ubuntu)
content += `\nTypes: deb\nURIs: http://security.debian.org/debian-security\nSuites: ${codename}-security\nComponents: main\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg\n`;
}
}
return mirrorWriteCommand(
file,
rewriteMirrorSources(content, target),
S,
update,
);
}
export async function updateLinuxMirrorFile(mirror: string): Promise<string> {
-28
View File
@@ -1,28 +0,0 @@
import { EventEmitter } from 'events';
import { database } from '.';
import { DatabaseClient } from './databaseClient';
// Keep the existing Keyv serialization and namespace. Only its SQLite transport
// changes; old cached authentication records remain readable in either version.
export class CacheStore extends EventEmitter {
readonly opts = { dialect: 'sqlite' };
namespace?: string;
constructor(private client: () => DatabaseClient = () => database) {
super();
}
get(key: string): Promise<string | undefined> {
return this.client().request('cache', 'get', key);
}
async set(key: string, value: string) {
return this.client().request('cache', 'set', key, value);
}
async delete(key: string): Promise<boolean> {
return this.client().request('cache', 'delete', key);
}
async clear(): Promise<void> {
throw new Error('Clearing all authentication namespaces is not supported');
}
async disconnect(): Promise<void> {
/* The shared database client owns the worker. */
}
}
+36 -3
View File
@@ -1,4 +1,5 @@
import { Repository, RecordRow } from './repository';
import { sequelize } from '.';
import { DataTypes, Model, ModelDefined } from 'sequelize';
export class Crontab {
name?: string;
@@ -61,5 +62,37 @@ export enum CrontabStatus {
'queued' = 3,
}
export type CronInstance = RecordRow<Crontab>;
export const CrontabModel = new Repository<Crontab>('Crontab');
export interface CronInstance extends Model<Crontab, Crontab>, Crontab {}
export const CrontabModel = sequelize.define<CronInstance>('Crontab', {
name: {
unique: 'compositeIndex',
type: DataTypes.STRING,
},
command: {
unique: 'compositeIndex',
type: DataTypes.STRING,
},
schedule: {
unique: 'compositeIndex',
type: DataTypes.STRING,
},
timestamp: DataTypes.STRING,
saved: DataTypes.BOOLEAN,
status: DataTypes.NUMBER,
isSystem: DataTypes.NUMBER,
pid: DataTypes.NUMBER,
isDisabled: DataTypes.NUMBER,
isPinned: DataTypes.NUMBER,
log_path: DataTypes.STRING,
queued_token: DataTypes.STRING,
labels: DataTypes.JSON,
last_running_time: DataTypes.NUMBER,
last_execution_time: DataTypes.NUMBER,
sub_id: { type: DataTypes.NUMBER, allowNull: true },
extra_schedules: DataTypes.JSON,
task_before: DataTypes.STRING,
task_after: DataTypes.STRING,
log_name: DataTypes.STRING,
allow_multiple_instances: DataTypes.NUMBER,
work_dir: DataTypes.STRING,
});
+48 -3
View File
@@ -1,4 +1,5 @@
import { Repository, RecordRow } from './repository';
import { DataTypes, Model } from 'sequelize';
import { sequelize } from '.';
export class CrontabStat {
id?: number;
@@ -22,5 +23,49 @@ export class CrontabStat {
}
}
export type CrontabStatInstance = RecordRow<CrontabStat>;
export const CrontabStatModel = new Repository<CrontabStat>('CrontabStat');
export interface CrontabStatInstance extends Model<CrontabStat, CrontabStat>, CrontabStat {}
export const CrontabStatModel = sequelize.define<CrontabStatInstance>(
'CrontabStat',
{
id: {
type: DataTypes.INTEGER,
primaryKey: true,
autoIncrement: true,
},
ref_id: {
type: DataTypes.NUMBER,
allowNull: false,
},
date: {
type: DataTypes.STRING,
allowNull: false,
},
run_count: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
success_count: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
fail_count: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
total_time: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
max_time: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
},
{
indexes: [
{ unique: true, fields: ['ref_id', 'date'] },
{ fields: ['date'] },
],
},
);
+23 -3
View File
@@ -1,4 +1,5 @@
import { Repository, RecordRow } from './repository';
import { sequelize } from '.';
import { DataTypes, Model } from 'sequelize';
export enum CronViewType {
'系统' = 1,
@@ -38,5 +39,24 @@ export class CrontabView {
}
}
export type CronViewInstance = RecordRow<CrontabView>;
export const CrontabViewModel = new Repository<CrontabView>('CrontabView');
export interface CronViewInstance
extends Model<CrontabView, CrontabView>,
CrontabView {}
export const CrontabViewModel = sequelize.define<CronViewInstance>(
'CrontabView',
{
name: {
unique: 'name',
type: DataTypes.STRING,
},
position: DataTypes.NUMBER,
isDisabled: DataTypes.NUMBER,
filters: DataTypes.JSON,
sorts: DataTypes.JSON,
filterRelation: {
type: DataTypes.STRING,
allowNull: true,
},
type: DataTypes.NUMBER,
},
);
-325
View File
@@ -1,325 +0,0 @@
import { AsyncLocalStorage } from 'async_hooks';
import { ChildProcess, fork } from 'child_process';
import { join } from 'path';
import { setTimeout, clearTimeout } from 'timers';
import { DatabaseError } from './databaseErrors';
type Scope = { active: boolean; child: boolean; token: number };
type Reply = {
id: number;
value?: any;
error?: { code: string; message: string };
};
type Limits = { timeoutMs?: number; queueLimit?: number };
// One worker per backend process; SQLite itself coordinates independent processes.
// A transaction owns this client's queue until its callback and commit finish.
export class DatabaseClient {
private worker?: ChildProcess;
private stopped: Promise<void> = Promise.resolve();
private failed?: Error;
private closed = false;
private nextId = 0;
private queued = 0;
private tail: Promise<void> = Promise.resolve();
private pending = new Map<
number,
{
resolve(value: any): void;
reject(error: Error): void;
timer: NodeJS.Timeout;
}
>();
private scope = new AsyncLocalStorage<Scope>();
private readonly timeoutMs: number;
private readonly queueLimit: number;
constructor(
public readonly storage: string,
private cacheStorage?: string,
limits: Limits = {},
) {
this.timeoutMs =
limits.timeoutMs ?? (process.arch === 'arm' ? 300000 : 60000);
this.queueLimit = limits.queueLimit ?? 256;
if (
!Number.isSafeInteger(this.timeoutMs) ||
this.timeoutMs < 1 ||
!Number.isSafeInteger(this.queueLimit) ||
this.queueLimit < 1
)
throw new Error('Invalid database worker limits');
}
private abort(error: Error) {
this.failed ||= error;
for (const request of this.pending.values()) {
clearTimeout(request.timer);
request.reject(this.failed);
}
this.pending.clear();
this.unref();
this.worker?.kill('SIGKILL');
}
private unref() {
this.worker?.unref();
this.worker?.channel?.unref();
}
private start() {
if (this.worker) return this.worker;
const [major, minor] = process.versions.node.split('.').map(Number);
if (major < 22 || (major === 22 && minor < 17))
throw new DatabaseError(
'Qinglong requires Node >=22.17; keep the previous installation until Node is upgraded',
'UNSUPPORTED_NODE',
);
// ts-node's CLI places its executable in execArgv, rather than using -r.
// Source workers register their own compiler; compiled workers need none.
const preloads = __filename.endsWith('.ts')
? ['--require', require.resolve('ts-node/register/transpile-only')]
: [];
const armv6 =
process.arch === 'arm' &&
Number(
(process.config.variables as { arm_version?: string | number })
.arm_version,
) === 6;
const worker = fork(require.resolve('./databaseProcess'), [], {
serialization: 'advanced',
stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
execArgv: [
...preloads,
...(armv6 ? ['--jitless'] : []),
'--max-old-space-size=128',
],
env: {
...process.env,
NODE_OPTIONS: '',
...(__filename.endsWith('.ts')
? { TS_NODE_PROJECT: join(__dirname, 'tsconfig.worker.json') }
: {}),
},
});
this.stopped = new Promise((resolve) =>
worker.once('close', () => resolve()),
);
const onOwnerExit = () => worker.kill('SIGKILL');
process.once('exit', onOwnerExit);
worker.once('close', () => process.removeListener('exit', onOwnerExit));
this.worker = worker;
worker.on('message', (reply: Reply & { type?: string }) => {
if (reply.type === 'fatal' && reply.error) {
this.abort(new DatabaseError(reply.error.message, reply.error.code));
return;
}
const request = this.pending.get(reply.id);
if (!request) return;
this.pending.delete(reply.id);
clearTimeout(request.timer);
if (!this.pending.size) this.unref();
if (reply.error)
request.reject(
new DatabaseError(reply.error.message, reply.error.code),
);
else request.resolve(reply.value);
});
const fail = () =>
this.abort(
new DatabaseError(
'Database worker stopped; restart the service before retrying a write',
'DATABASE_WORKER_EXIT',
),
);
worker.on('error', fail);
worker.on('exit', () => {
if (!this.closed) fail();
});
worker.send({
storage: this.storage,
cacheStorage: this.cacheStorage,
ownerPid: process.pid,
});
this.unref();
return worker;
}
private send(
operation: string,
args: any[] = [],
token?: number,
): Promise<any> {
if (this.failed) return Promise.reject(this.failed);
if (this.closed)
return Promise.reject(
new DatabaseError('Database is closed', 'DATABASE_CLOSED'),
);
return new Promise((resolve, reject) => {
try {
const worker = this.start();
const id = ++this.nextId;
const timer = setTimeout(
() =>
this.abort(
new DatabaseError(
'Database worker timed out; writes are never automatically replayed',
'DATABASE_WORKER_TIMEOUT',
),
),
this.timeoutMs,
);
timer.unref();
this.pending.set(id, { resolve, reject, timer });
worker.ref();
worker.channel?.ref();
try {
worker.send({ id, operation, args, token }, (error) => {
if (error)
this.abort(
new DatabaseError(
'Database IPC disconnected',
'DATABASE_WORKER_EXIT',
),
);
});
} catch (error) {
this.pending.delete(id);
clearTimeout(timer);
if (!this.pending.size) this.unref();
reject(error);
}
} catch (error) {
reject(error);
}
});
}
private exclusive<T>(
operation: () => Promise<T>,
shutdown = false,
): Promise<T> {
if (!shutdown && this.queued >= this.queueLimit)
return Promise.reject(
new DatabaseError('Database request queue is full', 'DATABASE_BUSY'),
);
this.queued++;
const previous = this.tail;
let release!: () => void;
this.tail = new Promise<void>((resolve) => {
release = resolve;
});
return previous.then(operation).finally(() => {
this.queued--;
release();
});
}
request<T = any>(operation: string, ...args: any[]): Promise<T> {
const scope = this.scope.getStore();
if (scope) {
if (!scope.active || scope.child)
return Promise.reject(
new DatabaseError(
'Transaction scope is no longer available',
'TRANSACTION_SCOPE',
),
);
if (this.pending.size >= this.queueLimit)
return Promise.reject(
new DatabaseError('Database request queue is full', 'DATABASE_BUSY'),
);
return this.send(operation, args, scope.token);
}
return this.exclusive(() => this.send(operation, args));
}
async transaction<T>(operation: () => Promise<T>): Promise<T> {
const parent = this.scope.getStore();
if (parent && (!parent.active || parent.child))
throw new DatabaseError(
'Concurrent or expired nested transaction',
'TRANSACTION_SCOPE',
);
const run = async () => {
const token = ++this.nextId;
if (parent) parent.child = true;
const scope: Scope = { active: true, child: false, token };
let began = false;
let timer: NodeJS.Timeout | undefined;
try {
await this.send('begin', [], token);
began = true;
const deadline = new Promise<never>((_, reject) => {
timer = setTimeout(
() =>
reject(
new DatabaseError(
'Database transaction timed out',
'DATABASE_TRANSACTION_TIMEOUT',
),
),
this.timeoutMs,
);
});
const value = await Promise.race([
this.scope.run(scope, operation),
deadline,
]);
scope.active = false;
await this.send('commit', [], token);
return value;
} catch (error) {
scope.active = false;
if (began) {
try {
await this.send('rollback', [], token);
} catch {
this.abort(
new DatabaseError(
'Transaction rollback failed; restart the service',
'DATABASE_ROLLBACK_FAILED',
),
);
}
}
throw error;
} finally {
clearTimeout(timer);
if (parent) parent.child = false;
}
};
return parent ? run() : this.exclusive(run);
}
query<T = Record<string, unknown>>(
sql: string,
params: unknown[] = [],
): Promise<T[]> {
return this.request('query', sql, params);
}
get healthy() {
return !this.failed && !this.closed;
}
async close() {
if (this.scope.getStore())
throw new DatabaseError(
'Cannot close within a transaction',
'TRANSACTION_SCOPE',
);
await this.exclusive(async () => {
if (this.closed) return;
try {
if (this.worker && !this.failed) await this.send('close');
} finally {
this.closed = true;
this.worker?.ref();
this.worker?.kill('SIGKILL');
await this.stopped;
}
}, true);
}
}
-36
View File
@@ -1,36 +0,0 @@
// Error details (SQL, bindings, credentials) stay inside the database worker.
export class DatabaseError extends Error {
constructor(message: string, public readonly code: string) {
super(message);
this.name = 'DatabaseError';
}
}
export function databaseStartupFailure(value: unknown) {
const error = (value && typeof value === 'object' ? value : {}) as Error & { code?: string };
const missing = /Cannot find (?:module|package) ['"]([^'"]+)['"]/.exec(
error.message || '',
)?.[1];
const importer = /Require stack:\r?\n-\s+([^\r\n]+)/.exec(
error.message || '',
)?.[1];
const dependency =
/MODULE_NOT_FOUND/.test(error.code || '') &&
Boolean(
missing &&
((!missing.startsWith('.') &&
!missing.startsWith('/') &&
!missing.startsWith('node:')) ||
/node_modules[/\\]/.test(missing) ||
/node_modules[/\\]/.test(importer || '')),
);
return dependency
? {
code: 'MODULE_NOT_FOUND',
message: "Cannot find module '@mikro-orm/sql'",
}
: {
code: 'DATABASE_WORKER_EXIT',
message: 'Database process failed to start',
};
}
-61
View File
@@ -1,61 +0,0 @@
import { databaseStartupFailure } from './databaseErrors';
import { Worker } from 'worker_threads';
// IPC disconnect cannot run while DatabaseSync is executing SQL. Start this
// independent, allocation-light monitor before opening SQLite so a SIGKILLed
// backend cannot leave an orphan holding the old service's database write lock.
async function monitorOwner(ownerPid: number) {
if (!Number.isSafeInteger(ownerPid) || ownerPid < 1)
throw new Error('Invalid database owner');
const monitor = new Worker(
`
const {parentPort,workerData}=require('node:worker_threads');
process.on('uncaughtException',()=>process.kill(process.pid,'SIGKILL'));
const check=()=>{
if(process.ppid!==workerData.ownerPid)process.kill(process.pid,'SIGKILL');
};
check();
setInterval(check,500);
parentPort.postMessage('ready');
`,
{
eval: true,
workerData: { ownerPid },
// V8's process-wide --jitless remains active on ARMv6. Do not load the
// source worker's ts-node compiler into this tiny, plain-JavaScript monitor.
execArgv: [],
resourceLimits: {
maxOldGenerationSizeMb: 8,
maxYoungGenerationSizeMb: 2,
stackSizeMb: 1,
},
},
);
monitor.once('exit', () => process.exit(1));
await new Promise<void>((resolve, reject) => {
monitor.once('message', () => resolve());
monitor.once('error', reject);
});
monitor.unref();
}
// The bootstrap catches dependency import errors before the ORM module loads.
// The advanced IPC serializer preserves Date, Buffer and undefined values.
process.once('disconnect', () => process.exit(1));
process.once('message', async (settings) => {
const pending: unknown[] = [];
const buffer = (request: unknown) => pending.push(request);
// Keep a listener throughout asynchronous monitor startup. Node may flush
// several buffered IPC messages while delivering the initial settings.
process.on('message', buffer);
try {
await monitorOwner((settings as { ownerPid: number }).ownerPid);
require('./databaseWorker').startDatabaseWorker(settings);
process.removeListener('message', buffer);
for (const request of pending) process.emit('message', request, undefined);
} catch (error) {
process.send?.({ type: 'fatal', error: databaseStartupFailure(error) });
} finally {
process.removeListener('message', buffer);
}
});
-280
View File
@@ -1,280 +0,0 @@
import { mkdirSync } from 'fs';
import { dirname } from 'path';
import { EntitySchema, Type } from '@mikro-orm/core';
import { MikroORM, SqliteDriver, NodeSqliteDialect, raw } from '@mikro-orm/sql';
import { entityDefinitions } from './schema';
// Sequelize DATE readers expect TEXT. Numeric epoch dates would break rollback.
class LegacyDateType extends Type<Date, string> {
convertToDatabaseValue(value: Date) {
return value == null
? value
: value.toISOString().replace('T', ' ').replace('Z', ' +00:00');
}
convertToJSValue(value: string | Date) {
return value == null
? value
: value instanceof Date
? value
: new Date(value);
}
getColumnType() {
return 'DATETIME';
}
compareAsType() {
return 'string' as const;
}
get runtimeType() {
return 'Date';
}
}
const entities = Object.values(entityDefinitions).map(
(definition) =>
new EntitySchema({
...definition,
properties: Object.fromEntries(
Object.entries(definition.properties).map(([name, field]) => [
name,
{
...field,
type:
field.type === 'legacyDate' ? new LegacyDateType() : field.type,
},
]),
),
} as any),
);
let orm: Awaited<ReturnType<typeof MikroORM.init<SqliteDriver>>>;
let cache: any;
const transactions: number[] = [];
let ready: Promise<void>;
let workerData: { storage: string; cacheStorage?: string; ownerPid: number };
async function initialize() {
if (workerData.storage !== ':memory:')
mkdirSync(dirname(workerData.storage), { recursive: true });
orm = await MikroORM.init<SqliteDriver>({
driver: SqliteDriver,
driverOptions: new NodeSqliteDialect(workerData.storage),
dbName: workerData.storage,
entities,
debug: false,
allowGlobalContext: false,
discovery: { warnWhenNoEntities: false },
});
await orm.em.getConnection().execute('PRAGMA busy_timeout=10000');
}
function filter(name: keyof typeof entityDefinitions, value: any): any {
if (value === undefined) throw new Error('Undefined database predicate');
if (value == null || typeof value !== 'object' || value instanceof Date)
return value;
if (Array.isArray(value)) return value.map((part) => filter(name, part));
const properties = entityDefinitions[name].properties;
return Object.fromEntries(
Object.entries(value).map(([key, item]: [string, any]) => {
if (item === undefined) throw new Error('Undefined database predicate');
// LIKE searches JSON's serialized text, matching the pre-migration behavior.
if (
key in properties &&
(properties as any)[key].type === 'json' &&
item &&
typeof item === 'object' &&
'$like' in item
)
return [raw(`cast("${key}" as text)`), item];
return [key, filter(name, item)];
}),
);
}
function values(
name: keyof typeof entityDefinitions,
payload: Record<string, any>,
insert = false,
) {
const properties = entityDefinitions[name].properties;
const data = Object.fromEntries(
Object.entries(payload).filter(
([key, value]) => key in properties && value !== undefined,
),
);
for (const [key, value] of Object.entries(data)) {
if ((properties as any)[key].type === 'json' && value !== null)
data[key] = JSON.parse(JSON.stringify(value));
}
const now = new Date();
if (insert && !data.createdAt) data.createdAt = now;
data.updatedAt = now;
return data;
}
function plain(name: keyof typeof entityDefinitions, row: any) {
if (!row) return null;
return Object.fromEntries(
Object.keys(entityDefinitions[name].properties)
.filter((key) => row[key] !== undefined)
.map((key) => [key, row[key]]),
);
}
async function execute(
operation: string,
args: any[],
token?: number,
): Promise<any> {
await ready;
// Reject queued work after owner death; the monitor interrupts running SQL.
if (process.ppid !== workerData.ownerPid) process.exit(1);
const connection = orm.em.getConnection();
if (operation === 'begin') {
await connection.execute(
transactions.length ? `SAVEPOINT "ql_${token}"` : 'BEGIN IMMEDIATE',
);
transactions.push(token!);
return;
}
if (token !== transactions[transactions.length - 1])
throw new Error('Database transaction ownership mismatch');
if (operation === 'commit' || operation === 'rollback') {
if (!transactions.length) throw new Error('No active database transaction');
const nested = transactions.length > 1;
if (operation === 'rollback' && nested)
await connection.execute(`ROLLBACK TO SAVEPOINT "ql_${token}"`);
await connection.execute(
nested ? `RELEASE SAVEPOINT "ql_${token}"` : operation.toUpperCase(),
);
transactions.pop();
return;
}
if (operation === 'close') {
if (transactions.length)
throw new Error('Cannot close an active transaction');
cache?.close();
await orm.close();
return;
}
if (operation === 'query') return connection.execute(args[0], args[1]);
if (operation === 'cache') {
if (!workerData.cacheStorage)
throw new Error('No cache database configured');
if (!cache) {
const { DatabaseSync } = require('node:sqlite');
mkdirSync(dirname(workerData.cacheStorage), { recursive: true });
cache = new DatabaseSync(workerData.cacheStorage);
cache.exec(
'PRAGMA busy_timeout=10000;CREATE TABLE IF NOT EXISTS keyv(key VARCHAR(255) PRIMARY KEY,value TEXT)',
);
}
const [action, key, value] = args;
switch (action) {
case 'get':
return cache.prepare('SELECT value FROM keyv WHERE key=?').get(key)
?.value;
case 'set':
cache
.prepare(
'INSERT INTO keyv(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value',
)
.run(key, value);
return true;
case 'delete':
return (
cache.prepare('DELETE FROM keyv WHERE key=?').run(key).changes > 0
);
default:
throw new Error('Unsupported cache operation');
}
}
const [name, condition, options = {}] = args as [
keyof typeof entityDefinitions,
any,
any,
];
if (!(name in entityDefinitions)) throw new Error('Unknown entity');
const em = orm.em.fork();
const entityName = entities.find((entity) => entity.meta.className === name)!;
if (operation !== 'insert' && condition == null)
throw new Error('Missing database predicate');
const query = operation === 'insert' ? {} : filter(name, condition);
const readOptions = {
...options,
orderBy: options.orderBy?.map((order: Record<string, string>) =>
Object.fromEntries(
Object.entries(order).map(([field, direction]) => [
name === 'Env' && field === 'isPinned'
? raw('COALESCE("isPinned",0)')
: field,
direction,
]),
),
),
};
switch (operation) {
case 'find':
return (await em.find(entityName, query, readOptions)).map((row) =>
plain(name, row),
);
case 'findOne':
return plain(name, await em.findOne(entityName, query, readOptions));
case 'count':
return em.count(entityName, query);
case 'insert': {
const id = await em.insert(entityName, values(name, condition, true));
return plain(name, await em.findOneOrFail(entityName, { id }));
}
case 'nativeUpdate':
return em.nativeUpdate(entityName, query, values(name, options));
case 'nativeDelete':
return em.nativeDelete(entityName, query);
default:
throw new Error('Unsupported database operation');
}
}
export function startDatabaseWorker(settings: typeof workerData) {
workerData = settings;
ready = initialize();
void ready.catch(() => {});
let queue = Promise.resolve();
process.on('message', ({ id, operation, args, token }) => {
// Continue after a failed statement so the owner can still roll back.
queue = queue.then(async () => {
try {
process.send?.({
id,
value: await execute(operation, args, token),
});
} catch (error: any) {
const code =
error.name === 'UniqueConstraintViolationException'
? 'UNIQUE_CONSTRAINT'
: error.name === 'TableNotFoundException'
? 'DATABASE_SCHEMA_MISSING'
: (
{
5: 'SQLITE_BUSY',
6: 'SQLITE_LOCKED',
8: 'SQLITE_READONLY',
10: 'SQLITE_IOERR',
11: 'SQLITE_CORRUPT',
13: 'SQLITE_FULL',
14: 'SQLITE_CANTOPEN',
26: 'SQLITE_NOTADB',
} as Record<number, string>
)[Number(error.errcode) & 255] || 'DATABASE_QUERY_FAILED';
process.send?.({
id,
error: {
code,
message:
code === 'UNIQUE_CONSTRAINT'
? 'Resource already exists'
: `Database operation failed (${code})`,
},
});
}
});
});
}
+16 -3
View File
@@ -1,4 +1,5 @@
import { Repository, RecordRow } from './repository';
import { sequelize } from '.';
import { DataTypes, Model, ModelDefined } from 'sequelize';
export class Dependence {
timestamp?: string;
@@ -46,5 +47,17 @@ export enum versionDependenceCommandTypes {
'=',
}
export type DependenceInstance = RecordRow<Dependence>;
export const DependenceModel = new Repository<Dependence>('Dependence');
export interface DependenceInstance
extends Model<Dependence, Dependence>,
Dependence {}
export const DependenceModel = sequelize.define<DependenceInstance>(
'Dependence',
{
name: DataTypes.STRING,
type: DataTypes.NUMBER,
timestamp: DataTypes.STRING,
status: DataTypes.NUMBER,
log: DataTypes.JSON,
remark: DataTypes.STRING,
},
);
+13 -3
View File
@@ -1,4 +1,5 @@
import { Repository, RecordRow } from './repository';
import { DataTypes, Model } from 'sequelize';
import { sequelize } from '.';
export class Env {
value?: string;
@@ -37,5 +38,14 @@ export const initPosition = 4500000000000000;
export const stepPosition = 10000000000;
export const minPosition = 100;
export type EnvInstance = RecordRow<Env>;
export const EnvModel = new Repository<Env>('Env');
export interface EnvInstance extends Model<Env, Env>, Env {}
export const EnvModel = sequelize.define<EnvInstance>('Env', {
value: { type: DataTypes.STRING, unique: 'compositeIndex' },
timestamp: DataTypes.STRING,
status: DataTypes.NUMBER,
position: DataTypes.NUMBER,
name: { type: DataTypes.STRING, unique: 'compositeIndex' },
remarks: DataTypes.STRING,
isPinned: DataTypes.NUMBER,
labels: DataTypes.JSON,
});
+18 -5
View File
@@ -1,10 +1,23 @@
import { Sequelize, Transaction } from 'sequelize';
import config from '../config/index';
import { join } from 'path';
import { DatabaseClient } from './databaseClient';
export const database = new DatabaseClient(
join(config.dbPath, 'database.sqlite'),
join(config.dbPath, 'keyv.sqlite'),
);
export const sequelize = new Sequelize({
dialect: 'sqlite',
storage: join(config.dbPath, 'database.sqlite'),
logging: false,
retry: {
max: 10,
match: ['SQLITE_BUSY: database is locked'],
},
pool: {
max: 5,
min: 2,
idle: 30000,
acquire: 30000,
evict: 10000,
},
transactionType: Transaction.TYPES.IMMEDIATE,
});
export type ResponseType<T> = { code: number; data?: T; message?: string };
+11 -11
View File
@@ -1,4 +1,5 @@
import { Repository, RecordRow } from './repository';
import { sequelize } from '.';
import { DataTypes, Model, ModelDefined } from 'sequelize';
export class App {
name: string;
@@ -23,14 +24,13 @@ export interface AppToken {
expiration: number;
}
export type AppScope =
| 'envs'
| 'crons'
| 'configs'
| 'scripts'
| 'logs'
| 'system'
| 'dashboard';
export type AppScope = 'envs' | 'crons' | 'configs' | 'scripts' | 'logs' | 'system' | 'dashboard';
export type AppInstance = RecordRow<App>;
export const AppModel = new Repository<App>('App');
export interface AppInstance extends Model<App, App>, App {}
export const AppModel = sequelize.define<AppInstance>('App', {
name: { type: DataTypes.STRING, unique: 'name' },
scopes: DataTypes.JSON,
client_id: DataTypes.STRING,
client_secret: DataTypes.STRING,
tokens: DataTypes.JSON,
});
-25
View File
@@ -1,25 +0,0 @@
export type Query = Record<string, any>;
export type QuerySpec = {
where: Query;
orderBy?: Record<string, string>[];
offset?: number;
limit?: number;
};
export function orderBy(order: string[][]): Record<string, string>[] {
return order.map(([field, direction]) => {
if (!/^(asc|desc)$/i.test(direction))
throw new Error('Invalid sort direction');
return { [field]: direction.toLowerCase() };
});
}
export function textSearch(fields: string[], text: string): Query {
return {
$or: fields.flatMap((field) =>
[...new Set([text, encodeURI(text)])].map((value) => ({
[field]: { $like: `%${value}%` },
})),
),
};
}
-54
View File
@@ -1,54 +0,0 @@
import { database } from '.';
import { DatabaseClient } from './databaseClient';
export type RecordRow<T> = T & { id: number; createdAt: Date; updatedAt: Date };
export type Query = Record<string, any>;
export type ReadOptions = {
fields?: string[];
orderBy?: Record<string, string>[];
limit?: number;
offset?: number;
};
// A small IPC boundary with MikroORM's native query/operation names. No ORM
// entities or request identity maps escape the single database worker.
export class Repository<T> {
constructor(
private name: string,
private client: () => DatabaseClient = () => database,
) {}
find(where: Query = {}, options: ReadOptions = {}): Promise<RecordRow<T>[]> {
return this.client().request('find', this.name, where, options);
}
findOne(
where: Query,
options: ReadOptions = {},
): Promise<RecordRow<T> | null> {
return this.client().request('findOne', this.name, where, options);
}
count(where: Query = {}): Promise<number> {
return this.client().request('count', this.name, where);
}
insert(data: Partial<T>): Promise<RecordRow<T>> {
return this.client().request('insert', this.name, data);
}
nativeUpdate(where: Query, data: Partial<T>): Promise<number> {
return this.client().request('nativeUpdate', this.name, where, data);
}
nativeDelete(where: Query): Promise<number> {
return this.client().request('nativeDelete', this.name, where);
}
async insertMany(data: Partial<T>[]): Promise<RecordRow<T>[]> {
return this.client().transaction(async () => {
const rows = [];
for (const item of data) rows.push(await this.insert(item));
return rows;
});
}
async getOrCreate(where: Query): Promise<RecordRow<T>> {
return this.client().transaction(
async () =>
(await this.findOne(where)) || this.insert(where as Partial<T>),
);
}
}
+38 -3
View File
@@ -1,4 +1,5 @@
import { Repository, RecordRow } from './repository';
import { sequelize } from '.';
import { DataTypes, Model } from 'sequelize';
export enum InstanceStatus {
'running' = 0,
@@ -40,7 +41,41 @@ export class RunningInstance {
}
}
export type RunningInstanceModel = RecordRow<RunningInstanceAttributes>;
export const RunningInstanceModel = new Repository<RunningInstanceAttributes>(
export interface RunningInstanceModel
extends Model<RunningInstanceAttributes, RunningInstanceAttributes>,
RunningInstanceAttributes {}
export const RunningInstanceModel = sequelize.define<RunningInstanceModel>(
'RunningInstance',
{
cron_id: {
type: DataTypes.NUMBER,
allowNull: false,
},
pid: {
type: DataTypes.NUMBER,
allowNull: true,
},
log_path: {
type: DataTypes.STRING,
allowNull: true,
},
started_at: {
type: DataTypes.NUMBER,
allowNull: false,
},
finished_at: {
type: DataTypes.NUMBER,
allowNull: true,
},
status: {
type: DataTypes.NUMBER,
allowNull: false,
defaultValue: InstanceStatus.running,
},
exit_code: {
type: DataTypes.NUMBER,
allowNull: true,
},
},
);
-178
View File
@@ -1,178 +0,0 @@
// Persisted SQLite names and types are part of the rollback contract.
type Field = {
type: string;
fieldName: string;
nullable?: boolean;
primary?: boolean;
autoincrement?: boolean;
};
function entity<Name extends string>(
name: Name,
tableName: string,
groups: Record<string, string[]>,
required: string[] = [],
) {
const properties: Record<string, Field> = {
id: { type: 'number', fieldName: 'id', primary: true, autoincrement: true },
};
for (const [type, fields] of Object.entries(groups))
for (const field of fields)
properties[field] = {
type,
fieldName: field,
nullable: !required.includes(field),
};
for (const field of ['createdAt', 'updatedAt'])
properties[field] = {
type: 'legacyDate',
fieldName: field,
nullable: false,
};
return { name, tableName, properties };
}
export const entityDefinitions = {
App: entity(
'App',
'Apps',
{
string: ['name', 'client_id', 'client_secret'],
json: ['scopes', 'tokens'],
},
[],
),
Auth: entity('Auth', 'Auths', { string: ['ip', 'type'], json: ['info'] }, []),
CrontabStat: entity(
'CrontabStat',
'CrontabStats',
{
number: [
'ref_id',
'run_count',
'success_count',
'fail_count',
'total_time',
'max_time',
],
string: ['date'],
},
['ref_id', 'date'],
),
CrontabView: entity(
'CrontabView',
'CrontabViews',
{
string: ['name', 'filterRelation'],
number: ['position', 'isDisabled', 'type'],
json: ['filters', 'sorts'],
},
[],
),
Crontab: entity(
'Crontab',
'Crontabs',
{
string: [
'name',
'command',
'schedule',
'timestamp',
'log_path',
'queued_token',
'task_before',
'task_after',
'log_name',
'work_dir',
],
boolean: ['saved'],
number: [
'status',
'isSystem',
'pid',
'isDisabled',
'isPinned',
'last_running_time',
'last_execution_time',
'sub_id',
'allow_multiple_instances',
],
json: ['labels', 'extra_schedules'],
},
[],
),
Dependence: entity(
'Dependence',
'Dependences',
{
string: ['name', 'timestamp', 'remark'],
number: ['type', 'status'],
json: ['log'],
},
[],
),
Env: entity(
'Env',
'Envs',
{
string: ['value', 'timestamp', 'name', 'remarks'],
number: ['status', 'position', 'isPinned'],
json: ['labels'],
},
[],
),
RunningInstance: entity(
'RunningInstance',
'RunningInstances',
{
number: [
'cron_id',
'pid',
'started_at',
'finished_at',
'status',
'exit_code',
],
string: ['log_path'],
},
['cron_id', 'started_at', 'status'],
),
Subscription: entity(
'Subscription',
'Subscriptions',
{
string: [
'name',
'url',
'schedule',
'type',
'whitelist',
'blacklist',
'dependences',
'extensions',
'sub_before',
'sub_after',
'branch',
'pull_type',
'log_path',
'schedule_type',
'alias',
'proxy',
],
json: ['interval_schedule', 'pull_option'],
number: ['status', 'pid', 'is_disabled', 'autoAddCron', 'autoDelCron'],
},
[],
),
};
export const initialSchema = [
'CREATE TABLE IF NOT EXISTS `Apps` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `scopes` JSON, `client_id` VARCHAR(255), `client_secret` VARCHAR(255), `tokens` JSON, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`name`))',
'CREATE TABLE IF NOT EXISTS `Auths` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `ip` VARCHAR(255), `type` VARCHAR(255), `info` JSON, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL)',
'CREATE TABLE IF NOT EXISTS `CrontabStats` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `ref_id` NUMBER NOT NULL, `date` VARCHAR(255) NOT NULL, `run_count` NUMBER DEFAULT 0, `success_count` NUMBER DEFAULT 0, `fail_count` NUMBER DEFAULT 0, `total_time` NUMBER DEFAULT 0, `max_time` NUMBER DEFAULT 0, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL)',
'CREATE TABLE IF NOT EXISTS `CrontabViews` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `position` NUMBER, `isDisabled` NUMBER, `filters` JSON, `sorts` JSON, `filterRelation` VARCHAR(255), `type` NUMBER, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`name`))',
'CREATE TABLE IF NOT EXISTS `Crontabs` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `command` VARCHAR(255), `schedule` VARCHAR(255), `timestamp` VARCHAR(255), `saved` TINYINT(1), `status` NUMBER, `isSystem` NUMBER, `pid` NUMBER, `isDisabled` NUMBER, `isPinned` NUMBER, `log_path` VARCHAR(255), `queued_token` VARCHAR(255), `labels` JSON, `last_running_time` NUMBER, `last_execution_time` NUMBER, `sub_id` NUMBER, `extra_schedules` JSON, `task_before` VARCHAR(255), `task_after` VARCHAR(255), `log_name` VARCHAR(255), `allow_multiple_instances` NUMBER, `work_dir` VARCHAR(255), `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`name`, `command`, `schedule`))',
'CREATE TABLE IF NOT EXISTS `Dependences` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `type` NUMBER, `timestamp` VARCHAR(255), `status` NUMBER, `log` JSON, `remark` VARCHAR(255), `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL)',
'CREATE TABLE IF NOT EXISTS `Envs` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `value` VARCHAR(255), `timestamp` VARCHAR(255), `status` NUMBER, `position` NUMBER, `name` VARCHAR(255), `remarks` VARCHAR(255), `isPinned` NUMBER, `labels` JSON, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`value`, `name`))',
'CREATE TABLE IF NOT EXISTS `RunningInstances` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `cron_id` NUMBER NOT NULL, `pid` NUMBER, `log_path` VARCHAR(255), `started_at` NUMBER NOT NULL, `finished_at` NUMBER, `status` NUMBER NOT NULL DEFAULT 0, `exit_code` NUMBER, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL)',
'CREATE TABLE IF NOT EXISTS `Subscriptions` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `url` VARCHAR(255), `schedule` VARCHAR(255), `interval_schedule` JSON, `type` VARCHAR(255), `whitelist` VARCHAR(255), `blacklist` VARCHAR(255), `status` NUMBER, `dependences` VARCHAR(255), `extensions` VARCHAR(255), `sub_before` VARCHAR(255), `sub_after` VARCHAR(255), `branch` VARCHAR(255), `pull_type` VARCHAR(255), `pull_option` JSON, `pid` NUMBER, `is_disabled` NUMBER, `log_path` VARCHAR(255), `schedule_type` VARCHAR(255), `alias` VARCHAR(255), `proxy` VARCHAR(255), `autoAddCron` NUMBER, `autoDelCron` NUMBER, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`name`, `url`, `schedule`, `interval_schedule`), UNIQUE (`alias`))',
'CREATE INDEX IF NOT EXISTS `crontab_stats_date` ON `CrontabStats` (`date`)',
'CREATE UNIQUE INDEX IF NOT EXISTS `crontab_stats_ref_id_date` ON `CrontabStats` (`ref_id`, `date`)',
];
-45
View File
@@ -1,45 +0,0 @@
import { database } from '.';
// Aggregate SQL stays explicit and bound; public input is never used as an SQL
// identifier. These queries preserve the dashboard response names.
export const statistics = {
async totals(date: string) {
const rows = await database.query(
`SELECT SUM(run_count) AS total_runs,
SUM(success_count) AS total_success, SUM(fail_count) AS total_fail,
SUM(total_time) AS total_time FROM CrontabStats WHERE date=?`,
[date],
);
return rows[0];
},
results(date: string, column: 'success_count' | 'fail_count') {
if (!['success_count', 'fail_count'].includes(column))
throw new Error('Invalid statistic');
return database.query(
`SELECT ref_id,SUM(${column}) AS result_count FROM CrontabStats
WHERE date=? AND ${column}>0 GROUP BY ref_id ORDER BY result_count DESC,ref_id ASC`,
[date],
);
},
trend(dates: string[]) {
if (!dates.length) return Promise.resolve([]);
return database.query(
`SELECT date,SUM(run_count) AS total_runs,
SUM(success_count) AS total_success,SUM(fail_count) AS total_fail FROM CrontabStats
WHERE date IN (${dates
.map(() => '?')
.join(',')}) GROUP BY date ORDER BY date ASC`,
dates,
);
},
ranking(date: string, metric: 'total_time' | 'run_count') {
if (!['total_time', 'run_count'].includes(metric))
throw new Error('Invalid statistic');
return database.query(
`SELECT ref_id,SUM(total_time) AS total_time,SUM(run_count) AS run_count,
SUM(success_count) AS success_count,MAX(max_time) AS max_time FROM CrontabStats
WHERE date=? AND run_count>0 GROUP BY ref_id ORDER BY SUM(${metric}) DESC LIMIT 5`,
[date],
);
},
};
+45 -3
View File
@@ -1,4 +1,5 @@
import { Repository, RecordRow } from './repository';
import { sequelize } from '.';
import { DataTypes, Model, ModelDefined } from 'sequelize';
import { SimpleIntervalSchedule } from 'toad-scheduler';
type SimpleIntervalScheduleUnit = keyof SimpleIntervalSchedule;
@@ -69,5 +70,46 @@ export enum SubscriptionStatus {
'queued',
}
export type SubscriptionInstance = RecordRow<Subscription>;
export const SubscriptionModel = new Repository<Subscription>('Subscription');
export interface SubscriptionInstance
extends Model<Subscription, Subscription>,
Subscription {}
export const SubscriptionModel = sequelize.define<SubscriptionInstance>(
'Subscription',
{
name: {
unique: 'compositeIndex',
type: DataTypes.STRING,
},
url: {
unique: 'compositeIndex',
type: DataTypes.STRING,
},
schedule: {
unique: 'compositeIndex',
type: DataTypes.STRING,
},
interval_schedule: {
unique: 'compositeIndex',
type: DataTypes.JSON,
},
type: DataTypes.STRING,
whitelist: DataTypes.STRING,
blacklist: DataTypes.STRING,
status: DataTypes.NUMBER,
dependences: DataTypes.STRING,
extensions: DataTypes.STRING,
sub_before: DataTypes.STRING,
sub_after: DataTypes.STRING,
branch: DataTypes.STRING,
pull_type: DataTypes.STRING,
pull_option: DataTypes.JSON,
pid: DataTypes.NUMBER,
is_disabled: DataTypes.NUMBER,
log_path: DataTypes.STRING,
schedule_type: DataTypes.STRING,
alias: { type: DataTypes.STRING, unique: 'alias' },
proxy: { type: DataTypes.STRING, allowNull: true },
autoAddCron: { type: DataTypes.NUMBER, allowNull: true },
autoDelCron: { type: DataTypes.NUMBER, allowNull: true },
},
);
+13 -19
View File
@@ -1,4 +1,5 @@
import { Repository, RecordRow } from './repository';
import { sequelize } from '.';
import { DataTypes, Model, ModelDefined } from 'sequelize';
import { NotificationInfo } from './notify';
export class SystemInfo {
@@ -27,12 +28,8 @@ export enum AuthDataType {
'removeLogFrequency' = 'removeLogFrequency',
'systemConfig' = 'systemConfig',
'authConfig' = 'authConfig',
'notifyLog' = 'notifyLog',
}
export const NotifyStatus = LoginStatus;
export type NotifyStatus = LoginStatus;
export interface SystemConfigInfo {
lang?: string;
panelTitle?: string;
@@ -56,17 +53,6 @@ export interface LoginLogInfo {
status?: LoginStatus;
}
export interface NotifyLogInfo {
timestamp?: number;
title?: string;
content?: string;
status?: NotifyStatus;
notifyType?: string;
source?: 'system' | 'script';
scriptName?: string;
taskId?: string;
}
export interface TokenInfo {
value: string;
timestamp: number;
@@ -102,8 +88,16 @@ export interface AuthInfo {
export type SystemModelInfo = SystemConfigInfo &
Partial<NotificationInfo> &
LoginLogInfo &
Partial<NotifyLogInfo> &
Partial<AuthInfo>;
export type SystemInstance = RecordRow<SystemInfo>;
export const SystemModel = new Repository<SystemInfo>('Auth');
export interface SystemInstance
extends Model<SystemInfo, SystemInfo>,
SystemInfo {}
export const SystemModel = sequelize.define<SystemInstance>('Auth', {
ip: DataTypes.STRING,
type: DataTypes.STRING,
info: {
type: DataTypes.JSON,
allowNull: true,
},
});
-11
View File
@@ -1,11 +0,0 @@
{
"extends": "../tsconfig.json",
"compilerOptions": {
"module": "node20",
"moduleResolution": "node16",
"rootDir": "..",
"outDir": "../../static/build"
},
"include": ["./databaseWorker.ts", "./schema.ts"],
"exclude": ["node_modules"]
}
+22 -6
View File
@@ -1,14 +1,30 @@
import Logger from './logger';
import { database } from '../data';
import { initialSchema } from '../data/schema';
import { EnvModel } from '../data/env';
import { CrontabModel } from '../data/cron';
import { DependenceModel } from '../data/dependence';
import { AppModel } from '../data/open';
import { SystemModel } from '../data/system';
import { SubscriptionModel } from '../data/subscription';
import { CrontabViewModel } from '../data/cronView';
import { CrontabStatModel } from '../data/cronStats';
import { RunningInstanceModel } from '../data/runningInstance';
import { sequelize } from '../data';
import { migrateSchema } from '../shared/schemaMigrations';
export default async () => {
try {
await database.transaction(async () => {
for (const sql of initialSchema) await database.query(sql);
await migrateSchema(database);
});
await CrontabModel.sync();
await DependenceModel.sync();
await AppModel.sync();
await SystemModel.sync();
await EnvModel.sync();
await SubscriptionModel.sync();
await CrontabViewModel.sync();
await CrontabStatModel.sync();
await RunningInstanceModel.sync();
await migrateSchema(sequelize);
Logger.info('[boot] DB loaded');
} catch (error) {
Logger.error('[boot] DB load failed', error);
+34 -22
View File
@@ -6,7 +6,7 @@ import config from '../config';
import { UnauthorizedError, expressjwt } from 'express-jwt';
import { getPlatform, getToken } from '../config/util';
import rewrite from 'express-urlrewrite';
import validationErrors from '../middlewares/validationErrors';
import { errors } from 'celebrate';
import { serveEnv } from '../config/serverEnv';
import { shareStore } from '../shared/store';
import { isValidToken, isDefaultAuthInfo } from '../shared/auth';
@@ -14,8 +14,8 @@ import { AuthInfo } from '../data/system';
import path from 'path';
import { t } from '../shared/i18n';
import { AppScope } from '../data/open';
import uniqueConstraintError from '../middlewares/uniqueConstraintError';
import protectedPathCase from '../middlewares/protectedPathCase';
function resolveTrustProxy(value = process.env.QL_TRUST_PROXY) {
const setting = value?.trim();
if (!setting) {
@@ -29,20 +29,24 @@ function resolveTrustProxy(value = process.env.QL_TRUST_PROXY) {
}
return setting;
}
export default ({ app }: { app: Application }) => {
// Security: Enable strict routing to prevent case-insensitive path bypass
app.set('case sensitive routing', true);
app.set('strict routing', true);
app.set('trust proxy', resolveTrustProxy());
app.use(cors());
// Security: Reject case variations under protected API namespaces before
// authentication checks can interpret the request differently from routing.
app.use(protectedPathCase);
// Rewrite URLs to strip baseUrl prefix if configured
// This allows the rest of the app to work without baseUrl awareness
if (config.baseUrl) {
app.use(rewrite(`${config.baseUrl}/*`, '/$1'));
}
app.get(`${config.api.prefix}/env.js`, serveEnv);
app.use(
`${config.api.prefix}/static`,
@@ -53,6 +57,7 @@ export default ({ app }: { app: Application }) => {
},
}),
);
const credentialPaths = ['/api', '/open'].flatMap((prefix) =>
['/user/login', '/user/init', '/user/two-factor/login'].map(
(route) => `${prefix}${route}`,
@@ -63,10 +68,13 @@ export default ({ app }: { app: Application }) => {
bodyParser.json({ limit: '16kb' }),
bodyParser.urlencoded({ limit: '16kb', extended: false }),
);
app.use(bodyParser.json({ limit: '50mb' }));
app.use(bodyParser.urlencoded({ limit: '50mb', extended: true }));
const frontendPath = path.join(config.rootPath, 'static/dist');
app.use(express.static(frontendPath));
app.use(
expressjwt({
secret: config.jwt.secret,
@@ -75,6 +83,7 @@ export default ({ app }: { app: Application }) => {
path: [...config.apiWhiteList, /^(\/(?!api\/).*)$/i],
}),
);
app.use((req: Request, res, next) => {
if (!req.headers) {
req.platform = 'desktop';
@@ -84,11 +93,13 @@ export default ({ app }: { app: Application }) => {
}
return next();
});
app.use(async (req: Request, res, next) => {
const pathLower = req.path.toLowerCase();
if (!['/open/', '/api/'].some((x) => pathLower.startsWith(x))) {
return next();
}
const headerToken = getToken(req);
if (pathLower.startsWith('/open/')) {
const apps = await shareStore.getApps();
@@ -99,12 +110,14 @@ export default ({ app }: { app: Application }) => {
const currentToken = doc.tokens.find((x) => x.value === headerToken);
const keyMatch = pathLower.match(/\/open\/([a-z]+)\/*/);
const key = keyMatch && keyMatch[1];
if (!doc.scopes.includes(key as AppScope)) {
const err = new UnauthorizedError('credentials_bad_scheme', {
message: t('暂无权限'),
});
return next(err);
}
if (
!currentToken ||
currentToken.expiration < Math.round(Date.now() / 1000)
@@ -114,9 +127,11 @@ export default ({ app }: { app: Application }) => {
});
return next(err);
}
return next();
}
}
const originPath = `${req.baseUrl}${pathLower === '/' ? '' : pathLower}`;
if (
!headerToken &&
@@ -125,15 +140,18 @@ export default ({ app }: { app: Application }) => {
) {
return next();
}
const authInfo = await shareStore.getAuthInfo();
if (isValidToken(authInfo, headerToken, req.platform, config.jwt.secret)) {
return next();
}
const errorCode = headerToken ? 'invalid_token' : 'credentials_required';
const errorMessage = headerToken ? t('Token 已失效') : t('请先登录');
const err = new UnauthorizedError(errorCode, { message: errorMessage });
next(err);
});
app.use(async (req, res, next) => {
const pathLower = req.path.toLowerCase();
if (
@@ -147,23 +165,20 @@ export default ({ app }: { app: Application }) => {
return next();
}
const authInfo = (await shareStore.getAuthInfo()) || ({} as AuthInfo);
let isInitialized = !isDefaultAuthInfo(authInfo);
if (isInitialized) {
return res.send({ code: 450, message: t('未知错误') });
} else {
return next();
}
});
app.use(rewrite('/open/*', '/api/$1'));
app.use(config.api.prefix, routes());
app.get('*', (req, res, next) => {
// Don't serve index.html for API routes
if (
req.path === config.api.prefix ||
req.path.startsWith(`${config.api.prefix}/`)
) {
return res.status(404).json({ code: 404, message: 'Not Found' });
}
app.get('*', (_, res, next) => {
const indexPath = path.join(frontendPath, 'index.html');
res.sendFile(indexPath, (err) => {
if (err) {
@@ -173,13 +188,12 @@ export default ({ app }: { app: Application }) => {
}
});
});
app.use(validationErrors);
app.use(uniqueConstraintError);
app.use(errors());
app.use(
(
err: Error & {
status: number;
},
err: Error & { status: number },
req: Request,
res: Response,
next: NextFunction,
@@ -193,16 +207,15 @@ export default ({ app }: { app: Application }) => {
return next(err);
},
);
app.use(
(
err: Error & {
errors: any[];
},
err: Error & { errors: any[] },
req: Request,
res: Response,
next: NextFunction,
) => {
if (err.name === 'DatabaseError') {
if (err.name.includes('Sequelize')) {
return res
.status(500)
.send({
@@ -215,11 +228,10 @@ export default ({ app }: { app: Application }) => {
return next(err);
},
);
app.use(
(
err: Error & {
status: number;
},
err: Error & { status: number },
req: Request,
res: Response,
next: NextFunction,
-33
View File
@@ -1,33 +0,0 @@
import type express from 'express';
import config from '../config';
export async function createHttpApplication(): Promise<express.Application> {
// Only the HTTP worker needs an Express application and its middleware.
const [
{ default: createExpress },
{ default: helmet },
{ default: cors },
{ default: compression },
{ monitoringMiddleware },
] = await Promise.all([
import('express'),
import('helmet'),
import('cors'),
import('compression'),
import('../middlewares/monitoring'),
]);
const app = createExpress();
app.use((req, res, next) => {
if (req.query.t) delete req.query.t;
next();
});
app.use(
helmet({
contentSecurityPolicy: false,
}),
);
app.use(cors(config.cors));
app.use(compression());
app.use(monitoringMiddleware);
return app;
}
+108 -77
View File
@@ -1,5 +1,3 @@
import { textSearch } from '../data/queries';
import { Query, QuerySpec, orderBy } from '../data/queries';
import DependenceService from '../services/dependence';
import { exec } from 'child_process';
import { Container } from 'typedi';
@@ -7,6 +5,7 @@ import { Crontab, CrontabModel, CrontabStatus } from '../data/cron';
import CronService from '../services/cron';
import EnvService from '../services/env';
import { DependenceModel, DependenceStatus } from '../data/dependence';
import { Op } from 'sequelize';
import config from '../config';
import { CrontabViewModel, CronViewType } from '../data/cronView';
import { initPosition } from '../data/env';
@@ -14,13 +13,15 @@ import { AuthDataType, SystemModel } from '../data/system';
import SystemService from '../services/system';
import UserService from '../services/user';
import { writeFile, readFile } from 'fs/promises';
import { fileExist, isDemoEnv, safeJSONParse } from '../config/util';
import { createRandomString, fileExist, isDemoEnv, safeJSONParse } from '../config/util';
import OpenService from '../services/open';
import { shareStore } from '../shared/store';
import Logger from './logger';
import cronClient from '../schedule/client';
import { AppModel } from '../data/open';
import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
import { setLang, systemLang } from '../shared/i18n';
export default async () => {
const cronService = Container.get(CronService);
const envService = Container.get(EnvService);
@@ -28,14 +29,34 @@ export default async () => {
const systemService = Container.get(SystemService);
const userService = Container.get(UserService);
const openService = Container.get(OpenService);
// 初始化增加系统配置
await openService.initializeSystemApp();
const systemConfig = await SystemModel.getOrCreate({
type: AuthDataType.systemConfig,
let systemApp = (
await AppModel.findOne({
where: { name: 'system' },
})
)?.get({ plain: true });
if (!systemApp) {
systemApp = await AppModel.create({
name: 'system',
scopes: ['crons', 'system', 'dashboard'],
client_id: createRandomString(12, 12),
client_secret: createRandomString(24, 24),
});
} else if (!systemApp.scopes.includes('dashboard')) {
await AppModel.update(
{ scopes: [...systemApp.scopes, 'dashboard'] },
{ where: { name: 'system' } },
);
}
const [systemConfig] = await SystemModel.findOrCreate({
where: { type: AuthDataType.systemConfig },
});
await SystemModel.getOrCreate({ type: AuthDataType.notification });
const authConfig = await SystemModel.getOrCreate({
type: AuthDataType.authConfig,
await SystemModel.findOrCreate({
where: { type: AuthDataType.notification },
});
const [authConfig] = await SystemModel.findOrCreate({
where: { type: AuthDataType.authConfig },
});
if (!authConfig?.info || isDemoEnv()) {
let authInfo = {
@@ -51,34 +72,36 @@ export default async () => {
} catch (error) {
Logger.warn('Failed to read auth config file, using default credentials');
}
await SystemModel.nativeUpdate(
{ id: authConfig.id },
{
info: authInfo,
type: AuthDataType.authConfig,
},
);
await SystemModel.upsert({
id: authConfig?.id,
info: authInfo,
type: AuthDataType.authConfig,
});
}
const installDependencies = async () => {
const docs = await DependenceModel.find(
{},
{
orderBy: orderBy([
['type', 'DESC'],
['createdAt', 'DESC'],
]),
},
);
await DependenceModel.nativeUpdate(
{ id: docs.map((x) => x.id!) },
const docs = await DependenceModel.findAll({
where: {},
order: [
['type', 'DESC'],
['createdAt', 'DESC'],
],
raw: true,
});
await DependenceModel.update(
{ status: DependenceStatus.queued, log: [] },
{ where: { id: docs.map((x) => x.id!) } },
);
setTimeout(async () => {
await dependenceService.installDependenceOneByOne(docs);
const bootAfterLoader = await import('./bootAfter');
bootAfterLoader.default();
}, 5000);
};
// 初始化更新 linux/python/nodejs 镜像源配置
if (systemConfig.info?.pythonMirror) {
systemService.updatePythonMirror({
@@ -101,40 +124,39 @@ export default async () => {
nodeMirror: systemConfig.info?.nodeMirror,
});
}
// 初始化新增默认全部任务视图
CrontabViewModel.find({ type: CronViewType.系统, name: '全部任务' }, {}).then(
(docs) => {
if (docs.length === 0) {
CrontabViewModel.insert({
name: '全部任务',
type: CronViewType.系统,
position: initPosition / 2,
});
}
},
);
CrontabViewModel.findAll({
where: { type: CronViewType.系统, name: '全部任务' },
raw: true,
}).then((docs) => {
if (docs.length === 0) {
CrontabViewModel.create({
name: '全部任务',
type: CronViewType.系统,
position: initPosition / 2,
});
}
});
// 初始化更新所有任务状态为空闲
await CrontabModel.nativeUpdate({}, { status: CrontabStatus.idle });
await CrontabModel.update({ status: CrontabStatus.idle }, { where: {} });
// 清空所有运行中的实例记录(服务重启后进程已不存在)
await RunningInstanceModel.nativeUpdate(
{ status: InstanceStatus.running },
{
status: InstanceStatus.stopped,
finished_at: Math.floor(Date.now() / 1000),
// The previous process's exit code cannot be recovered after a restart.
},
await RunningInstanceModel.update(
{ status: InstanceStatus.stopped },
{ where: { status: InstanceStatus.running } },
);
// 初始化时执行一次所有的 ql repo 任务
CrontabModel.find(
{
isDisabled: { ['$ne']: 1 },
$or: [
...textSearch(['command'], 'ql repo').$or,
...textSearch(['command'], 'ql raw').$or,
],
CrontabModel.findAll({
where: {
isDisabled: { [Op.ne]: 1 },
command: {
[Op.or]: [{ [Op.like]: `%ql repo%` }, { [Op.like]: `%ql raw%` }],
},
},
{},
).then((docs) => {
}).then((docs) => {
for (let i = 0; i < docs.length; i++) {
const doc = docs[i];
if (doc) {
@@ -142,79 +164,88 @@ export default async () => {
}
}
});
// 更新2.11.3以前的脚本路径
CrontabModel.find(
{
$or: ['scripts', 'config', 'log', 'db'].flatMap(
(folder) =>
textSearch(['command'], `/${config.rootPath}/${folder}/`).$or,
),
CrontabModel.findAll({
where: {
command: {
[Op.or]: [
{ [Op.like]: `%\/${config.rootPath}\/scripts\/%` },
{ [Op.like]: `%\/${config.rootPath}\/config\/%` },
{ [Op.like]: `%\/${config.rootPath}\/log\/%` },
{ [Op.like]: `%\/${config.rootPath}\/db\/%` },
],
},
},
{},
).then(async (docs) => {
}).then(async (docs) => {
for (let i = 0; i < docs.length; i++) {
const doc = docs[i];
if (doc) {
if (doc.command.includes(`${config.rootPath}/scripts/`)) {
await CrontabModel.nativeUpdate(
{ id: doc.id },
await CrontabModel.update(
{ command: doc.command.replace(`${config.rootPath}/scripts/`, '') },
{ where: { id: doc.id } },
);
}
if (doc.command.includes(`${config.rootPath}/log/`)) {
await CrontabModel.nativeUpdate(
{ id: doc.id },
await CrontabModel.update(
{
command: `${config.dataPath}/log/${doc.command.replace(
`${config.rootPath}/log/`,
'',
)}`,
},
{ where: { id: doc.id } },
);
}
if (doc.command.includes(`${config.rootPath}/config/`)) {
await CrontabModel.nativeUpdate(
{ id: doc.id },
await CrontabModel.update(
{
command: `${config.dataPath}/config/${doc.command.replace(
`${config.rootPath}/config/`,
'',
)}`,
},
{ where: { id: doc.id } },
);
}
if (doc.command.includes(`${config.rootPath}/db/`)) {
await CrontabModel.nativeUpdate(
{ id: doc.id },
await CrontabModel.update(
{
command: `${config.dataPath}/db/${doc.command.replace(
`${config.rootPath}/db/`,
'',
)}`,
},
{ where: { id: doc.id } },
);
}
}
}
});
// 初始化语言(必须在 autosave_crontab 之前)
const lang = systemConfig.info?.lang || systemLang();
setLang(lang);
// 确保 lang_env.sh 存在
try {
const langEnvExist = await fileExist(config.langEnvFile);
if (!langEnvExist) {
await writeFile(config.langEnvFile, `export QL_LANG='${lang}'\n`);
}
} catch {}
} catch { }
// 初始化保存一次ck和定时任务数据
cronClient.readiness.configure(async () => {
await cronService.autosave_crontab(true);
await systemService.restoreCronConcurrency();
});
cronClient.readiness.configure(() => cronService.autosave_crontab(true));
await cronClient.readiness.recover();
await envService.set_envs();
const authInfo = await userService.getAuthInfo();
const apps = await openService.findApps();
await shareStore.updateAuthInfo(authInfo);
await openService.refreshApps();
if (apps?.length) {
await shareStore.updateApps(apps);
}
};
+8 -10
View File
@@ -7,23 +7,23 @@ import config from '../config';
import { fileExist } from '../config/util';
import { join } from 'path';
import { t } from '../shared/i18n';
import Logger from './logger';
import { assertSubscriptionAlias } from '../shared/subscriptionPath';
export default async () => {
const systemService = Container.get(SystemService);
const scheduleService = Container.get(ScheduleService);
const subscriptionService = Container.get(SubscriptionService);
const sshKeyService = Container.get(SshKeyService);
// 生成内置token
let tokenCommand = `ts-node-transpile-only ${join(
config.rootPath,
'back/token.ts',
)}`;
const tokenFile = join(config.rootPath, 'static/build/token.js');
if (await fileExist(tokenFile)) {
tokenCommand = `node ${tokenFile}`;
}
tokenCommand += ' --renew';
const cron = {
id: NaN,
name: t('生成token'),
@@ -38,6 +38,7 @@ export default async () => {
},
true,
);
// 运行删除日志任务
const data = await systemService.getSystemConfig();
if (data && data.info) {
@@ -57,21 +58,18 @@ export default async () => {
true,
);
}
systemService.updateTimezone(data.info);
// Apply global SSH key if configured
if (data.info.globalSshKey) {
await sshKeyService.addGlobalSSHKey(data.info.globalSshKey, 'global');
}
}
await subscriptionService.setSshConfig();
const subs = await subscriptionService.list();
for (const sub of subs) {
try {
const doc = sub;
assertSubscriptionAlias(doc.alias);
await subscriptionService.handleTask(doc, !sub.is_disabled);
} catch (error) {
Logger.warn(t('跳过订阅 %s 的无效定时配置: %s'), sub.id, error);
}
subscriptionService.handleTask(sub.get({ plain: true }), !sub.is_disabled);
}
};
-13
View File
@@ -1,13 +0,0 @@
import { ErrorRequestHandler } from 'express';
import { DatabaseError } from '../data/databaseErrors';
import { t } from '../shared/i18n';
// Handle the database constraint itself so concurrent writes are covered too.
const uniqueConstraintError: ErrorRequestHandler = (err, req, res, next) => {
if (!(err instanceof DatabaseError) || err.code !== 'UNIQUE_CONSTRAINT')
return next(err);
// Constraint details can contain environment values or application credentials.
res
.status(409)
.json({ code: 409, message: t('资源已存在,请检查重复的名称或值') });
};
export default uniqueConstraintError;
-18
View File
@@ -1,18 +0,0 @@
import { errors, isCelebrateError } from 'celebrate';
import { ErrorRequestHandler } from 'express';
import { t } from '../shared/i18n';
const respond = errors();
// Schemas are compiled once at startup; resolve their messages per request so
// changing the system language also changes errors from those existing schemas.
const validationErrors: ErrorRequestHandler = (error, req, res, next) => {
if (isCelebrateError(error)) {
for (const detail of error.details.values()) {
detail.message = detail.details.map((item) => t(item.message)).join('. ');
}
}
return respond(error, req, res, next);
};
export default validationErrors;
-10
View File
@@ -260,15 +260,6 @@ message SystemNotifyRequest {
optional NotificationInfo notificationInfo = 3;
}
message RecordNotifyLogRequest {
string title = 1;
string content = 2;
bool success = 3;
string notifyType = 4;
string scriptName = 5;
string taskId = 6;
}
service Api {
rpc GetEnvs(GetEnvsRequest) returns (EnvsResponse) {}
rpc CreateEnv(CreateEnvRequest) returns (EnvsResponse) {}
@@ -280,7 +271,6 @@ service Api {
rpc UpdateEnvNames(UpdateEnvNamesRequest) returns (Response) {}
rpc GetEnvById(GetEnvByIdRequest) returns (EnvResponse) {}
rpc SystemNotify(SystemNotifyRequest) returns (Response) {}
rpc RecordNotifyLog(RecordNotifyLogRequest) returns (Response) {}
rpc GetCronDetail(CronDetailRequest) returns (CronDetailResponse) {}
rpc CreateCron(CreateCronRequest) returns (CronResponse) {}
rpc UpdateCron(UpdateCronRequest) returns (CronResponse) {}
+1 -175
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-ts_proto. DO NOT EDIT.
// versions:
// protoc-gen-ts_proto v2.6.1
// protoc v6.31.1
// protoc v3.17.3
// source: back/protos/api.proto
/* eslint-disable */
@@ -417,15 +417,6 @@ export interface SystemNotifyRequest {
notificationInfo?: NotificationInfo | undefined;
}
export interface RecordNotifyLogRequest {
title: string;
content: string;
success: boolean;
notifyType: string;
scriptName: string;
taskId: string;
}
function createBaseEnvItem(): EnvItem {
return {
id: undefined,
@@ -4305,146 +4296,6 @@ export const SystemNotifyRequest: MessageFns<SystemNotifyRequest> = {
},
};
function createBaseRecordNotifyLogRequest(): RecordNotifyLogRequest {
return { title: "", content: "", success: false, notifyType: "", scriptName: "", taskId: "" };
}
export const RecordNotifyLogRequest: MessageFns<RecordNotifyLogRequest> = {
encode(message: RecordNotifyLogRequest, writer: BinaryWriter = new BinaryWriter()): BinaryWriter {
if (message.title !== "") {
writer.uint32(10).string(message.title);
}
if (message.content !== "") {
writer.uint32(18).string(message.content);
}
if (message.success !== false) {
writer.uint32(24).bool(message.success);
}
if (message.notifyType !== "") {
writer.uint32(34).string(message.notifyType);
}
if (message.scriptName !== "") {
writer.uint32(42).string(message.scriptName);
}
if (message.taskId !== "") {
writer.uint32(50).string(message.taskId);
}
return writer;
},
decode(input: BinaryReader | Uint8Array, length?: number): RecordNotifyLogRequest {
const reader = input instanceof BinaryReader ? input : new BinaryReader(input);
let end = length === undefined ? reader.len : reader.pos + length;
const message = createBaseRecordNotifyLogRequest();
while (reader.pos < end) {
const tag = reader.uint32();
switch (tag >>> 3) {
case 1: {
if (tag !== 10) {
break;
}
message.title = reader.string();
continue;
}
case 2: {
if (tag !== 18) {
break;
}
message.content = reader.string();
continue;
}
case 3: {
if (tag !== 24) {
break;
}
message.success = reader.bool();
continue;
}
case 4: {
if (tag !== 34) {
break;
}
message.notifyType = reader.string();
continue;
}
case 5: {
if (tag !== 42) {
break;
}
message.scriptName = reader.string();
continue;
}
case 6: {
if (tag !== 50) {
break;
}
message.taskId = reader.string();
continue;
}
}
if ((tag & 7) === 4 || tag === 0) {
break;
}
reader.skip(tag & 7);
}
return message;
},
fromJSON(object: any): RecordNotifyLogRequest {
return {
title: isSet(object.title) ? globalThis.String(object.title) : "",
content: isSet(object.content) ? globalThis.String(object.content) : "",
success: isSet(object.success) ? globalThis.Boolean(object.success) : false,
notifyType: isSet(object.notifyType) ? globalThis.String(object.notifyType) : "",
scriptName: isSet(object.scriptName) ? globalThis.String(object.scriptName) : "",
taskId: isSet(object.taskId) ? globalThis.String(object.taskId) : "",
};
},
toJSON(message: RecordNotifyLogRequest): unknown {
const obj: any = {};
if (message.title !== "") {
obj.title = message.title;
}
if (message.content !== "") {
obj.content = message.content;
}
if (message.success !== false) {
obj.success = message.success;
}
if (message.notifyType !== "") {
obj.notifyType = message.notifyType;
}
if (message.scriptName !== "") {
obj.scriptName = message.scriptName;
}
if (message.taskId !== "") {
obj.taskId = message.taskId;
}
return obj;
},
create<I extends Exact<DeepPartial<RecordNotifyLogRequest>, I>>(base?: I): RecordNotifyLogRequest {
return RecordNotifyLogRequest.fromPartial(base ?? ({} as any));
},
fromPartial<I extends Exact<DeepPartial<RecordNotifyLogRequest>, I>>(object: I): RecordNotifyLogRequest {
const message = createBaseRecordNotifyLogRequest();
message.title = object.title ?? "";
message.content = object.content ?? "";
message.success = object.success ?? false;
message.notifyType = object.notifyType ?? "";
message.scriptName = object.scriptName ?? "";
message.taskId = object.taskId ?? "";
return message;
},
};
export type ApiService = typeof ApiService;
export const ApiService = {
getEnvs: {
@@ -4537,15 +4388,6 @@ export const ApiService = {
responseSerialize: (value: Response) => Buffer.from(Response.encode(value).finish()),
responseDeserialize: (value: Buffer) => Response.decode(value),
},
recordNotifyLog: {
path: "/com.ql.api.Api/RecordNotifyLog",
requestStream: false,
responseStream: false,
requestSerialize: (value: RecordNotifyLogRequest) => Buffer.from(RecordNotifyLogRequest.encode(value).finish()),
requestDeserialize: (value: Buffer) => RecordNotifyLogRequest.decode(value),
responseSerialize: (value: Response) => Buffer.from(Response.encode(value).finish()),
responseDeserialize: (value: Buffer) => Response.decode(value),
},
getCronDetail: {
path: "/com.ql.api.Api/GetCronDetail",
requestStream: false,
@@ -4640,7 +4482,6 @@ export interface ApiServer extends UntypedServiceImplementation {
updateEnvNames: handleUnaryCall<UpdateEnvNamesRequest, Response>;
getEnvById: handleUnaryCall<GetEnvByIdRequest, EnvResponse>;
systemNotify: handleUnaryCall<SystemNotifyRequest, Response>;
recordNotifyLog: handleUnaryCall<RecordNotifyLogRequest, Response>;
getCronDetail: handleUnaryCall<CronDetailRequest, CronDetailResponse>;
createCron: handleUnaryCall<CreateCronRequest, CronResponse>;
updateCron: handleUnaryCall<UpdateCronRequest, CronResponse>;
@@ -4803,21 +4644,6 @@ export interface ApiClient extends Client {
options: Partial<CallOptions>,
callback: (error: ServiceError | null, response: Response) => void,
): ClientUnaryCall;
recordNotifyLog(
request: RecordNotifyLogRequest,
callback: (error: ServiceError | null, response: Response) => void,
): ClientUnaryCall;
recordNotifyLog(
request: RecordNotifyLogRequest,
metadata: Metadata,
callback: (error: ServiceError | null, response: Response) => void,
): ClientUnaryCall;
recordNotifyLog(
request: RecordNotifyLogRequest,
metadata: Metadata,
options: Partial<CallOptions>,
callback: (error: ServiceError | null, response: Response) => void,
): ClientUnaryCall;
getCronDetail(
request: CronDetailRequest,
callback: (error: ServiceError | null, response: CronDetailResponse) => void,
-4
View File
@@ -5,7 +5,6 @@ package com.ql.cron;
service Cron {
rpc addCron(AddCronRequest) returns (AddCronResponse);
rpc delCron(DeleteCronRequest) returns (DeleteCronResponse);
rpc setConcurrency(SetConcurrencyRequest) returns (AddCronResponse);
}
message ISchedule { string schedule = 1; }
@@ -28,6 +27,3 @@ message AddCronResponse {}
message DeleteCronRequest { repeated string ids = 1; }
message DeleteCronResponse {}
// Zero restores the default limit.
message SetConcurrencyRequest { int32 concurrency = 1; }
+1 -89
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-ts_proto. DO NOT EDIT.
// versions:
// protoc-gen-ts_proto v2.6.1
// protoc v5.28.3
// protoc v3.17.3
// source: back/protos/cron.proto
/* eslint-disable */
@@ -48,11 +48,6 @@ export interface DeleteCronRequest {
export interface DeleteCronResponse {
}
/** Zero restores the default limit. */
export interface SetConcurrencyRequest {
concurrency: number;
}
function createBaseISchedule(): ISchedule {
return { schedule: "" };
}
@@ -457,64 +452,6 @@ export const DeleteCronResponse: MessageFns<DeleteCronResponse> = {
},
};
function createBaseSetConcurrencyRequest(): SetConcurrencyRequest {
return { concurrency: 0 };
}
export const SetConcurrencyRequest: MessageFns<SetConcurrencyRequest> = {
encode(message: SetConcurrencyRequest, writer: BinaryWriter = new BinaryWriter()): BinaryWriter {
if (message.concurrency !== 0) {
writer.uint32(8).int32(message.concurrency);
}
return writer;
},
decode(input: BinaryReader | Uint8Array, length?: number): SetConcurrencyRequest {
const reader = input instanceof BinaryReader ? input : new BinaryReader(input);
let end = length === undefined ? reader.len : reader.pos + length;
const message = createBaseSetConcurrencyRequest();
while (reader.pos < end) {
const tag = reader.uint32();
switch (tag >>> 3) {
case 1: {
if (tag !== 8) {
break;
}
message.concurrency = reader.int32();
continue;
}
}
if ((tag & 7) === 4 || tag === 0) {
break;
}
reader.skip(tag & 7);
}
return message;
},
fromJSON(object: any): SetConcurrencyRequest {
return { concurrency: isSet(object.concurrency) ? globalThis.Number(object.concurrency) : 0 };
},
toJSON(message: SetConcurrencyRequest): unknown {
const obj: any = {};
if (message.concurrency !== 0) {
obj.concurrency = Math.round(message.concurrency);
}
return obj;
},
create<I extends Exact<DeepPartial<SetConcurrencyRequest>, I>>(base?: I): SetConcurrencyRequest {
return SetConcurrencyRequest.fromPartial(base ?? ({} as any));
},
fromPartial<I extends Exact<DeepPartial<SetConcurrencyRequest>, I>>(object: I): SetConcurrencyRequest {
const message = createBaseSetConcurrencyRequest();
message.concurrency = object.concurrency ?? 0;
return message;
},
};
export type CronService = typeof CronService;
export const CronService = {
addCron: {
@@ -535,21 +472,11 @@ export const CronService = {
responseSerialize: (value: DeleteCronResponse) => Buffer.from(DeleteCronResponse.encode(value).finish()),
responseDeserialize: (value: Buffer) => DeleteCronResponse.decode(value),
},
setConcurrency: {
path: "/com.ql.cron.Cron/setConcurrency",
requestStream: false,
responseStream: false,
requestSerialize: (value: SetConcurrencyRequest) => Buffer.from(SetConcurrencyRequest.encode(value).finish()),
requestDeserialize: (value: Buffer) => SetConcurrencyRequest.decode(value),
responseSerialize: (value: AddCronResponse) => Buffer.from(AddCronResponse.encode(value).finish()),
responseDeserialize: (value: Buffer) => AddCronResponse.decode(value),
},
} as const;
export interface CronServer extends UntypedServiceImplementation {
addCron: handleUnaryCall<AddCronRequest, AddCronResponse>;
delCron: handleUnaryCall<DeleteCronRequest, DeleteCronResponse>;
setConcurrency: handleUnaryCall<SetConcurrencyRequest, AddCronResponse>;
}
export interface CronClient extends Client {
@@ -583,21 +510,6 @@ export interface CronClient extends Client {
options: Partial<CallOptions>,
callback: (error: ServiceError | null, response: DeleteCronResponse) => void,
): ClientUnaryCall;
setConcurrency(
request: SetConcurrencyRequest,
callback: (error: ServiceError | null, response: AddCronResponse) => void,
): ClientUnaryCall;
setConcurrency(
request: SetConcurrencyRequest,
metadata: Metadata,
callback: (error: ServiceError | null, response: AddCronResponse) => void,
): ClientUnaryCall;
setConcurrency(
request: SetConcurrencyRequest,
metadata: Metadata,
options: Partial<CallOptions>,
callback: (error: ServiceError | null, response: AddCronResponse) => void,
): ClientUnaryCall;
}
export const CronClient = makeGenericClientConstructor(CronService, "com.ql.cron.Cron") as unknown as {
+4 -8
View File
@@ -5,7 +5,7 @@ import { isValidCronSchedule } from '../shared/cronSchedule';
import { scheduleStacks } from './data';
import { runCron } from '../shared/runCron';
import Logger from '../loaders/logger';
import { t, tf } from '../shared/i18n';
import { tf } from '../shared/i18n';
// Validate the entire batch before replacing any existing jobs.
const isValidCronField = (cron: string): boolean => {
@@ -67,12 +67,8 @@ const addCron = (
createCronJob(
schedule,
async () => {
Logger.info(t('[schedule][准备运行任务] 命令: %s'), item.command);
await runCron(
item.command,
item,
() => scheduleStacks.get(item.id) === jobs,
);
Logger.info('[schedule][准备运行任务] 命令: %s', item.command);
await runCron(item.command, item);
},
{
name: `${item.id}: ${item.name || ''}`,
@@ -104,7 +100,7 @@ const addCron = (
scheduleStacks.set(id, jobs);
jobs.forEach((job) => job.start());
Logger.info(
t('[schedule][创建定时任务] 任务ID: %s, 规则数: %s'),
'[schedule][创建定时任务] 任务ID: %s, 规则数: %s',
id,
jobs.length,
);
+38 -23
View File
@@ -16,7 +16,6 @@ import {
MoveEnvRequest,
Response,
SystemNotifyRequest,
RecordNotifyLogRequest,
UpdateEnvNamesRequest,
UpdateEnvRequest,
} from '../protos/api';
@@ -39,7 +38,10 @@ import {
RunCronsRequest,
} from '../protos/api';
import { NotificationInfo } from '../data/notify';
import { Model } from 'sequelize';
Container.set('logger', LoggerInstance);
export const getEnvs = async (
call: ServerUnaryCall<GetEnvsRequest, EnvsResponse>,
callback: sendUnaryData<EnvsResponse>,
@@ -59,6 +61,7 @@ export const getEnvs = async (
});
}
};
export const createEnv = async (
call: ServerUnaryCall<CreateEnvRequest, EnvsResponse>,
callback: sendUnaryData<EnvsResponse>,
@@ -71,6 +74,7 @@ export const createEnv = async (
callback(e);
}
};
export const updateEnv = async (
call: ServerUnaryCall<UpdateEnvRequest, EnvResponse>,
callback: sendUnaryData<EnvResponse>,
@@ -83,6 +87,7 @@ export const updateEnv = async (
message: 'id parameter is required',
});
}
const envService = Container.get(EnvService);
const data = await envService.update(
pick(call.request.env, ['id', 'name', 'value', 'remarks']) as EnvItem,
@@ -92,6 +97,7 @@ export const updateEnv = async (
callback(e);
}
};
export const deleteEnvs = async (
call: ServerUnaryCall<DeleteEnvsRequest, Response>,
callback: sendUnaryData<Response>,
@@ -103,6 +109,7 @@ export const deleteEnvs = async (
message: 'ids parameter is required',
});
}
const envService = Container.get(EnvService);
await envService.remove(call.request.ids);
callback(null, { code: 200 });
@@ -110,6 +117,7 @@ export const deleteEnvs = async (
callback(e);
}
};
export const moveEnv = async (
call: ServerUnaryCall<MoveEnvRequest, EnvResponse>,
callback: sendUnaryData<EnvResponse>,
@@ -122,6 +130,7 @@ export const moveEnv = async (
message: 'id parameter is required',
});
}
const envService = Container.get(EnvService);
const data = await envService.move(call.request.id, {
fromIndex: call.request.fromIndex,
@@ -132,6 +141,7 @@ export const moveEnv = async (
callback(e);
}
};
export const disableEnvs = async (
call: ServerUnaryCall<DisableEnvsRequest, Response>,
callback: sendUnaryData<Response>,
@@ -143,6 +153,7 @@ export const disableEnvs = async (
message: 'ids parameter is required',
});
}
const envService = Container.get(EnvService);
await envService.disabled(call.request.ids);
callback(null, { code: 200 });
@@ -150,6 +161,7 @@ export const disableEnvs = async (
callback(e);
}
};
export const enableEnvs = async (
call: ServerUnaryCall<EnableEnvsRequest, Response>,
callback: sendUnaryData<Response>,
@@ -161,6 +173,7 @@ export const enableEnvs = async (
message: 'ids parameter is required',
});
}
const envService = Container.get(EnvService);
await envService.enabled(call.request.ids);
callback(null, { code: 200 });
@@ -168,6 +181,7 @@ export const enableEnvs = async (
callback(e);
}
};
export const updateEnvNames = async (
call: ServerUnaryCall<UpdateEnvNamesRequest, Response>,
callback: sendUnaryData<Response>,
@@ -179,6 +193,7 @@ export const updateEnvNames = async (
message: 'ids parameter is required',
});
}
const envService = Container.get(EnvService);
await envService.updateNames({
ids: call.request.ids,
@@ -189,6 +204,7 @@ export const updateEnvNames = async (
callback(e);
}
};
export const getEnvById = async (
call: ServerUnaryCall<GetEnvByIdRequest, EnvResponse>,
callback: sendUnaryData<EnvResponse>,
@@ -201,6 +217,7 @@ export const getEnvById = async (
message: 'id parameter is required',
});
}
const envService = Container.get(EnvService);
const data = await envService.getDb({ id: call.request.id });
callback(null, {
@@ -211,6 +228,7 @@ export const getEnvById = async (
callback(e);
}
};
export const systemNotify = async (
call: ServerUnaryCall<SystemNotifyRequest, Response>,
callback: sendUnaryData<Response>,
@@ -220,36 +238,18 @@ export const systemNotify = async (
const data = await systemService.notify({
title: call.request.title,
content: call.request.content,
notificationInfo: call.request
.notificationInfo as unknown as NotificationInfo,
notificationInfo: call.request.notificationInfo as unknown as NotificationInfo,
});
callback(null, data);
} catch (e: any) {
callback(e);
}
};
export const recordNotifyLog = async (
call: ServerUnaryCall<RecordNotifyLogRequest, Response>,
callback: sendUnaryData<Response>,
) => {
try {
await Container.get(SystemService).recordNotifyLog({
title: call.request.title,
content: call.request.content,
status: call.request.success ? 0 : 1,
notifyType: call.request.notifyType,
source: 'script',
scriptName: call.request.scriptName,
taskId: call.request.taskId,
});
callback(null, { code: 200 });
} catch (error: any) {
callback(error);
}
};
const normalizeCronData = (data: CronItem | null): CronItem | undefined => {
if (!data) return undefined;
const cron = data;
// create() returns a Sequelize instance; spreading it omits attribute getters.
const cron = data instanceof Model ? (data.get({ plain: true }) as CronItem) : data;
return {
...cron,
labels: cron.labels ?? [],
@@ -260,6 +260,7 @@ const normalizeCronData = (data: CronItem | null): CronItem | undefined => {
task_after: cron.task_after ?? undefined,
};
};
export const getCronDetail = async (
call: ServerUnaryCall<CronDetailRequest, CronDetailResponse>,
callback: sendUnaryData<CronDetailResponse>,
@@ -281,6 +282,7 @@ export const getCronDetail = async (
callback(e);
}
};
export const createCron = async (
call: ServerUnaryCall<CreateCronRequest, CronResponse>,
callback: sendUnaryData<CronResponse>,
@@ -293,6 +295,7 @@ export const createCron = async (
callback(e);
}
};
export const updateCron = async (
call: ServerUnaryCall<UpdateCronRequest, CronResponse>,
callback: sendUnaryData<CronResponse>,
@@ -300,6 +303,7 @@ export const updateCron = async (
try {
const cronService = Container.get(CronService);
const { id, ...fields } = call.request;
const updateRequest = {
id,
...Object.entries(fields).reduce((acc: any, [key, value]) => {
@@ -309,12 +313,14 @@ export const updateCron = async (
return acc;
}, {}),
} as UpdateCronRequest;
const data = (await cronService.update(updateRequest)) as CronItem;
callback(null, { code: 200, data: normalizeCronData(data) });
} catch (e: any) {
callback(e);
}
};
export const deleteCrons = async (
call: ServerUnaryCall<DeleteCronsRequest, Response>,
callback: sendUnaryData<Response>,
@@ -327,6 +333,7 @@ export const deleteCrons = async (
callback(e);
}
};
export const getCrons = async (
call: ServerUnaryCall<GetCronsRequest, CronsResponse>,
callback: sendUnaryData<CronsResponse>,
@@ -354,6 +361,7 @@ export const getCrons = async (
});
}
};
export const getCronById = async (
call: ServerUnaryCall<GetCronByIdRequest, CronResponse>,
callback: sendUnaryData<CronResponse>,
@@ -366,6 +374,7 @@ export const getCronById = async (
message: 'id parameter is required',
});
}
const cronService = Container.get(CronService);
const data = (await cronService.getDb({ id: call.request.id })) as CronItem;
callback(null, { code: 200, data: normalizeCronData(data) });
@@ -377,6 +386,7 @@ export const getCronById = async (
});
}
};
export const enableCrons = async (
call: ServerUnaryCall<EnableCronsRequest, Response>,
callback: sendUnaryData<Response>,
@@ -388,6 +398,7 @@ export const enableCrons = async (
message: 'ids parameter is required',
});
}
const cronService = Container.get(CronService);
await cronService.enabled(call.request.ids);
callback(null, { code: 200 });
@@ -395,6 +406,7 @@ export const enableCrons = async (
callback(e);
}
};
export const disableCrons = async (
call: ServerUnaryCall<DisableCronsRequest, Response>,
callback: sendUnaryData<Response>,
@@ -406,6 +418,7 @@ export const disableCrons = async (
message: 'ids parameter is required',
});
}
const cronService = Container.get(CronService);
await cronService.disabled(call.request.ids);
callback(null, { code: 200 });
@@ -413,6 +426,7 @@ export const disableCrons = async (
callback(e);
}
};
export const runCrons = async (
call: ServerUnaryCall<RunCronsRequest, Response>,
callback: sendUnaryData<Response>,
@@ -424,6 +438,7 @@ export const runCrons = async (
message: 'ids parameter is required',
});
}
const cronService = Container.get(CronService);
await cronService.run(call.request.ids);
callback(null, { code: 200 });
-19
View File
@@ -62,25 +62,6 @@ class Client {
return this._client;
}
async setConcurrency(concurrency: number): Promise<void> {
await this.waitForReady(2000);
return new Promise((resolve, reject) => {
this.client.setConcurrency(
{ concurrency }, new Metadata(), { deadline: Date.now() + 5000 },
(err) => {
if (err) {
if (err.code === status.UNAVAILABLE || err.code === status.DEADLINE_EXCEEDED) {
this.readiness.invalidate();
Object.assign(err, { status: 503 });
}
return reject(err);
}
resolve();
},
);
});
}
async addCron(
request: AddCronRequest['crons'],
replace = false
+15 -16
View File
@@ -1,4 +1,3 @@
import { t } from '../shared/i18n';
import { ServerUnaryCall, sendUnaryData } from '@grpc/grpc-js';
import { DeleteCronRequest, DeleteCronResponse } from '../protos/cron';
import { scheduleStacks } from './data';
@@ -10,24 +9,24 @@ const delCron = (
) => {
for (const id of call.request.ids) {
if (scheduleStacks.has(id)) {
Logger.info(t('[schedule][取消定时任务] 任务ID: %s'), id);
Logger.info(
'[schedule][取消定时任务] 任务ID: %s',
id,
);
// 防御性过滤历史调度栈中的空任务,
// 否则对 null 调 cancel() 会让整个取消流程抛出 UNKNOWN 错误,
// 进而导致 HTTP 端的 remove() 跳过 setCrontab(),造成 crontab.list 残留。
scheduleStacks
.get(id)
?.filter((x) => x != null)
.forEach((x) => {
try {
x.cancel();
} catch (error: any) {
Logger.warn(
t('[schedule][取消任务失败] 任务ID: %s, 错误: %s'),
id,
error?.message || error,
);
}
});
scheduleStacks.get(id)?.filter((x) => x != null).forEach((x) => {
try {
x.cancel();
} catch (error: any) {
Logger.warn(
'[schedule][取消任务失败] 任务ID: %s, 错误: %s',
id,
error?.message || error,
);
}
});
scheduleStacks.delete(id);
}
}
+1 -2
View File
@@ -4,7 +4,6 @@ import path from 'path';
import { request } from 'undici';
import { HealthCheckRequest, HealthCheckResponse } from '../protos/health';
import config from '../config';
import { database } from '../data';
function formatError(error: unknown): string {
if (!(error instanceof Error)) {
@@ -55,7 +54,7 @@ const check = async (
switch (call.request.service) {
// Local scheduler liveness only: never call HTTP from this probe.
case 'scheduler':
return callback(null, { status: database.healthy ? 1 : 2 });
return callback(null, { status: 1 });
case 'cron': {
const healthUrl = `http://localhost:${config.port}${
config.baseUrl || ''
-26
View File
@@ -1,26 +0,0 @@
import { ServerUnaryCall, sendUnaryData, status } from '@grpc/grpc-js';
import os from 'os';
import { AddCronResponse, SetConcurrencyRequest } from '../protos/cron';
import taskLimit from '../shared/pLimit';
export async function setConcurrency(
call: ServerUnaryCall<SetConcurrencyRequest, AddCronResponse>,
callback: sendUnaryData<AddCronResponse>,
) {
const limit = call.request.concurrency;
if (!Number.isInteger(limit) || limit < 0 || limit > 2147483647) {
callback(
Object.assign(new Error('Invalid concurrency'), {
code: status.INVALID_ARGUMENT,
}),
null,
);
return;
}
try {
await taskLimit.setCustomLimit(limit || Math.max(os.cpus().length, 4));
callback(null, {});
} catch (error) {
callback(error as Error, null);
}
}
+4 -3
View File
@@ -12,12 +12,13 @@ export default class ConfigService {
public async getFile(filePath: string, res: Response) {
let content = '';
if (!filePath || filePath.startsWith('data/scripts/')) {
if (!filePath) {
return res.send({ code: 403, message: t('文件无法访问') });
}
const scriptFile = filePath.startsWith('data/scripts/');
const resolved = resolveFileAccess(
config.configPath,
[filePath],
scriptFile ? config.scriptPath : config.configPath,
[scriptFile ? filePath.slice('data/scripts/'.length) : filePath],
config.blackFileList,
);
if (!resolved) {
+292 -248
View File
File diff suppressed because it is too large Load Diff
+31 -19
View File
@@ -1,4 +1,3 @@
import { Query, QuerySpec, orderBy } from '../data/queries';
import { Service, Inject } from 'typedi';
import winston from 'winston';
import { CrontabView, CrontabViewModel } from '../data/cronView';
@@ -8,12 +7,12 @@ import {
minPosition,
stepPosition,
} from '../data/env';
import { FindOptions } from 'sequelize';
@Service()
export default class CronViewService {
constructor(
@Inject('logger')
private logger: winston.Logger,
) {}
constructor(@Inject('logger') private logger: winston.Logger) {}
public async create(payload: CrontabView): Promise<CrontabView> {
let position = initPosition;
const views = await this.list();
@@ -23,51 +22,61 @@ export default class CronViewService {
position = position / 2;
const tab = new CrontabView({ ...payload, position });
const doc = await this.insert(tab);
await this.checkPosition(tab.position!);
return doc;
}
public async insert(payload: CrontabView): Promise<CrontabView> {
return await CrontabViewModel.insert(payload);
return await CrontabViewModel.create(payload, { returning: true });
}
public async update(payload: CrontabView): Promise<CrontabView> {
const doc = await this.getDb({ id: payload.id });
const tab = new CrontabView({ ...doc, ...payload });
const newDoc = await this.updateDb(tab);
return newDoc;
}
public async updateDb(payload: CrontabView): Promise<CrontabView> {
await CrontabViewModel.nativeUpdate({ id: payload.id }, payload);
await CrontabViewModel.update(payload, { where: { id: payload.id } });
return await this.getDb({ id: payload.id });
}
public async remove(ids: number[]) {
await CrontabViewModel.nativeDelete({ id: ids });
await CrontabViewModel.destroy({ where: { id: ids } });
}
public async list(): Promise<CrontabView[]> {
try {
const result = await CrontabViewModel.find(
{},
{
orderBy: orderBy([['position', 'DESC']]),
},
);
const result = await CrontabViewModel.findAll({
where: {},
order: [['position', 'DESC']],
});
return result;
} catch (error) {
throw error;
}
}
public async getDb(query: QuerySpec['where']): Promise<CrontabView> {
const doc: any = await CrontabViewModel.findOne({ ...query }, {});
public async getDb(
query: FindOptions<CrontabView>['where'],
): Promise<CrontabView> {
const doc: any = await CrontabViewModel.findOne({ where: { ...query } });
if (!doc) {
throw new Error(`CronView ${JSON.stringify(query)} not found`);
}
return doc;
return doc.get({ plain: true });
}
public async disabled(ids: number[]) {
await CrontabViewModel.nativeUpdate({ id: ids }, { isDisabled: 1 });
await CrontabViewModel.update({ isDisabled: 1 }, { where: { id: ids } });
}
public async enabled(ids: number[]) {
await CrontabViewModel.nativeUpdate({ id: ids }, { isDisabled: 0 });
await CrontabViewModel.update({ isDisabled: 0 }, { where: { id: ids } });
}
private async checkPosition(position: number) {
const precisionPosition = parseFloat(position.toPrecision(16));
if (precisionPosition < minPosition || precisionPosition > maxPosition) {
@@ -79,9 +88,11 @@ export default class CronViewService {
}
}
}
private getPrecisionPosition(position: number): number {
return parseFloat(position.toPrecision(16));
}
public async move({
id,
fromIndex,
@@ -107,6 +118,7 @@ export default class CronViewService {
id,
position: this.getPrecisionPosition(targetPosition),
});
await this.checkPosition(targetPosition);
return newDoc;
}
+114 -114
View File
@@ -1,5 +1,3 @@
import { textSearch } from '../data/queries';
import { Query, QuerySpec, orderBy } from '../data/queries';
import { Service, Inject } from 'typedi';
import winston from 'winston';
import config from '../config';
@@ -12,6 +10,7 @@ import {
} from '../data/dependence';
import { spawn } from 'cross-spawn';
import SockService from './sock';
import { FindOptions, Op } from 'sequelize';
import {
fileExist,
getPid,
@@ -21,23 +20,22 @@ import {
getInstallCommand,
getUninstallCommand,
getGetCommand,
isPythonSourceDependency,
} from '../config/util';
import dayjs from 'dayjs';
import taskLimit from '../shared/pLimit';
import { detectOS } from '../config/util';
import { LINUX_DEPENDENCE_COMMAND } from '../config/const';
import { t, tf } from '../shared/i18n';
@Service()
export default class DependenceService {
constructor(
@Inject('logger')
private logger: winston.Logger,
@Inject('logger') private logger: winston.Logger,
private sockService: SockService,
) {}
) { }
public async create(payloads: Dependence[]): Promise<Dependence[]> {
const tabs = payloads.map((x) => {
getInstallCommand(x.type, x.name);
const tab = new Dependence({ ...x, status: DependenceStatus.queued });
return tab;
});
@@ -45,14 +43,14 @@ export default class DependenceService {
this.installDependenceOneByOne(docs);
return docs;
}
public async insert(payloads: Dependence[]): Promise<Dependence[]> {
const docs = await DependenceModel.insertMany(payloads);
const docs = await DependenceModel.bulkCreate(payloads);
return docs;
}
public async update(
payload: Dependence & {
id: string;
},
payload: Dependence & { id: string },
): Promise<Dependence> {
const { id, ...other } = payload;
const doc = await this.getDb({ id });
@@ -61,43 +59,44 @@ export default class DependenceService {
...other,
status: DependenceStatus.queued,
});
getInstallCommand(tab.type, tab.name);
const newDoc = await this.updateDb(tab);
this.installDependenceOneByOne([newDoc]);
return newDoc;
}
private async updateDb(payload: Dependence): Promise<Dependence> {
await DependenceModel.nativeUpdate({ id: payload.id }, payload);
await DependenceModel.update(payload, { where: { id: payload.id } });
return await this.getDb({ id: payload.id });
}
public async remove(ids: number[], force = false): Promise<Dependence[]> {
const docs = await DependenceModel.find({ id: ids }, {});
const docs = await DependenceModel.findAll({ where: { id: ids } });
for (const doc of docs) {
taskLimit.removeQueuedDependency(doc);
}
const unInstalledDeps = docs.filter(
(x) => x.status !== DependenceStatus.installed,
);
const installedDeps = docs.filter(
(x) =>
x.status === DependenceStatus.installed ||
// Older list refreshes marked unprobeable sources as failed. They may
// still have installed distributions (including partial pip installs).
(x.status === DependenceStatus.installFailed &&
x.type === DependenceTypes.python3 &&
isPythonSourceDependency(x.name)),
(x) => x.status === DependenceStatus.installed,
);
const unInstalledDeps = docs.filter((x) => !installedDeps.includes(x));
await this.removeDb(unInstalledDeps.map((x) => x.id!));
if (installedDeps.length) {
await DependenceModel.nativeUpdate(
{ id: installedDeps.map((x) => x.id!) },
await DependenceModel.update(
{ status: DependenceStatus.queued, log: [] },
{ where: { id: ids } },
);
this.installDependenceOneByOne(installedDeps, false, force);
this.installDependenceOneByOne(docs, false, force);
}
return docs;
}
public async removeDb(ids: number[]) {
await DependenceModel.nativeDelete({ id: ids });
await DependenceModel.destroy({ where: { id: ids } });
}
public async dependencies(
{
searchValue,
@@ -124,7 +123,13 @@ export default class DependenceService {
condition.status = status.split(',').map(Number);
}
if (searchValue) {
Object.assign(condition, textSearch(['name'], searchValue));
const encodeText = encodeURI(searchValue);
condition.name = {
[Op.or]: [
{ [Op.like]: `%${searchValue}%` },
{ [Op.like]: `%${encodeText}%` },
],
};
}
try {
return await this.find(condition, sort);
@@ -132,6 +137,7 @@ export default class DependenceService {
throw error;
}
}
private async refreshInstalledStatuses(type?: DependenceTypes) {
const cacheDependenceTypes = [
DependenceTypes.nodejs,
@@ -140,43 +146,36 @@ export default class DependenceService {
if (type !== undefined && !cacheDependenceTypes.includes(type)) {
return;
}
const docs = await DependenceModel.find(
{
const docs = await DependenceModel.findAll({
where: {
status: DependenceStatus.installed,
type: type === undefined ? { ['$in']: cacheDependenceTypes } : type,
type: type === undefined ? { [Op.in]: cacheDependenceTypes } : type,
},
{},
);
});
const checks = await concurrentRun(
docs
.filter(
(doc) =>
doc.type !== DependenceTypes.python3 ||
!isPythonSourceDependency(doc.name),
)
.map((doc) => async () => {
return (await this.isDependenceInstalled(doc)) ? undefined : doc.id;
}),
docs.map((doc) => async () => {
return (await this.isDependenceInstalled(doc)) ? undefined : doc.id;
}),
5,
);
const missingIds = (checks || []).filter(
(id): id is number => id !== undefined,
);
if (missingIds.length) {
await DependenceModel.nativeUpdate(
{ id: missingIds },
await DependenceModel.update(
{ status: DependenceStatus.installFailed },
{ where: { id: missingIds } },
);
}
}
private async isDependenceInstalled(dependency: Dependence) {
let depName = dependency.name.trim();
const depVersionStr = versionDependenceCommandTypes[dependency.type];
let depVersion = '';
if (
dependency.type !== DependenceTypes.python3 &&
depName.includes(depVersionStr)
) {
if (depName.includes(depVersionStr)) {
const symbolRegx = new RegExp(
`(.*)${depVersionStr}([0-9\\.\\-\\+a-zA-Z]*)`,
);
@@ -186,6 +185,7 @@ export default class DependenceService {
depVersion = parsedVersion;
}
}
const depInfo = (
await promiseExecSuccess(getGetCommand(dependency.type, depName))
)
@@ -195,10 +195,12 @@ export default class DependenceService {
(dependency.type === DependenceTypes.nodejs &&
depInfo.split(' ')?.[0] === depName) ||
dependency.type === DependenceTypes.python3;
return Boolean(
depInfo && nameMatches && (!depVersion || depInfo.includes(depVersion)),
);
}
public installDependenceOneByOne(
docs: Dependence[],
isInstall: boolean = true,
@@ -207,27 +209,32 @@ export default class DependenceService {
docs.forEach((dep) => {
this.installOrUninstallDependency(dep, isInstall, force);
});
return taskLimit.waitDependencyQueueDone();
}
public async reInstall(ids: number[]): Promise<Dependence[]> {
await DependenceModel.nativeUpdate(
{ id: ids },
await DependenceModel.update(
{ status: DependenceStatus.queued, log: [] },
{ where: { id: ids } },
);
const docs = await DependenceModel.find({ id: ids }, {});
const docs = await DependenceModel.findAll({ where: { id: ids } });
for (const doc of docs) {
taskLimit.removeQueuedDependency(doc);
}
this.installDependenceOneByOne(docs, true, true);
return docs;
}
public async cancel(ids: number[]) {
const docs = await DependenceModel.find({ id: ids }, {});
const docs = await DependenceModel.findAll({ where: { id: ids } });
for (const doc of docs) {
taskLimit.removeQueuedDependency(doc);
let depInstallCommand = getInstallCommand(doc.type, doc.name);
let depUnInstallCommand = getUninstallCommand(doc.type, doc.name);
const isLinuxDependence = doc.type === DependenceTypes.linux;
if (isLinuxDependence) {
const osType = await detectOS();
if (!osType) {
@@ -245,63 +252,50 @@ export default class DependenceService {
pid && (await killTask(pid));
}
}
await DependenceModel.nativeUpdate(
{ id: ids },
await DependenceModel.update(
{ status: DependenceStatus.cancelled },
{ where: { id: ids } },
);
}
private async find(query: any, sort: any = []): Promise<Dependence[]> {
const docs = await DependenceModel.find(
{ ...query },
{
orderBy: orderBy([...sort, ['createdAt', 'DESC']]),
},
);
const docs = await DependenceModel.findAll({
where: { ...query },
order: [...sort, ['createdAt', 'DESC']],
});
return docs;
}
public async getDb(query: QuerySpec['where']): Promise<Dependence> {
const doc: any = await DependenceModel.findOne({ ...query }, {});
public async getDb(
query: FindOptions<Dependence>['where'],
): Promise<Dependence> {
const doc: any = await DependenceModel.findOne({ where: { ...query } });
if (!doc) {
throw new Error(`Dependency ${JSON.stringify(query)} not found`);
}
return doc;
return doc.get({ plain: true });
}
private async updateLog(ids: number[], log: string): Promise<void> {
taskLimit.updateDepLog(async () => {
const docs = await DependenceModel.find({ id: ids }, {});
const docs = await DependenceModel.findAll({ where: { id: ids } });
for (const doc of docs) {
const newLog = doc?.log ? [...doc.log, log] : [log];
await DependenceModel.nativeUpdate({ id: doc.id }, { log: newLog });
await DependenceModel.update(
{ log: newLog },
{ where: { id: doc.id } },
);
}
return null;
});
}
public installOrUninstallDependency(
dependency: Dependence,
isInstall: boolean = true,
force: boolean = false,
) {
return taskLimit.runDependeny(dependency, () => {
let validatedCommand: string;
try {
validatedCommand = isInstall
? getInstallCommand(dependency.type, dependency.name.trim())
: getUninstallCommand(dependency.type, dependency.name.trim());
} catch (error) {
taskLimit.removeQueuedDependency(dependency);
if (!isInstall && force) {
return this.removeDb([dependency.id!]).then(() => null);
}
return DependenceModel.nativeUpdate(
{ id: dependency.id },
{
status: isInstall
? DependenceStatus.installFailed
: DependenceStatus.removeFailed,
log: [String(error)],
},
).then(() => null);
}
return new Promise(async (resolve) => {
if (taskLimit.firstDependencyId !== dependency.id) {
return resolve(null);
@@ -320,9 +314,9 @@ export default class DependenceService {
taskLimit.removeQueuedDependency(dependency);
if (isLinuxDependence) {
if (!osType) {
await DependenceModel.nativeUpdate(
{ id: depIds },
await DependenceModel.update(
{ status: DependenceStatus.installFailed },
{ where: { id: depIds } },
);
const startTime = dayjs();
const message = tf(
@@ -345,17 +339,22 @@ export default class DependenceService {
}
linuxCommand = LINUX_DEPENDENCE_COMMAND[osType];
}
const status = isInstall
? DependenceStatus.installing
: DependenceStatus.removing;
await DependenceModel.nativeUpdate({ id: depIds }, { status });
let command = validatedCommand;
await DependenceModel.update({ status }, { where: { id: depIds } });
let command = isInstall
? getInstallCommand(dependency.type, depName)
: getUninstallCommand(dependency.type, depName);
if (isLinuxDependence) {
command = isInstall
? `${linuxCommand.install} ${depName.trim()}`
: `${linuxCommand.uninstall} ${depName.trim()}`;
}
const startTime = dayjs();
const message = tf(
'开始%s依赖 %s,开始时间 %s\n\n',
actionText,
@@ -369,21 +368,17 @@ export default class DependenceService {
status,
});
this.updateLog(depIds, message);
// 判断是否已经安装过依赖
// pip must resolve optional dependencies even if the base distribution
// is already present; its version alone cannot prove extras are ready.
if (
isInstall &&
!force &&
!(isPythonDependence && depName.includes('['))
) {
if (isInstall && !force) {
let getCommand = getGetCommand(dependency.type, depName);
const depVersionStr = versionDependenceCommandTypes[dependency.type];
if (isLinuxDependence) {
getCommand = `${linuxCommand.info} ${depName}`;
}
let depVersion = '';
if (!isPythonDependence && depName.includes(depVersionStr)) {
if (depName.includes(depVersionStr)) {
const symbolRegx = new RegExp(
`(.*)${depVersionStr}([0-9\\.\\-\\+a-zA-Z]*)`,
);
@@ -396,6 +391,7 @@ export default class DependenceService {
const depInfo = (await promiseExecSuccess(getCommand))
.replace(/\s{2,}/, ' ')
.replace(/\s+$/, '');
if (
depInfo &&
((isNodeDependence && depInfo.split(' ')?.[0] === depName) ||
@@ -420,9 +416,9 @@ export default class DependenceService {
status: DependenceStatus.installed,
});
this.updateLog(depIds, _message);
await DependenceModel.nativeUpdate(
{ id: depIds },
await DependenceModel.update(
{ status: DependenceStatus.installed },
{ where: { id: depIds } },
);
return resolve(null);
}
@@ -436,6 +432,7 @@ export default class DependenceService {
const cp = spawn(`${proxyStr} ${command}`, {
shell: '/bin/bash',
});
cp.stdout.on('data', async (data) => {
this.sockService.sendMessage({
type: socketMessageType,
@@ -445,6 +442,7 @@ export default class DependenceService {
});
this.updateLog(depIds, data.toString());
});
cp.stderr.on('data', async (data) => {
this.sockService.sendMessage({
type: socketMessageType,
@@ -454,6 +452,7 @@ export default class DependenceService {
});
this.updateLog(depIds, data.toString());
});
cp.on('error', async (err) => {
this.sockService.sendMessage({
type: socketMessageType,
@@ -463,26 +462,23 @@ export default class DependenceService {
});
this.updateLog(depIds, JSON.stringify(err));
});
cp.on('exit', async (code) => {
const endTime = dayjs();
const isSucceed = code === 0;
const resultText = isSucceed ? t('成功') : t('失败');
const message =
'\n' +
tf(
'依赖%s%s,结束时间 %s,耗时 %s 秒',
actionText,
resultText,
endTime.format('YYYY-MM-DD HH:mm:ss'),
String(endTime.diff(startTime, 'second')),
);
tf('依赖%s%s,结束时间 %s,耗时 %s 秒',
actionText,
resultText,
endTime.format('YYYY-MM-DD HH:mm:ss'),
String(endTime.diff(startTime, 'second')),
);
const exitStatus = isSucceed
? isInstall
? DependenceStatus.installed
: DependenceStatus.removed
: isInstall
? DependenceStatus.installFailed
: DependenceStatus.removeFailed;
? (isInstall ? DependenceStatus.installed : DependenceStatus.removed)
: (isInstall ? DependenceStatus.installFailed : DependenceStatus.removeFailed);
this.sockService.sendMessage({
type: socketMessageType,
message,
@@ -490,20 +486,24 @@ export default class DependenceService {
status: exitStatus,
});
this.updateLog(depIds, message);
const docs = await DependenceModel.find({ id: depIds }, {});
const docs = await DependenceModel.findAll({ where: { id: depIds } });
const _docIds = docs
.filter((x) => x.status !== DependenceStatus.cancelled)
.map((x) => x.id!);
if (_docIds.length > 0) {
await DependenceModel.nativeUpdate(
{ id: _docIds },
await DependenceModel.update(
{ status: exitStatus },
{ where: { id: _docIds } },
);
}
// 如果删除依赖成功或者强制删除
if ((isSucceed || force) && !isInstall) {
this.removeDb(depIds);
}
resolve(null);
});
});
+86 -41
View File
@@ -1,6 +1,5 @@
import { textSearch } from '../data/queries';
import { Query, QuerySpec, orderBy } from '../data/queries';
import groupBy from 'lodash/groupBy';
import { FindOptions, Op } from 'sequelize';
import { Inject, Service } from 'typedi';
import winston from 'winston';
import config from '../config';
@@ -14,13 +13,12 @@ import {
stepPosition,
} from '../data/env';
import { writeFileWithLock } from '../shared/utils';
import { database } from '../data';
import { sequelize } from '../data';
@Service()
export default class EnvService {
constructor(
@Inject('logger')
private logger: winston.Logger,
) {}
constructor(@Inject('logger') private logger: winston.Logger) { }
public async create(payloads: Env[]): Promise<Env[]> {
const envs = await this.envs();
let position = initPosition;
@@ -41,14 +39,16 @@ export default class EnvService {
await this.checkPosition(tabs[tabs.length - 1].position!);
return docs;
}
public async insert(payloads: Env[]): Promise<Env[]> {
const result: Env[] = [];
for (const env of payloads) {
const doc = await EnvModel.insert(env);
const doc = await EnvModel.create(env, { returning: true });
result.push(doc);
}
return result;
}
public async update(payload: Env): Promise<Env> {
const doc = await this.getDb({ id: payload.id });
const tab = new Env({ ...doc, ...payload });
@@ -56,14 +56,17 @@ export default class EnvService {
await this.set_envs();
return newDoc;
}
private async updateDb(payload: Env): Promise<Env> {
await EnvModel.nativeUpdate({ id: payload.id }, { ...payload });
await EnvModel.update({ ...payload }, { where: { id: payload.id } });
return await this.getDb({ id: payload.id });
}
public async remove(ids: number[]) {
await EnvModel.nativeDelete({ id: ids });
await EnvModel.destroy({ where: { id: ids } });
await this.set_envs();
}
public async move(
id: number,
{
@@ -86,13 +89,16 @@ export default class EnvService {
? (envs[toIndex].position! + envs[toIndex - 1].position!) / 2
: (envs[toIndex].position! + envs[toIndex + 1].position!) / 2;
}
const newDoc = await this.update({
id,
position: this.getPrecisionPosition(targetPosition),
});
await this.checkPosition(targetPosition, envs[toIndex].position!);
return newDoc;
}
private async checkPosition(position: number, edge: number = 0) {
const precisionPosition = this.getPrecisionPosition(position);
if (
@@ -108,20 +114,40 @@ export default class EnvService {
}
}
}
private getPrecisionPosition(position: number): number {
return Math.trunc(parseFloat(position.toPrecision(16)));
}
public async envs(searchText: string = '', query: any = {}): Promise<Env[]> {
let condition = { ...query };
if (searchText) {
const encodeText = encodeURI(searchText);
const reg = {
[Op.or]: [
{ [Op.like]: `%${searchText}%` },
{ [Op.like]: `%${encodeText}%` },
],
};
condition = {
...condition,
...textSearch(['name', 'value', 'remarks'], searchText),
[Op.or]: [
{
name: reg,
},
{
value: reg,
},
{
remarks: reg,
},
],
};
}
try {
const result = await this.find(condition, [
['isPinned', 'DESC'],
[sequelize.literal('COALESCE(`isPinned`, 0)'), 'DESC'],
['position', 'DESC'],
['createdAt', 'ASC'],
]);
@@ -130,73 +156,90 @@ export default class EnvService {
throw error;
}
}
private async find(query: any, sort: any = []): Promise<Env[]> {
const docs = await EnvModel.find(
{ ...query },
{
orderBy: orderBy([...sort]),
},
);
return docs.map((x) => x);
const docs = await EnvModel.findAll({
where: { ...query },
order: [...sort],
});
return docs.map((x) => x.get({ plain: true }));
}
public async getDb(query: QuerySpec['where']): Promise<Env> {
const doc: any = await EnvModel.findOne({ ...query }, {});
public async getDb(query: FindOptions<Env>['where']): Promise<Env> {
const doc: any = await EnvModel.findOne({ where: { ...query } });
if (!doc) {
throw new Error(`Env ${JSON.stringify(query)} not found`);
}
return doc;
return doc.get({ plain: true });
}
public async disabled(ids: number[]) {
await EnvModel.nativeUpdate({ id: ids }, { status: EnvStatus.disabled });
await EnvModel.update(
{ status: EnvStatus.disabled },
{ where: { id: ids } },
);
await this.set_envs();
}
public async enabled(ids: number[]) {
await EnvModel.nativeUpdate({ id: ids }, { status: EnvStatus.normal });
await EnvModel.update({ status: EnvStatus.normal }, { where: { id: ids } });
await this.set_envs();
}
public async updateNames({ ids, name }: { ids: number[]; name: string }) {
await EnvModel.nativeUpdate({ id: ids }, { name });
await EnvModel.update({ name }, { where: { id: ids } });
await this.set_envs();
}
public async pin(ids: number[]) {
await EnvModel.nativeUpdate({ id: ids }, { isPinned: 1 });
await EnvModel.update({ isPinned: 1 }, { where: { id: ids } });
}
public async unPin(ids: number[]) {
await EnvModel.nativeUpdate({ id: ids }, { isPinned: 0 });
await EnvModel.update({ isPinned: 0 }, { where: { id: ids } });
}
public async addLabels(ids: number[], labels: string[]) {
await database.transaction(async () => {
const docs = await EnvModel.find({ id: ids }, {});
await sequelize.transaction(async (transaction) => {
const docs = await EnvModel.findAll({
where: { id: ids },
transaction,
});
for (const doc of docs) {
const env = doc;
await EnvModel.nativeUpdate(
{ id: env.id },
const env = doc.get({ plain: true });
await EnvModel.update(
{ labels: Array.from(new Set([...(env.labels || []), ...labels])) },
{ where: { id: env.id }, transaction },
);
}
});
return await this.find({ id: ids });
}
public async removeLabels(ids: number[], labels: string[]) {
await database.transaction(async () => {
const docs = await EnvModel.find({ id: ids }, {});
await sequelize.transaction(async (transaction) => {
const docs = await EnvModel.findAll({
where: { id: ids },
transaction,
});
for (const doc of docs) {
const env = doc;
await EnvModel.nativeUpdate(
{ id: env.id },
const env = doc.get({ plain: true });
await EnvModel.update(
{
labels: (env.labels || []).filter(
(label: string) => !labels.includes(label),
),
},
{ where: { id: env.id }, transaction },
);
}
});
return await this.find({ id: ids });
}
public async set_envs() {
const envs = await this.envs('', {
name: { ['$ne']: null },
name: { [Op.not]: null },
status: EnvStatus.normal,
});
const groups = groupBy(envs, 'name');
@@ -206,6 +249,7 @@ export default class EnvService {
for (const key in groups) {
if (Object.prototype.hasOwnProperty.call(groups, key)) {
const group = groups[key];
// 忽略不符合bash要求的环境变量名称
if (/^[a-zA-Z_][0-9a-zA-Z_]*$/.test(key)) {
let value = group
@@ -218,9 +262,10 @@ export default class EnvService {
.map((x) => x.value)
.join('&')
.replace(/\\/g, '\\\\')}`;
js_env_string += `process.env[${JSON.stringify(
key,
)}]=${JSON.stringify(group.map((x) => x.value).join('&'))};\n`;
js_env_string += `process.env.${key}=\`${_env_value.replace(
/\`/g,
'\\`',
)}\`;\n`;
py_env_string += `os.environ['${key}']='''${_env_value.replace(
/\'/g,
"\\'",
+3 -4
View File
@@ -4,7 +4,6 @@ import { HealthService } from '../protos/health';
import { ApiService } from '../protos/api';
import { addCron } from '../schedule/addCron';
import { delCron } from '../schedule/delCron';
import { setConcurrency } from '../schedule/setConcurrency';
import { check } from '../schedule/health';
import * as Api from '../schedule/api';
import Logger from '../loaders/logger';
@@ -28,7 +27,7 @@ export class GrpcServerService {
async initialize() {
try {
this.server.addService(HealthService, { check });
this.server.addService(CronService, { addCron, delCron, setConcurrency });
this.server.addService(CronService, { addCron, delCron });
this.server.addService(ApiService, Api);
const tlsConfig = await initGrpcCerts();
@@ -50,13 +49,13 @@ export class GrpcServerService {
for (const host of hostsToTry) {
try {
const address = this.formatGrpcAddress(host, grpcPort);
const boundPort = await bindAsync(address, credentials);
await bindAsync(address, credentials);
Logger.debug(`[boot] gRPC service started successfully on ${address}`);
metricsService.record('grpc_service_start', 1, {
port: grpcPort.toString(),
host
});
return boundPort;
return grpcPort;
} catch (err) {
lastError = err as Error;
Logger.warn(`Failed to bind gRPC on ${host}:${grpcPort}, trying next...`, err);
+3 -8
View File
@@ -1,17 +1,13 @@
import { database } from '../data';
import { Service } from 'typedi';
import Logger from '../loaders/logger';
import cronClient from '../schedule/client';
import { HttpServerService } from './http';
import { runtimeBuild } from '../shared/runtimeBuild';
interface HealthStatus {
build: typeof runtimeBuild;
status: 'ok' | 'error';
services: {
http: boolean;
grpc: boolean;
database: boolean;
};
metrics: {
uptime: number;
@@ -26,16 +22,16 @@ interface HealthStatus {
export class HealthService {
private startTime = Date.now();
constructor(private httpServerService: HttpServerService) {}
constructor(
private httpServerService: HttpServerService,
) {}
async check(): Promise<HealthStatus> {
const status: HealthStatus = {
build: runtimeBuild,
status: 'ok',
services: {
http: true,
grpc: true,
database: database.healthy,
},
metrics: {
uptime: Math.floor((Date.now() - this.startTime) / 1000),
@@ -69,7 +65,6 @@ export class HealthService {
Logger.error('gRPC server check failed:', err);
}
if (!status.services.database) status.status = 'error';
return status;
}
}
+6 -1
View File
@@ -1,4 +1,5 @@
import { resolveFileAccess } from '../shared/fileAccess';
import path from 'path';
import { Inject, Service } from 'typedi';
import winston from 'winston';
import config from '../config';
@@ -8,6 +9,10 @@ export default class LogService {
constructor(@Inject('logger') private logger: winston.Logger) {}
public checkFilePath(filePath: string, fileName: string) {
return resolveFileAccess(config.logPath, [filePath || '', fileName]);
return resolveFileAccess(
config.logPath,
[filePath || '', fileName],
config.blackFileList,
);
}
}
+1 -2
View File
@@ -1,4 +1,3 @@
import { t } from '../shared/i18n';
import { performance } from 'perf_hooks';
import Logger from '../loaders/logger';
@@ -127,7 +126,7 @@ export class MetricsService {
timestamp: Date.now(),
metrics: this.getMetrics(),
};
Logger.info(t('性能指标报告:'), report);
Logger.info('性能指标报告:', report);
return report;
}
}
+26 -42
View File
@@ -1,4 +1,3 @@
import { persistNotificationAudit } from '../shared/notificationAudit';
import crypto from 'crypto';
import nodemailer from 'nodemailer';
import { Inject, Service } from 'typedi';
@@ -56,33 +55,27 @@ export default class NotificationService {
content: string,
notificationInfo?: NotificationInfo,
): Promise<boolean | undefined> {
let type: string | undefined;
let success = false;
try {
const saved = await this.userService.getNotificationMode();
type = notificationInfo?.type || saved.type;
if (type) {
this.title = title;
this.content = content;
const { type: _, ...overrides } = notificationInfo || {};
const { type: _saved, ...rest } = saved;
this.params = { ...rest, ...overrides };
try {
success = !!(await this.modeMap.get(type)?.call(this));
} catch (error) {
console.error(error);
}
}
return success;
} finally {
await persistNotificationAudit({
title,
content,
notifyType: type,
source: 'system',
status: success ? 0 : 1,
});
let { type, ...rest } = await this.userService.getNotificationMode();
if (notificationInfo?.type) {
type = notificationInfo?.type;
}
if (type) {
this.title = title;
this.content = content;
let params = rest;
if (notificationInfo) {
const { type: _, ...others } = notificationInfo;
params = { ...rest, ...others };
}
this.params = params;
const notificationModeAction = this.modeMap.get(type);
try {
return await notificationModeAction?.call(this);
} catch (error: any) {
console.error(error);
}
}
return false;
}
public async testNotify(
@@ -91,23 +84,14 @@ export default class NotificationService {
content: string,
) {
const { type, ...rest } = info;
if (!type) return true;
let success = false;
try {
if (type) {
this.title = title;
this.content = content;
this.params = rest;
success = !!(await this.modeMap.get(type)?.call(this));
return success;
} finally {
await persistNotificationAudit({
title,
content,
notifyType: type,
source: 'system',
status: success ? 0 : 1,
});
const notificationModeAction = this.modeMap.get(type);
return await notificationModeAction?.call(this);
}
return true;
}
private parseMailRecipients(value?: string) {
@@ -937,8 +921,7 @@ export default class NotificationService {
private async openiLink() {
const { openiLinkAppToken, openiLinkHubUrl, openiLinkContextToken } =
this.params;
const baseUrl =
openiLinkHubUrl?.replace(/\/$/, '') || 'https://hub.openilink.com';
const baseUrl = openiLinkHubUrl?.replace(/\/$/, '') || 'https://hub.openilink.com';
const url = `${baseUrl}/bot/v1/message/send`;
const body: Record<string, string> = {
type: 'text',
@@ -991,4 +974,5 @@ export default class NotificationService {
throw new Error(error.response ? error.response.body : error);
}
}
}
+72 -126
View File
@@ -1,81 +1,36 @@
import { textSearch } from '../data/queries';
import { Service, Inject } from 'typedi';
import winston from 'winston';
import { createRandomString } from '../config/util';
import { App, AppModel } from '../data/open';
import { v4 as uuidV4 } from 'uuid';
import sequelize, { Op } from 'sequelize';
import { shareStore } from '../shared/store';
import { t } from '../shared/i18n';
import { lock } from 'proper-lockfile';
import path from 'path';
import os from 'os';
import { writeFile, rename, rm, stat, chmod } from 'fs/promises';
import config from '../config';
@Service()
export default class OpenService {
constructor(
@Inject('logger')
private logger: winston.Logger,
) {}
// Every database mutation and full authentication-cache publication shares
// this cross-process lock. Lock a separate marker, not token.json itself.
private async withAppsLock<T>(operation: () => Promise<T>): Promise<T> {
const target = path.join(config.configPath, '.apps-state');
const release = await lock(target, {
realpath: false,
stale: 10000,
retries: { retries: 10, factor: 2, minTimeout: 100, maxTimeout: 3000 },
});
try {
return await operation();
} finally {
await release();
}
}
public async initializeSystemApp(): Promise<void> {
await this.withAppsLock(async () => {
const doc = await AppModel.findOne({ name: 'system' }, {});
const app = doc;
if (!app) {
await this.insert({
name: 'system',
scopes: ['crons', 'system', 'dashboard'],
client_id: createRandomString(12, 12),
client_secret: createRandomString(24, 24),
});
} else if (!app.scopes.includes('dashboard')) {
await AppModel.nativeUpdate(
{ id: app.id },
{ scopes: [...app.scopes, 'dashboard'] },
);
}
await shareStore.updateApps(await this.find({}));
});
}
public async refreshApps(): Promise<void> {
await this.withAppsLock(async () => {
await shareStore.updateApps(await this.find({}));
});
}
constructor(@Inject('logger') private logger: winston.Logger) {}
public async findApps(): Promise<App[] | null> {
const docs = await this.find({});
return docs;
}
public async create(payload: App): Promise<App> {
return this.withAppsLock(async () => {
const tab = { ...payload };
tab.client_id = createRandomString(12, 12);
tab.client_secret = createRandomString(24, 24);
const doc = await this.insert(tab);
const apps = await this.find({});
await shareStore.updateApps(apps);
return { ...doc, tokens: [] };
});
const tab = { ...payload };
tab.client_id = createRandomString(12, 12);
tab.client_secret = createRandomString(24, 24);
const doc = await this.insert(tab);
const apps = await this.find({});
await shareStore.updateApps(apps);
return { ...doc, tokens: [] };
}
private async insert(payload: App): Promise<App> {
const doc = await AppModel.insert(payload);
return doc;
public async insert(payload: App): Promise<App> {
const doc = await AppModel.create(payload, { returning: true });
return doc.get({ plain: true });
}
public async update(payload: App): Promise<App> {
const newDoc = await this.updateDb({
name: payload.name,
@@ -84,28 +39,28 @@ export default class OpenService {
} as App);
return { ...newDoc, tokens: [] };
}
private async updateDb(payload: Partial<App>): Promise<App> {
return this.withAppsLock(async () => {
await AppModel.nativeUpdate({ id: payload.id }, payload);
const apps = await this.find({});
await shareStore.updateApps(apps);
return apps?.find((x) => x.id === payload.id) as App;
});
await AppModel.update(payload, { where: { id: payload.id } });
const apps = await this.find({});
await shareStore.updateApps(apps);
return apps?.find((x) => x.id === payload.id) as App;
}
public async getDb(query: Record<string, any>): Promise<App> {
const doc = await AppModel.findOne(query, {});
const doc = await AppModel.findOne({ where: query });
if (!doc) {
throw new Error(`App ${JSON.stringify(query)} not found`);
}
return doc;
return doc.get({ plain: true });
}
public async remove(ids: number[]) {
return this.withAppsLock(async () => {
await AppModel.nativeDelete({ id: ids });
const apps = await this.find({});
await shareStore.updateApps(apps);
});
await AppModel.destroy({ where: { id: ids } });
const apps = await this.find({});
await shareStore.updateApps(apps);
}
public async resetSecret(id: number): Promise<App> {
const tab: Partial<App> = {
client_secret: createRandomString(24, 24),
@@ -121,6 +76,7 @@ export default class OpenService {
const newDoc = await this.updateDb(tab);
return newDoc;
}
public async list(
searchText: string = '',
sort: any = {},
@@ -128,7 +84,18 @@ export default class OpenService {
): Promise<App[]> {
let condition = { ...query };
if (searchText) {
condition = { ...condition, ...textSearch(['name'], searchText) };
const encodeText = encodeURI(searchText);
const reg = {
[Op.or]: [
{ [Op.like]: `%${searchText}%` },
{ [Op.like]: `%${encodeText}%` },
],
};
condition = {
...condition,
name: reg,
};
}
try {
const result = await this.find(condition);
@@ -139,18 +106,13 @@ export default class OpenService {
throw error;
}
}
private async find(query: Record<string, any>, sort?: any): Promise<App[]> {
const docs = await AppModel.find({ ...query }, {});
return docs.map((x) => x);
const docs = await AppModel.findAll({ where: { ...query } });
return docs.map((x) => x.get({ plain: true }));
}
public async authToken(credentials: {
client_id: string;
client_secret: string;
}): Promise<any> {
return this.withAppsLock(() => this.issueToken(credentials));
}
// Caller holds withAppsLock through database update and cache publication.
private async issueToken({
public async authToken({
client_id,
client_secret,
}: {
@@ -159,7 +121,7 @@ export default class OpenService {
}): Promise<any> {
let token = uuidV4();
const expiration = Math.round(Date.now() / 1000) + 2592000; // 2592000 30天
const doc = await AppModel.findOne({ client_id, client_secret }, {});
const doc = await AppModel.findOne({ where: { client_id, client_secret } });
if (doc) {
const timestamp = Math.round(Date.now() / 1000);
const invalidTokens = (doc.tokens || []).filter(
@@ -175,7 +137,10 @@ export default class OpenService {
} else {
tokens = [...invalidTokens, { value: token, expiration }];
}
await AppModel.nativeUpdate({ client_id, client_secret }, { tokens });
await AppModel.update(
{ tokens },
{ where: { client_id, client_secret } },
);
const apps = await this.find({});
await shareStore.updateApps(apps);
return {
@@ -190,47 +155,28 @@ export default class OpenService {
return { code: 400, message: t('client_id 或 client_seret 有误') };
}
}
public async generateSystemToken(renew = false): Promise<{
public async generateSystemToken(): Promise<{
value: string;
expiration: number;
}> {
return this.withAppsLock(async () => {
// Read after acquiring the lock; a concurrent issuer may have renewed.
const systemApp = await this.getDb({ name: 'system' });
const now = Math.round(Date.now() / 1000);
let appToken = systemApp.tokens
?.filter((token) => token.expiration > now)
.sort((left, right) => right.expiration - left.expiration)[0];
if (renew || !appToken) {
const { data } = await this.issueToken({
client_id: systemApp.client_id,
client_secret: systemApp.client_secret,
});
appToken = { value: data.token, expiration: data.expiration };
} else {
// Recover a prior database commit whose cache publication failed.
await shareStore.updateApps(await this.find({}));
}
const tokenFile = path.join(config.configPath, 'token.json');
const temporaryFile = `${tokenFile}.${process.pid}.tmp`;
const mode = await stat(tokenFile).then(
(file) => file.mode & 0o777,
(error: NodeJS.ErrnoException) => {
if (error.code === 'ENOENT') return 0o600;
throw error;
},
);
try {
await writeFile(temporaryFile, `${JSON.stringify(appToken)}${os.EOL}`, {
mode,
});
await chmod(temporaryFile, mode);
// Shell readers see a complete old or new document, never a partial one.
await rename(temporaryFile, tokenFile);
} finally {
await rm(temporaryFile, { force: true });
}
return appToken;
const apps = await shareStore.getApps();
const systemApp = apps?.find((x) => x.name === 'system');
if (!systemApp) {
throw new Error('system app not found');
}
const now = Math.round(Date.now() / 1000);
const currentToken = systemApp.tokens?.find((x) => x.expiration > now);
if (currentToken) {
return currentToken;
}
const { data } = await this.authToken({
client_id: systemApp.client_id,
client_secret: systemApp.client_secret,
});
return {
...data,
value: data.token,
};
}
}
+41 -25
View File
@@ -1,10 +1,11 @@
import fs from 'fs/promises';
import path from 'path';
import dayjs from 'dayjs';
import { Op } from 'sequelize';
import { Service } from 'typedi';
import config from '../config';
import { CrontabStatModel } from '../data/cronStats';
import { database } from '../data';
import { sequelize } from '../data';
import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
import { AuthDataType, SystemModel } from '../data/system';
import {
@@ -14,50 +15,54 @@ import {
normalizeRetentionPolicy,
RetentionPolicy,
} from '../shared/retention';
export interface StorageCleanupRequest extends RetentionPolicy {
dependenceCacheTypes?: DependenceCacheType[];
compactDatabase?: boolean;
}
function runningInstanceWhere(days: number) {
const cutoff = dayjs().subtract(days, 'day').unix();
return {
status: { ['$ne']: InstanceStatus.running },
['$or']: [
{ finished_at: { ['$lt']: cutoff } },
status: { [Op.ne]: InstanceStatus.running },
[Op.or]: [
{ finished_at: { [Op.lt]: cutoff } },
{
finished_at: { ['$eq']: null },
started_at: { ['$lt']: cutoff },
finished_at: { [Op.is]: null },
started_at: { [Op.lt]: cutoff },
},
],
};
}
function cronStatWhere(days: number) {
return {
date: { ['$lt']: dayjs().subtract(days, 'day').format('YYYY-MM-DD') },
date: { [Op.lt]: dayjs().subtract(days, 'day').format('YYYY-MM-DD') },
};
}
@Service()
export default class RetentionService {
public async updatePolicy(policy: Partial<RetentionPolicy>) {
const normalized = normalizeRetentionPolicy(policy);
const systemConfig = await SystemModel.findOne(
{ type: AuthDataType.systemConfig },
{},
);
const systemConfig = await SystemModel.findOne({
where: { type: AuthDataType.systemConfig },
});
if (!systemConfig) {
throw new Error('System config not found');
}
await SystemModel.nativeUpdate(
{ id: systemConfig.id },
await SystemModel.update(
{
info: {
...systemConfig.info,
...normalized,
},
},
{ where: { id: systemConfig.id } },
);
return normalized;
}
public async preview(request: StorageCleanupRequest) {
const policy = normalizeRetentionPolicy(request);
const dependenceCacheTypes = (request.dependenceCacheTypes || []).filter(
@@ -65,12 +70,14 @@ export default class RetentionService {
);
const [runningInstances, cronStats, dependenceCaches] = await Promise.all([
policy.runningInstanceRetentionDays > 0
? RunningInstanceModel.count(
runningInstanceWhere(policy.runningInstanceRetentionDays),
)
? RunningInstanceModel.count({
where: runningInstanceWhere(policy.runningInstanceRetentionDays),
})
: 0,
policy.cronStatRetentionDays > 0
? CrontabStatModel.count(cronStatWhere(policy.cronStatRetentionDays))
? CrontabStatModel.count({
where: cronStatWhere(policy.cronStatRetentionDays),
})
: 0,
Promise.all(
dependenceCacheTypes.map(async (type) => ({
@@ -81,6 +88,7 @@ export default class RetentionService {
})),
),
]);
return {
policy,
runningInstances,
@@ -93,23 +101,29 @@ export default class RetentionService {
compactDatabase: Boolean(request.compactDatabase),
};
}
public async cleanup(request: StorageCleanupRequest) {
const preview = await this.preview(request);
const deleted = await database.transaction(async () => {
const deleted = await sequelize.transaction(async (transaction) => {
const runningInstances =
preview.policy.runningInstanceRetentionDays > 0
? await RunningInstanceModel.nativeDelete(
runningInstanceWhere(preview.policy.runningInstanceRetentionDays),
)
? await RunningInstanceModel.destroy({
where: runningInstanceWhere(
preview.policy.runningInstanceRetentionDays,
),
transaction,
})
: 0;
const cronStats =
preview.policy.cronStatRetentionDays > 0
? await CrontabStatModel.nativeDelete(
cronStatWhere(preview.policy.cronStatRetentionDays),
)
? await CrontabStatModel.destroy({
where: cronStatWhere(preview.policy.cronStatRetentionDays),
transaction,
})
: 0;
return { runningInstances, cronStats };
});
const dependenceCaches = [];
for (const cache of preview.dependenceCaches) {
await fs.rm(path.join(config.dependenceCachePath, cache.type), {
@@ -118,12 +132,14 @@ export default class RetentionService {
});
dependenceCaches.push(cache);
}
if (
request.compactDatabase &&
(deleted.runningInstances || deleted.cronStats)
) {
await database.query('VACUUM');
await sequelize.query('VACUUM');
}
return {
deleted: {
...deleted,
+12 -13
View File
@@ -1,4 +1,3 @@
import { t } from '../shared/i18n';
import { Service, Inject } from 'typedi';
import winston from 'winston';
import { createCronJob, CronJob } from '../shared/cronScheduler';
@@ -84,7 +83,7 @@ export default class ScheduleService {
let cp: ChildProcessWithoutNullStreams | undefined;
let result: ProcessResult = { code: null, signal: null };
try {
this.logger.info(t('[panel][开始执行任务] 任务ID: %s'), others.id);
this.logger.info('[panel][开始执行任务] 任务ID: %s', others.id);
await callbacks.onBefore?.(startTime);
cp = spawn(command, { shell: '/bin/bash' });
const child = cp;
@@ -104,7 +103,7 @@ export default class ScheduleService {
if (result.error) {
this.logger.error(
t('[panel][执行任务失败] 任务ID: %s, 错误: %s'),
'[panel][执行任务失败] 任务ID: %s, 错误: %s',
others.id,
result.error.message,
);
@@ -112,7 +111,7 @@ export default class ScheduleService {
await callbacks.onError?.(result.error.message);
} catch (error) {
this.logger.error(
t('[panel][任务错误回调失败] %s'),
'[panel][任务错误回调失败] %s',
asError(error).message,
);
}
@@ -128,12 +127,12 @@ export default class ScheduleService {
} catch (error) {
result.error ??= asError(error);
this.logger.error(
t('[panel][任务结束回调失败] %s'),
'[panel][任务结束回调失败] %s',
asError(error).message,
);
}
this.logger.info(
t('[panel][执行任务结束] 任务ID: %s, 退出码: %j'),
'[panel][执行任务结束] 任务ID: %s, 退出码: %j',
others.id,
result.code,
);
@@ -142,7 +141,7 @@ export default class ScheduleService {
).catch((error) => {
// Queue/setup failures must not become unhandled rejections in detached callers.
rejectStart(asError(error));
this.logger.error(t('[panel][任务队列失败] %s'), asError(error).message);
this.logger.error('[panel][任务队列失败] %s', asError(error).message);
return { ...others, code: null, signal: null, error: asError(error) };
});
@@ -157,7 +156,7 @@ export default class ScheduleService {
) {
const _id = this.formatId(id);
this.logger.info(
t('[panel][创建cron任务] 任务ID: %s, cron: %s, 任务名: %s, 执行命令: %s'),
'[panel][创建cron任务] 任务ID: %s, cron: %s, 任务名: %s, 执行命令: %s',
_id,
schedule,
name,
@@ -184,7 +183,7 @@ export default class ScheduleService {
} catch (error) {
// A persisted invalid subscription must not become an unhandled startup rejection.
this.logger.warn(
t('[panel][跳过无效定时任务] 任务ID: %s, cron: %s, 错误: %s'),
'[panel][跳过无效定时任务] 任务ID: %s, cron: %s, 错误: %s',
_id,
schedule,
asError(error).message,
@@ -205,7 +204,7 @@ export default class ScheduleService {
async cancelCronTask({ id = 0, name }: ScheduleTaskType) {
const _id = this.formatId(id);
this.logger.info(t('[panel][取消定时任务] 任务名: %s'), name);
this.logger.info('[panel][取消定时任务] 任务名: %s', name);
if (this.scheduleStacks.has(_id)) {
this.scheduleStacks.get(_id)?.cancel();
this.scheduleStacks.delete(_id);
@@ -220,7 +219,7 @@ export default class ScheduleService {
) {
const _id = this.formatId(id);
this.logger.info(
t('[panel][创建interval任务] 任务ID: %s, 任务名: %s, 执行命令: %s'),
'[panel][创建interval任务] 任务ID: %s, 任务名: %s, 执行命令: %s',
_id,
name,
command,
@@ -237,7 +236,7 @@ export default class ScheduleService {
},
(err) => {
this.logger.error(
t('[panel][执行任务失败] 命令: %s, 错误信息: %j'),
'[panel][执行任务失败] 命令: %s, 错误信息: %j',
command,
err,
);
@@ -265,7 +264,7 @@ export default class ScheduleService {
async cancelIntervalTask({ id = 0, name }: ScheduleTaskType) {
const _id = this.formatId(id);
this.logger.info(
t('[panel][取消interval任务] 任务ID: %s, 任务名: %s'),
'[panel][取消interval任务] 任务ID: %s, 任务名: %s',
_id,
name,
);
+5 -1
View File
@@ -66,7 +66,11 @@ export default class ScriptService {
}
public checkFilePath(filePath: string, fileName: string) {
return resolveFileAccess(config.scriptPath, [filePath || '', fileName]);
return resolveFileAccess(
config.scriptPath,
[filePath || '', fileName],
config.blackFileList,
);
}
public async getFile(filePath: string, fileName: string) {
-12
View File
@@ -1,12 +0,0 @@
import path from 'path';
import { Service } from 'typedi';
import config from '../config';
import { ScriptHistory } from '../shared/scriptHistory';
/** Panel configuration and lifecycle; the storage core stays independently testable. */
@Service()
export default class ScriptHistoryService extends ScriptHistory {
constructor() {
super(config.scriptPath, path.join(config.dataPath, 'script-history'));
}
}
+21 -52
View File
@@ -1,4 +1,3 @@
import { t } from '../shared/i18n';
import { Service, Inject } from 'typedi';
import winston from 'winston';
import fs from 'fs/promises';
@@ -9,22 +8,10 @@ import { formatUrl } from '../config/subscription';
import config from '../config';
import { fileExist, rmPath } from '../config/util';
import { writeFileWithLock } from '../shared/utils';
import {
getSubscriptionSshAlias,
resolveSubscriptionPath,
} from '../shared/subscriptionPath';
@Service()
export default class SshKeyService {
// OpenSSH uses the effective user's passwd entry, not the HOME override.
private get homedir(): string {
try {
return os.userInfo().homedir;
} catch {
// Custom container UIDs may have no passwd entry. Keep startup working.
return os.homedir();
}
}
private homedir = os.homedir();
private sshPath = config.sshdPath;
private sshConfigFilePath = path.resolve(this.homedir, '.ssh', 'config');
private sshConfigHeader = `Include ${path.join(this.sshPath, '*.config')}`;
@@ -34,10 +21,6 @@ export default class SshKeyService {
}
private async initSshConfigFile() {
await fs.mkdir(path.dirname(this.sshConfigFilePath), {
recursive: true,
mode: 0o700,
});
let config = '';
const _exist = await fileExist(this.sshConfigFilePath);
if (_exist) {
@@ -59,22 +42,22 @@ export default class SshKeyService {
key: string,
): Promise<void> {
try {
const filePath = resolveSubscriptionPath(this.sshPath, alias);
const filePath = path.join(this.sshPath, alias);
try {
await rmPath(filePath);
} catch {}
} catch { }
await writeFileWithLock(filePath, `${key}${os.EOL}`, { mode: '400' });
} catch (error) {
this.logger.error(t('生成私钥文件失败'), error);
this.logger.error('生成私钥文件失败', error);
}
}
private async removePrivateKeyFile(alias: string): Promise<void> {
try {
const filePath = resolveSubscriptionPath(this.sshPath, alias);
const filePath = path.join(this.sshPath, alias);
await rmPath(filePath);
} catch (error) {
this.logger.error(t('删除私钥文件失败'), error);
this.logger.error('删除私钥文件失败', error);
}
}
@@ -83,26 +66,18 @@ export default class SshKeyService {
host: string,
proxy?: string,
) {
if (
!/^[a-zA-Z0-9.-]+$/.test(host) ||
(proxy && !/^[a-zA-Z0-9.:[\]-]+$/.test(proxy))
) {
throw Object.assign(new Error('Invalid SSH host or proxy'), {
status: 400,
});
}
if (host === 'github.com') {
host = `ssh.github.com\n Port 443\n HostkeyAlgorithms +ssh-rsa`;
}
const proxyStr = proxy
? ` ProxyCommand nc -v -x ${proxy} %h %p 2>/dev/null\n`
: '';
const config = `Host ${alias}\n Hostname ${host}\n IdentityFile ${resolveSubscriptionPath(
const config = `Host ${alias}\n Hostname ${host}\n IdentityFile ${path.join(
this.sshPath,
alias,
)}\n StrictHostKeyChecking no\n${proxyStr}`;
await writeFileWithLock(
`${resolveSubscriptionPath(this.sshPath, alias, '.config')}`,
`${path.join(this.sshPath, `${alias}.config`)}`,
config,
{
encoding: 'utf8',
@@ -113,10 +88,10 @@ export default class SshKeyService {
private async removeSshConfig(alias: string) {
try {
const filePath = resolveSubscriptionPath(this.sshPath, alias, '.config');
const filePath = path.join(this.sshPath, `${alias}.config`);
await rmPath(filePath);
} catch (error) {
this.logger.error(t('删除ssh配置文件%s失败'), alias, error);
this.logger.error(`删除ssh配置文件${alias}失败`, error);
}
}
@@ -126,7 +101,6 @@ export default class SshKeyService {
host: string,
proxy?: string,
): Promise<void> {
alias = getSubscriptionSshAlias(alias);
await this.generatePrivateKeyFile(alias, key);
await this.generateSingleSshConfig(alias, host, proxy);
}
@@ -136,7 +110,6 @@ export default class SshKeyService {
host: string,
proxy?: string,
): Promise<void> {
alias = getSubscriptionSshAlias(alias);
await this.removePrivateKeyFile(alias);
await this.removeSshConfig(alias);
}
@@ -144,19 +117,15 @@ export default class SshKeyService {
public async setSshConfig(docs: Subscription[]) {
for (const doc of docs) {
if (doc.type === 'private-repo' && doc.pull_type === 'ssh-key') {
try {
const alias = getSubscriptionSshAlias(doc.alias);
const { host } = formatUrl(doc);
await this.removePrivateKeyFile(alias);
await this.removeSshConfig(alias);
await this.generatePrivateKeyFile(
alias,
(doc.pull_option as any).private_key,
);
await this.generateSingleSshConfig(alias, host, doc.proxy);
} catch (error) {
this.logger.warn(t('跳过订阅 %s 的无效 SSH 配置: %s'), doc.id, error);
}
const { alias, proxy } = doc;
const { host } = formatUrl(doc);
await this.removePrivateKeyFile(alias);
await this.removeSshConfig(alias);
await this.generatePrivateKeyFile(
alias,
(doc.pull_option as any).private_key,
);
await this.generateSingleSshConfig(alias, host, proxy);
}
}
}
@@ -175,12 +144,12 @@ export default class SshKeyService {
private async generateGlobalSshConfig(alias: string) {
// Create a config that matches all hosts, making this key globally available
const config = `Host *\n IdentityFile ${resolveSubscriptionPath(
const config = `Host *\n IdentityFile ${path.join(
this.sshPath,
alias,
)}\n StrictHostKeyChecking no\n`;
await writeFileWithLock(
`${resolveSubscriptionPath(this.sshPath, alias, '.config')}`,
`${path.join(this.sshPath, `${alias}.config`)}`,
config,
{
encoding: 'utf8',
+119 -163
View File
@@ -1,5 +1,3 @@
import { textSearch } from '../data/queries';
import { Query, QuerySpec, orderBy } from '../data/queries';
import { Service, Inject } from 'typedi';
import winston from 'winston';
import config from '../config';
@@ -21,7 +19,8 @@ import {
rmPath,
} from '../config/util';
import fs from 'fs/promises';
import path from 'path';
import { FindOptions, Op } from 'sequelize';
import path, { join } from 'path';
import ScheduleService, { TaskCallbacks } from './schedule';
import { SimpleIntervalSchedule } from 'toad-scheduler';
import SockService from './sock';
@@ -31,28 +30,21 @@ import dayjs from 'dayjs';
import { LOG_END_SYMBOL } from '../config/const';
import { formatCommand, formatUrl } from '../config/subscription';
import { CrontabModel } from '../data/cron';
import { assertSubscriptionAlias } from '../shared/subscriptionPath';
import { resolveLogPath } from '../shared/logPath';
import {
getSubscriptionStorageCandidates,
isSubscriptionStorageName,
resolveSubscriptionStoragePath,
} from '../shared/subscriptionStorage';
import { withSubscriptionMutation } from '../shared/subscriptionMutationLock';
import CrontabService from './cron';
import taskLimit from '../shared/pLimit';
import { logStreamManager } from '../shared/logStreamManager';
import { LogReadOptions, readLogChunk } from '../shared/logReader';
@Service()
export default class SubscriptionService {
constructor(
@Inject('logger')
private logger: winston.Logger,
@Inject('logger') private logger: winston.Logger,
private scheduleService: ScheduleService,
private sockService: SockService,
private sshKeyService: SshKeyService,
private crontabService: CrontabService,
) {}
public async list(
searchText?: string,
ids?: string,
@@ -60,33 +52,46 @@ export default class SubscriptionService {
let query = {};
const subIds = JSON.parse(ids || '[]');
if (searchText) {
query = textSearch(['name', 'url'], searchText);
const reg = {
[Op.or]: [
{ [Op.like]: `%${searchText}%` },
{ [Op.like]: `%${encodeURI(searchText)}%` },
],
};
query = {
[Op.or]: [
{
name: reg,
},
{
url: reg,
},
],
};
}
try {
const result = await SubscriptionModel.find(
{ ...query, ...(ids ? { id: subIds } : undefined) },
{
orderBy: orderBy([
['is_disabled', 'ASC'],
['createdAt', 'DESC'],
]),
},
);
const result = await SubscriptionModel.findAll({
where: { ...query, ...(ids ? { id: subIds } : undefined) },
order: [
['is_disabled', 'ASC'],
['createdAt', 'DESC'],
],
});
return result;
} catch (error) {
throw error;
}
}
public async handleTask(
doc: Subscription,
needCreate = true,
runImmediately = false,
) {
// Cancelling by ID must also work for invalid historical subscriptions.
if (needCreate) {
const { url } = formatUrl(doc);
doc.command = formatCommand(doc, url as string);
}
const { url } = formatUrl(doc);
doc.command = formatCommand(doc, url as string);
if (doc.schedule_type === 'crontab') {
this.scheduleService.cancelCronTask(doc as any);
needCreate &&
@@ -107,34 +112,34 @@ export default class SubscriptionService {
));
}
}
public async setSshConfig() {
const docs = await SubscriptionModel.find({}, {});
const docs = await SubscriptionModel.findAll();
await this.sshKeyService.setSshConfig(docs);
}
private taskCallbacks(doc: Subscription): TaskCallbacks {
return {
onBefore: async (startTime) => {
const logTime = startTime.format('YYYY-MM-DD-HH-mm-ss');
const logPath = `${doc.alias}/${logTime}.log`;
await SubscriptionModel.nativeUpdate(
{ id: doc.id },
await SubscriptionModel.update(
{
status: SubscriptionStatus.running,
log_path: logPath,
},
{ where: { id: doc.id } },
);
const absolutePath = await handleLogPath(
logPath as string,
tf('## 开始执行... %s\n', startTime.format('YYYY-MM-DD HH:mm:ss')),
);
// 执行sub_before
let beforeStr = '';
try {
if (doc.sub_before) {
await logStreamManager.write(
absolutePath,
`\n## ${t('执行before命令...')}\n\n`,
);
await logStreamManager.write(absolutePath, `\n## ${t('执行before命令...')}\n\n`);
beforeStr = await promiseExec(doc.sub_before);
}
} catch (error: any) {
@@ -146,11 +151,11 @@ export default class SubscriptionService {
}
},
onStart: async (cp: ChildProcessWithoutNullStreams, startTime) => {
await SubscriptionModel.nativeUpdate(
{ id: doc.id },
await SubscriptionModel.update(
{
pid: cp.pid,
},
{ where: { id: doc.id } },
);
},
onEnd: async (cp, endTime, diff) => {
@@ -158,6 +163,7 @@ export default class SubscriptionService {
try {
const sub = await this.getDb({ id: doc.id });
absolutePath = await handleLogPath(sub.log_path as string);
// 执行 sub_after
let afterStr = '';
try {
@@ -170,12 +176,12 @@ export default class SubscriptionService {
}
} catch (error: any) {
afterStr =
(error.stderr && error.stderr.toString()) ||
JSON.stringify(error);
(error.stderr && error.stderr.toString()) || JSON.stringify(error);
}
if (afterStr) {
await logStreamManager.write(absolutePath, `${afterStr}\n`);
}
await logStreamManager.write(
absolutePath,
'\n' +
@@ -190,10 +196,10 @@ export default class SubscriptionService {
try {
if (absolutePath) await logStreamManager.closeStream(absolutePath);
} finally {
await SubscriptionModel.nativeUpdate({ id: doc.id }, {
status: SubscriptionStatus.idle,
pid: null,
} as any);
await SubscriptionModel.update(
{ status: SubscriptionStatus.idle, pid: null } as any,
{ where: { id: doc.id } },
);
this.sockService.sendMessage({
type: 'runSubscriptionEnd',
message: t('订阅执行完成'),
@@ -214,45 +220,33 @@ export default class SubscriptionService {
},
};
}
public async create(payload: Subscription): Promise<Subscription> {
assertSubscriptionAlias(payload.alias);
return withSubscriptionMutation(async () => {
const tab = new Subscription(payload);
const doc = await this.insert(tab);
await this.handleTask(doc);
await this.setSshConfig();
return doc;
});
const tab = new Subscription(payload);
const doc = await this.insert(tab);
await this.handleTask(doc.get({ plain: true }));
await this.setSshConfig();
return doc;
}
public async insert(payload: Subscription): Promise<SubscriptionInstance> {
return await SubscriptionModel.insert(payload);
return await SubscriptionModel.create(payload, { returning: true });
}
public async update(payload: Subscription): Promise<Subscription> {
assertSubscriptionAlias(payload.alias);
return withSubscriptionMutation(async () => {
const doc = await this.getDb({ id: payload.id });
const tab = new Subscription({ ...doc, ...payload });
const newDoc = await this.updateDb(tab);
await this.handleTask(newDoc, !newDoc.is_disabled);
await this.removeSshConfigForSubscription(doc);
await this.setSshConfig();
return newDoc;
});
}
private async removeSshConfigForSubscription(doc: Subscription) {
if (doc.type !== 'private-repo' || doc.pull_type !== 'ssh-key') return;
// Invalid historical rows must remain deletable without deriving unsafe paths.
try {
assertSubscriptionAlias(doc.alias);
} catch {
return;
}
await this.sshKeyService.removeSSHKey(doc.alias, '');
const doc = await this.getDb({ id: payload.id });
const tab = new Subscription({ ...doc, ...payload });
const newDoc = await this.updateDb(tab);
await this.handleTask(newDoc, !newDoc.is_disabled);
await this.setSshConfig();
return newDoc;
}
public async updateDb(payload: Subscription): Promise<Subscription> {
await SubscriptionModel.nativeUpdate({ id: payload.id }, payload);
await SubscriptionModel.update(payload, { where: { id: payload.id } });
return await this.getDb({ id: payload.id });
}
public async status({
ids,
status,
@@ -268,7 +262,6 @@ export default class SubscriptionService {
last_running_time: number;
last_execution_time: number;
}) {
if (log_path) resolveLogPath(config.logPath, log_path);
const options: any = {
status,
pid,
@@ -278,112 +271,66 @@ export default class SubscriptionService {
if (last_running_time > 0) {
options.last_running_time = last_running_time;
}
return await SubscriptionModel.nativeUpdate({ id: ids }, { ...options });
return await SubscriptionModel.update(
{ ...options },
{ where: { id: ids } },
);
}
public async remove(
ids: number[],
query: {
force?: boolean;
},
) {
return withSubscriptionMutation(async () => {
const docs = await SubscriptionModel.find({ id: ids }, {});
const storagePaths: string[] = [];
const retainedPaths: string[] = [];
const collectPaths = (
doc: Subscription,
targets: string[],
deleting = false,
) => {
for (const names of getSubscriptionStorageCandidates(doc)) {
// A URL accepted by one engine can produce an invalid name in another.
// Validate each candidate independently, retaining the safe candidate.
if (!Object.values(names).every(isSubscriptionStorageName)) continue;
const resolve = deleting
? resolveSubscriptionStoragePath
: path.resolve;
const scriptPath = resolve(config.scriptPath, names.script);
if (scriptPath) targets.push(scriptPath);
if (names.repo) {
const repoPath = resolve(config.repoPath, names.repo);
if (repoPath) targets.push(repoPath);
}
if (names.raw) {
const rawPath = resolve(
path.join(config.dataPath, 'raw'),
names.raw,
);
if (rawPath) targets.push(rawPath);
}
}
};
// Validate all deletion targets before changing rows or removing any files.
if (query?.force === true) {
for (const doc of docs) {
assertSubscriptionAlias(doc.alias);
resolveSubscriptionStoragePath(config.scriptPath, doc.alias);
resolveSubscriptionStoragePath(config.repoPath, doc.alias);
collectPaths(doc, storagePaths, true);
}
const remaining = await SubscriptionModel.find({}, {});
for (const doc of remaining) {
if (!ids.includes(doc.id!)) collectPaths(doc, retainedPaths);
}
public async remove(ids: number[], query: { force?: boolean }) {
const docs = await SubscriptionModel.findAll({ where: { id: ids } });
for (const doc of docs) {
await this.handleTask(doc.get({ plain: true }), false);
}
await SubscriptionModel.destroy({ where: { id: ids } });
await this.setSshConfig();
if (query?.force === true) {
const crons = await CrontabModel.findAll({ where: { sub_id: ids } });
if (crons?.length) {
await this.crontabService.remove(crons.map((x) => x.id!));
}
for (const doc of docs) {
await this.handleTask(doc, false);
const filePath = join(config.scriptPath, doc.alias);
const repoPath = join(config.repoPath, doc.alias);
await rmPath(filePath);
await rmPath(repoPath);
}
await SubscriptionModel.nativeDelete({ id: ids });
for (const doc of docs) {
await this.removeSshConfigForSubscription(doc);
}
await this.setSshConfig();
if (query?.force === true) {
const crons = await CrontabModel.find({ sub_id: ids }, {});
if (crons?.length) {
await this.crontabService.remove(crons.map((x) => x.id!));
}
for (const storagePath of new Set(storagePaths)) {
// rm is recursive: an ancestor must also be retained if a surviving
// subscription uses a nested branch, and vice versa.
const shared = retainedPaths.some(
(retained) =>
retained === storagePath ||
retained.startsWith(`${storagePath}${path.sep}`) ||
storagePath.startsWith(`${retained}${path.sep}`),
);
if (!shared) await rmPath(storagePath);
}
}
});
}
}
public async getDb(query: QuerySpec['where']): Promise<Subscription> {
const doc = await SubscriptionModel.findOne({ ...query }, {});
public async getDb(
query: FindOptions<Subscription>['where'],
): Promise<Subscription> {
const doc = await SubscriptionModel.findOne({ where: { ...query } });
if (!doc) {
throw new Error(`Subscription ${JSON.stringify(query)} not found`);
}
return doc;
return doc.get({ plain: true });
}
public async run(ids: number[]) {
await SubscriptionModel.nativeUpdate(
{ id: ids },
await SubscriptionModel.update(
{ status: SubscriptionStatus.queued },
{ where: { id: ids } },
);
ids.forEach((id) => {
this.runSingle(id);
});
}
public async stop(ids: number[]) {
const docs = await SubscriptionModel.find({ id: ids }, {});
const docs = await SubscriptionModel.findAll({ where: { id: ids } });
let failure: unknown;
for (const doc of docs) {
try {
if (doc.pid) {
await killTask(doc.pid, true);
}
await SubscriptionModel.nativeUpdate(
{ id: doc.id, pid: doc.pid ?? null },
await SubscriptionModel.update(
{ status: SubscriptionStatus.idle, pid: null } as any,
{ where: { id: doc.id, pid: doc.pid ?? null } },
);
} catch (error) {
this.logger.error(error);
@@ -392,12 +339,15 @@ export default class SubscriptionService {
}
if (failure) throw failure;
}
private async runSingle(subscriptionId: number) {
const subscription = await this.getDb({ id: subscriptionId });
if (subscription.status !== SubscriptionStatus.queued) {
return;
}
const command = formatCommand(subscription);
this.scheduleService.runTask(command, this.taskCallbacks(subscription), {
name: subscription.name,
schedule: subscription.schedule,
@@ -406,22 +356,25 @@ export default class SubscriptionService {
runOrigin: 'subscription',
});
}
public async disabled(ids: number[]) {
await SubscriptionModel.nativeUpdate({ id: ids }, { is_disabled: 1 });
const docs = await SubscriptionModel.find({ id: ids }, {});
await SubscriptionModel.update({ is_disabled: 1 }, { where: { id: ids } });
const docs = await SubscriptionModel.findAll({ where: { id: ids } });
await this.setSshConfig();
for (const doc of docs) {
await this.handleTask(doc, false);
await this.handleTask(doc.get({ plain: true }), false);
}
}
public async enabled(ids: number[]) {
await SubscriptionModel.nativeUpdate({ id: ids }, { is_disabled: 0 });
const docs = await SubscriptionModel.find({ id: ids }, {});
await SubscriptionModel.update({ is_disabled: 0 }, { where: { id: ids } });
const docs = await SubscriptionModel.findAll({ where: { id: ids } });
await this.setSshConfig();
for (const doc of docs) {
await this.handleTask(doc);
await this.handleTask(doc.get({ plain: true }));
}
}
public async log(id: number, options: LogReadOptions = {}) {
const doc = await this.getDb({ id });
if (!doc || !doc.log_path) {
@@ -433,17 +386,20 @@ export default class SubscriptionService {
truncated: false,
};
}
const absolutePath = await handleLogPath(doc.log_path as string);
return await readLogChunk(absolutePath, options);
}
public async logs(id: number) {
const doc = await this.getDb({ id });
if (!doc) {
return [];
}
if (doc.log_path) {
const relativeDir = path.dirname(`${doc.log_path}`);
const dir = path.dirname(resolveLogPath(config.logPath, doc.log_path));
const dir = path.resolve(config.logPath, relativeDir);
const _exist = await fileExist(dir);
if (_exist) {
let files = await fs.readdir(dir);
+136 -191
View File
@@ -1,12 +1,7 @@
import { Query, QuerySpec, orderBy } from '../data/queries';
import { persistNotificationAudit } from '../shared/notificationAudit';
import { spawn } from 'cross-spawn';
import { execFile } from 'child_process';
import { promisify } from 'util';
import { shellQuote } from '../shared/shellQuote';
import { Response } from 'express';
import fs from 'fs';
import { request } from 'undici';
import { Agent, request } from 'undici';
import path from 'path';
import { Inject, Service } from 'typedi';
import winston from 'winston';
@@ -34,12 +29,8 @@ import {
SystemInstance,
SystemModel,
SystemModelInfo,
NotifyStatus,
NotifyLogInfo,
} from '../data/system';
import taskLimit from '../shared/pLimit';
import cronClient from '../schedule/client';
import { withSchedulerMutation } from '../shared/schedulerMutationLock';
import NotificationService from './notify';
import ScheduleService, { TaskCallbacks } from './schedule';
import SockService from './sock';
@@ -47,16 +38,18 @@ import os from 'os';
import dayjs from 'dayjs';
import { t, setLang } from '../shared/i18n';
import { updateLinuxMirrorFile } from '../config/util';
@Service()
export default class SystemService {
@Inject((type) => NotificationService)
private notificationService!: NotificationService;
constructor(
@Inject('logger')
private logger: winston.Logger,
@Inject('logger') private logger: winston.Logger,
private scheduleService: ScheduleService,
private sockService: SockService,
) {}
) { }
public async getSystemConfig() {
const doc = await this.getDb({ type: AuthDataType.systemConfig });
return {
@@ -64,19 +57,22 @@ export default class SystemService {
info: { ...doc.info, timezone: doc.info?.timezone || 'Asia/Shanghai' },
};
}
private async updateAuthDb(payload: SystemInfo): Promise<SystemInfo> {
const { id, ...others } = payload;
await SystemModel.nativeUpdate({ id }, others);
await SystemModel.update(others, { where: { id } });
const doc = await this.getDb({ id });
return doc;
}
public async getDb(query: any): Promise<SystemInfo> {
const doc = await SystemModel.findOne(query, {});
const doc = await SystemModel.findOne({ where: query });
if (!doc) {
throw new Error(`System ${JSON.stringify(query)} not found`);
}
return doc;
return doc.get({ plain: true });
}
public async updateNotificationMode(notificationInfo: NotificationInfo) {
const code = Math.random().toString().slice(-6);
const isSuccess = await this.notificationService.testNotify(
@@ -94,6 +90,7 @@ export default class SystemService {
return { code: 400, message: t('通知发送失败,请检查参数') };
}
}
public async updateLogRemoveFrequency(info: SystemModelInfo) {
const oDoc = await this.getSystemConfig();
const result = await this.updateAuthDb({
@@ -120,30 +117,19 @@ export default class SystemService {
}
return { code: 200, data: info };
}
public async updateCronConcurrency(info: SystemModelInfo) {
const limit = info.cronConcurrency ?? 0;
if (!Number.isInteger(limit) || limit < 0 || limit > 2147483647) {
throw Object.assign(new Error(t('参数错误')), { status: 400 });
const oDoc = await this.getSystemConfig();
await this.updateAuthDb({
...oDoc,
info: { ...oDoc.info, ...info },
});
if (info.cronConcurrency) {
await taskLimit.setCustomLimit(info.cronConcurrency);
}
return withSchedulerMutation(async () => {
const oDoc = await this.getSystemConfig();
await this.updateAuthDb({
...oDoc,
info: { ...oDoc.info, ...info, cronConcurrency: limit },
});
await taskLimit.setCustomLimit(limit || Math.max(os.cpus().length, 4));
await cronClient.setConcurrency(limit);
return { code: 200, data: info };
});
}
public async restoreCronConcurrency() {
await withSchedulerMutation(async () => {
const doc = await this.getSystemConfig();
const limit = doc.info?.cronConcurrency ?? 0;
await taskLimit.setCustomLimit(limit || Math.max(os.cpus().length, 4));
await cronClient.setConcurrency(limit);
});
return { code: 200, data: info };
}
public async updateDependenceProxy(info: SystemModelInfo) {
const oDoc = await this.getSystemConfig();
await this.updateAuthDb({
@@ -153,15 +139,14 @@ export default class SystemService {
if (info.dependenceProxy) {
await fs.promises.writeFile(
config.dependenceProxyFile,
`export http_proxy=${shellQuote(
info.dependenceProxy,
)}\nexport https_proxy=${shellQuote(info.dependenceProxy)}`,
`export http_proxy="${info.dependenceProxy}"\nexport https_proxy="${info.dependenceProxy}"`,
);
} else {
await fs.promises.rm(config.dependenceProxyFile);
}
return { code: 200, data: info };
}
public async updateNodeMirror(info: SystemModelInfo, res?: Response) {
const oDoc = await this.getSystemConfig();
await this.updateAuthDb({
@@ -170,16 +155,15 @@ export default class SystemService {
});
let cmd = 'pnpm config delete registry';
if (info.nodeMirror) {
cmd = `pnpm config set registry ${shellQuote(info.nodeMirror)}`;
cmd = `pnpm config set registry ${info.nodeMirror}`;
}
let command = `cd && ${cmd}`;
const docs = await DependenceModel.find(
{
const docs = await DependenceModel.findAll({
where: {
type: DependenceTypes.nodejs,
status: DependenceStatus.installed,
},
{},
);
});
if (docs.length > 0) {
command += ` && pnpm i -g`;
}
@@ -211,6 +195,7 @@ export default class SystemService {
},
);
}
public async updatePythonMirror(info: SystemModelInfo) {
const oDoc = await this.getSystemConfig();
await this.updateAuthDb({
@@ -219,11 +204,12 @@ export default class SystemService {
});
let cmd = 'pip config unset global.index-url';
if (info.pythonMirror) {
cmd = `pip3 config set global.index-url ${shellQuote(info.pythonMirror)}`;
cmd = `pip3 config set global.index-url ${info.pythonMirror}`;
}
await promiseExec(cmd);
return { code: 200, data: info };
}
public async updateLinuxMirror(
info: SystemModelInfo,
res?: Response,
@@ -234,6 +220,7 @@ export default class SystemService {
return;
}
const command = await updateLinuxMirrorFile(info.linuxMirror || '');
let hasError = false;
this.scheduleService.runTask(
command,
{
@@ -241,15 +228,14 @@ export default class SystemService {
res?.setHeader('QL-Task-Pid', `${cp.pid}`);
res?.end();
},
onEnd: async (cp) => {
onEnd: async () => {
this.sockService.sendMessage({
type: 'updateLinuxMirror',
message: 'update linux mirror end',
status: 'completed',
});
onEnd?.();
// Tools such as sudo and apt may write warnings to stderr on success.
if (cp?.exitCode === 0 && cp.signalCode === null) {
if (!hasError) {
await this.updateAuthDb({
...oDoc,
info: { ...oDoc.info, ...info },
@@ -257,6 +243,7 @@ export default class SystemService {
}
},
onError: async (message: string) => {
hasError = true;
this.sockService.sendMessage({ type: 'updateLinuxMirror', message });
},
onLog: async (message: string) => {
@@ -270,25 +257,30 @@ export default class SystemService {
},
);
}
public async checkUpdate() {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 15000);
try {
const currentVersionContent = await parseVersion(config.versionFile);
const { body, statusCode } = await request(
`${config.lastVersionFile}?t=${Date.now()}`,
{ signal: controller.signal },
);
const text = await body.text();
if (statusCode !== 200) throw new Error('Version request failed');
const lastVersionContent = parseContentVersion(text);
if (
!lastVersionContent ||
typeof lastVersionContent.version !== 'string' ||
!/^\d+\.\d+\.\d+$/.test(lastVersionContent.version)
) {
throw new Error('Invalid version response');
let lastVersionContent;
try {
const { body } = await request(
`${config.lastVersionFile}?t=${Date.now()}`,
{
dispatcher: new Agent({
keepAliveTimeout: 30000,
keepAliveMaxTimeout: 30000,
}),
},
);
const text = await body.text();
lastVersionContent = parseContentVersion(text);
} catch (error) { }
if (!lastVersionContent) {
lastVersionContent = currentVersionContent;
}
return {
code: 200,
data: {
@@ -304,12 +296,11 @@ export default class SystemService {
} catch (error: any) {
return {
code: 400,
message: t('检查更新失败,请检查网络后重试'),
message: error.message,
};
} finally {
clearTimeout(timeout);
}
}
private checkHasNewVersion(curVersion: string, lastVersion: string) {
const curArr = curVersion.split('.').map((x) => parseInt(x, 10));
const lastArr = lastVersion.split('.').map((x) => parseInt(x, 10));
@@ -328,49 +319,45 @@ export default class SystemService {
}
return false;
}
public async updateSystem() {
return this.startUpgradeJob('download');
}
private upgradeJobs() {
return require(path.join(
config.rootPath,
'cli/src/internal/maintenance/panelUpgrade.cjs',
));
}
public async getUpgradeStatus() {
return {
code: 200,
data: await this.upgradeJobs().getStatus(config.tmpPath),
};
}
private async startUpgradeJob(action: 'download' | 'apply') {
try {
const data = await this.upgradeJobs().startJob(
{
root: config.rootPath,
tmp: config.tmpPath,
env: {
...process.env,
dir_data: config.dataPath,
dir_config: config.configPath,
QlPort: String(config.port),
QlBaseUrl: config.baseUrl,
},
},
action,
);
return { code: 200, data };
} catch (error: any) {
this.logger.error('[upgrade] %s', error.message);
return { code: 400, message: t('升级任务启动失败,请检查日志后重试') };
}
const cp = spawn('real_time=true ql update false', { shell: '/bin/bash' });
cp.stdout.on('data', (data) => {
this.sockService.sendMessage({
type: 'updateSystemVersion',
message: data.toString(),
});
});
cp.stderr.on('data', (data) => {
this.sockService.sendMessage({
type: 'updateSystemVersion',
message: data.toString(),
});
});
cp.on('error', (err) => {
this.sockService.sendMessage({
type: 'updateSystemVersion',
message: JSON.stringify(err),
status: 'failed',
});
});
cp.on('exit', (code) => {
this.sockService.sendMessage({
type: 'updateSystemVersion',
message: '',
status: code === 0 ? 'success' : 'failed',
});
});
return { code: 200 };
}
public async reloadSystem(target?: 'system' | 'data') {
if (target && !['system', 'data'].includes(target)) {
return { code: 400, message: t('参数错误') };
}
if (target === 'system') return this.startUpgradeJob('apply');
const cmd = `real_time=true ql reload ${shellQuote(target || '')}`;
const cmd = `real_time=true ql reload ${target || ''}`;
const cp = spawn(cmd, {
shell: '/bin/bash',
detached: true,
@@ -382,6 +369,7 @@ export default class SystemService {
});
return { code: 200 };
}
public async notify({
title,
content,
@@ -409,51 +397,29 @@ export default class SystemService {
return { code: 400, message: t('通知发送失败,请检查系统设置/通知配置') };
}
}
public async recordNotifyLog(info: NotifyLogInfo): Promise<void> {
await persistNotificationAudit(info);
}
public async getNotifyLog(): Promise<Array<NotifyLogInfo>> {
const docs = await SystemModel.find(
{ type: AuthDataType.notifyLog },
{
orderBy: orderBy([['id', 'DESC']]),
limit: 200,
},
);
return docs.map((x) => ({ ...x.info, id: x.id }));
}
public async run(
{
command,
logPath,
}: {
command: string;
logPath?: string;
},
callback: TaskCallbacks,
) {
public async run({ command, logPath }: { command: string; logPath?: string }, callback: TaskCallbacks) {
if (!command.startsWith(TASK_COMMAND)) {
command = `${TASK_COMMAND} ${command}`;
}
const logPathPrefix = logPath ? `real_log_path=${logPath}` : '';
this.scheduleService.runTask(
`${logPathPrefix} real_time=true ${command}`,
callback,
{
command,
id: command.replace(/ /g, '-'),
runOrigin: 'system',
},
);
const logPathPrefix = logPath ? `real_log_path=${logPath}` : ''
this.scheduleService.runTask(`${logPathPrefix} real_time=true ${command}`, callback, {
command,
id: command.replace(/ /g, '-'),
runOrigin: 'system',
});
}
public async stop({ command, pid }: { command: string; pid: number }) {
if (!pid && !command) {
return { code: 400, message: t('参数错误') };
}
if (pid) {
await killTask(pid);
return { code: 200 };
}
if (!command.startsWith(TASK_COMMAND)) {
command = `${TASK_COMMAND} ${command}`;
}
@@ -465,47 +431,24 @@ export default class SystemService {
return { code: 400, message: t('任务未找到') };
}
}
public async exportData(res: Response, type?: string[]) {
try {
let dataDirs = ['db', 'upload'];
if (type && type.length) {
dataDirs = dataDirs.concat(type.filter((x) => x !== 'base'));
}
const allowed = new Set([
'db',
'upload',
'config',
'scripts',
'log',
'deps',
'syslog',
'dep_cache',
'raw',
'repo',
'ssh.d',
'script-history',
]);
if (dataDirs.some((dir) => !allowed.has(dir))) {
return res.status(400).send({ code: 400, message: t('参数错误') });
}
if (dataDirs.includes('scripts')) dataDirs.push('script-history');
// History is created lazily; old/fresh installations must still export.
if (!fs.existsSync(path.join(config.dataPath, 'script-history'))) {
dataDirs = dataDirs.filter((dir) => dir !== 'script-history');
}
const dataPaths = [...new Set(dataDirs)].map((dir) => `data/${dir}`);
await promisify(execFile)(
'tar',
['-zcf', config.dataTgzFile, '--', ...dataPaths],
{
cwd: path.dirname(config.dataPath),
},
const dataPaths = dataDirs.map((dir) => `data/${dir}`);
await promiseExec(
`cd ${config.dataPath} && cd ../ && tar -zcvf ${config.dataTgzFile
} ${dataPaths.join(' ')}`,
);
res.download(config.dataTgzFile);
} catch (error: any) {
return res.send({ code: 400, message: error.message });
}
}
public async importData() {
try {
await promiseExec(`rm -rf ${path.join(config.tmpPath, 'data')}`);
@@ -517,6 +460,7 @@ export default class SystemService {
return { code: 400, message: error.message };
}
}
public async getSystemLog(
res: Response,
query: {
@@ -536,6 +480,7 @@ export default class SystemService {
.reverse()
.filter((x) => x.title.endsWith('.log'))
.filter((x) => x.createTime >= startTime && x.createTime <= endTime);
const limit = Math.min(
Math.max(Number(query.limit) || 1024 * 1024, 1),
1024 * 1024,
@@ -543,10 +488,7 @@ export default class SystemService {
const total = logs.reduce((size, log) => size + (log.size || 0), 0);
let remaining = limit;
const selected: Array<
(typeof logs)[number] & {
start: number;
length: number;
}
(typeof logs)[number] & { start: number; length: number }
> = [];
for (let index = logs.length - 1; index >= 0 && remaining > 0; index--) {
const log = logs[index];
@@ -557,6 +499,7 @@ export default class SystemService {
remaining -= length;
}
}
const contentLength = selected.reduce((size, log) => size + log.length, 0);
res.set({
'Content-Length': contentLength,
@@ -568,6 +511,7 @@ export default class SystemService {
res.end();
return;
}
const currentLog = fileNames.shift();
if (currentLog) {
const currentFileStream = fs.createReadStream(
@@ -584,23 +528,15 @@ export default class SystemService {
}
})(res, selected);
}
public async deleteSystemLog() {
const result = await readDirs(config.systemLogPath, config.systemLogPath);
const logs = result.reverse().filter((x) => x.title.endsWith('.log'));
for (const log of logs) {
// All workers keep append handles open; unlinking strands future writes
// in a deleted inode until rotation. Preserve the file while clearing it.
const file = await fs.promises.open(
path.join(config.systemLogPath, log.title),
fs.constants.O_WRONLY | fs.constants.O_NOFOLLOW,
);
try {
await file.truncate(0);
} finally {
await file.close();
}
await rmPath(path.join(config.systemLogPath, log.title));
}
}
public async updateTimezone(info: SystemModelInfo) {
if (!info.timezone) {
info.timezone = 'Asia/Shanghai';
@@ -617,6 +553,7 @@ export default class SystemService {
return { code: 400, message: t('设置时区失败') };
}
}
public async updateLanguage(info: SystemModelInfo) {
const oDoc = await this.getSystemConfig();
const lang = info.lang || 'zh';
@@ -628,7 +565,7 @@ export default class SystemService {
try {
await fs.promises.writeFile(
config.langEnvFile,
`export QL_LANG=${shellQuote(lang)}\n`,
`export QL_LANG='${lang}'\n`,
);
} catch (error) {
this.logger.error(`Failed to write lang_env.sh: ${error}`);
@@ -636,6 +573,7 @@ export default class SystemService {
setLang(lang);
return { code: 200, data: { lang } };
}
public async updatePanelTitle(info: SystemModelInfo) {
const oDoc = await this.getSystemConfig();
const panelTitle = info.panelTitle?.trim() || '';
@@ -645,36 +583,43 @@ export default class SystemService {
});
return { code: 200, data: { panelTitle } };
}
public async updateGlobalSshKey(info: SystemModelInfo) {
const oDoc = await this.getSystemConfig();
const result = await this.updateAuthDb({
...oDoc,
info: { ...oDoc.info, ...info },
});
// Apply the global SSH key
const SshKeyService = require('./sshKey').default;
const Container = require('typedi').Container;
const sshKeyService = Container.get(SshKeyService);
if (info.globalSshKey) {
await sshKeyService.addGlobalSSHKey(info.globalSshKey, 'global');
} else {
await sshKeyService.removeGlobalSSHKey('global');
}
return { code: 200, data: result };
}
public async cleanDependence(type: 'node' | 'python3') {
if (!type || !['node', 'python3'].includes(type)) {
return { code: 400, message: t('参数错误') };
}
const finalPath = path.join(config.dependenceCachePath, type);
await fs.promises.rm(finalPath, { recursive: true, force: true });
await DependenceModel.nativeUpdate(
{
type:
type === 'node' ? DependenceTypes.nodejs : DependenceTypes.python3,
status: DependenceStatus.installed,
},
await DependenceModel.update(
{ status: DependenceStatus.installFailed },
{
where: {
type:
type === 'node' ? DependenceTypes.nodejs : DependenceTypes.python3,
status: DependenceStatus.installed,
},
},
);
return { code: 200 };
}
+85 -37
View File
@@ -33,28 +33,26 @@ import {
verifyPassword,
} from '../shared/password';
import { serializeAuthMutation } from '../shared/authMutation';
import { LoginThrottle } from '../shared/loginThrottle';
@Service()
export default class UserService {
private loginThrottle = new LoginThrottle();
@Inject((type) => NotificationService)
private notificationService!: NotificationService;
constructor(
@Inject('logger')
private logger: winston.Logger,
@Inject('logger') private logger: winston.Logger,
private scheduleService: ScheduleService,
private sockService: SockService,
) {}
@serializeAuthMutation
public async login(
payloads: {
username: string;
password: string;
},
payloads: { username: string; password: string },
req: Request,
): Promise<any> {
return this.authenticate(payloads, req);
}
private async authenticate(
payloads: {
username: string;
@@ -89,6 +87,7 @@ export default class UserService {
platform,
blockedIps = [],
} = content;
if (
ip &&
blockedIps.some((blockedIp) => normalizeClientIp(blockedIp) === ip)
@@ -106,16 +105,22 @@ export default class UserService {
this.getLoginLog();
return { code: 403, message: t('该 IP 已被列入黑名单') };
}
const waitTime = this.loginThrottle.retryAfter(ip, timestamp);
if (waitTime > 0) {
const retriesTime = Math.pow(3, retries) * 1000;
if (retries > 2 && timestamp - lastlogon < retriesTime) {
const waitTime = Math.ceil(
(retriesTime - (timestamp - lastlogon)) / 1000,
);
return {
code: 410,
message: tf('失败次数过多,请%s秒后重试', waitTime),
data: waitTime,
};
}
const passwordMatches =
username === cUsername && (await verifyPassword(password, cPassword));
if (passwordMatches && twoFactorActivated && needTwoFactor) {
await this.updateAuthInfo(content, {
isTwoFactorChecking: true,
@@ -126,14 +131,15 @@ export default class UserService {
message: '',
};
}
if (passwordMatches) {
this.loginThrottle.reset(ip);
const data = createRandomString(50, 100);
const expiration = twoFactorActivated ? '60d' : '20d';
let token = jwt.sign({ data }, config.jwt.secret, {
expiresIn: config.jwt.expiresIn || expiration,
algorithm: 'HS384',
});
const tokenInfo: TokenInfo = {
value: token,
timestamp,
@@ -141,11 +147,13 @@ export default class UserService {
address,
platform: req.platform,
};
const updatedTokens = this.addTokenToList(
tokens,
req.platform,
tokenInfo,
);
await this.updateAuthInfo(content, {
password: isPasswordHash(cPassword)
? cPassword
@@ -165,7 +173,7 @@ export default class UserService {
t('你于') +
dayjs(timestamp).format('YYYY-MM-DD HH:mm:ss') +
t('在') +
t(address) +
address +
' ' +
req.platform +
t('端') +
@@ -198,9 +206,8 @@ export default class UserService {
},
};
} else {
const failures = this.loginThrottle.fail(ip, timestamp);
await this.updateAuthInfo(content, {
retries: failures,
retries: retries + 1,
lastlogon: timestamp,
lastip: ip,
lastaddr: address,
@@ -211,7 +218,7 @@ export default class UserService {
t('你于') +
dayjs(timestamp).format('YYYY-MM-DD HH:mm:ss') +
t('在') +
t(address) +
address +
' ' +
req.platform +
t('端') +
@@ -232,8 +239,8 @@ export default class UserService {
},
});
this.getLoginLog();
if (failures > 3) {
const waitTime = this.loginThrottle.retryAfter(ip, timestamp);
if (retries > 2) {
const waitTime = Math.round(Math.pow(3, retries + 1));
return {
code: 410,
message: tf('失败次数过多,请%s秒后重试', waitTime),
@@ -244,13 +251,16 @@ export default class UserService {
}
}
}
@serializeAuthMutation
public async logout(platform: string, tokenValue: string): Promise<any> {
if (!platform || !tokenValue) {
this.logger.warn('Invalid logout parameters - empty platform or token');
return;
}
const authInfo = await this.getAuthInfo();
// Verify the token exists before attempting to remove it
const tokenExists = this.findTokenInList(
authInfo.tokens,
@@ -264,36 +274,45 @@ export default class UserService {
);
return;
}
const updatedTokens = this.removeTokenFromList(
authInfo.tokens,
platform,
tokenValue,
);
await this.updateAuthInfo(authInfo, {
token: authInfo.token === tokenValue ? '' : authInfo.token,
tokens: updatedTokens,
});
}
public async getLoginLog(): Promise<Array<SystemModelInfo | undefined>> {
const docs = await SystemModel.find({ type: AuthDataType.loginLog }, {});
const docs = await SystemModel.findAll({
where: { type: AuthDataType.loginLog },
});
if (docs && docs.length > 0) {
const result = docs.sort(
(a, b) => b.info!.timestamp! - a.info!.timestamp!,
);
if (result.length > 100) {
const ids = result.slice(100).map((x) => x.id!);
await SystemModel.nativeDelete({ id: ids });
await SystemModel.destroy({
where: { id: ids },
});
}
return result.map((x) => x.info);
}
return [];
}
public async getIpBlacklist(): Promise<string[]> {
const authInfo = await this.getAuthInfo();
return uniq((authInfo.blockedIps || []).map(normalizeClientIp)).filter(
Boolean,
);
}
@serializeAuthMutation
public async blockIp(ip: string): Promise<string[]> {
const authInfo = await this.getAuthInfo();
@@ -304,6 +323,7 @@ export default class UserService {
await this.updateAuthInfo(authInfo, { blockedIps });
return blockedIps;
}
@serializeAuthMutation
public async unblockIp(ip: string): Promise<string[]> {
const authInfo = await this.getAuthInfo();
@@ -314,10 +334,12 @@ export default class UserService {
await this.updateAuthInfo(authInfo, { blockedIps });
return blockedIps;
}
private async insertDb(payload: SystemInfo): Promise<SystemInfo> {
const doc = await SystemModel.insert({ ...payload });
const doc = await SystemModel.create({ ...payload }, { returning: true });
return doc;
}
@serializeAuthMutation
public async initializeUser({
username,
@@ -343,6 +365,7 @@ export default class UserService {
});
return { code: 200, message: t('更新成功') };
}
@serializeAuthMutation
public async updateUsernameAndPassword({
username,
@@ -365,12 +388,14 @@ export default class UserService {
});
return { code: 200, message: t('更新成功') };
}
@serializeAuthMutation
public async updateAvatar(avatar: string) {
const authInfo = await this.getAuthInfo();
await this.updateAuthInfo(authInfo, { avatar });
return { code: 200, data: avatar, message: t('更新成功') };
}
@serializeAuthMutation
public async initTwoFactor() {
const secret = authenticator.generateSecret();
@@ -382,6 +407,7 @@ export default class UserService {
await this.updateAuthInfo(authInfo, { twoFactorSecret: secret });
return { secret, url: otpauth };
}
@serializeAuthMutation
public async activeTwoFactor(code: string) {
const authInfo = await this.getAuthInfo();
@@ -400,24 +426,21 @@ export default class UserService {
}
return isValid;
}
@serializeAuthMutation
public async twoFactorLogin(
{
username,
password,
code,
}: {
username: string;
password: string;
code: string;
},
}: { username: string; password: string; code: string },
req: any,
) {
const authInfo = await this.getAuthInfo();
const { isTwoFactorChecking, twoFactorSecret } = authInfo;
const now = Date.now();
const source = getClientIp(req);
if (this.loginThrottle.retryAfter(source, now) > 0) {
const retries = authInfo.retries || 0;
if (retries > 2 && now - authInfo.lastlogon < Math.pow(3, retries) * 1000) {
return { code: 410, message: t('失败次数过多,请稍后重试') };
}
if (
@@ -449,8 +472,9 @@ export default class UserService {
.join(' ');
}
await this.updateAuthInfo(authInfo, {
retries: this.loginThrottle.fail(source, now),
retries: retries + 1,
lastlogon: now,
isTwoFactorChecking: retries + 1 < 5,
lastip: ip,
lastaddr: address,
platform: req.platform,
@@ -458,6 +482,7 @@ export default class UserService {
return { code: 430, message: t('验证失败') };
}
}
@serializeAuthMutation
public async deactivateTwoFactor() {
const authInfo = await this.getAuthInfo();
@@ -471,6 +496,7 @@ export default class UserService {
});
return true;
}
public async getAuthInfo() {
const authInfo = await shareStore.getAuthInfo();
if (authInfo) {
@@ -479,6 +505,7 @@ export default class UserService {
const doc = await this.getDb({ type: AuthDataType.authConfig });
return (doc.info || {}) as AuthInfo;
}
private async updateAuthInfo(authInfo: AuthInfo, info: Partial<AuthInfo>) {
const result = { ...authInfo, ...info };
await shareStore.updateAuthInfo(result);
@@ -490,27 +517,34 @@ export default class UserService {
this.sockService.getClients().forEach((conn) => conn.close('401'));
}
}
public async getNotificationMode(): Promise<NotificationInfo> {
const doc = await this.getDb({ type: AuthDataType.notification });
return (doc.info || {}) as NotificationInfo;
}
private async updateAuthDb(payload: SystemInfo): Promise<any> {
let doc = await SystemModel.findOne({ type: payload.type }, {});
let doc = await SystemModel.findOne({ where: { type: payload.type } });
if (doc) {
await SystemModel.nativeUpdate({ id: doc.id }, payload);
doc = await SystemModel.findOne({ id: doc.id });
const updateResult = await SystemModel.update(payload, {
where: { id: doc.id },
returning: true,
});
doc = updateResult[1][0];
} else {
doc = await SystemModel.insert(payload);
doc = await SystemModel.create(payload, { returning: true });
}
return doc;
}
public async getDb(query: any): Promise<SystemInfo> {
const doc = await SystemModel.findOne({ ...query }, {});
const doc = await SystemModel.findOne({ where: { ...query } });
if (!doc) {
throw new Error(`${JSON.stringify(query)} not found`);
}
return doc;
return doc.get({ plain: true });
}
public async updateNotificationMode(notificationInfo: NotificationInfo) {
const code = Math.random().toString().slice(-6);
const isSuccess = await this.notificationService.testNotify(
@@ -528,10 +562,12 @@ export default class UserService {
return { code: 400, message: t('通知发送失败,请检查参数') };
}
}
private normalizeTokens(
tokens: Record<string, string | TokenInfo[]>,
): Record<string, TokenInfo[]> {
const normalized: Record<string, TokenInfo[]> = {};
for (const [platform, value] of Object.entries(tokens)) {
if (typeof value === 'string') {
// Legacy format: convert string token to TokenInfo array
@@ -553,8 +589,10 @@ export default class UserService {
normalized[platform] = value || [];
}
}
return normalized;
}
private addTokenToList(
tokens: Record<string, string | TokenInfo[]>,
platform: string,
@@ -568,12 +606,16 @@ export default class UserService {
);
maxTokensPerPlatform = config.maxTokensPerPlatform;
}
const normalized = this.normalizeTokens(tokens);
if (!normalized[platform]) {
normalized[platform] = [];
}
// Add new token
normalized[platform].unshift(tokenInfo);
// Limit the number of active tokens per platform
if (normalized[platform].length > maxTokensPerPlatform) {
normalized[platform] = normalized[platform].slice(
@@ -581,32 +623,40 @@ export default class UserService {
maxTokensPerPlatform,
);
}
return normalized;
}
private removeTokenFromList(
tokens: Record<string, string | TokenInfo[]>,
platform: string,
tokenValue: string,
): Record<string, TokenInfo[]> {
const normalized = this.normalizeTokens(tokens);
if (normalized[platform]) {
normalized[platform] = normalized[platform].filter(
(t) => t.value !== tokenValue,
);
}
return normalized;
}
private findTokenInList(
tokens: Record<string, string | TokenInfo[]>,
platform: string,
tokenValue: string,
): TokenInfo | undefined {
const normalized = this.normalizeTokens(tokens);
if (normalized[platform]) {
return normalized[platform].find((t) => t.value === tokenValue);
}
return undefined;
}
@serializeAuthMutation
public async resetAuthInfo(info: Partial<AuthInfo>) {
const { retries, twoFactorActivated, password, username } = info;
@@ -623,6 +673,7 @@ export default class UserService {
},
(x) => !isNil(x),
);
if (password !== undefined) {
payload.password = await hashPassword(password);
}
@@ -639,8 +690,5 @@ export default class UserService {
});
}
await this.updateAuthInfo(authInfo, payload);
if (retries === 0 || password !== undefined || username !== undefined) {
this.loginThrottle.clear();
}
}
}
-11
View File
@@ -1,11 +0,0 @@
// Match command text literally: looking up a task must never execute its input.
export function findCronId(crontab: string, command: string): string {
if (!command) return '';
for (const line of crontab.split('\n')) {
const id = /\bID=(\d+)\s/.exec(line);
if (id && line.includes(command, id.index + id[0].length)) {
return id[1];
}
}
return '';
}
+11 -70
View File
@@ -22,7 +22,6 @@ export interface CronSchedule {
// Validation and registration repeatedly see identical rules in large snapshots.
// Cache only immutable syntax, never dates or running tasks, and bound user input.
const scheduleCache = new Map<string, CronSchedule>();
const daysInMonth = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
// Canonicalize legacy shorthand, aliases, names and numeric-start steps before
// node-cron sees them. Extend legacy aliases explicitly; reject H and bare /N.
@@ -59,73 +58,17 @@ export function parseCronSchedule(schedule: unknown): CronSchedule {
const expression = CronExpressionParser.parse(parts.join(' '));
if (!expression.hasNext())
throw new Error('Cron schedule has no next execution');
let normalized = parts.map((part) => part.replace(/\?/g, '*'));
// cron-parser includes Sunday in stepped ranges ending in 7 even when the
// step does not land on 7. node-cron accepts those ranges with different days.
if (/(^|,)(?:\d+|[a-z]{3})-0*7\/\d+(?=,|$)/i.test(normalized[5])) {
normalized[5] = expression.fields.dayOfWeek.values.join(',');
}
// Most rules are already supported; only inspect individual fields on failure.
if (!validate(normalized.join(' '))) {
const fields = [
expression.fields.second,
expression.fields.minute,
expression.fields.hour,
expression.fields.dayOfMonth,
expression.fields.month,
expression.fields.dayOfWeek,
];
normalized = normalized.map((value, index) => {
// Preserve supported lists, ranges and names. Expand only fields node-cron
// cannot consume, such as numeric/name-start steps; never compress them.
const probe = ['0', '0', '0', '*', '*', '*'];
probe[index] = value;
if (validate(probe.join(' '))) return value;
const values = fields[index].values;
if (index === 5 && expression.fields.dayOfWeek.nthDay) {
// The legacy normalizer treated *#N / ?#N as an unrestricted weekday.
if (values.length === 8) return '*';
// # applies to a single weekday; normalize its numeric spelling only.
return values.length === 1
? `${values[0]}#${expression.fields.dayOfWeek.nthDay}`
: value;
}
return values.join(',');
});
}
const months = expression.fields.month.values;
// cron-parser 4 clipped dates outside a single selected month's maximum
// before deciding whether the day field covered the whole month.
const dayCount = expression.fields.dayOfMonth.values.filter(
(day) =>
months.length !== 1 ||
typeof day !== 'number' ||
day <= daysInMonth[months[0] - 1],
).length;
const normalized = expression.stringify(true).replace(/\?/g, '*').split(' ');
const dayCount = expression.fields.dayOfMonth.values.length;
const weekCount = expression.fields.dayOfWeek.values.length;
// Preserve the legacy count-based wildcard rules, including repeated Sunday
// and L tokens: an ignored day field must not remain an AND constraint.
const weekdayPattern = [
...normalized.slice(0, 3),
'*',
...normalized.slice(4),
].join(' ');
const patterns =
weekCount === 8
? [[...normalized.slice(0, 5), '*'].join(' ')]
: dayCount < 31
? [[...normalized.slice(0, 5), '*'].join(' '), weekdayPattern]
: [weekdayPattern];
const valid = patterns.map((pattern) => validate(pattern));
if (patterns.length === 2 && !valid[0] && valid[1]) {
// A date branch can be impossible for all chosen months while the weekday
// branch still runs. Drop it only if its syntax is valid with any month.
if (validate([...normalized.slice(0, 4), '*', '*'].join(' '))) {
patterns.shift();
valid.shift();
}
}
if (!valid.every(Boolean)) {
dayCount < 31 && weekCount < 8
? [
[...normalized.slice(0, 5), '*'].join(' '),
[...normalized.slice(0, 3), '*', ...normalized.slice(4)].join(' '),
]
: [normalized.join(' ')];
if (!patterns.every((pattern) => validate(pattern))) {
throw new Error('Unsupported cron schedule');
}
const parsed = {
@@ -135,10 +78,8 @@ export function parseCronSchedule(schedule: unknown): CronSchedule {
};
Object.freeze(patterns);
Object.freeze(parsed);
if (scheduleCache.size >= 512) {
const oldest = scheduleCache.keys().next().value;
if (oldest !== undefined) scheduleCache.delete(oldest);
}
if (scheduleCache.size >= 512)
scheduleCache.delete(scheduleCache.keys().next().value);
scheduleCache.set(cacheKey, parsed);
return parsed;
}
+2 -3
View File
@@ -1,4 +1,3 @@
import { t } from './i18n';
import { createTask, ScheduledTask } from 'node-cron';
import { parseCronSchedule } from './cronSchedule';
@@ -37,7 +36,7 @@ export function createCronJob(
if (cancelled || !started || !accepts(index, date)) return;
if (missed) {
options.logger.warn(
t('[schedule][补执行迟到任务] 任务: %s, 计划时间: %s'),
'[schedule][补执行迟到任务] 任务: %s, 计划时间: %s',
options.name,
date.toISOString(),
);
@@ -46,7 +45,7 @@ export function createCronJob(
await callback(date);
} catch (error) {
options.logger.error(
t('[schedule][定时回调失败] 任务: %s, 错误: %s'),
'[schedule][定时回调失败] 任务: %s, 错误: %s',
options.name,
error instanceof Error ? error.message : String(error),
);
+91 -224
View File
@@ -1,234 +1,111 @@
import { shareStore } from './store';
const messages: Record<string, Record<string, string>> = {
zh: {},
en: {
'升级任务启动失败,请检查日志后重试':
'Failed to start the update. Check the logs and try again.',
'[schedule][停止已运行任务] 任务ID: %s, PID: %s':
'[schedule][Stopping running task] Task ID: %s, PID: %s',
'接口已下线,请使用 /configs/detail 接口':
'This endpoint has been retired. Use /configs/detail instead.',
'接口已下线,请使用 /logs/detail 接口':
'This endpoint has been retired. Use /logs/detail instead.',
'接口已下线,请使用 /scripts/detail 接口':
'This endpoint has been retired. Use /scripts/detail instead.',
'仅支持 PNG、JPEG、GIF、WebP、AVIF 图片':
'Only PNG, JPEG, GIF, WebP and AVIF images are supported',
请先初始化: 'Please initialize the panel first',
请先关闭两步验证: 'Please disable two-factor authentication first',
'失败次数过多,请稍后重试':
'Too many failed attempts. Please try again later.',
配置更新失败: 'Failed to update configuration',
'trust proxy 配置格式无效': 'Invalid trust proxy configuration',
'代理层数不能超过 20': 'Proxy hop count cannot exceed 20',
'环境变量 QL_TRUST_PROXY 已生效,系统设置不可覆盖':
'QL_TRUST_PROXY is active and cannot be overridden by system settings',
系统配置不存在: 'System configuration does not exist',
'历史版本仅支持不超过 1 MiB 的 UTF-8 文本脚本':
'Version history only supports UTF-8 text scripts up to 1 MiB',
脚本不存在: 'Script does not exist',
请选择脚本文件: 'Please select a script file',
'历史版本读取失败,请检查历史文件':
'Failed to read version history. Check the history file.',
历史版本不存在或已清理:
'This history version does not exist or has been removed',
'文件已变化,请刷新历史版本后重试':
'The file has changed. Refresh version history and try again.',
'历史版本容量不足,无法安全保存':
'Version history storage is full. The file cannot be saved safely.',
'跳过订阅 %s 的无效定时配置: %s':
'Skipping invalid schedule for subscription %s: %s',
'[schedule][准备运行任务] 命令: %s':
'[schedule][Preparing task] Command: %s',
'[schedule][创建定时任务] 任务ID: %s, 规则数: %s':
'[schedule][Creating scheduled task] Task ID: %s, schedules: %s',
'[schedule][取消定时任务] 任务ID: %s':
'[schedule][Cancelling scheduled task] Task ID: %s',
'[schedule][取消任务失败] 任务ID: %s, 错误: %s':
'[schedule][Failed to cancel task] Task ID: %s, error: %s',
'[panel][停止任务失败] PID: %s, 错误: %s':
'[panel][Failed to stop task] PID: %s, error: %s',
'[panel][停止实例失败] 实例ID: %s, PID: %s, 错误: %s':
'[panel][Failed to stop instance] Instance ID: %s, PID: %s, error: %s',
'[panel][执行任务失败] 任务ID: %s, 错误: %s':
'[panel][Task failed] Task ID: %s, error: %s',
'[panel][执行任务结束] 任务ID: %s, 退出码: %j':
'[panel][Task finished] Task ID: %s, exit code: %j',
'[panel][创建任务失败] 任务ID: %s, 错误: %s':
'[panel][Failed to create task] Task ID: %s, error: %s',
'[panel][关闭任务日志失败] %s': '[panel][Failed to close task log] %s',
'[panel][清理任务状态失败] %s': '[panel][Failed to clear task status] %s',
'性能指标报告:': 'Performance metrics:',
'[panel][开始执行任务] 任务ID: %s': '[panel][Starting task] Task ID: %s',
'[panel][任务错误回调失败] %s': '[panel][Task error callback failed] %s',
'[panel][任务结束回调失败] %s':
'[panel][Task completion callback failed] %s',
'[panel][任务队列失败] %s': '[panel][Task queue failed] %s',
'[panel][创建cron任务] 任务ID: %s, cron: %s, 任务名: %s, 执行命令: %s':
'[panel][Creating cron task] Task ID: %s, cron: %s, name: %s, command: %s',
'[panel][跳过无效定时任务] 任务ID: %s, cron: %s, 错误: %s':
'[panel][Skipping invalid schedule] Task ID: %s, cron: %s, error: %s',
'[panel][取消定时任务] 任务名: %s':
'[panel][Cancelling scheduled task] Name: %s',
'[panel][创建interval任务] 任务ID: %s, 任务名: %s, 执行命令: %s':
'[panel][Creating interval task] Task ID: %s, name: %s, command: %s',
'[panel][执行任务失败] 命令: %s, 错误信息: %j':
'[panel][Task failed] Command: %s, error: %j',
'[panel][取消interval任务] 任务ID: %s, 任务名: %s':
'[panel][Cancelling interval task] Task ID: %s, name: %s',
生成私钥文件失败: 'Failed to generate private key file',
删除私钥文件失败: 'Failed to delete private key file',
'删除ssh配置文件%s失败': 'Failed to delete SSH configuration file %s',
'跳过订阅 %s 的无效 SSH 配置: %s':
'Skipping invalid SSH configuration for subscription %s: %s',
'[schedule][补执行迟到任务] 任务: %s, 计划时间: %s':
'[schedule][Running late task] Task: %s, scheduled time: %s',
'[schedule][定时回调失败] 任务: %s, 错误: %s':
'[schedule][Scheduled callback failed] Task: %s, error: %s',
'[schedule][任务队列] 空闲中...': '[schedule][Task queue] Idle...',
'[schedule][任务加入队列] 运行中任务数: %s, 等待中任务数: %s':
'[schedule][Task queued] Running: %s, waiting: %s',
'[schedule][开始处理任务] 运行中任务数: %s, 等待中任务数: %s':
'[schedule][Processing task] Running: %s, waiting: %s',
'[schedule][任务处理成功] 参数 %s':
'[schedule][Task succeeded] Arguments: %s',
'[schedule][任务处理错误] 参数 %s': '[schedule][Task failed] Arguments: %s',
'[schedule][任务处理结束] 运行中任务数: %s, 等待中任务数: %s':
'[schedule][Task finished] Running: %s, waiting: %s',
'[schedule][任务重复运行] 通知失败 %s':
'[schedule][Duplicate task] Notification failed: %s',
'[schedule][任务重复运行] 参数 %s':
'[schedule][Duplicate task] Arguments: %s',
'[schedule][停止已运行任务] 任务ID: %s':
'[schedule][Stopping running task] Task ID: %s',
'[schedule][检查已运行任务失败] 任务ID: %s, 错误: %s':
'[schedule][Failed to check running task] Task ID: %s, error: %s',
'[schedule][开始执行任务] 参数 %s':
'[schedule][Starting task] Arguments: %s',
'[schedule][任务标准错误] 命令: %s, 信息: %s':
'[schedule][Task stderr] Command: %s, message: %s',
'[schedule][执行任务失败] 命令: %s, 错误: %s':
'[schedule][Task failed] Command: %s, error: %s',
'[schedule][执行任务结束] 任务ID: %s, 退出码: %j':
'[schedule][Task finished] Task ID: %s, exit code: %j',
'[schedule][创建任务失败] 命令: %s, 错误: %s':
'[schedule][Failed to create task] Command: %s, error: %s',
'检查更新失败,请检查网络后重试':
'Failed to check for updates. Check your network and try again.',
'资源已存在,请检查重复的名称或值':
'Resource already exists; check for duplicate names or values',
暂无权限: 'Access denied',
参数错误: 'Invalid parameter',
参数不正确: 'Invalid parameter',
文件无法访问: 'File not accessible',
文件不存在: 'File not found',
路径不存在: 'Path does not exist',
路径不正确: 'Invalid path',
'暂无权限': 'Access denied',
'参数错误': 'Invalid parameter',
'参数不正确': 'Invalid parameter',
'文件无法访问': 'File not accessible',
'文件不存在': 'File not found',
'路径不存在': 'Path does not exist',
'路径不正确': 'Invalid path',
'通知发送失败,请检查参数': 'Notification failed, check parameters',
'通知发送失败,请检查系统设置/通知配置':
'Notification failed, check system settings',
通知发送成功: 'Notification sent successfully',
设置时区失败: 'Failed to set timezone',
任务未找到: 'Task not found',
密码不能设置为admin: 'Password cannot be admin',
更新成功: 'Update successful',
未知错误: 'Unknown error',
验证失败: 'Verification failed',
实例不存在或已停止: 'Instance does not exist or stopped',
实例已停止: 'Instance stopped',
确认停止实例: 'Confirm to stop instance',
确认停止运行实例: 'Confirm to stop running instance',
确认停止: 'Confirm to stop',
确认停止定时任务: 'Confirm to stop scheduled task',
确认删除: 'Confirm to delete',
确认删除定时任务: 'Confirm to delete scheduled task',
确认删除选中的定时任务吗: 'Confirm to delete selected tasks?',
确认运行: 'Confirm to run',
确认运行定时任务: 'Confirm to run scheduled task',
确认保存: 'Confirm to save',
确认保存文件: 'Confirm to save file',
确认重启: 'Confirm restart',
确认启用: 'Confirm to enable',
确认禁用: 'Confirm to disable',
确认: 'Confirm',
删除成功: 'Deleted successfully',
操作成功: 'Operation successful',
参数不完整: 'Incomplete parameters',
默认路径不支持删除: 'Default path cannot be deleted',
必须在日志目录下: 'Must be within log directory',
'通知发送失败,请检查系统设置/通知配置': 'Notification failed, check system settings',
'通知发送成功': 'Notification sent successfully',
'设置时区失败': 'Failed to set timezone',
'任务未找到': 'Task not found',
'密码不能设置为admin': 'Password cannot be admin',
'更新成功': 'Update successful',
'未知错误': 'Unknown error',
'验证失败': 'Verification failed',
'实例不存在或已停止': 'Instance does not exist or stopped',
'实例已停止': 'Instance stopped',
'确认停止实例': 'Confirm to stop instance',
'确认停止运行实例': 'Confirm to stop running instance',
'确认停止': 'Confirm to stop',
'确认停止定时任务': 'Confirm to stop scheduled task',
'确认删除': 'Confirm to delete',
'确认删除定时任务': 'Confirm to delete scheduled task',
'确认删除选中的定时任务吗': 'Confirm to delete selected tasks?',
'确认运行': 'Confirm to run',
'确认运行定时任务': 'Confirm to run scheduled task',
'确认保存': 'Confirm to save',
'确认保存文件': 'Confirm to save file',
'确认重启': 'Confirm restart',
'确认启用': 'Confirm to enable',
'确认禁用': 'Confirm to disable',
'确认': 'Confirm',
'删除成功': 'Deleted successfully',
'操作成功': 'Operation successful',
'参数不完整': 'Incomplete parameters',
'默认路径不支持删除': 'Default path cannot be deleted',
'必须在日志目录下': 'Must be within log directory',
'备份数据上传成功,确认覆盖数据': 'Backup uploaded, confirm overwrite',
'如果恢复失败,可进入容器执行': 'If restore fails, run in container:',
系统将在: 'System will',
秒后自动刷新: 'refresh in seconds',
'系统将在': 'System will',
'秒后自动刷新': 'refresh in seconds',
'生成数据中...': 'Generating data...',
'每条数据 name 或者 value 字段不能为空,参考导出文件格式':
'Each entry must have name and value, see export format',
不支持当前依赖类型: 'Unsupported dependency type',
依赖已存在: 'Dependency already exists',
依赖不存在: 'Dependency does not exist',
该脚本正在运行中: 'Script is running',
该脚本未在运行中: 'Script is not running',
文件内容为空: 'File content is empty',
文件名不能为空: 'File name cannot be empty',
标签不能为空: 'Label cannot be empty',
名称不能为空: 'Name cannot be empty',
名称不能为保留关键字: 'Name cannot be reserved keyword',
名称已存在: 'Name already exists',
密码错误: 'Incorrect password',
用户不存在: 'User does not exist',
请输入用户名和密码: 'Please enter username and password',
无权访问: 'Access denied',
登录成功: 'Login successful',
退出成功: 'Logout successful',
'每条数据 name 或者 value 字段不能为空,参考导出文件格式': 'Each entry must have name and value, see export format',
'不支持当前依赖类型': 'Unsupported dependency type',
'依赖已存在': 'Dependency already exists',
'依赖不存在': 'Dependency does not exist',
'该脚本正在运行中': 'Script is running',
'该脚本未在运行中': 'Script is not running',
'文件内容为空': 'File content is empty',
'文件名不能为空': 'File name cannot be empty',
'标签不能为空': 'Label cannot be empty',
'名称不能为空': 'Name cannot be empty',
'名称不能为保留关键字': 'Name cannot be reserved keyword',
'名称已存在': 'Name already exists',
'密码错误': 'Incorrect password',
'用户不存在': 'User does not exist',
'请输入用户名和密码': 'Please enter username and password',
'无权访问': 'Access denied',
'登录成功': 'Login successful',
'退出成功': 'Logout successful',
'Token 已失效': 'Token expired',
'Token 无效': 'Invalid token',
两步骤验证已开启: '2FA enabled',
两步骤验证已关闭: '2FA disabled',
验证码错误: 'Invalid verification code',
验证码已过期: 'Verification code expired',
请先开启两步骤验证: 'Please enable 2FA first',
两步骤验证密钥不能为空: '2FA secret cannot be empty',
用户已存在: 'User already exists',
用户名不能为admin: 'Username cannot be admin',
不能删除自己: 'Cannot delete yourself',
不能禁用自己: 'Cannot disable yourself',
文件路径无效: 'Invalid file path',
保存成功: 'Saved successfully',
'两步骤验证已开启': '2FA enabled',
'两步骤验证已关闭': '2FA disabled',
'验证码错误': 'Invalid verification code',
'验证码已过期': 'Verification code expired',
'请先开启两步骤验证': 'Please enable 2FA first',
'两步骤验证密钥不能为空': '2FA secret cannot be empty',
'用户已存在': 'User already exists',
'用户名不能为admin': 'Username cannot be admin',
'不能删除自己': 'Cannot delete yourself',
'不能禁用自己': 'Cannot disable yourself',
'文件路径无效': 'Invalid file path',
'保存成功': 'Saved successfully',
'client_id 或 client_seret 有误': 'Invalid client_id or client_secret',
订阅执行完成: 'Subscription completed',
'wxPusher 服务的 TopicIds 和 Uids 至少配置一个才行':
'wxPusher requires at least one of TopicIds or Uids',
'订阅执行完成': 'Subscription completed',
'wxPusher 服务的 TopicIds 和 Uids 至少配置一个才行': 'wxPusher requires at least one of TopicIds or Uids',
'wxPusher SPT 不能为空': 'wxPusher SPT cannot be empty',
'wxPusher SPT 最多支持 10 个': 'wxPusher SPT supports at most 10 tokens',
'Url 或者 Body 中必须包含 $title': 'Url or Body must contain $title',
'绝对路径必须在日志目录内或使用 /dev/null':
'Absolute path must be within log directory or use /dev/null',
请先登录: 'Please login first',
'请先登录': 'Please login first',
'运行中...': 'Running...',
'日志不存在...': 'Log does not exist...',
未分类: 'Uncategorized',
任务重复运行: 'Duplicate task execution',
日志设置为忽略: 'Log set to ignore',
定时规则不能为空: 'Schedule rule cannot be empty',
无效的定时规则: 'Invalid schedule rule',
定时规则必须是字符串: 'Schedule rule must be a string',
额外定时规则必须是对象: 'Extra schedule must be an object',
额外定时规则必须是数组: 'Extra schedules must be an array',
'未分类': 'Uncategorized',
'任务重复运行': 'Duplicate task execution',
'日志设置为忽略': 'Log set to ignore',
'定时规则不能为空': 'Schedule rule cannot be empty',
'无效的定时规则': 'Invalid schedule rule',
'日志名称只能包含中文、字母、数字、下划线、连字符、点和路径分隔符':
'Log name can only contain Chinese characters, letters, numbers, underscores, hyphens, dots, and path separators',
日志名称不能超过100个字符: 'Log name cannot exceed 100 characters',
'错误的用户名密码,请重试':
'Incorrect username or password, please try again',
'日志名称不能超过100个字符': 'Log name cannot exceed 100 characters',
'错误的用户名密码,请重试': 'Incorrect username or password, please try again',
'该 IP 已被列入黑名单': 'This IP address has been blocked',
'已加入 IP 黑名单': 'IP address added to the blacklist',
'已移出 IP 黑名单': 'IP address removed from the blacklist',
青龙快讯: 'QingLong',
登录通知: 'Login Notification',
你于: 'You at ',
在: ' in ',
端: ' client',
内网IP: 'Private IP',
'青龙快讯': 'QingLong',
'登录通知': 'Login Notification',
'你于': 'You at ',
'在': ' in ',
端: '',
登录失败: 'login failed',
',ip地址': ', IP: ',
'任务#%s': 'Task#%s',
@@ -253,13 +130,12 @@ const messages: Record<string, Record<string, string>> = {
青龙: 'QingLong',
'【蛟龙】测试通知 https://t.me/jiao_long':
'[JiaoLong] Test notification https://t.me/jiao_long',
生成token: 'Generate token',
删除日志: 'Delete logs',
'生成token': 'Generate token',
'删除日志': 'Delete logs',
'## 开始执行... %s\n': '## Start executing... %s\n',
'执行before命令...': 'Execute before command...',
'执行after命令...': 'Execute after command...',
'## 执行结束... %s 耗时 %s 秒':
'## Execution finished... %s elapsed %s seconds',
'## 执行结束... %s 耗时 %s 秒': '## Execution finished... %s elapsed %s seconds',
'调度器注册失败,任务创建已回滚':
'Scheduler registration failed, task creation rolled back',
'调度器注册失败,任务更新已回滚':
@@ -275,18 +151,9 @@ const messages: Record<string, Record<string, string>> = {
let currentLang: string = 'zh';
export function setLang(lang: string, persist = true) {
export function setLang(lang: string) {
currentLang = lang || 'zh';
if (!persist) return;
// Translation is also used by standalone scheduler and CLI helpers. Load
// persistent storage only when changing the language, not when reading it.
const { shareStore } = require('./store') as typeof import('./store');
const language = currentLang;
return shareStore.setLang(language).then(() => {
if (currentLang === language && process.connected && process.send) {
process.send({ type: 'language-changed', lang: language }, () => {});
}
});
shareStore.setLang(currentLang);
}
/** 系统默认语言:Intl 检测 → 'zh'(仅返回 zh/en) */
-1
View File
@@ -18,7 +18,6 @@ export interface IDependencyFn<T> {
export interface ICronFn<T> {
(): Promise<T>;
cron?: TCron;
isCurrent?: () => boolean;
}
export interface ISchedule {
-32
View File
@@ -1,32 +0,0 @@
import cluster from 'cluster';
import { setLang, systemLang } from './i18n';
function languageMessage(
message: unknown,
): message is { type: string; lang: string } {
const value = message as { type?: string; lang?: string } | null;
return (
value?.type === 'language-changed' && ['zh', 'en'].includes(value.lang!)
);
}
// Each worker owns a translation cache. Relay saved settings to live workers;
// a replacement worker restores the persisted language before serving requests.
export async function initializeLanguageSync() {
if (cluster.isPrimary) {
cluster.on('message', (_worker, message) => {
if (!languageMessage(message)) return;
setLang(message.lang, false);
for (const worker of Object.values(cluster.workers || {})) {
if (worker?.isConnected()) worker.send(message, () => {});
}
});
return;
} else {
process.on('message', (message) => {
if (languageMessage(message)) setLang(message.lang, false);
});
}
const { shareStore } = await import('./store');
setLang((await shareStore.getLang()) || systemLang(), false);
}
-121
View File
@@ -1,121 +0,0 @@
import { shellQuote } from './shellQuote';
const repositoryNames =
/^(debian(?:-security)?|ubuntu(?:-ports)?|alpine|armbian)$/;
export function normalizeMirror(mirror: string): URL {
if (/\s/.test(mirror)) throw new Error('Invalid Linux mirror URL');
const url = new URL(mirror);
if (
!['http:', 'https:'].includes(url.protocol) ||
url.username ||
url.password ||
url.search ||
url.hash
) {
throw new Error(
'Linux mirror must be an HTTP(S) URL without credentials, query or fragment',
);
}
return url;
}
export function replaceMirrorUrl(current: string, mirror: string): string {
const target = normalizeMirror(mirror);
const previous = new URL(current);
const oldSegments = previous.pathname.split('/').filter(Boolean);
const newSegments = target.pathname.split('/').filter(Boolean);
const repositoryIndex = oldSegments.findIndex((part) =>
repositoryNames.test(part),
);
const targetIndex = newSegments.findIndex((part) =>
repositoryNames.test(part),
);
const prefix =
targetIndex >= 0 ? newSegments.slice(0, targetIndex) : newSegments;
if (repositoryIndex >= 0) {
const repository = oldSegments[repositoryIndex];
let suffix = oldSegments.slice(repositoryIndex + 1);
// Repair paths already corrupted by repeated authority-only replacement.
while (suffix[0] === repository) suffix.shift();
return `${target.origin}/${[...prefix, repository, ...suffix].join('/')}`;
}
// Unknown repository paths: a full URL replaces the old root, a hostname
// preserves it. Never append the newly entered root to the existing root.
return `${target.origin}/${(newSegments.length
? newSegments
: oldSegments
).join('/')}`;
}
export function rewriteMirrorSources(content: string, mirror: string): string {
normalizeMirror(mirror);
return content
.split(/(\r?\n[ \t]*\r?\n)/)
.map((stanza) => {
const signedBy = stanza.match(/^Signed-By:\s*([^\r\n]*)/im)?.[1];
return stanza
.split('\n')
.map((line) => {
if (!/^\s*(?:URIs:|deb(?:-src)?\s|https?:\/\/)/.test(line))
return line;
const legacy = /^\s*deb(?:-src)?\s/.test(line);
const key = legacy
? line.match(/\bsigned-by=([^\]\s]+)/i)?.[1]
: signedBy;
// In .list files a # starts an inline comment; its URLs are metadata.
const comment = legacy ? line.indexOf('#') : -1;
const source = comment < 0 ? line : line.slice(0, comment);
const rewritten = source.replace(/https?:\/\/[^\s#]+/g, (url) =>
isSystemRepository(url, key) ? replaceMirrorUrl(url, mirror) : url,
);
return rewritten + (comment < 0 ? '' : line.slice(comment));
})
.join('\n');
})
.join('');
}
function isSystemRepository(url: string, signedBy?: string): boolean {
const keys =
signedBy
?.trim()
.split(/[\s,]+/)
.filter(Boolean) || [];
const archiveKeyring =
keys.length > 0 &&
keys.every((key) =>
/\/(?:debian|ubuntu|armbian)-archive-keyring\.(?:gpg|asc)$/.test(key),
);
// A separate signing key identifies a third-party archive, even if its
// repository happens to be called /debian or /ubuntu.
if (keys.length > 0 && !archiveKeyring) return false;
const segments = new URL(url).pathname.split('/').filter(Boolean);
const repositoryIndex = segments.findIndex((part) =>
repositoryNames.test(part),
);
if (repositoryIndex < 0) return false;
// Without archive metadata, accept only conventional archive roots. A
// vendor path such as Docker's /linux/debian is not the Debian archive.
return (
archiveKeyring ||
repositoryIndex === 0 ||
segments
.slice(0, repositoryIndex)
.every((part) => /^mirrors?$/.test(part)) ||
segments[repositoryIndex] === 'alpine'
);
}
export function mirrorWriteCommand(
file: string,
content: string,
prefix: string,
update: string,
): string {
return `${prefix}mkdir -p ${shellQuote(
file.slice(0, file.lastIndexOf('/')),
)} && printf '%s' ${shellQuote(content)} | ${prefix}tee ${shellQuote(
file,
)} >/dev/null && ${update}`;
}
-11
View File
@@ -1,11 +0,0 @@
import { resolveFileAccess } from './fileAccess';
/** Check stored paths too: old database rows are not trusted input. */
export function resolveLogPath(root: string, value: string): string {
const target = resolveFileAccess(root, [value]);
if (!target)
throw Object.assign(new Error('Log path is outside the log directory'), {
status: 400,
});
return target;
}
-47
View File
@@ -1,47 +0,0 @@
const RETRY_TTL = 15 * 60 * 1000;
const MAX_SOURCES = 10000;
const MAX_RETRY_SECONDS = 5 * 60;
/** Source-specific, bounded backoff; persisted account history is not a lock. */
export class LoginThrottle {
private sources = new Map<
string,
{ failures: number; lastFailure: number }
>();
private get(source: string, now: number) {
const state = this.sources.get(source);
if (state && now - state.lastFailure >= RETRY_TTL) {
this.sources.delete(source);
return undefined;
}
return state;
}
retryAfter(source: string, now: number): number {
const state = this.get(source, now);
if (!state || state.failures < 3) return 0;
const seconds = Math.min(3 ** state.failures, MAX_RETRY_SECONDS);
return Math.max(
0,
Math.ceil((seconds * 1000 - (now - state.lastFailure)) / 1000),
);
}
fail(source: string, now: number): number {
const failures = Math.min((this.get(source, now)?.failures || 0) + 1, 6);
this.sources.delete(source);
if (this.sources.size >= MAX_SOURCES) {
this.sources.delete(this.sources.keys().next().value!);
}
this.sources.set(source, { failures, lastFailure: now });
return failures;
}
reset(source: string) {
this.sources.delete(source);
}
clear() {
this.sources.clear();
}
}
-47
View File
@@ -1,47 +0,0 @@
import { Query, QuerySpec, orderBy } from '../data/queries';
import {
SystemModel,
AuthDataType,
NotifyStatus,
NotifyLogInfo,
} from '../data/system';
// Both backend sends and script reports use the same bounded, best-effort sink.
export async function persistNotificationAudit(
info: NotifyLogInfo,
): Promise<void> {
try {
await SystemModel.insert({
type: AuthDataType.notifyLog,
info: {
timestamp: Date.now(),
title: String(info.title || '').slice(0, 512),
content: String(info.content || '').slice(0, 8192),
status:
info.status === NotifyStatus.success
? NotifyStatus.success
: NotifyStatus.fail,
notifyType: String(info.notifyType || '').slice(0, 64),
source: info.source === 'script' ? 'script' : 'system',
scriptName: String(info.scriptName || '').slice(0, 512),
taskId: /^\d+$/.test(info.taskId || '') ? info.taskId : undefined,
},
});
const oldest = await SystemModel.find(
{ type: AuthDataType.notifyLog },
{
fields: ['id'],
orderBy: orderBy([['id', 'DESC']]),
offset: 200,
limit: 1,
},
);
if (oldest[0]?.id) {
await SystemModel.nativeDelete({
type: AuthDataType.notifyLog,
id: { ['$lte']: oldest[0].id },
});
}
} catch (error) {
console.error('Failed to persist notification audit log');
}
}
+26 -48
View File
@@ -50,9 +50,11 @@ class TaskLimit {
Buffer.from(tlsConfig.clientCert),
)
: credentials.createInsecure();
this._client = new ApiClient(`localhost:${config.grpcPort}`, creds, {
'grpc.enable_http_proxy': 0,
});
this._client = new ApiClient(
`localhost:${config.grpcPort}`,
creds,
{ 'grpc.enable_http_proxy': 0 },
);
}
return this._client;
}
@@ -79,36 +81,29 @@ class TaskLimit {
private handleEvents() {
this.cronLimit.on('add', () => {
Logger.info(
t('[schedule][任务加入队列] 运行中任务数: %s, 等待中任务数: %s'),
this.cronLimitActiveCount,
this.cronLimitPendingCount,
`[schedule][任务加入队列] 运行中任务数: ${this.cronLimitActiveCount}, 等待中任务数: ${this.cronLimitPendingCount}`,
);
});
this.cronLimit.on('active', () => {
Logger.info(
t('[schedule][开始处理任务] 运行中任务数: %s, 等待中任务数: %s'),
this.cronLimitActiveCount + 1,
this.cronLimitPendingCount,
`[schedule][开始处理任务] 运行中任务数: ${
this.cronLimitActiveCount + 1
}, 等待中任务数: ${this.cronLimitPendingCount}`,
);
});
this.cronLimit.on('completed', (param) => {
Logger.info(t('[schedule][任务处理成功] 参数 %s'), JSON.stringify(param));
Logger.info(`[schedule][任务处理成功] 参数 ${JSON.stringify(param)}`);
});
this.cronLimit.on('error', (error) => {
Logger.error(
t('[schedule][任务处理错误] 参数 %s'),
JSON.stringify(error),
);
Logger.error(`[schedule][任务处理错误] 参数 ${JSON.stringify(error)}`);
});
this.cronLimit.on('next', () => {
Logger.info(
t('[schedule][任务处理结束] 运行中任务数: %s, 等待中任务数: %s'),
this.cronLimitActiveCount,
this.cronLimitPendingCount,
`[schedule][任务处理结束] 运行中任务数: ${this.cronLimitActiveCount}, 等待中任务数: ${this.cronLimitPendingCount}`,
);
});
this.cronLimit.on('idle', () => {
Logger.info(t(`[schedule][任务队列] 空闲中...`));
Logger.info(`[schedule][任务队列] 空闲中...`);
});
}
@@ -118,15 +113,12 @@ class TaskLimit {
}
}
public removeQueuedCron(id: string, completed?: ICronFn<any>) {
public removeQueuedCron(id: string) {
if (this.queuedCrons.has(id)) {
const runs = this.queuedCrons.get(id);
if (runs && runs.length > 0) {
const remaining = completed
? runs.filter((run) => run !== completed)
: runs.slice(0, -1);
if (remaining.length) this.queuedCrons.set(id, remaining);
else this.queuedCrons.delete(id);
runs.pop();
this.queuedCrons.set(id, runs);
}
}
}
@@ -137,20 +129,13 @@ class TaskLimit {
this.manualCronoLimit.concurrency = limit;
return;
}
try {
const doc = await SystemModel.findOne({
type: AuthDataType.systemConfig,
});
if (doc?.info?.cronConcurrency) {
this.cronLimit.concurrency = doc.info.cronConcurrency;
this.manualCronoLimit.concurrency = doc.info.cronConcurrency;
}
} catch (error) {
// Imports can precede the isolated initializer on a fresh installation.
// Use defaults until that initializer creates Auths; never create schema here.
if ((error as { code?: string }).code !== 'DATABASE_SCHEMA_MISSING')
throw error;
await SystemModel.sync();
const doc = await SystemModel.findOne({
where: { type: AuthDataType.systemConfig },
});
if (doc?.info?.cronConcurrency) {
this.cronLimit.concurrency = doc.info.cronConcurrency;
this.manualCronoLimit.concurrency = doc.info.cronConcurrency;
}
}
@@ -158,15 +143,9 @@ class TaskLimit {
cron: TCron,
fn: ICronFn<T>,
options?: Partial<QueueAddOptions>,
isCurrent: () => boolean = () => true,
): Promise<T | void> {
if (!isCurrent()) return;
fn.cron = cron;
fn.isCurrent = isCurrent;
// Invalidated snapshots must not consume the new revision's repeat budget.
const runs = this.queuedCrons
.get(cron.id)
?.filter((run) => run.isCurrent?.() !== false);
let runs = this.queuedCrons.get(cron.id);
const result = runs?.length ? [...runs, fn] : [fn];
const repeatTimes = this.repeatCronNotifyMap.get(cron.id) || 0;
if (result?.length > 5) {
@@ -186,14 +165,13 @@ class TaskLimit {
(err, res) => {
if (err) {
Logger.error(
t('[schedule][任务重复运行] 通知失败 %s'),
JSON.stringify(err),
`[schedule][任务重复运行] 通知失败 ${JSON.stringify(err)}`,
);
}
},
);
}
Logger.warn(t('[schedule][任务重复运行] 参数 %s'), JSON.stringify(cron));
Logger.warn(`[schedule][任务重复运行] 参数 ${JSON.stringify(cron)}`);
return;
}
this.queuedCrons.set(cron.id, result);
+22 -37
View File
@@ -1,4 +1,3 @@
import { t } from './i18n';
import { spawn } from 'cross-spawn';
import taskLimit from './pLimit';
import Logger from '../loaders/logger';
@@ -9,23 +8,15 @@ import { RunningInstanceModel, InstanceStatus } from '../data/runningInstance';
import dayjs from 'dayjs';
import { observeChildProcess, asError } from './childProcess';
export function runCron(
cmd: string,
cron: ICron,
isCurrent: () => boolean = () => true,
): Promise<number | void> {
const execute = async () => {
export function runCron(cmd: string, cron: ICron): Promise<number | void> {
return taskLimit.runWithCronLimit(cron, async () => {
try {
if (!isCurrent()) return;
// Check if the cron is already running and stop it (only if multiple instances are not allowed)
try {
const existingCron = await CrontabModel.findOne({
id: Number(cron.id),
where: { id: Number(cron.id) },
});
// A mutation may commit while this queued run is reading the database.
if (!isCurrent() || !existingCron || existingCron.isDisabled) return;
// Default to single instance mode (0) for backward compatibility
const allowSingleInstances =
existingCron?.allow_multiple_instances === 0;
@@ -38,50 +29,45 @@ export function runCron(
existingCron.status === CrontabStatus.queued)
) {
Logger.info(
t('[schedule][停止已运行任务] 任务ID: %s, PID: %s'),
cron.id,
existingCron.pid,
`[schedule][停止已运行任务] 任务ID: ${cron.id}, PID: ${existingCron.pid}`,
);
await killTask(existingCron.pid, true);
// Mark old running instances as stopped
const stoppedAt = dayjs().unix();
await RunningInstanceModel.nativeUpdate(
{
cron_id: Number(cron.id),
status: InstanceStatus.running,
},
await RunningInstanceModel.update(
{ status: InstanceStatus.stopped, finished_at: stoppedAt },
{
where: {
cron_id: Number(cron.id),
status: InstanceStatus.running,
},
},
);
// Update the status to idle after killing
await CrontabModel.nativeUpdate(
{ id: Number(cron.id) },
await CrontabModel.update(
{ status: CrontabStatus.idle, pid: undefined },
{ where: { id: Number(cron.id) } },
);
}
} catch (error) {
Logger.error(
t('[schedule][检查已运行任务失败] 任务ID: %s, 错误: %s'),
cron.id,
error,
`[schedule][检查已运行任务失败] 任务ID: ${cron.id}, 错误: ${error}`,
);
throw error;
}
// In particular, invalidation can happen while stopping a previous run.
if (!isCurrent()) return;
Logger.info(
t('[schedule][开始执行任务] 参数 %s'),
JSON.stringify({
`[schedule][开始执行任务] 参数 ${JSON.stringify({
...cron,
command: cmd,
}),
})}`,
);
const cp = spawn(cmd, { shell: '/bin/bash' });
const { completed } = observeChildProcess(cp, {
onStderr: async (message) => {
Logger.info(
t('[schedule][任务标准错误] 命令: %s, 信息: %s'),
'[schedule][任务标准错误] 命令: %s, 信息: %s',
cmd,
message,
);
@@ -90,26 +76,25 @@ export function runCron(
const result = await completed;
if (result.error) {
Logger.error(
t('[schedule][执行任务失败] 命令: %s, 错误: %s'),
'[schedule][执行任务失败] 命令: %s, 错误: %s',
cmd,
result.error.message,
);
}
Logger.info(
t('[schedule][执行任务结束] 任务ID: %s, 退出码: %j'),
'[schedule][执行任务结束] 任务ID: %s, 退出码: %j',
cron.id,
result.code,
);
return { ...cron, command: cmd, pid: cp.pid, ...result } as any;
} catch (error) {
Logger.error(
t('[schedule][创建任务失败] 命令: %s, 错误: %s'),
'[schedule][创建任务失败] 命令: %s, 错误: %s',
cmd,
asError(error).message,
);
} finally {
taskLimit.removeQueuedCron(cron.id, execute);
taskLimit.removeQueuedCron(cron.id);
}
};
return taskLimit.runWithCronLimit(cron, execute, undefined, isCurrent);
});
}
-41
View File
@@ -1,41 +0,0 @@
import { readFileSync } from 'fs';
import path from 'path';
import { createHash, randomUUID } from 'crypto';
function read(filename: string): Buffer | undefined {
try {
return readFileSync(filename);
} catch {
return undefined;
}
}
function manifest(): { sourceCommit?: string } {
try {
const value = JSON.parse(
read(path.resolve(__dirname, '../../build-info.json'))?.toString() ||
'{}',
);
return value && typeof value === 'object' ? value : {};
} catch {
return {};
}
}
// Capture before serving requests. Reading the installed files on each request
// would let an old process claim the identity of a newly copied build.
const backend = read(path.resolve(__dirname, '../app.js'));
const root = process.env.QL_DIR || path.resolve(__dirname, '../../..');
const version = read(path.join(root, 'version.yaml'))
?.toString()
.match(/^\uFEFF?version:\s*["']?(\d+\.\d+\.\d+)/m)?.[1];
export const runtimeBuild = Object.freeze({
instanceId: randomUUID(),
parentPid: process.ppid,
version,
sourceCommit: manifest().sourceCommit,
backendSha256: backend
? createHash('sha256').update(backend).digest('hex')
: undefined,
});
+15 -5
View File
@@ -1,4 +1,4 @@
import { DatabaseClient } from '../data/databaseClient';
import { QueryTypes, Sequelize } from 'sequelize';
// Append new entries; IDs are persisted and must not be renumbered or reused.
const columns = [
@@ -27,19 +27,25 @@ const columns = [
{ table: 'Crontabs', column: 'queued_token', type: 'VARCHAR(255)' },
];
export async function migrateSchema(database: DatabaseClient): Promise<void> {
await database.transaction(async () => {
export async function migrateSchema(database: Sequelize): Promise<void> {
await database.transaction(async (transaction) => {
await database.query(
'CREATE TABLE IF NOT EXISTS "SchemaMigrations" ("id" TEXT PRIMARY KEY, "applied_at" TEXT NOT NULL)',
{ transaction },
);
const applied = await database.query<{ id: string }>(
'SELECT "id" FROM "SchemaMigrations"',
{ type: QueryTypes.SELECT, transaction },
);
const appliedIds = new Set(applied.map(({ id }) => id));
for (const { table, column, type } of columns) {
const id = `add-${table}-${column}`;
const fields = await database.query<{ name: string }>(
`PRAGMA table_info("${table}")`,
{
type: QueryTypes.SELECT,
transaction,
},
);
if (fields.length === 0)
throw new Error(`Migration table is missing: ${table}`);
@@ -47,12 +53,16 @@ export async function migrateSchema(database: DatabaseClient): Promise<void> {
// table/column/type come only from the static migration manifest above.
await database.query(
`ALTER TABLE "${table}" ADD COLUMN "${column}" ${type}`,
{ transaction },
);
}
if (!appliedIds.has(id)) {
await database.query(
'INSERT INTO "SchemaMigrations" ("id", "applied_at") VALUES (?, ?)',
[id, new Date().toISOString()],
'INSERT INTO "SchemaMigrations" ("id", "applied_at") VALUES (:id, :appliedAt)',
{
replacements: { id, appliedAt: new Date().toISOString() },
transaction,
},
);
}
}
-467
View File
@@ -1,467 +0,0 @@
import * as fs from 'fs/promises';
import { createReadStream } from 'fs';
import path from 'path';
import { createHash, randomUUID } from 'crypto';
import { promisify } from 'util';
import { brotliCompress, brotliDecompress, constants } from 'zlib';
import { lock } from 'proper-lockfile';
import { resolveFileAccess } from './fileAccess';
import { getUniqueLockPath } from './utils';
const compress = promisify(brotliCompress);
const decompress = promisify(brotliDecompress);
const hash = (content: string) =>
createHash('sha256').update(content).digest('hex');
export const SCRIPT_HISTORY_LIMITS = {
versions: 20,
fileBytes: 1024 * 1024,
unpackedBytes: 8 * 1024 * 1024,
archiveBytes: 2 * 1024 * 1024,
totalBytes: 32 * 1024 * 1024,
files: 1000,
};
type Source = 'baseline' | 'save' | 'restore' | 'external';
interface Version {
id: string;
createdAt: string;
source: Source;
hash: string;
content: string;
}
interface Archive {
format: 1;
path: string;
versions: Version[];
head?: { hash: string; createdAt: string; source: Source };
}
export class ScriptHistoryError extends Error {
constructor(public status: number, message: string) {
super(message);
}
}
export class HistoryUnavailableError extends ScriptHistoryError {
constructor(public currentHash: string) {
super(413, '历史版本仅支持不超过 1 MiB 的 UTF-8 文本脚本');
}
}
interface SaveOptions {
skipHistory?: boolean;
expectedHash?: string;
}
/** Lazy, bounded storage: no watcher, database, daemon, or in-memory cache.
* Full snapshots share a Brotli window, so similar revisions compress together
* without a patch chain. All mutations use the existing script lock and a
* cross-process archive lock. History is durable before overwriting the script. */
export class ScriptHistory {
constructor(
private scriptRoot: string,
private historyRoot: string,
private blacklist: string[] = [],
private limits = SCRIPT_HISTORY_LIMITS,
) {}
private async target(directory: string, filename: string) {
const candidate = resolveFileAccess(
this.scriptRoot,
[directory, filename],
this.blacklist,
);
if (!candidate) throw new ScriptHistoryError(403, '暂无权限');
try {
const file = await fs.realpath(candidate);
const root = await fs.realpath(this.scriptRoot);
const relative = path.relative(root, file);
const key = hash(relative);
return {
file,
relative,
archive: path.join(this.historyRoot, key + '.br'),
};
} catch (error: any) {
if (error.code === 'ENOENT')
throw new ScriptHistoryError(404, '脚本不存在');
throw error;
}
}
private async readText(file: string) {
const handle = await fs.open(file, 'r');
try {
const stat = await handle.stat();
if (!stat.isFile()) throw new ScriptHistoryError(400, '请选择脚本文件');
if (stat.size > this.limits.fileBytes)
throw new ScriptHistoryError(
413,
'历史版本仅支持不超过 1 MiB 的 UTF-8 文本脚本',
);
// Bounded read even if an external process grows the file after stat.
const bytes = Buffer.alloc(this.limits.fileBytes + 1);
let size = 0;
while (size < bytes.length) {
const { bytesRead } = await handle.read(
bytes,
size,
bytes.length - size,
null,
);
if (!bytesRead) break;
size += bytesRead;
}
const buffer = bytes.subarray(0, size);
const content = buffer.toString('utf8');
if (
size > this.limits.fileBytes ||
content.includes('\0') ||
!Buffer.from(content).equals(buffer)
) {
throw new ScriptHistoryError(
413,
'历史版本仅支持不超过 1 MiB 的 UTF-8 文本脚本',
);
}
return content;
} finally {
await handle.close();
}
}
private async readArchive(file: string, relative: string): Promise<Archive> {
try {
const stat = await fs.lstat(file);
if (!stat.isFile() || stat.size > this.limits.archiveBytes)
throw new Error('Invalid archive');
const bytes = await decompress(await fs.readFile(file), {
maxOutputLength: this.limits.unpackedBytes,
});
const archive: Archive = JSON.parse(bytes.toString('utf8'));
if (
archive.format !== 1 ||
archive.path !== relative ||
!Array.isArray(archive.versions) ||
archive.versions.length > this.limits.versions ||
archive.versions.some(
(v) =>
typeof v.content !== 'string' ||
typeof v.id !== 'string' ||
typeof v.createdAt !== 'string' ||
!['baseline', 'save', 'restore', 'external'].includes(v.source) ||
Buffer.byteLength(v.content) > this.limits.fileBytes ||
hash(v.content) !== v.hash,
)
) {
throw new Error('Invalid archive');
}
return archive;
} catch (error: any) {
if (error.code === 'ENOENT')
return { format: 1, path: relative, versions: [] };
throw new ScriptHistoryError(500, '历史版本读取失败,请检查历史文件');
}
}
async list(directory: string, filename: string) {
const target = await this.target(directory, filename);
const [current, archive] = await Promise.all([
this.readText(target.file),
this.readArchive(target.archive, target.relative),
]);
const currentHash = hash(current);
const versions = archive.versions
.slice()
.reverse()
.map(({ content, ...v }) => ({
...v,
size: Buffer.byteLength(content),
identical: v.hash === currentHash,
}));
return { versions, currentHash, limit: this.limits.versions };
}
async detail(directory: string, filename: string, id: string) {
const target = await this.target(directory, filename);
const [current, archive] = await Promise.all([
this.readText(target.file),
this.readArchive(target.archive, target.relative),
]);
const version = archive.versions.find((v) => v.id === id);
if (!version) throw new ScriptHistoryError(404, '历史版本不存在或已清理');
return { version, current, currentHash: hash(current) };
}
async save(
directory: string,
filename: string,
content: string,
options: SaveOptions = {},
) {
return this.mutate(directory, filename, { content, ...options });
}
async restore(
directory: string,
filename: string,
id: string,
expectedHash: string,
) {
return this.mutate(directory, filename, { id, expectedHash });
}
private async mutate(
directory: string,
filename: string,
change:
| ({ content: string } & SaveOptions)
| { id: string; expectedHash: string },
) {
const target = await this.target(directory, filename);
const releaseFile = await lock(target.file, {
lockfilePath: getUniqueLockPath(target.file),
retries: { retries: 20, minTimeout: 50, maxTimeout: 250 },
});
try {
let current: string;
try {
current = await this.readText(target.file);
if (
'content' in change &&
(Buffer.byteLength(change.content) > this.limits.fileBytes ||
change.content.includes('\0'))
) {
throw new ScriptHistoryError(
413,
'历史版本仅支持不超过 1 MiB 的 UTF-8 文本脚本',
);
}
} catch (error) {
if (
!('content' in change) ||
!(error instanceof ScriptHistoryError) ||
error.status !== 413
)
throw error;
// Hash oversized or non-UTF-8 files without reading them all into memory.
const digest = createHash('sha256');
for await (const chunk of createReadStream(target.file))
digest.update(chunk);
const currentHash = digest.digest('hex');
if (!change.skipHistory) throw new HistoryUnavailableError(currentHash);
if (change.expectedHash !== currentHash)
throw new ScriptHistoryError(409, '文件已变化,请刷新历史版本后重试');
await this.preparedWrite(target.file, change.content, (commit) =>
commit(),
);
return {
content: change.content,
changed: true,
historyRecorded: false,
};
}
if (
('id' in change || change.skipHistory) &&
hash(current) !== change.expectedHash
) {
throw new ScriptHistoryError(409, '文件已变化,请刷新历史版本后重试');
}
if ('content' in change && change.content === current)
return { content: current, changed: false };
await fs.mkdir(this.historyRoot, { recursive: true, mode: 0o700 });
const releaseHistory = await lock(this.historyRoot, {
retries: { retries: 20, minTimeout: 50, maxTimeout: 250 },
});
try {
const archive = await this.readArchive(target.archive, target.relative);
const restored =
'id' in change
? archive.versions.find((v) => v.id === change.id)
: undefined;
if ('id' in change && !restored)
throw new ScriptHistoryError(404, '历史版本不存在或已清理');
const content =
'content' in change ? change.content : restored!.content;
if (content === current) return { content, changed: false };
if (
Buffer.byteLength(content) > this.limits.fileBytes ||
content.includes('\0')
) {
throw new ScriptHistoryError(
413,
'历史版本仅支持不超过 1 MiB 的 UTF-8 文本脚本',
);
}
const currentHash = hash(current);
if (
archive.versions[archive.versions.length - 1]?.hash !== currentHash
) {
const head =
archive.head?.hash === currentHash ? archive.head : undefined;
archive.versions.push({
id: randomUUID(),
createdAt: head?.createdAt || new Date().toISOString(),
source:
head?.source ||
(archive.versions.length ? 'external' : 'baseline'),
hash: currentHash,
content: current,
});
}
// Only prior contents are stored. Head metadata labels the next snapshot,
// but is used only if its hash matches the file. A failed file write can
// never introduce a phantom content version or lose the old snapshot.
archive.head = {
hash: hash(content),
createdAt: new Date().toISOString(),
source: 'id' in change ? 'restore' : 'save',
};
const packed = await this.pack(archive);
const cleanupPending = await this.preparedWrite(
target.file,
content,
(commit) => this.store(target.archive, packed, commit),
);
return {
content,
changed: true,
historyRecorded: true,
cleanupPending,
};
} finally {
await releaseHistory();
}
} finally {
await releaseFile();
}
}
private async pack(archive: Archive) {
archive.versions = archive.versions.slice(-this.limits.versions);
while (true) {
const serialized = Buffer.from(JSON.stringify(archive));
if (serialized.length <= this.limits.unpackedBytes) {
const compressed = await compress(serialized, {
params: {
[constants.BROTLI_PARAM_QUALITY]: 4,
[constants.BROTLI_PARAM_LGWIN]: 22,
},
});
if (compressed.length <= this.limits.archiveBytes) return compressed;
}
if (archive.versions.length <= 1)
throw new ScriptHistoryError(413, '历史版本容量不足,无法安全保存');
archive.versions.shift();
}
}
/** Stage the script before publishing history. A failed write/fsync never
* truncates the original. Preserve the existing file's ownership and mode. */
private async preparedWrite<T>(
file: string,
content: string,
run: (commit: () => Promise<void>) => Promise<T>,
): Promise<T> {
const stat = await fs.stat(file);
const temporary = path.join(
path.dirname(file),
`.ql-save-${randomUUID()}.tmp`,
);
try {
await fs.writeFile(temporary, content, {
encoding: 'utf8',
mode: 0o600,
flag: 'wx',
});
if (
typeof process.getuid === 'function' &&
typeof process.getgid === 'function' &&
(stat.uid !== process.getuid() || stat.gid !== process.getgid())
) {
await fs.chown(temporary, stat.uid, stat.gid);
}
await fs.chmod(temporary, stat.mode & 0o7777);
const handle = await fs.open(temporary, 'r');
try {
await handle.sync();
} finally {
await handle.close();
}
return await run(() => fs.rename(temporary, file));
} finally {
await fs.unlink(temporary).catch(() => undefined);
}
}
private async store(
file: string,
content: Buffer,
commit: () => Promise<void>,
) {
const entries = [];
for (const name of await fs.readdir(this.historyRoot)) {
if (!/^[a-f0-9]{64}\.br$/.test(name)) continue;
const fullPath = path.join(this.historyRoot, name);
if (fullPath === file) continue;
const stat = await fs.lstat(fullPath);
entries.push({ file: fullPath, size: stat.size, time: stat.mtimeMs });
}
entries.sort((a, b) => a.time - b.time);
let total = entries.reduce((sum, v) => sum + v.size, content.length);
const victims: string[] = [];
while (
entries.length &&
(total > this.limits.totalBytes || entries.length >= this.limits.files)
) {
const oldest = entries.shift()!;
victims.push(oldest.file);
total -= oldest.size;
}
if (total > this.limits.totalBytes)
throw new ScriptHistoryError(413, '历史版本容量不足,无法安全保存');
const previous = await fs.readFile(file).catch((error) => {
if (error.code === 'ENOENT') return undefined;
throw error;
});
const temporary = path.join(this.historyRoot, '.pending.tmp');
const publish = async (bytes: Buffer) => {
await fs.unlink(temporary).catch((error) => {
if (error.code !== 'ENOENT') throw error;
});
await fs.writeFile(temporary, bytes, { mode: 0o600, flag: 'wx' });
const handle = await fs.open(temporary, 'r');
try {
await handle.sync();
} finally {
await handle.close();
}
await fs.rename(temporary, file);
};
try {
await publish(content);
try {
await commit();
} catch (error) {
// The original script is still intact. Restore its exact history before
// returning an error; no unrelated archive has been removed yet.
if (previous) await publish(previous);
else await fs.unlink(file);
throw error;
}
// Quota eviction is post-commit maintenance. Report cleanup failures as a
// warning, not a failed save (which would encourage a misleading retry).
let cleanupPending = false;
for (const victim of victims) {
try {
await fs.unlink(victim);
} catch (error: any) {
if (error.code !== 'ENOENT') cleanupPending = true;
}
}
return cleanupPending;
} finally {
await fs.unlink(temporary).catch(() => undefined);
}
}
}
-5
View File
@@ -1,5 +0,0 @@
/** Quote one literal POSIX shell argument, including quotes and substitutions. */
export function shellQuote(value: string): string {
if (value.includes('\0')) throw new Error('Invalid shell argument');
return "'" + value.replace(/'/g, "'\\''") + "'";
}
+4 -2
View File
@@ -1,7 +1,9 @@
import { AuthInfo } from '../data/system';
import { App } from '../data/open';
import Keyv from 'keyv';
import { CacheStore } from '../data/cacheStore';
import KeyvSqlite from '@keyv/sqlite';
import config from '../config';
import path from 'path';
export enum EKeyv {
'apps' = 'apps',
@@ -15,7 +17,7 @@ export interface IKeyvStore {
lang: string;
}
const keyvSqlite = new CacheStore();
const keyvSqlite = new KeyvSqlite(path.join(config.dbPath, 'keyv.sqlite'));
export const keyvStore = new Keyv<IKeyvStore>({ store: keyvSqlite });
export const shareStore = {
-30
View File
@@ -1,30 +0,0 @@
import lockfile from 'proper-lockfile';
import path from 'path';
import config from '../config';
// Protect subscription storage ownership from the first database read through
// cleanup. Creation and URL/branch updates must use the same cross-process lock.
export async function withSubscriptionMutation<T>(
operation: () => Promise<T>,
): Promise<T> {
const target = `${path.resolve(config.scriptPath)}.subscriptions`;
let release: () => Promise<void>;
try {
release = await lockfile.lock(target, {
realpath: false,
stale: 30000,
update: 10000,
retries: { retries: 50, factor: 1, minTimeout: 100, maxTimeout: 100 },
});
} catch (cause) {
throw Object.assign(
new Error('Subscription configuration is busy', { cause }),
{ status: 503 },
);
}
try {
return await operation();
} finally {
await release();
}
}
-45
View File
@@ -1,45 +0,0 @@
import { resolveFileAccess } from './fileAccess';
import { createHash } from 'crypto';
export function assertSubscriptionAlias(alias: string): void {
if (
typeof alias !== 'string' ||
alias.length > 200 ||
!/^[^\s\\\p{C}]+$/u.test(alias) ||
alias.startsWith('~') ||
alias.split('/').some((part) => !part || part === '.' || part === '..')
) {
throw Object.assign(new Error('Invalid subscription alias'), {
status: 400,
});
}
}
/** Keep ordinary SSH names and use a reserved namespace for path-like aliases.
* The database alias and its script/log directories remain unchanged. */
export function getSubscriptionSshAlias(alias: string): string {
assertSubscriptionAlias(alias);
if (/^[\p{L}\p{N}_.+-]+$/u.test(alias) && !alias.endsWith('.config')) {
return alias;
}
return `~sub_${createHash('sha256').update(alias).digest('hex')}`;
}
export function resolveSubscriptionPath(
root: string,
alias: string,
suffix = '',
): string {
// Internal SSH filenames cannot collide with user-supplied aliases.
if (!/^~sub_[a-f0-9]{64}$/.test(alias)) {
assertSubscriptionAlias(
alias.startsWith('~global_') ? alias.slice(8) : alias,
);
}
const target = resolveFileAccess(root, [alias + suffix]);
if (!target)
throw Object.assign(new Error('Invalid subscription path'), {
status: 400,
});
return target;
}
-106
View File
@@ -1,106 +0,0 @@
import type { Subscription } from '../data/subscription';
import { formatUrl } from '../config/subscription';
import path from 'path';
import fs from 'fs';
import { resolveFileAccess } from './fileAccess';
// Storage namespaces come from URLs and branches, not user aliases. In
// particular, a legal Git directory can exceed the alias's 200-character limit.
export function isSubscriptionStorageName(name: string): boolean {
return (
typeof name === 'string' &&
!!name &&
!path.isAbsolute(name) &&
!/[\0\r\n\\]/.test(name) &&
!name.split('/').some((part) => !part || part === '.' || part === '..')
);
}
export function resolveSubscriptionStoragePath(
root: string,
name: string,
): string | undefined {
const target = isSubscriptionStorageName(name)
? resolveFileAccess(root, [name])
: '';
if (!target) {
if (isSubscriptionStorageName(name)) {
try {
fs.lstatSync(path.resolve(root, name));
} catch (error: any) {
// A long query can make the Shell candidate unrepresentable while
// the CLI's pathname-based file exists. Use the actual filesystem
// limit: macOS permits Unicode names exceeding 255 UTF-8 bytes.
if (error.code === 'ENAMETOOLONG') return undefined;
}
}
throw Object.assign(new Error('Invalid subscription path'), {
status: 400,
});
}
return target;
}
/** Match shell/update.sh get_uniq_path and update_raw, including SCP URLs. */
export function getSubscriptionStorageNames(
doc: Subscription,
engine: 'shell' | 'cli' = 'shell',
): {
script: string;
repo?: string;
raw?: string;
} {
// Old imported rows may only have an alias and no URL.
if (!doc.url) return { script: doc.alias, repo: doc.alias };
const url = formatUrl(doc).url!;
const trimmed = url.endsWith('/') ? url.slice(0, -1) : url;
const basename = trimmed.slice(trimmed.lastIndexOf('/') + 1);
const dot = basename.lastIndexOf('.');
const repo = dot < 0 ? basename : basename.slice(0, dot);
const parentUrl = engine === 'cli' ? trimmed : url;
const slash = parentUrl.lastIndexOf('/');
const parent =
engine === 'shell' && slash < 0 ? url : parentUrl.slice(0, slash);
const owner = parent.slice(parent.lastIndexOf('/') + 1);
const author = owner.slice(owner.lastIndexOf(':') + 1);
const stem = `${author.slice(author.lastIndexOf('.') + 1)}_${repo}`;
if (doc.type === 'file') {
const suffix =
engine === 'cli'
? path.extname(new URL(url).pathname)
: `.${url.slice(url.lastIndexOf('.') + 1)}`;
const raw = `${stem}${suffix}`;
return { script: `raw_${raw}`, raw };
}
const directory = doc.branch ? `${stem}_${doc.branch}` : stem;
return { script: directory, repo: directory };
}
// A subscription may have run through either engine before being deleted.
export function getSubscriptionStorageCandidates(doc: Subscription) {
const candidates: ReturnType<typeof getSubscriptionStorageNames>[] = [];
for (const engine of ['shell', 'cli'] as const) {
try {
candidates.push(getSubscriptionStorageNames(doc, engine));
} catch {
// Historical rows may have an invalid SSH alias or missing credentials.
// Retain their URL-derived storage without needing an executable SSH URL.
try {
candidates.push(
getSubscriptionStorageNames(
{
url: doc.url,
alias: doc.alias,
branch: doc.branch,
type: doc.type === 'private-repo' ? 'public-repo' : doc.type,
},
engine,
),
);
} catch {
// One engine's invalid URL must not hide the other engine's candidate.
}
}
}
return candidates;
}

Some files were not shown because too many files have changed in this diff Show More