mirror of
https://github.com/whyour/qinglong.git
synced 2026-11-18 00:57:53 +08:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d9e9e083ab | ||
|
|
8a9e70e90b | ||
|
|
3b6a873520 | ||
|
|
2ddb592800 | ||
|
|
b838d4bb42 | ||
|
|
73ad00ba32 | ||
|
|
27b322a10d | ||
|
|
1cfc0c8e2e | ||
|
|
cae580e02f | ||
|
|
76b285c3db | ||
|
|
13e0ff55d4 | ||
|
|
2bf814e562 | ||
|
|
41910b5236 | ||
|
|
442ab2c616 | ||
|
|
a5f22f8d44 | ||
|
|
f2c2f5e4f8 | ||
|
|
2efc5e4503 | ||
|
|
b92621bdb8 | ||
|
|
da9c6a6bbb | ||
|
|
d895fbba36 | ||
|
|
6862ad4f6c | ||
|
|
b8110c52eb | ||
|
|
99a4d0eff8 | ||
|
|
b486989399 | ||
|
|
77fe551425 | ||
|
|
2669f251ac | ||
|
|
3f02b04dff | ||
|
|
e8b6a30bd9 | ||
|
|
a36a77086a | ||
|
|
cb5dfec2b5 | ||
|
|
286088a32c | ||
|
|
a1104ef47c | ||
|
|
1a152575cc | ||
|
|
ef348b445b | ||
|
|
8305825ffd | ||
|
|
e555fcbb0b | ||
|
|
83e8f6bccd | ||
|
|
5801f735bc | ||
|
|
53e1129a12 | ||
|
|
13c0a4d21b | ||
|
|
126fe8eb67 | ||
|
|
73c41eed22 | ||
|
|
40149b6f17 | ||
|
|
6532775a25 | ||
|
|
6469792546 | ||
|
|
0aa521a288 | ||
|
|
1ac04a8062 | ||
|
|
e435f6e886 | ||
|
|
db88d037db | ||
|
|
a0c09046c6 | ||
|
|
5a9e53c771 | ||
|
|
be877cfddc | ||
|
|
fdeffebba8 | ||
|
|
3c61e933a1 | ||
|
|
cf236f8324 | ||
|
|
7334566d3c | ||
|
|
4cd6c6ad2e | ||
|
|
81a60254a2 | ||
|
|
1503f07c75 | ||
|
|
70636900b2 | ||
|
|
331f4f9138 | ||
|
|
b06cb72334 | ||
|
|
89c6355635 | ||
|
|
68ffa106f8 | ||
|
|
9d5f58ef99 | ||
|
|
ef1c54a620 | ||
|
|
0218404d9f | ||
|
|
24f7d1a46f | ||
|
|
fa8910b3d6 | ||
|
|
2cab112391 | ||
|
|
2ab9c702e6 | ||
|
|
6dd98a8565 | ||
|
|
06c8d8b112 | ||
|
|
a5406bedca | ||
|
|
3d2b98a442 | ||
|
|
93d7ec69dc | ||
|
|
a303e33b4e | ||
|
|
44129ca088 | ||
|
|
def4917447 | ||
|
|
bc0f35e3f7 | ||
|
|
f168efcdae | ||
|
|
0af546cd1d | ||
|
|
0d33d2321a | ||
|
|
79964f149c | ||
|
|
bcb2471768 | ||
|
|
fcf0ee619b | ||
|
|
f3ec352066 | ||
|
|
023d0f1cc4 | ||
|
|
f526d3f972 | ||
|
|
e28f746294 |
@@ -21,3 +21,6 @@ JWT_EXPIRES_IN=
|
|||||||
QINIU_AK=
|
QINIU_AK=
|
||||||
QINIU_SK=
|
QINIU_SK=
|
||||||
QINIU_SCOPE=
|
QINIU_SCOPE=
|
||||||
|
|
||||||
|
# Python 虚拟环境路径,留空则使用系统 Python
|
||||||
|
# PYTHON_VENV_DIR=./.venv
|
||||||
|
|||||||
@@ -2,23 +2,60 @@ name: Build And Push Docker Image
|
|||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
paths-ignore:
|
paths:
|
||||||
- "*.md"
|
- "**"
|
||||||
|
- "!*.md"
|
||||||
|
# CLI is verified separately; its shared Bash upgrade engine ships in images.
|
||||||
|
- "!cli/**"
|
||||||
|
- "cli/src/internal/maintenance/upgradeArtifacts.cjs"
|
||||||
|
- "cli/src/internal/maintenance/platformBundle.cjs"
|
||||||
|
- "cli/src/internal/maintenance/upgradePrestart.cjs"
|
||||||
|
- "cli/src/internal/maintenance/panelUpgrade.cjs"
|
||||||
|
- "cli/src/internal/maintenance/nodeRuntime.cjs"
|
||||||
|
- "!.github/workflows/cli-package.yml"
|
||||||
branches:
|
branches:
|
||||||
- "master"
|
- "master"
|
||||||
- "develop"
|
- "develop"
|
||||||
tags:
|
tags:
|
||||||
- "v*"
|
- "v*"
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
static_sync_only:
|
||||||
|
description: Synchronize an existing static snapshot to GitLab and verify Gitee
|
||||||
|
type: boolean
|
||||||
|
default: false
|
||||||
|
static_ref:
|
||||||
|
description: Static branch or version name, for example v2.22.0
|
||||||
|
type: string
|
||||||
|
default: ''
|
||||||
|
static_ref_type:
|
||||||
|
description: Synchronize a branch, or a version branch and Git tag
|
||||||
|
type: choice
|
||||||
|
options: [branch, tag]
|
||||||
|
default: branch
|
||||||
|
npm_rc_version:
|
||||||
|
description: Publish the verified main npm RC from develop, for example 2.25.0-rc.0
|
||||||
|
type: string
|
||||||
|
default: ''
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
|
|
||||||
|
# Keep master image tags and their bundle indexes in one serialized release.
|
||||||
|
# A rerun may intentionally replace a version, including from the same commit.
|
||||||
|
# Develop image builds and their shared bundle index serialize separately.
|
||||||
|
# Git-tag builds and static synchronization remain independent.
|
||||||
|
concurrency:
|
||||||
|
group: version-images-${{ github.ref == 'refs/heads/master' && 'master' || (github.ref == 'refs/heads/develop' && !inputs.static_sync_only && 'develop') || format('{0}-{1}', github.run_id, github.run_attempt) }}
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
validate:
|
validate:
|
||||||
|
if: ${{ !inputs.static_sync_only }}
|
||||||
uses: ./.github/workflows/validate.yml
|
uses: ./.github/workflows/validate.yml
|
||||||
|
|
||||||
code_gitlab:
|
code_gitlab:
|
||||||
|
if: ${{ !inputs.static_sync_only }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v6
|
||||||
@@ -35,37 +72,8 @@ jobs:
|
|||||||
git push --force --all gitlab 2>&1 || echo "::warning::GitLab push failed"
|
git push --force --all gitlab 2>&1 || echo "::warning::GitLab push failed"
|
||||||
git push --force --tags gitlab 2>&1 || echo "::warning::GitLab tags push failed"
|
git push --force --tags gitlab 2>&1 || echo "::warning::GitLab tags push failed"
|
||||||
|
|
||||||
code_gitee:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v6
|
|
||||||
with:
|
|
||||||
fetch-depth: 0
|
|
||||||
- name: Push to Gitee
|
|
||||||
env:
|
|
||||||
GITEE_TOKEN: ${{ secrets.GITEE_TOKEN }}
|
|
||||||
run: |
|
|
||||||
set +e
|
|
||||||
mkdir -p ~/.ssh
|
|
||||||
printf '%s\n' "${{ secrets.GITLAB_SSH_PK }}" > ~/.ssh/id_rsa
|
|
||||||
chmod 600 ~/.ssh/id_rsa
|
|
||||||
ssh-keyscan -T 10 gitee.com >> ~/.ssh/known_hosts 2>/dev/null
|
|
||||||
git remote add gitee git@gitee.com:whyour/qinglong.git 2>/dev/null
|
|
||||||
|
|
||||||
if git push --force --all gitee 2>&1; then
|
|
||||||
echo "::notice::Gitee push --all succeeded"
|
|
||||||
else
|
|
||||||
echo "::warning::Push failed, trying to create repo via API..."
|
|
||||||
curl -sS --connect-timeout 30 --max-time 60 \
|
|
||||||
-X POST "https://gitee.com/api/v5/user/repos" \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{"name":"qinglong","private":"false"}' \
|
|
||||||
"?access_token=$GITEE_TOKEN" 2>/dev/null
|
|
||||||
git push --force --all gitee 2>&1 && echo "::notice::Gitee push succeeded after repo creation" || echo "::warning::Gitee push failed after retry"
|
|
||||||
fi
|
|
||||||
git push --force --tags gitee 2>&1 || echo "::warning::Gitee tags push failed"
|
|
||||||
|
|
||||||
build-static:
|
build-static:
|
||||||
|
if: ${{ !inputs.static_sync_only }}
|
||||||
needs: validate
|
needs: validate
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
@@ -76,7 +84,7 @@ jobs:
|
|||||||
|
|
||||||
- uses: actions/setup-node@v6
|
- uses: actions/setup-node@v6
|
||||||
with:
|
with:
|
||||||
node-version: "20"
|
node-version-file: ".nvmrc"
|
||||||
cache: "pnpm"
|
cache: "pnpm"
|
||||||
cache-dependency-path: pnpm-lock.yaml
|
cache-dependency-path: pnpm-lock.yaml
|
||||||
|
|
||||||
@@ -87,6 +95,9 @@ jobs:
|
|||||||
pnpm build:back
|
pnpm build:back
|
||||||
pnpm build:info
|
pnpm build:info
|
||||||
|
|
||||||
|
- name: Pack the npm panel archive from the verified build
|
||||||
|
run: node scripts/verify-panel-package.cjs pack "$RUNNER_TEMP/npm-panel"
|
||||||
|
|
||||||
- uses: actions/upload-artifact@v6
|
- uses: actions/upload-artifact@v6
|
||||||
with:
|
with:
|
||||||
name: qinglong-static-${{ github.sha }}
|
name: qinglong-static-${{ github.sha }}
|
||||||
@@ -94,70 +105,137 @@ jobs:
|
|||||||
if-no-files-found: error
|
if-no-files-found: error
|
||||||
include-hidden-files: true
|
include-hidden-files: true
|
||||||
|
|
||||||
|
- uses: actions/upload-artifact@v6
|
||||||
|
with:
|
||||||
|
name: qinglong-npm-${{ github.sha }}
|
||||||
|
path: ${{ runner.temp }}/npm-panel/
|
||||||
|
if-no-files-found: error
|
||||||
|
retention-days: 14
|
||||||
|
|
||||||
- name: copy to static repo
|
- name: copy to static repo
|
||||||
env:
|
env:
|
||||||
GITHUB_REPO: github.com/${{ github.repository_owner }}/qinglong-static
|
GITHUB_REPO: github.com/${{ github.repository_owner }}/qinglong-static
|
||||||
GITHUB_BRANCH: ${{ github.ref_name }}
|
GITHUB_BRANCH: ${{ github.ref_name }}
|
||||||
|
GITHUB_REF_TYPE: ${{ github.ref_type }}
|
||||||
|
STATIC_TOKEN: ${{ secrets.API_TOKEN }}
|
||||||
run: |
|
run: |
|
||||||
mkdir -p tmp
|
: "${STATIC_TOKEN:?Missing static repository token}"
|
||||||
cd ./tmp
|
bash scripts/publish-static.sh "https://x-access-token:${STATIC_TOKEN}@${GITHUB_REPO}.git" "$GITHUB_BRANCH" "$GITHUB_REF_TYPE"
|
||||||
cp -rf ../static/* ./
|
|
||||||
git init -b ${GITHUB_BRANCH} && git add .
|
- name: publish verified upgrade download sizes
|
||||||
git config --local user.name 'github-actions[bot]'
|
env:
|
||||||
git config --local user.email 'github-actions[bot]@users.noreply.github.com'
|
GITHUB_REPO: github.com/${{ github.repository_owner }}/qinglong-static
|
||||||
git commit --allow-empty -m "copy static at $(date +'%Y-%m-%d %H:%M:%S')"
|
GITHUB_BRANCH: ${{ github.ref_name }}
|
||||||
git push --force --quiet "https://${{ secrets.API_TOKEN }}@${GITHUB_REPO}.git" ${GITHUB_BRANCH}:${GITHUB_BRANCH}
|
STATIC_TOKEN: ${{ secrets.API_TOKEN }}
|
||||||
|
run: |
|
||||||
|
: "${STATIC_TOKEN:?Missing static repository token}"
|
||||||
|
node scripts/publish-upgrade-info.cjs "https://x-access-token:${STATIC_TOKEN}@${GITHUB_REPO}.git" "$GITHUB_BRANCH" "$GITHUB_SHA"
|
||||||
|
|
||||||
static_gitlab:
|
static_gitlab:
|
||||||
needs: build-static
|
needs: build-static
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
concurrency:
|
||||||
|
group: static-gitlab-${{ github.ref_name }}
|
||||||
|
cancel-in-progress: false
|
||||||
steps:
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
- name: Push qinglong-static to GitLab
|
- name: Push qinglong-static to GitLab
|
||||||
|
env:
|
||||||
|
STATIC_BRANCH: ${{ github.ref_name }}
|
||||||
|
STATIC_REF_TYPE: ${{ github.ref_type }}
|
||||||
|
MIRROR_SSH_KEY: ${{ secrets.GITLAB_SSH_PK }}
|
||||||
run: |
|
run: |
|
||||||
set +e
|
set -euo pipefail
|
||||||
mkdir -p ~/.ssh
|
mkdir -p ~/.ssh
|
||||||
printf '%s\n' "${{ secrets.GITLAB_SSH_PK }}" > ~/.ssh/id_rsa
|
printf '%s\n' "$MIRROR_SSH_KEY" > ~/.ssh/id_rsa
|
||||||
chmod 600 ~/.ssh/id_rsa
|
chmod 600 ~/.ssh/id_rsa
|
||||||
ssh-keyscan -T 10 gitlab.com >> ~/.ssh/known_hosts 2>/dev/null
|
ssh-keyscan -T 10 gitlab.com >> ~/.ssh/known_hosts 2>/dev/null
|
||||||
git clone --depth=1 --single-branch https://github.com/whyour/qinglong-static.git static-mirror
|
bash scripts/sync-static.sh git@gitlab.com:whyour/qinglong-static.git "$STATIC_BRANCH" "$STATIC_REF_TYPE"
|
||||||
cd static-mirror
|
|
||||||
git remote add gitlab git@gitlab.com:whyour/qinglong-static.git 2>/dev/null
|
|
||||||
git push --force --all gitlab 2>&1 || echo "::warning::GitLab static push failed"
|
|
||||||
git push --force --tags gitlab 2>&1 || echo "::warning::GitLab static tags push failed"
|
|
||||||
|
|
||||||
static_gitee:
|
verify-gitee-mirrors:
|
||||||
needs: build-static
|
needs: build-static
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 14
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
repository: [qinglong, qinglong-static]
|
||||||
steps:
|
steps:
|
||||||
- name: Push qinglong-static to Gitee
|
- uses: actions/checkout@v6
|
||||||
|
# Gitee Pull mirrors and GitHub push webhooks own synchronization.
|
||||||
|
# Verification runs alongside Docker builds and needs no Gitee credentials.
|
||||||
|
- name: Wait for Gitee Pull mirror
|
||||||
|
if: ${{ matrix.repository == 'qinglong' }}
|
||||||
env:
|
env:
|
||||||
GITEE_TOKEN: ${{ secrets.GITEE_TOKEN }}
|
GH_TOKEN: ${{ github.token }}
|
||||||
|
QL_MIRROR_NOTIFY_REPO: whyour/qinglong
|
||||||
|
MIRROR_REPOSITORY: ${{ matrix.repository }}
|
||||||
|
MIRROR_REF: ${{ github.ref_name }}
|
||||||
|
MIRROR_REF_TYPE: ${{ github.ref_type }}
|
||||||
|
QL_MIRROR_INCLUDE_VERSION_BRANCH: ${{ matrix.repository == 'qinglong-static' }}
|
||||||
|
QL_MIRROR_NOTIFY_ON_START: 'false'
|
||||||
run: |
|
run: |
|
||||||
set +e
|
bash scripts/wait-for-mirror.sh "https://github.com/whyour/$MIRROR_REPOSITORY.git" "https://gitee.com/whyour/$MIRROR_REPOSITORY.git" "$MIRROR_REF" "$MIRROR_REF_TYPE"
|
||||||
mkdir -p ~/.ssh
|
- name: Notify and wait for Gitee static mirror
|
||||||
printf '%s\n' "${{ secrets.GITLAB_SSH_PK }}" > ~/.ssh/id_rsa
|
if: ${{ matrix.repository == 'qinglong-static' }}
|
||||||
chmod 600 ~/.ssh/id_rsa
|
env:
|
||||||
ssh-keyscan -T 10 gitee.com >> ~/.ssh/known_hosts 2>/dev/null
|
GH_TOKEN: ${{ secrets.API_TOKEN }}
|
||||||
git clone --depth=1 --single-branch https://github.com/whyour/qinglong-static.git static-mirror
|
QL_MIRROR_NOTIFY_REPO: whyour/qinglong-static
|
||||||
cd static-mirror
|
MIRROR_REF: ${{ github.ref_name }}
|
||||||
git remote add gitee git@gitee.com:whyour/qinglong-static.git 2>/dev/null
|
MIRROR_REF_TYPE: ${{ github.ref_type }}
|
||||||
|
QL_MIRROR_INCLUDE_VERSION_BRANCH: 'true'
|
||||||
|
run: |
|
||||||
|
bash scripts/wait-for-mirror.sh https://github.com/whyour/qinglong-static.git https://gitee.com/whyour/qinglong-static.git "$MIRROR_REF" "$MIRROR_REF_TYPE"
|
||||||
|
|
||||||
if git push --force --all gitee 2>&1; then
|
sync-existing-static:
|
||||||
echo "::notice::Gitee static push succeeded"
|
if: ${{ github.event_name == 'workflow_dispatch' && inputs.static_sync_only }}
|
||||||
else
|
runs-on: ubuntu-latest
|
||||||
echo "::warning::Push failed, trying to create repo via API..."
|
strategy:
|
||||||
curl -sS --connect-timeout 30 --max-time 60 \
|
fail-fast: false
|
||||||
-X POST "https://gitee.com/api/v5/user/repos" \
|
matrix:
|
||||||
-H "Content-Type: application/json" \
|
mirror: [gitlab]
|
||||||
-d '{"name":"qinglong-static","private":"false"}' \
|
concurrency:
|
||||||
"?access_token=$GITEE_TOKEN" 2>/dev/null
|
group: static-${{ matrix.mirror }}-${{ inputs.static_ref }}
|
||||||
git push --force --all gitee 2>&1 && echo "::notice::Gitee static push succeeded after repo creation" || echo "::warning::Gitee static push failed after retry"
|
cancel-in-progress: false
|
||||||
fi
|
steps:
|
||||||
git push --force --tags gitee 2>&1 || echo "::warning::Gitee static tags push failed"
|
- uses: actions/checkout@v6
|
||||||
|
- name: Synchronize existing static snapshot
|
||||||
|
env:
|
||||||
|
STATIC_REF: ${{ inputs.static_ref }}
|
||||||
|
STATIC_REF_TYPE: ${{ inputs.static_ref_type }}
|
||||||
|
MIRROR_HOST: ${{ matrix.mirror }}.com
|
||||||
|
MIRROR_SSH_KEY: ${{ secrets.GITLAB_SSH_PK }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
mkdir -p ~/.ssh
|
||||||
|
printf '%s\n' "$MIRROR_SSH_KEY" > ~/.ssh/id_rsa
|
||||||
|
chmod 600 ~/.ssh/id_rsa
|
||||||
|
ssh-keyscan -T 10 "$MIRROR_HOST" >> ~/.ssh/known_hosts 2>/dev/null
|
||||||
|
bash scripts/sync-static.sh "git@$MIRROR_HOST:whyour/qinglong-static.git" "$STATIC_REF" "$STATIC_REF_TYPE"
|
||||||
|
|
||||||
|
verify-existing-static-gitee:
|
||||||
|
if: ${{ github.event_name == 'workflow_dispatch' && inputs.static_sync_only }}
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 14
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
- name: Wait for existing Gitee static snapshot
|
||||||
|
env:
|
||||||
|
GH_TOKEN: ${{ secrets.API_TOKEN }}
|
||||||
|
QL_MIRROR_NOTIFY_REPO: whyour/qinglong-static
|
||||||
|
MIRROR_REF: ${{ inputs.static_ref }}
|
||||||
|
MIRROR_REF_TYPE: ${{ inputs.static_ref_type }}
|
||||||
|
QL_MIRROR_INCLUDE_VERSION_BRANCH: 'true'
|
||||||
|
run: |
|
||||||
|
bash scripts/wait-for-mirror.sh https://github.com/whyour/qinglong-static.git https://gitee.com/whyour/qinglong-static.git "$MIRROR_REF" "$MIRROR_REF_TYPE"
|
||||||
|
|
||||||
build-alpine:
|
build-alpine:
|
||||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||||
needs: build-static
|
needs: build-static
|
||||||
|
outputs:
|
||||||
|
image-digest: ${{ steps.image.outputs.digest }}
|
||||||
runs-on: ubuntu-22.04
|
runs-on: ubuntu-22.04
|
||||||
permissions:
|
permissions:
|
||||||
packages: write
|
packages: write
|
||||||
@@ -173,7 +251,7 @@ jobs:
|
|||||||
version: "8.3.1"
|
version: "8.3.1"
|
||||||
- uses: actions/setup-node@v6
|
- uses: actions/setup-node@v6
|
||||||
with:
|
with:
|
||||||
node-version: "20"
|
node-version: "24"
|
||||||
cache: "pnpm"
|
cache: "pnpm"
|
||||||
cache-dependency-path: pnpm-lock.yaml
|
cache-dependency-path: pnpm-lock.yaml
|
||||||
|
|
||||||
@@ -223,6 +301,7 @@ jobs:
|
|||||||
uses: docker/setup-buildx-action@v4
|
uses: docker/setup-buildx-action@v4
|
||||||
|
|
||||||
- name: Build and push (Alpine)
|
- name: Build and push (Alpine)
|
||||||
|
id: image
|
||||||
uses: docker/build-push-action@v7
|
uses: docker/build-push-action@v7
|
||||||
with:
|
with:
|
||||||
build-args: |
|
build-args: |
|
||||||
@@ -244,6 +323,8 @@ jobs:
|
|||||||
build-debian:
|
build-debian:
|
||||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||||
needs: build-static
|
needs: build-static
|
||||||
|
outputs:
|
||||||
|
image-digest: ${{ steps.image.outputs.digest }}
|
||||||
runs-on: ubuntu-22.04
|
runs-on: ubuntu-22.04
|
||||||
permissions:
|
permissions:
|
||||||
packages: write
|
packages: write
|
||||||
@@ -259,7 +340,7 @@ jobs:
|
|||||||
version: "8.3.1"
|
version: "8.3.1"
|
||||||
- uses: actions/setup-node@v6
|
- uses: actions/setup-node@v6
|
||||||
with:
|
with:
|
||||||
node-version: "20"
|
node-version: "24"
|
||||||
cache: "pnpm"
|
cache: "pnpm"
|
||||||
cache-dependency-path: pnpm-lock.yaml
|
cache-dependency-path: pnpm-lock.yaml
|
||||||
|
|
||||||
@@ -308,6 +389,7 @@ jobs:
|
|||||||
uses: docker/setup-buildx-action@v4
|
uses: docker/setup-buildx-action@v4
|
||||||
|
|
||||||
- name: Build and push (Debian)
|
- name: Build and push (Debian)
|
||||||
|
id: image
|
||||||
uses: docker/build-push-action@v7
|
uses: docker/build-push-action@v7
|
||||||
with:
|
with:
|
||||||
build-args: |
|
build-args: |
|
||||||
@@ -316,7 +398,7 @@ jobs:
|
|||||||
SOURCE_COMMIT=${{ github.sha }}
|
SOURCE_COMMIT=${{ github.sha }}
|
||||||
network: host
|
network: host
|
||||||
# Keep s390x on Debian: npm can hang under QEMU with Alpine (nodejs/docker-node#1973).
|
# Keep s390x on Debian: npm can hang under QEMU with Alpine (nodejs/docker-node#1973).
|
||||||
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
|
platforms: linux/amd64,linux/arm64,linux/ppc64le,linux/s390x
|
||||||
context: .
|
context: .
|
||||||
file: ./docker/Dockerfile.debian
|
file: ./docker/Dockerfile.debian
|
||||||
push: true
|
push: true
|
||||||
@@ -327,175 +409,78 @@ jobs:
|
|||||||
cache-from: type=registry,ref=whyour/qinglong:cache-debian
|
cache-from: type=registry,ref=whyour/qinglong:cache-debian
|
||||||
cache-to: type=registry,ref=whyour/qinglong:cache-debian,mode=max
|
cache-to: type=registry,ref=whyour/qinglong:cache-debian,mode=max
|
||||||
|
|
||||||
build-alpine310:
|
# One unmodified amd64 package for migration acceptance only. This does not
|
||||||
if: ${{ github.ref_name == 'master' }}
|
# publish bundles or advance development/formal channel indexes.
|
||||||
needs: build-static
|
build-npm-migration-bundle:
|
||||||
runs-on: ubuntu-22.04
|
needs: build-debian
|
||||||
permissions:
|
runs-on: ubuntu-latest
|
||||||
packages: write
|
timeout-minutes: 20
|
||||||
contents: read
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v6
|
||||||
- uses: actions/download-artifact@v7
|
- uses: docker/setup-buildx-action@v4
|
||||||
with:
|
- name: Export the official platform package from this run's image digest
|
||||||
name: qinglong-static-${{ github.sha }}
|
env:
|
||||||
path: static/
|
IMAGE_DIGEST: ${{ needs.build-debian.outputs.image-digest }}
|
||||||
- uses: pnpm/action-setup@v6
|
|
||||||
with:
|
|
||||||
version: "8.3.1"
|
|
||||||
- uses: actions/setup-node@v6
|
|
||||||
with:
|
|
||||||
node-version: "20"
|
|
||||||
cache: "pnpm"
|
|
||||||
cache-dependency-path: pnpm-lock.yaml
|
|
||||||
|
|
||||||
- name: Read version from version.yaml
|
|
||||||
id: version
|
|
||||||
run: |
|
run: |
|
||||||
VERSION=$(grep '^version:' version.yaml | awk '{print $2}')
|
set -euo pipefail
|
||||||
echo "version=$VERSION" >> $GITHUB_OUTPUT
|
[[ "$IMAGE_DIGEST" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||||
|
image="ghcr.io/${GITHUB_REPOSITORY}@${IMAGE_DIGEST}"
|
||||||
- name: Setup timezone
|
docker buildx build --platform linux/amd64 --build-arg "BASE_IMAGE=$image" --file docker/Dockerfile.upgrade-bundle --output "type=local,dest=$RUNNER_TEMP/npm-migration-bundle" .
|
||||||
run: sudo timedatectl set-timezone Asia/Shanghai
|
- uses: actions/upload-artifact@v6
|
||||||
|
|
||||||
- name: Login to DockerHub
|
|
||||||
uses: docker/login-action@v4
|
|
||||||
with:
|
with:
|
||||||
username: ${{ secrets.DOCKER_USERNAME }}
|
name: npm-migration-bundle-${{ github.sha }}
|
||||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
path: ${{ runner.temp }}/npm-migration-bundle/*
|
||||||
|
if-no-files-found: error
|
||||||
|
retention-days: 3
|
||||||
|
|
||||||
- name: Login to GHCR
|
verify-npm-package:
|
||||||
uses: docker/login-action@v4
|
needs: [build-static, build-npm-migration-bundle]
|
||||||
with:
|
runs-on: ubuntu-latest
|
||||||
registry: ghcr.io
|
timeout-minutes: 30
|
||||||
username: ${{ github.repository_owner }}
|
strategy:
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
- name: Set up QEMU
|
node: ['20.18.1', '24']
|
||||||
uses: docker/setup-qemu-action@v4
|
container: node:${{ matrix.node }}-bookworm
|
||||||
with:
|
|
||||||
cache-image: false
|
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
|
||||||
uses: docker/setup-buildx-action@v4
|
|
||||||
|
|
||||||
- name: Build and push (Alpine Python 3.10)
|
|
||||||
uses: docker/build-push-action@v7
|
|
||||||
with:
|
|
||||||
build-args: |
|
|
||||||
QL_MAINTAINER=${{ github.repository_owner }}
|
|
||||||
QL_BRANCH=${{ github.ref_name }}
|
|
||||||
SOURCE_COMMIT=${{ github.sha }}
|
|
||||||
network: host
|
|
||||||
platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x,linux/386
|
|
||||||
context: .
|
|
||||||
file: ./docker/Dockerfile.310
|
|
||||||
push: true
|
|
||||||
provenance: mode=max
|
|
||||||
sbom: true
|
|
||||||
tags: |
|
|
||||||
whyour/qinglong:python3.10
|
|
||||||
whyour/qinglong:${{ steps.version.outputs.version }}-python3.10
|
|
||||||
cache-from: type=registry,ref=whyour/qinglong:cache-alpine-python3.10
|
|
||||||
cache-to: type=registry,ref=whyour/qinglong:cache-alpine-python3.10,mode=max
|
|
||||||
|
|
||||||
build-debian310:
|
|
||||||
if: ${{ github.ref_name == 'master' }}
|
|
||||||
needs: build-static
|
|
||||||
runs-on: ubuntu-22.04
|
|
||||||
permissions:
|
|
||||||
packages: write
|
|
||||||
contents: read
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v6
|
||||||
- uses: actions/download-artifact@v7
|
- uses: actions/download-artifact@v6
|
||||||
with:
|
with:
|
||||||
name: qinglong-static-${{ github.sha }}
|
name: qinglong-npm-${{ github.sha }}
|
||||||
path: static/
|
path: ${{ runner.temp }}/npm-panel/
|
||||||
- uses: pnpm/action-setup@v6
|
- uses: actions/download-artifact@v6
|
||||||
with:
|
with:
|
||||||
version: "8.3.1"
|
name: npm-migration-bundle-${{ github.sha }}
|
||||||
- uses: actions/setup-node@v6
|
path: ${{ runner.temp }}/npm-migration-bundle/
|
||||||
with:
|
- name: Install isolated service test tools
|
||||||
node-version: "20"
|
|
||||||
cache: "pnpm"
|
|
||||||
cache-dependency-path: pnpm-lock.yaml
|
|
||||||
|
|
||||||
- name: Read version from version.yaml
|
|
||||||
id: version
|
|
||||||
run: |
|
run: |
|
||||||
VERSION=$(grep '^version:' version.yaml | awk '{print $2}')
|
apt-get update
|
||||||
echo "version=$VERSION" >> $GITHUB_OUTPUT
|
apt-get install -y --no-install-recommends python3 openssl curl jq nginx procps
|
||||||
|
# The published legacy launcher resets PATH to standard directories.
|
||||||
- name: Setup timezone
|
# /usr/local is confined to this disposable job container; PM2_HOME
|
||||||
run: sudo timedatectl set-timezone Asia/Shanghai
|
# remains private to each verification run.
|
||||||
|
npm install --global --prefix /usr/local --no-audit --no-fund pm2@6.0.14
|
||||||
- name: Login to DockerHub
|
- name: Install and upgrade the exact npm archive
|
||||||
uses: docker/login-action@v4
|
env:
|
||||||
|
QL_PANEL_PACKAGE_TEST_ISOLATED: '1'
|
||||||
|
QL_PANEL_TEST_PM2: /usr/local/bin/pm2
|
||||||
|
QL_PANEL_TEST_BUNDLE_DIR: ${{ runner.temp }}/npm-migration-bundle
|
||||||
|
run: node scripts/verify-panel-package.cjs verify "$RUNNER_TEMP/npm-panel" "$RUNNER_TEMP/npm-panel-results"
|
||||||
|
- uses: actions/upload-artifact@v6
|
||||||
with:
|
with:
|
||||||
username: ${{ secrets.DOCKER_USERNAME }}
|
name: qinglong-npm-verification-${{ matrix.node }}-${{ github.sha }}
|
||||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
path: ${{ runner.temp }}/npm-panel-results/
|
||||||
|
if-no-files-found: error
|
||||||
- name: Login to GHCR
|
retention-days: 14
|
||||||
uses: docker/login-action@v4
|
|
||||||
with:
|
|
||||||
registry: ghcr.io
|
|
||||||
username: ${{ github.repository_owner }}
|
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
|
|
||||||
- name: Set up QEMU
|
|
||||||
uses: docker/setup-qemu-action@v4
|
|
||||||
with:
|
|
||||||
cache-image: false
|
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
|
||||||
uses: docker/setup-buildx-action@v4
|
|
||||||
|
|
||||||
- name: Build and push (Debian Python 3.10)
|
|
||||||
uses: docker/build-push-action@v7
|
|
||||||
with:
|
|
||||||
build-args: |
|
|
||||||
QL_MAINTAINER=${{ github.repository_owner }}
|
|
||||||
QL_BRANCH=${{ github.ref_name }}
|
|
||||||
SOURCE_COMMIT=${{ github.sha }}
|
|
||||||
network: host
|
|
||||||
# Keep s390x on Debian: npm can hang under QEMU with Alpine (nodejs/docker-node#1973).
|
|
||||||
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
|
|
||||||
context: .
|
|
||||||
file: ./docker/Dockerfile.debian310
|
|
||||||
push: true
|
|
||||||
provenance: mode=max
|
|
||||||
sbom: true
|
|
||||||
tags: |
|
|
||||||
whyour/qinglong:debian-python3.10
|
|
||||||
whyour/qinglong:${{ steps.version.outputs.version }}-debian-python3.10
|
|
||||||
cache-from: type=registry,ref=whyour/qinglong:cache-debian-python3.10
|
|
||||||
cache-to: type=registry,ref=whyour/qinglong:cache-debian-python3.10,mode=max
|
|
||||||
|
|
||||||
publish:
|
publish:
|
||||||
if: ${{ github.repository == 'whyour/qinglong' && github.ref == 'refs/heads/master' }}
|
if: ${{ github.repository == 'whyour/qinglong' && (github.ref == 'refs/heads/master' || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/develop')) }}
|
||||||
needs: [build-alpine, build-debian, build-alpine310, build-debian310]
|
needs: [build-alpine, build-debian, verify-npm-package]
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
id-token: write
|
id-token: write
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
|
||||||
- uses: pnpm/action-setup@v3
|
|
||||||
with:
|
|
||||||
version: "8.3.1"
|
|
||||||
|
|
||||||
- uses: actions/setup-node@v3
|
|
||||||
with:
|
|
||||||
node-version: "20"
|
|
||||||
cache: "pnpm"
|
|
||||||
|
|
||||||
- name: build front and back
|
|
||||||
run: |
|
|
||||||
pnpm install --frozen-lockfile
|
|
||||||
pnpm build:front
|
|
||||||
pnpm build:back
|
|
||||||
|
|
||||||
- name: Set up Node for trusted publishing
|
- name: Set up Node for trusted publishing
|
||||||
uses: actions/setup-node@v6
|
uses: actions/setup-node@v6
|
||||||
with:
|
with:
|
||||||
@@ -503,18 +488,124 @@ jobs:
|
|||||||
registry-url: https://registry.npmjs.org
|
registry-url: https://registry.npmjs.org
|
||||||
package-manager-cache: false
|
package-manager-cache: false
|
||||||
|
|
||||||
|
- uses: actions/download-artifact@v6
|
||||||
|
with:
|
||||||
|
name: qinglong-npm-${{ github.sha }}
|
||||||
|
path: npm-panel/
|
||||||
|
|
||||||
- name: Publish npm package with OIDC
|
- name: Publish npm package with OIDC
|
||||||
shell: bash
|
shell: bash
|
||||||
|
env:
|
||||||
|
NPM_RC_VERSION: ${{ inputs.npm_rc_version }}
|
||||||
run: |
|
run: |
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
name=$(node -p 'require("./package.json").name')
|
shopt -s nullglob
|
||||||
version=$(node -p 'require("./package.json").version')
|
archives=(npm-panel/*.tgz)
|
||||||
|
if [ "${#archives[@]}" -ne 1 ]; then
|
||||||
|
echo '::error::Expected exactly one verified panel npm archive.'
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
archive="${archives[0]}"
|
||||||
|
metadata=$(tar -xOf "$archive" package/package.json)
|
||||||
|
name=$(jq -er '.name | select(. == "@whyour/qinglong")' <<< "$metadata")
|
||||||
|
version=$(jq -er '.version | select(type == "string")' <<< "$metadata")
|
||||||
|
if [[ "${GITHUB_EVENT_NAME:-}" == workflow_dispatch && "${GITHUB_REF:-}" == refs/heads/develop ]]; then
|
||||||
|
if [[ ! "${NPM_RC_VERSION:-}" =~ ^[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+$ || "$version" != "${NPM_RC_VERSION:-}" ]]; then
|
||||||
|
echo '::error::Develop npm publication requires the exact requested RC archive version.'
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
if npm view "$name@$version" version --json --registry=https://registry.npmjs.org > "$RUNNER_TEMP/npm-version.json" 2> "$RUNNER_TEMP/npm-version-error.log"; then
|
if npm view "$name@$version" version --json --registry=https://registry.npmjs.org > "$RUNNER_TEMP/npm-version.json" 2> "$RUNNER_TEMP/npm-version-error.log"; then
|
||||||
jq -e --arg version "$version" '. == $version' "$RUNNER_TEMP/npm-version.json" > /dev/null
|
jq -e --arg version "$version" '. == $version' "$RUNNER_TEMP/npm-version.json" > /dev/null
|
||||||
echo "::notice::$name@$version is already published; skipping npm publication for this image rebuild."
|
echo "::notice::$name@$version is already published; skipping npm publication for this image rebuild."
|
||||||
elif jq -e '.error.code == "E404"' "$RUNNER_TEMP/npm-version.json" > /dev/null; then
|
elif jq -e '.error.code == "E404"' "$RUNNER_TEMP/npm-version.json" > /dev/null; then
|
||||||
npm publish --access public --registry=https://registry.npmjs.org
|
if [[ "${GITHUB_EVENT_NAME:-}" == workflow_dispatch && "${GITHUB_REF:-}" == refs/heads/develop ]]; then
|
||||||
|
npm publish "./$archive" --ignore-scripts --access public --tag rc --registry=https://registry.npmjs.org
|
||||||
|
else
|
||||||
|
npm publish "./$archive" --ignore-scripts --access public --registry=https://registry.npmjs.org
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
echo '::error::Could not check the published npm version; refusing to publish.'
|
echo '::error::Could not check the published npm version; refusing to publish.'
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
build-upgrade-bundles:
|
||||||
|
# Both channels publish packages from this run's immutable image digests.
|
||||||
|
if: ${{ !inputs.static_sync_only && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop') }}
|
||||||
|
needs: [build-alpine, build-debian]
|
||||||
|
# Node 24 can SIGILL while archiving ARM64 musl under x64 QEMU.
|
||||||
|
runs-on: ${{ matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-22.04' }}
|
||||||
|
timeout-minutes: 90
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
packages: write
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
include:
|
||||||
|
- { platform: linux/amd64, libc: musl, name: x64-musl }
|
||||||
|
- { platform: linux/arm/v6, libc: musl, name: armv6-musl }
|
||||||
|
- { platform: linux/arm/v7, libc: musl, name: armv7-musl }
|
||||||
|
- { platform: linux/arm64, libc: musl, name: arm64-musl }
|
||||||
|
- { platform: linux/ppc64le, libc: musl, name: ppc64-musl }
|
||||||
|
- { platform: linux/s390x, libc: musl, name: s390x-musl }
|
||||||
|
- { platform: linux/386, libc: musl, name: ia32-musl }
|
||||||
|
- { platform: linux/amd64, libc: glibc, name: x64-glibc }
|
||||||
|
- { platform: linux/arm64, libc: glibc, name: arm64-glibc }
|
||||||
|
- { platform: linux/ppc64le, libc: glibc, name: ppc64-glibc }
|
||||||
|
- { platform: linux/s390x, libc: glibc, name: s390x-glibc }
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
- uses: docker/setup-qemu-action@v4
|
||||||
|
if: ${{ matrix.platform != 'linux/arm64' }}
|
||||||
|
with:
|
||||||
|
cache-image: false
|
||||||
|
- uses: docker/setup-buildx-action@v4
|
||||||
|
- name: Build and probe the immutable platform package
|
||||||
|
env:
|
||||||
|
ALPINE_DIGEST: ${{ needs.build-alpine.outputs.image-digest }}
|
||||||
|
DEBIAN_DIGEST: ${{ needs.build-debian.outputs.image-digest }}
|
||||||
|
BUNDLE_LIBC: ${{ matrix.libc }}
|
||||||
|
BUNDLE_PLATFORM: ${{ matrix.platform }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
digest="$ALPINE_DIGEST"
|
||||||
|
if [ "$BUNDLE_LIBC" = glibc ]; then digest="$DEBIAN_DIGEST"; fi
|
||||||
|
test -n "$digest"
|
||||||
|
image="ghcr.io/${GITHUB_REPOSITORY}@${digest}"
|
||||||
|
docker buildx build --platform "$BUNDLE_PLATFORM" --build-arg "BASE_IMAGE=$image" --file docker/Dockerfile.upgrade-bundle --output type=local,dest=output/upgrade .
|
||||||
|
- uses: actions/upload-artifact@v6
|
||||||
|
with:
|
||||||
|
name: upgrade-${{ matrix.name }}-${{ github.sha }}
|
||||||
|
path: output/upgrade/*
|
||||||
|
if-no-files-found: error
|
||||||
|
|
||||||
|
publish-upgrade-bundles:
|
||||||
|
if: ${{ !inputs.static_sync_only && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop') }}
|
||||||
|
needs: build-upgrade-bundles
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
|
concurrency:
|
||||||
|
group: upgrade-channel-${{ github.ref_name }}
|
||||||
|
cancel-in-progress: false
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
- uses: actions/setup-node@v6
|
||||||
|
with:
|
||||||
|
node-version: '24'
|
||||||
|
- uses: pnpm/action-setup@v6
|
||||||
|
with:
|
||||||
|
version: '8.3.1'
|
||||||
|
- run: pnpm install --prod --frozen-lockfile
|
||||||
|
- run: npm --prefix cli ci
|
||||||
|
- uses: actions/download-artifact@v7
|
||||||
|
with:
|
||||||
|
pattern: upgrade-*-${{ github.sha }}
|
||||||
|
path: output/upgrade
|
||||||
|
merge-multiple: true
|
||||||
|
- run: node scripts/build-bundle-installer.cjs output/upgrade/install-bundle.cjs
|
||||||
|
- name: Publish complete packages before advancing the channel
|
||||||
|
env:
|
||||||
|
GH_TOKEN: ${{ github.token }}
|
||||||
|
run: node scripts/publish-upgrade-bundles.cjs output/upgrade "$GITHUB_SHA" "$GITHUB_REPOSITORY" "$GITHUB_REF_NAME"
|
||||||
|
|||||||
@@ -21,16 +21,27 @@ on:
|
|||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
inputs:
|
inputs:
|
||||||
publish:
|
publish:
|
||||||
description: Publish the verified CLI package (master only)
|
description: Publish the verified CLI package (master stable / develop prerelease)
|
||||||
type: boolean
|
type: boolean
|
||||||
default: false
|
default: false
|
||||||
|
prerelease_version:
|
||||||
|
description: Develop only, e.g. 0.1.5-beta.0 (alpha/beta/rc)
|
||||||
|
type: string
|
||||||
|
default: ''
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
group: cli-package-${{ github.workflow }}-${{ github.ref }}
|
# Keep validation runs separate from releases; never interrupt a running release.
|
||||||
cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
|
# Master push releases share the same group as manually requested releases.
|
||||||
|
group: >-
|
||||||
|
cli-package-${{ github.workflow }}-${{ github.ref }}-${{
|
||||||
|
((github.event_name == 'workflow_dispatch' && inputs.publish) ||
|
||||||
|
(github.event_name == 'push' && github.ref == 'refs/heads/master')) && 'publish' || 'check' }}
|
||||||
|
cancel-in-progress: >-
|
||||||
|
${{ github.ref != 'refs/heads/master' &&
|
||||||
|
!(github.event_name == 'workflow_dispatch' && inputs.publish) }}
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
package:
|
package:
|
||||||
@@ -52,6 +63,20 @@ jobs:
|
|||||||
sudo apt-get update
|
sudo apt-get update
|
||||||
sudo apt-get install -y --no-install-recommends bash ca-certificates curl git jq perl procps python3 unzip zip
|
sudo apt-get install -y --no-install-recommends bash ca-certificates curl git jq perl procps python3 unzip zip
|
||||||
npm install --global --ignore-scripts --no-audit --no-fund ts-node@10.9.2 typescript@5.2.2
|
npm install --global --ignore-scripts --no-audit --no-fund ts-node@10.9.2 typescript@5.2.2
|
||||||
|
- name: Set requested prerelease version before verification
|
||||||
|
if: github.event_name == 'workflow_dispatch' && inputs.prerelease_version != ''
|
||||||
|
working-directory: cli
|
||||||
|
env:
|
||||||
|
PRERELEASE_VERSION: ${{ inputs.prerelease_version }}
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
if [[ "$GITHUB_REF" != refs/heads/develop ||
|
||||||
|
! "$PRERELEASE_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+-(alpha|beta|rc)\.[0-9]+$ ]]; then
|
||||||
|
echo '::error::Prereleases require develop and an explicit alpha/beta/rc version.'
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
npm version "$PRERELEASE_VERSION" --no-git-tag-version --ignore-scripts
|
||||||
- run: npm ci --prefix cli --no-audit --no-fund
|
- run: npm ci --prefix cli --no-audit --no-fund
|
||||||
- run: npm run check:cli
|
- run: npm run check:cli
|
||||||
- run: npm run test:cli
|
- run: npm run test:cli
|
||||||
@@ -68,13 +93,33 @@ jobs:
|
|||||||
if-no-files-found: error
|
if-no-files-found: error
|
||||||
retention-days: 14
|
retention-days: 14
|
||||||
|
|
||||||
|
platform-smoke:
|
||||||
|
runs-on: ${{ matrix.os }}
|
||||||
|
timeout-minutes: 15
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
os: [windows-latest, ubuntu-24.04-arm]
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
- uses: actions/setup-node@v6
|
||||||
|
with:
|
||||||
|
node-version: '24'
|
||||||
|
cache: npm
|
||||||
|
cache-dependency-path: cli/package-lock.json
|
||||||
|
- run: npm ci --prefix cli --no-audit --no-fund
|
||||||
|
- run: npm run build:cli
|
||||||
|
- run: node --test cli/test/remote/platformSmoke.test.cjs
|
||||||
|
|
||||||
publish:
|
publish:
|
||||||
needs: package
|
needs: [package, platform-smoke]
|
||||||
if: >-
|
if: >-
|
||||||
github.repository == 'whyour/qinglong' &&
|
github.repository == 'whyour/qinglong' &&
|
||||||
github.ref == 'refs/heads/master' &&
|
((github.ref == 'refs/heads/master' &&
|
||||||
(github.event_name == 'push' ||
|
(github.event_name == 'push' ||
|
||||||
(github.event_name == 'workflow_dispatch' && inputs.publish))
|
(github.event_name == 'workflow_dispatch' && inputs.publish))) ||
|
||||||
|
(github.ref == 'refs/heads/develop' && github.event_name == 'workflow_dispatch' &&
|
||||||
|
inputs.publish && inputs.prerelease_version != ''))
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 5
|
timeout-minutes: 5
|
||||||
permissions:
|
permissions:
|
||||||
@@ -91,6 +136,8 @@ jobs:
|
|||||||
name: qinglong-cli-${{ github.sha }}
|
name: qinglong-cli-${{ github.sha }}
|
||||||
path: cli-package
|
path: cli-package
|
||||||
- name: Publish verified npm archive
|
- name: Publish verified npm archive
|
||||||
|
env:
|
||||||
|
PRERELEASE_VERSION: ${{ inputs.prerelease_version }}
|
||||||
shell: bash
|
shell: bash
|
||||||
run: |
|
run: |
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
@@ -103,13 +150,30 @@ jobs:
|
|||||||
archive="${archives[0]}"
|
archive="${archives[0]}"
|
||||||
metadata=$(tar -xOf "$archive" package/package.json)
|
metadata=$(tar -xOf "$archive" package/package.json)
|
||||||
name=$(jq -er '.name | select(. == "@whyour/qinglong-cli")' <<< "$metadata")
|
name=$(jq -er '.name | select(. == "@whyour/qinglong-cli")' <<< "$metadata")
|
||||||
version=$(jq -er '.version | select(type == "string" and test("^[0-9]+\\.[0-9]+\\.[0-9]+$"))' <<< "$metadata")
|
version=$(jq -er '.version | select(type == "string")' <<< "$metadata")
|
||||||
|
if [[ "$GITHUB_REF" == refs/heads/develop ]]; then
|
||||||
|
if [[ "$version" != "$PRERELEASE_VERSION" ||
|
||||||
|
! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+-(alpha|beta|rc)\.[0-9]+$ ]]; then
|
||||||
|
echo '::error::Archive version does not match the requested prerelease.'
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
tag="${BASH_REMATCH[1]}"
|
||||||
|
elif [[ "$GITHUB_REF" == refs/heads/master && "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||||
|
tag=latest
|
||||||
|
else
|
||||||
|
echo '::error::Refusing to publish this branch/version combination.'
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
if npm view "$name@$version" version --json --registry=https://registry.npmjs.org > version.json 2> version-error.log; then
|
if npm view "$name@$version" version --json --registry=https://registry.npmjs.org > version.json 2> version-error.log; then
|
||||||
jq -e --arg version "$version" '. == $version' version.json > /dev/null
|
jq -e --arg version "$version" '. == $version' version.json > /dev/null
|
||||||
|
if [[ "$tag" != latest ]]; then
|
||||||
|
echo '::error::Prerelease version already exists; choose a new version to test this build.'
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
echo "::notice::$name@$version is already published; bump cli/package.json and its lockfile to release changes."
|
echo "::notice::$name@$version is already published; bump cli/package.json and its lockfile to release changes."
|
||||||
exit 0
|
exit 0
|
||||||
elif ! jq -e '.error.code == "E404"' version.json > /dev/null; then
|
elif ! jq -e '.error.code == "E404"' version.json > /dev/null; then
|
||||||
echo '::error::Could not check the published CLI version; refusing to publish.'
|
echo '::error::Could not check the published CLI version; refusing to publish.'
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
npm publish "./$archive" --access public --tag latest --ignore-scripts --registry=https://registry.npmjs.org
|
npm publish "./$archive" --access public --tag "$tag" --ignore-scripts --registry=https://registry.npmjs.org
|
||||||
|
|||||||
@@ -0,0 +1,171 @@
|
|||||||
|
name: Release readiness
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
paths:
|
||||||
|
- '.github/workflows/release-readiness.yml'
|
||||||
|
- 'package.json'
|
||||||
|
- 'pnpm-lock.yaml'
|
||||||
|
- 'docker/Dockerfile*'
|
||||||
|
- 'shell/node-runtime-probe.cjs'
|
||||||
|
- 'scripts/verify-panel-package.cjs'
|
||||||
|
- 'scripts/verify-image-node-layers.py'
|
||||||
|
- 'test/back/node-runtime-probe.test.cjs'
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: release-readiness-${{ github.event.pull_request.number || github.ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
regression:
|
||||||
|
uses: ./.github/workflows/validate.yml
|
||||||
|
|
||||||
|
static:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 15
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
with:
|
||||||
|
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||||
|
- uses: pnpm/action-setup@v6
|
||||||
|
with:
|
||||||
|
version: '8.3.1'
|
||||||
|
- uses: actions/setup-node@v6
|
||||||
|
with:
|
||||||
|
node-version: '20'
|
||||||
|
cache: pnpm
|
||||||
|
- run: pnpm install --frozen-lockfile
|
||||||
|
- run: pnpm build:front && pnpm build:back && pnpm build:info
|
||||||
|
- uses: actions/upload-artifact@v6
|
||||||
|
with:
|
||||||
|
name: release-readiness-static
|
||||||
|
path: static/
|
||||||
|
include-hidden-files: true
|
||||||
|
if-no-files-found: error
|
||||||
|
- name: Pack the verified npm archive for isolated runtime validation
|
||||||
|
run: node scripts/verify-panel-package.cjs pack "$RUNNER_TEMP/npm-panel"
|
||||||
|
- uses: actions/upload-artifact@v6
|
||||||
|
with:
|
||||||
|
name: release-readiness-npm-panel
|
||||||
|
path: ${{ runner.temp }}/npm-panel/
|
||||||
|
if-no-files-found: error
|
||||||
|
|
||||||
|
images:
|
||||||
|
needs: [regression, static]
|
||||||
|
runs-on: ubuntu-22.04
|
||||||
|
timeout-minutes: 60
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
include:
|
||||||
|
- image: alpine
|
||||||
|
file: docker/Dockerfile
|
||||||
|
cache: alpine
|
||||||
|
platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x,linux/386
|
||||||
|
- image: debian
|
||||||
|
file: docker/Dockerfile.debian
|
||||||
|
cache: debian
|
||||||
|
platforms: linux/amd64,linux/arm64,linux/ppc64le,linux/s390x
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
with:
|
||||||
|
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||||
|
- name: Record source
|
||||||
|
id: source
|
||||||
|
env:
|
||||||
|
SOURCE_URL: ${{ github.event.pull_request.head.repo.clone_url || format('https://github.com/{0}.git', github.repository) }}
|
||||||
|
SOURCE_BRANCH: ${{ github.head_ref || github.ref_name }}
|
||||||
|
run: |
|
||||||
|
echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "url=$SOURCE_URL" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "branch=$SOURCE_BRANCH" >> "$GITHUB_OUTPUT"
|
||||||
|
- uses: actions/download-artifact@v7
|
||||||
|
with:
|
||||||
|
name: release-readiness-static
|
||||||
|
path: static/
|
||||||
|
- uses: docker/setup-qemu-action@v4
|
||||||
|
with:
|
||||||
|
cache-image: false
|
||||||
|
- uses: docker/setup-buildx-action@v4
|
||||||
|
- name: Build every release architecture
|
||||||
|
uses: docker/build-push-action@v7
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: ${{ matrix.file }}
|
||||||
|
platforms: ${{ matrix.platforms }}
|
||||||
|
build-args: |
|
||||||
|
QL_URL=${{ steps.source.outputs.url }}
|
||||||
|
QL_BRANCH=${{ steps.source.outputs.branch }}
|
||||||
|
SOURCE_COMMIT=${{ steps.source.outputs.sha }}
|
||||||
|
push: false
|
||||||
|
provenance: false
|
||||||
|
sbom: false
|
||||||
|
cache-from: type=registry,ref=whyour/qinglong:cache-${{ matrix.cache }}
|
||||||
|
- name: Load native image for runtime verification
|
||||||
|
uses: docker/build-push-action@v7
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: ${{ matrix.file }}
|
||||||
|
platforms: linux/amd64
|
||||||
|
build-args: |
|
||||||
|
QL_URL=${{ steps.source.outputs.url }}
|
||||||
|
QL_BRANCH=${{ steps.source.outputs.branch }}
|
||||||
|
SOURCE_COMMIT=${{ steps.source.outputs.sha }}
|
||||||
|
tags: qinglong-readiness:${{ matrix.image }}
|
||||||
|
load: true
|
||||||
|
push: false
|
||||||
|
provenance: false
|
||||||
|
sbom: false
|
||||||
|
- name: Verify native production runtime without external networking
|
||||||
|
run: |
|
||||||
|
docker run --rm --network none --entrypoint node \
|
||||||
|
qinglong-readiness:${{ matrix.image }} \
|
||||||
|
-e 'const assert=require("node:assert/strict"); const {createRequire}=require("node:module"); for(const [name,want] of Object.entries({"@grpc/grpc-js":"1.14.5",compression:"1.8.2",multer:"2.4.0",sequelize:"6.37.8"}))assert.equal(require(name+"/package.json").version,want); const expressRequire=createRequire(require.resolve("express")); assert.equal(expressRequire("proxy-addr/package.json").version,"2.0.8"); const sockRequire=createRequire(require.resolve("sockjs")); assert.equal(sockRequire("websocket-driver/package.json").version,"0.7.5"); require("sqlite3"); require("nodemailer"); require("undici"); console.log("Minimal production patches and runtime imports verified")'
|
||||||
|
- name: Load ARMv6 image for SQLite and Node regression checks
|
||||||
|
if: startsWith(matrix.image, 'alpine')
|
||||||
|
uses: docker/build-push-action@v7
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: ${{ matrix.file }}
|
||||||
|
platforms: linux/arm/v6
|
||||||
|
build-args: |
|
||||||
|
QL_URL=${{ steps.source.outputs.url }}
|
||||||
|
QL_BRANCH=${{ steps.source.outputs.branch }}
|
||||||
|
SOURCE_COMMIT=${{ steps.source.outputs.sha }}
|
||||||
|
tags: qinglong-readiness:${{ matrix.image }}-armv6
|
||||||
|
load: true
|
||||||
|
push: false
|
||||||
|
provenance: false
|
||||||
|
sbom: false
|
||||||
|
- name: Verify ARMv6 SQLite on an ARM1176 CPU without networking
|
||||||
|
if: startsWith(matrix.image, 'alpine')
|
||||||
|
run: |
|
||||||
|
docker run --rm --network none --platform linux/arm/v6 \
|
||||||
|
-e QEMU_CPU=arm1176 --entrypoint node \
|
||||||
|
qinglong-readiness:${{ matrix.image }}-armv6 --jitless \
|
||||||
|
-e 'const assert=require("node:assert/strict"); const sqlite3=require("sqlite3"); const db=new sqlite3.Database(":memory:"); db.get("SELECT 42 AS n",(error,row)=>{assert.ifError(error); assert.equal(row.n,42); db.close()});'
|
||||||
|
- name: Check Node binaries in every serialized image layer
|
||||||
|
if: startsWith(matrix.image, 'alpine')
|
||||||
|
run: |
|
||||||
|
docker save qinglong-readiness:${{ matrix.image }} -o "$RUNNER_TEMP/node-layers-amd64.tar"
|
||||||
|
docker save qinglong-readiness:${{ matrix.image }}-armv6 -o "$RUNNER_TEMP/node-layers-arm.tar"
|
||||||
|
python3 scripts/verify-image-node-layers.py "$RUNNER_TEMP/node-layers-amd64.tar" amd64
|
||||||
|
python3 scripts/verify-image-node-layers.py "$RUNNER_TEMP/node-layers-arm.tar" arm
|
||||||
|
rm "$RUNNER_TEMP/node-layers-amd64.tar" "$RUNNER_TEMP/node-layers-arm.tar"
|
||||||
|
- name: Export native image for isolated dev validation
|
||||||
|
if: matrix.image == 'debian' || matrix.image == 'alpine'
|
||||||
|
run: |
|
||||||
|
docker save qinglong-readiness:${{ matrix.image }} | gzip -1 > release-readiness-${{ matrix.image }}-image.tar.gz
|
||||||
|
sha256sum release-readiness-${{ matrix.image }}-image.tar.gz > release-readiness-${{ matrix.image }}-image.sha256
|
||||||
|
docker image inspect qinglong-readiness:${{ matrix.image }} > release-readiness-${{ matrix.image }}-image.json
|
||||||
|
- uses: actions/upload-artifact@v6
|
||||||
|
if: matrix.image == 'debian' || matrix.image == 'alpine'
|
||||||
|
with:
|
||||||
|
name: release-readiness-${{ matrix.image }}-image
|
||||||
|
path: release-readiness-${{ matrix.image }}-image.*
|
||||||
|
compression-level: 0
|
||||||
|
if-no-files-found: error
|
||||||
+10
@@ -29,6 +29,7 @@ __pycache__
|
|||||||
/shell/preload/*-notify.json
|
/shell/preload/*-notify.json
|
||||||
/shell/preload/__ql_notify__.*
|
/shell/preload/__ql_notify__.*
|
||||||
/shell/preload/lang_env.sh
|
/shell/preload/lang_env.sh
|
||||||
|
/shell/upgrade-lock.yaml
|
||||||
|
|
||||||
.deepseek/
|
.deepseek/
|
||||||
.claude/
|
.claude/
|
||||||
@@ -38,3 +39,12 @@ __pycache__
|
|||||||
|
|
||||||
# Local security audit artifacts
|
# Local security audit artifacts
|
||||||
/audit/
|
/audit/
|
||||||
|
|
||||||
|
# Local browser tests, screenshots, reports, and temporary test output
|
||||||
|
/.playwright-cli/
|
||||||
|
/output/
|
||||||
|
/playwright-report/
|
||||||
|
/test-results/
|
||||||
|
/blob-report/
|
||||||
|
/coverage/
|
||||||
|
/.nyc_output/
|
||||||
|
|||||||
-22
@@ -1,22 +0,0 @@
|
|||||||
/.tmp/
|
|
||||||
/.github/
|
|
||||||
/.vscode/
|
|
||||||
/.history/
|
|
||||||
/back/**/*.ts
|
|
||||||
/back/**/*.json
|
|
||||||
/cli/
|
|
||||||
/data/
|
|
||||||
/src/
|
|
||||||
/static/**/*.js.map
|
|
||||||
/static/**/*.gz
|
|
||||||
/.editorconfig
|
|
||||||
/.gitignore
|
|
||||||
/.prettierignore
|
|
||||||
/.prettierrc
|
|
||||||
/.umirc.ts
|
|
||||||
/nodemon.json
|
|
||||||
/pnpm-lock.yaml
|
|
||||||
/tsconfig.back.json
|
|
||||||
/tsconfig.json
|
|
||||||
/typings.d.ts
|
|
||||||
/.env
|
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
# Node.js 原生部署与 Python venv
|
||||||
|
|
||||||
|
设置 `QL_DIR` 为安装目录,`QL_DATA_DIR` 为以 `/data` 结尾的绝对持久化数据路径。
|
||||||
|
如需隔离 Python,请在启动进程的环境中显式设置 `PYTHON_VENV_DIR`(绝对路径或相对 `QL_DIR` 的路径),例如 `export PYTHON_VENV_DIR=/srv/qinglong/.venv`。未设置时保留现有 Python prefix 行为。
|
||||||
|
|
||||||
|
- `qinglong`(`shell/start.sh`)安装系统依赖和 Node 工具,并安装 Python requests,然后调用公共启动脚本。
|
||||||
|
- `npm run start:simple`(`shell/start-simplify.sh`)使用已准备的 Node、pm2、nginx、jq 等依赖,不安装系统包、全局 npm 包或 requests;显式指定的 venv 不存在时仍会创建。请先在 venv 中安装 requests 和任务所需依赖。
|
||||||
|
- 两种启动方式均保留已有 `.env`;使用 venv 时清除继承的 `PYTHONPATH`/`PYTHONUSERBASE`,依赖查询、安装和卸载使用该 venv 的 Python。
|
||||||
|
|
||||||
|
该模式需具备现有启动流程所要求的 nginx 配置和 pm2 服务权限。生产环境设置变量应通过 systemd 的 `Environment` 或相应进程配置完成。已有系统 Python 依赖不自动迁移至 venv,需在 venv 中重新安装。
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
import { NextFunction, Request, Response, Router } from 'express';
|
import { NextFunction, Request, Response, Router } from 'express';
|
||||||
import { celebrate, Joi } from 'celebrate';
|
import { celebrate, Joi } from 'celebrate';
|
||||||
|
import { t } from '../shared/i18n';
|
||||||
import {
|
import {
|
||||||
diagnoseClientIp,
|
diagnoseClientIp,
|
||||||
getTrustProxyConfig,
|
getTrustProxyConfig,
|
||||||
@@ -36,7 +37,7 @@ export default (app: Router) => {
|
|||||||
} catch (error) {
|
} catch (error) {
|
||||||
res.send({
|
res.send({
|
||||||
code: 400,
|
code: 400,
|
||||||
message: error instanceof Error ? error.message : '配置更新失败',
|
message: error instanceof Error ? error.message : t('配置更新失败'),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|||||||
+3
-7
@@ -4,7 +4,6 @@ import { Logger } from 'winston';
|
|||||||
import config from '../config';
|
import config from '../config';
|
||||||
import * as fs from 'fs/promises';
|
import * as fs from 'fs/promises';
|
||||||
import { celebrate, Joi } from 'celebrate';
|
import { celebrate, Joi } from 'celebrate';
|
||||||
import { join, basename } from 'path';
|
|
||||||
import { SAMPLE_FILES } from '../config/const';
|
import { SAMPLE_FILES } from '../config/const';
|
||||||
import { t } from '../shared/i18n';
|
import { t } from '../shared/i18n';
|
||||||
import ConfigService from '../services/config';
|
import ConfigService from '../services/config';
|
||||||
@@ -73,15 +72,12 @@ export default (app: Router) => {
|
|||||||
const logger: Logger = Container.get('logger');
|
const logger: Logger = Container.get('logger');
|
||||||
try {
|
try {
|
||||||
const { name, content } = req.body;
|
const { name, content } = req.body;
|
||||||
// Resolve path first to prevent traversal attacks
|
|
||||||
let basePath = config.configPath;
|
|
||||||
if (name.startsWith('data/scripts/')) {
|
if (name.startsWith('data/scripts/')) {
|
||||||
basePath = join(config.rootPath, 'data/scripts');
|
return res.send({ code: 403, message: t('文件无法访问') });
|
||||||
}
|
}
|
||||||
const cleanName = name.replace(/^data\/scripts\//, '');
|
|
||||||
const normalized = resolveFileAccess(
|
const normalized = resolveFileAccess(
|
||||||
basePath,
|
config.configPath,
|
||||||
[cleanName],
|
[name],
|
||||||
config.blackFileList,
|
config.blackFileList,
|
||||||
);
|
);
|
||||||
if (!normalized) {
|
if (!normalized) {
|
||||||
|
|||||||
+122
-11
@@ -1,4 +1,10 @@
|
|||||||
|
import { executionIdPattern, executionTriggerId, receiptFinishTime } from '../shared/executionIdentity';
|
||||||
|
import { readDailyResults } from '../shared/legacyDailyStatistics';
|
||||||
|
import { PendingCronInstall } from '../shared/errors';
|
||||||
|
import { triggerHistory } from '../shared/triggerHistory';
|
||||||
|
import { previewSchedule, withNextRun } from '../shared/cronPreview';
|
||||||
import { Router, Request, Response, NextFunction } from 'express';
|
import { Router, Request, Response, NextFunction } from 'express';
|
||||||
|
import { literal, Op } from 'sequelize';
|
||||||
import { Container } from 'typedi';
|
import { Container } from 'typedi';
|
||||||
import { Logger } from 'winston';
|
import { Logger } from 'winston';
|
||||||
import CronService from '../services/cron';
|
import CronService from '../services/cron';
|
||||||
@@ -164,14 +170,25 @@ export default (app: Router) => {
|
|||||||
const logger: Logger = Container.get('logger');
|
const logger: Logger = Container.get('logger');
|
||||||
try {
|
try {
|
||||||
const cronService = Container.get(CronService);
|
const cronService = Container.get(CronService);
|
||||||
const data = await cronService.crontabs(req.query as any);
|
const result = await cronService.crontabs(req.query as any, true);
|
||||||
return res.send({ code: 200, data });
|
return res.send({ code: 200, data: { ...result, data: result.data.map(withNextRun) } });
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
logger.error('🔥 error: %o', e);
|
logger.error('🔥 error: %o', e);
|
||||||
return next(e);
|
return next(e);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
route.get('/preview', celebrate({ query: Joi.object({
|
||||||
|
schedule: Joi.string().required(),
|
||||||
|
}).unknown(true) }), (req, res) => {
|
||||||
|
try {
|
||||||
|
const times = previewSchedule(String(req.query.schedule), 3, Date.now());
|
||||||
|
return res.send({ code: 200, data: { times, timezone: Intl.DateTimeFormat().resolvedOptions().timeZone } });
|
||||||
|
} catch {
|
||||||
|
return res.status(400).send({ code: 400, message: t('Cron表达式格式有误') });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
route.get(
|
route.get(
|
||||||
'/detail',
|
'/detail',
|
||||||
async (req: Request, res: Response, next: NextFunction) => {
|
async (req: Request, res: Response, next: NextFunction) => {
|
||||||
@@ -199,6 +216,11 @@ export default (app: Router) => {
|
|||||||
const data = await cronService.create(req.body);
|
const data = await cronService.create(req.body);
|
||||||
return res.send({ code: 200, data });
|
return res.send({ code: 200, data });
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
if (e instanceof PendingCronInstall) {
|
||||||
|
return res.status(e.status).send({
|
||||||
|
code: e.status, message: e.message, data: { id: e.cronId, saved: false },
|
||||||
|
});
|
||||||
|
}
|
||||||
return next(e);
|
return next(e);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -224,13 +246,14 @@ export default (app: Router) => {
|
|||||||
route.put(
|
route.put(
|
||||||
'/stop',
|
'/stop',
|
||||||
celebrate({
|
celebrate({
|
||||||
|
headers: Joi.object({ 'idempotency-key': Joi.string().guid().optional() }).unknown(true),
|
||||||
body: Joi.array().items(Joi.number().required()),
|
body: Joi.array().items(Joi.number().required()),
|
||||||
}),
|
}),
|
||||||
async (req: Request, res: Response, next: NextFunction) => {
|
async (req: Request, res: Response, next: NextFunction) => {
|
||||||
const logger: Logger = Container.get('logger');
|
const logger: Logger = Container.get('logger');
|
||||||
try {
|
try {
|
||||||
const cronService = Container.get(CronService);
|
const cronService = Container.get(CronService);
|
||||||
const data = await cronService.stop(req.body);
|
const data = await cronService.stop(req.body, req.get('Idempotency-Key'));
|
||||||
return res.send({ code: 200, data });
|
return res.send({ code: 200, data });
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
return next(e);
|
return next(e);
|
||||||
@@ -322,6 +345,11 @@ export default (app: Router) => {
|
|||||||
id: Joi.number().required(),
|
id: Joi.number().required(),
|
||||||
}),
|
}),
|
||||||
query: Joi.object({
|
query: Joi.object({
|
||||||
|
triggerId: Joi.string().max(256).optional(),
|
||||||
|
instanceId: Joi.number().integer().positive().optional(),
|
||||||
|
previousPath: Joi.string().optional(),
|
||||||
|
follow: Joi.boolean().optional(),
|
||||||
|
cursor: Joi.string().pattern(/^\d{1,16}\.[a-f0-9]{64}$/).optional(),
|
||||||
offset: Joi.number().integer().min(0).optional(),
|
offset: Joi.number().integer().min(0).optional(),
|
||||||
limit: Joi.number()
|
limit: Joi.number()
|
||||||
.integer()
|
.integer()
|
||||||
@@ -337,6 +365,11 @@ export default (app: Router) => {
|
|||||||
try {
|
try {
|
||||||
const cronService = Container.get(CronService);
|
const cronService = Container.get(CronService);
|
||||||
const result = await cronService.log(req.params.id, {
|
const result = await cronService.log(req.params.id, {
|
||||||
|
triggerId: req.query.triggerId as string | undefined,
|
||||||
|
instanceId: req.query.instanceId === undefined ? undefined : Number(req.query.instanceId),
|
||||||
|
previousPath: req.query.previousPath as string | undefined,
|
||||||
|
follow: req.query.follow as unknown as boolean,
|
||||||
|
cursor: req.query.cursor as string | undefined,
|
||||||
offset: req.query.offset as unknown as number,
|
offset: req.query.offset as unknown as number,
|
||||||
limit: req.query.limit as unknown as number,
|
limit: req.query.limit as unknown as number,
|
||||||
tail: req.query.tail as unknown as boolean,
|
tail: req.query.tail as unknown as boolean,
|
||||||
@@ -345,6 +378,10 @@ export default (app: Router) => {
|
|||||||
code: 200,
|
code: 200,
|
||||||
data: result.content,
|
data: result.content,
|
||||||
logStatus: result.status,
|
logStatus: result.status,
|
||||||
|
logPath: result.logPath,
|
||||||
|
triggerId: result.triggerId,
|
||||||
|
cursor: result.cursor,
|
||||||
|
reset: result.reset,
|
||||||
offset: result.offset,
|
offset: result.offset,
|
||||||
nextOffset: result.nextOffset,
|
nextOffset: result.nextOffset,
|
||||||
total: result.total,
|
total: result.total,
|
||||||
@@ -379,7 +416,10 @@ export default (app: Router) => {
|
|||||||
route.delete(
|
route.delete(
|
||||||
'/',
|
'/',
|
||||||
celebrate({
|
celebrate({
|
||||||
body: Joi.array().items(Joi.number().required()),
|
body: Joi.array().min(1).items(Joi.number().integer().positive(), Joi.object({
|
||||||
|
id: Joi.number().integer().positive().required(), script: Joi.string().required(),
|
||||||
|
revision: Joi.number().integer(), command: Joi.string(), sub_id: Joi.number().integer().allow(null),
|
||||||
|
})),
|
||||||
}),
|
}),
|
||||||
async (req: Request, res: Response, next: NextFunction) => {
|
async (req: Request, res: Response, next: NextFunction) => {
|
||||||
const logger: Logger = Container.get('logger');
|
const logger: Logger = Container.get('logger');
|
||||||
@@ -452,8 +492,8 @@ export default (app: Router) => {
|
|||||||
const logger: Logger = Container.get('logger');
|
const logger: Logger = Container.get('logger');
|
||||||
try {
|
try {
|
||||||
const cronService = Container.get(CronService);
|
const cronService = Container.get(CronService);
|
||||||
const data = await cronService.getDb({ id: req.params.id });
|
const data = await cronService.getDb({ id: req.params.id }, true);
|
||||||
return res.send({ code: 200, data });
|
return res.send({ code: 200, data: withNextRun(data) });
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
return next(e);
|
return next(e);
|
||||||
}
|
}
|
||||||
@@ -471,13 +511,19 @@ export default (app: Router) => {
|
|||||||
last_running_time: Joi.number().optional().allow(null),
|
last_running_time: Joi.number().optional().allow(null),
|
||||||
last_execution_time: Joi.number().optional().allow(null),
|
last_execution_time: Joi.number().optional().allow(null),
|
||||||
exit_code: Joi.number().optional().allow(null),
|
exit_code: Joi.number().optional().allow(null),
|
||||||
}),
|
trigger_stopped: Joi.boolean().optional(),
|
||||||
|
execution_id: Joi.string().pattern(executionIdPattern).optional(),
|
||||||
|
trigger_id: Joi.string().pattern(/^[a-z0-9-]{1,120}$/).optional(),
|
||||||
|
}).oxor('execution_id', 'trigger_id'),
|
||||||
}),
|
}),
|
||||||
async (req: Request, res: Response, next: NextFunction) => {
|
async (req: Request, res: Response, next: NextFunction) => {
|
||||||
try {
|
try {
|
||||||
const cronService = Container.get(CronService);
|
const cronService = Container.get(CronService);
|
||||||
|
const { execution_id, ...body } = req.body;
|
||||||
const data = await cronService.status({
|
const data = await cronService.status({
|
||||||
...req.body,
|
...body,
|
||||||
|
finished_at: receiptFinishTime(req.get('X-QL-Finished-At')),
|
||||||
|
...(execution_id && { trigger_id: executionTriggerId(execution_id) }),
|
||||||
status: req.body.status ? parseInt(req.body.status) : undefined,
|
status: req.body.status ? parseInt(req.body.status) : undefined,
|
||||||
pid: req.body.pid ? parseInt(req.body.pid) : undefined,
|
pid: req.body.pid ? parseInt(req.body.pid) : undefined,
|
||||||
});
|
});
|
||||||
@@ -488,22 +534,87 @@ export default (app: Router) => {
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
|
route.get('/:id/daily-results', celebrate({
|
||||||
|
params: Joi.object({ id: Joi.number().integer().positive().required() }),
|
||||||
|
}), async (req, res, next) => {
|
||||||
|
try {
|
||||||
|
const data = await readDailyResults(RunningInstanceModel.sequelize!, Number(req.params.id));
|
||||||
|
return res.send({ code: 200, data });
|
||||||
|
} catch (error) {
|
||||||
|
return next(error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
route.get('/:id/triggers', celebrate({
|
||||||
|
params: Joi.object({ id: Joi.number().integer().positive().required() }),
|
||||||
|
query: Joi.object({
|
||||||
|
offset: Joi.number().integer().min(0),
|
||||||
|
limit: Joi.number().integer().min(1).max(26).default(25),
|
||||||
|
ids: Joi.array().items(Joi.string().max(256)).min(1).max(25).single(),
|
||||||
|
beforeTime: Joi.number().integer().min(0),
|
||||||
|
beforeId: Joi.string().max(256),
|
||||||
|
}).and('beforeTime', 'beforeId').oxor('offset', 'beforeTime', 'ids').unknown(true),
|
||||||
|
}), async (req, res, next) => {
|
||||||
|
try {
|
||||||
|
const { offset, limit, beforeTime, beforeId, ids } = req.query;
|
||||||
|
const history = triggerHistory();
|
||||||
|
const before = beforeId ? { time: Number(beforeTime), id: String(beforeId) } : undefined;
|
||||||
|
const data = ids
|
||||||
|
? await history.visibleTriggers(Number(req.params.id), ids as string[])
|
||||||
|
: await history.recent(Number(req.params.id), Number(offset || 0), Number(limit), before);
|
||||||
|
if (req.query.withQueueWait === 'true') {
|
||||||
|
const now = Date.now();
|
||||||
|
return res.send({ code: 200, data: data.map(row => row.state === 'queued' && row.queued_at != null
|
||||||
|
? { ...row, queue_wait_ms: Math.max(0, (row.dequeued_at ?? now) - row.queued_at) } : row) });
|
||||||
|
}
|
||||||
|
return res.send({ code: 200, data });
|
||||||
|
} catch (err) { next(err); }
|
||||||
|
});
|
||||||
|
|
||||||
route.get(
|
route.get(
|
||||||
'/:id/instances',
|
'/:id/instances',
|
||||||
celebrate({
|
celebrate({
|
||||||
|
// This route historically ignored query parameters. Only validate and
|
||||||
|
// apply cursor options when the caller explicitly requests pagination.
|
||||||
|
query: Joi.alternatives().conditional(Joi.object({ paged: Joi.valid('true').required() }).unknown(), {
|
||||||
|
then: Joi.object({
|
||||||
|
limit: Joi.number().integer().min(1).max(100),
|
||||||
|
before: Joi.number().integer().positive(),
|
||||||
|
ids: Joi.string().pattern(/^\d+(,\d+){0,99}$/),
|
||||||
|
}).unknown(true).oxor('ids', 'limit').oxor('ids', 'before'),
|
||||||
|
otherwise: Joi.object().unknown(true),
|
||||||
|
}),
|
||||||
params: Joi.object({
|
params: Joi.object({
|
||||||
id: Joi.number().required(),
|
id: Joi.number().required(),
|
||||||
}),
|
}),
|
||||||
}),
|
}),
|
||||||
async (req: Request<{ id: number }>, res: Response, next: NextFunction) => {
|
async (req: Request<{ id: number }>, res: Response, next: NextFunction) => {
|
||||||
try {
|
try {
|
||||||
|
// The instance row may retain a provisional end for cleanup after a restart.
|
||||||
|
const pending = `EXISTS (SELECT 1 FROM CronTriggers t WHERE t.instance_id=RunningInstance.id
|
||||||
|
AND (t.exit_pending=1 OR (t.state='unknown' AND t.reason='restart')))`;
|
||||||
|
const paged = req.query.paged === 'true';
|
||||||
|
const ids = paged && req.query.ids ? String(req.query.ids).split(',').map(Number) : undefined;
|
||||||
|
const limit = paged && !ids ? Number(req.query.limit ?? 25) : undefined;
|
||||||
const instances = await RunningInstanceModel.findAll({
|
const instances = await RunningInstanceModel.findAll({
|
||||||
|
attributes: {
|
||||||
|
exclude: ['finished_at', 'exit_code'],
|
||||||
|
include: [
|
||||||
|
[literal(`CASE WHEN ${pending} THEN NULL ELSE finished_at END`), 'finished_at'],
|
||||||
|
[literal(`CASE WHEN ${pending} THEN NULL ELSE exit_code END`), 'exit_code'],
|
||||||
|
],
|
||||||
|
},
|
||||||
where: {
|
where: {
|
||||||
cron_id: req.params.id,
|
cron_id: req.params.id,
|
||||||
|
...(ids && { id: { [Op.in]: ids } }),
|
||||||
|
...(paged && req.query.before && { id: { [Op.lt]: Number(req.query.before) } }),
|
||||||
},
|
},
|
||||||
order: [['started_at', 'DESC']],
|
order: paged ? [['id', 'DESC']] : [['started_at', 'DESC'], ['id', 'DESC']],
|
||||||
|
...(limit && { limit: limit + 1 }),
|
||||||
raw: true,
|
raw: true,
|
||||||
});
|
});
|
||||||
|
if (limit) return res.send({ code: 200, data: instances.slice(0, limit),
|
||||||
|
next: instances.length > limit ? instances[limit - 1].id : null });
|
||||||
return res.send({ code: 200, data: instances });
|
return res.send({ code: 200, data: instances });
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
return next(e);
|
return next(e);
|
||||||
@@ -522,8 +633,8 @@ export default (app: Router) => {
|
|||||||
async (req: Request<{ id: number; instanceId: number }>, res: Response, next: NextFunction) => {
|
async (req: Request<{ id: number; instanceId: number }>, res: Response, next: NextFunction) => {
|
||||||
try {
|
try {
|
||||||
const cronService = Container.get(CronService);
|
const cronService = Container.get(CronService);
|
||||||
const data = await cronService.stopInstance(req.params.instanceId);
|
const data = await cronService.stopInstance(Number(req.params.id), Number(req.params.instanceId));
|
||||||
return res.send(data);
|
return res.status(data.code).send(data);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
return next(e);
|
return next(e);
|
||||||
}
|
}
|
||||||
|
|||||||
+43
-51
@@ -1,3 +1,7 @@
|
|||||||
|
import { executionIdPattern, executionTriggerId, receiptFinishTime } from '../shared/executionIdentity';
|
||||||
|
import { oldestQueued } from '../shared/queueDiagnostics';
|
||||||
|
import { containerResources } from '../shared/containerResources';
|
||||||
|
import { TriggerHistory } from '../shared/triggerHistory';
|
||||||
import { Router, Request, Response, NextFunction } from 'express';
|
import { Router, Request, Response, NextFunction } from 'express';
|
||||||
import { Container } from 'typedi';
|
import { Container } from 'typedi';
|
||||||
import { fn, col, where, Op } from 'sequelize';
|
import { fn, col, where, Op } from 'sequelize';
|
||||||
@@ -12,6 +16,11 @@ import os from 'os';
|
|||||||
import { isEmpty } from 'lodash';
|
import { isEmpty } from 'lodash';
|
||||||
import { t, tf } from '../shared/i18n';
|
import { t, tf } from '../shared/i18n';
|
||||||
|
|
||||||
|
function averageElapsed(totalTime: number, runs: number, untimed: number) {
|
||||||
|
const samples = Number(runs) - (Number(untimed) || 0);
|
||||||
|
return samples > 0 ? Math.round(Number(totalTime) / samples) : 0;
|
||||||
|
}
|
||||||
|
|
||||||
const route = Router();
|
const route = Router();
|
||||||
|
|
||||||
export default (app: Router) => {
|
export default (app: Router) => {
|
||||||
@@ -19,41 +28,24 @@ export default (app: Router) => {
|
|||||||
|
|
||||||
route.post('/record', async (req: Request, res: Response) => {
|
route.post('/record', async (req: Request, res: Response) => {
|
||||||
try {
|
try {
|
||||||
const { ref_id, code, elapsed } = req.body;
|
const { ref_id, code, elapsed, trigger_id, execution_id } = req.body;
|
||||||
if (!ref_id) return res.send({ code: 400, message: 'ref_id required' });
|
if (!ref_id) return res.send({ code: 400, message: 'ref_id required' });
|
||||||
|
|
||||||
const today = dayjs().format('YYYY-MM-DD');
|
if (trigger_id != null && (typeof trigger_id !== 'string' || !/^[a-z0-9-]{1,120}$/.test(trigger_id))) {
|
||||||
const isSuccess = code === 0 ? 1 : 0;
|
return res.send({ code: 400 });
|
||||||
const isFail = code !== 0 ? 1 : 0;
|
|
||||||
const elapsedMs = (Number(elapsed) || 0) * 1000;
|
|
||||||
|
|
||||||
const existing = await CrontabStatModel.findOne({
|
|
||||||
where: { ref_id: Number(ref_id), date: today },
|
|
||||||
});
|
|
||||||
|
|
||||||
if (existing) {
|
|
||||||
await CrontabStatModel.update(
|
|
||||||
{
|
|
||||||
run_count: (existing.run_count || 0) + 1,
|
|
||||||
success_count: (existing.success_count || 0) + isSuccess,
|
|
||||||
fail_count: (existing.fail_count || 0) + isFail,
|
|
||||||
total_time: (existing.total_time || 0) + elapsedMs,
|
|
||||||
max_time: Math.max(existing.max_time || 0, elapsedMs),
|
|
||||||
},
|
|
||||||
{ where: { id: existing.id } },
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
await CrontabStatModel.create({
|
|
||||||
ref_id: Number(ref_id),
|
|
||||||
date: today,
|
|
||||||
run_count: 1,
|
|
||||||
success_count: isSuccess,
|
|
||||||
fail_count: isFail,
|
|
||||||
total_time: elapsedMs,
|
|
||||||
max_time: elapsedMs,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
if (execution_id !== undefined && (trigger_id !== undefined ||
|
||||||
|
typeof execution_id !== 'string' || !executionIdPattern.test(execution_id))) return res.send({ code: 400 });
|
||||||
|
const id = execution_id ? executionTriggerId(execution_id) : trigger_id;
|
||||||
|
// Old wrappers also send a status receipt with the process identity.
|
||||||
|
// A stats-only retry cannot be distinguished from another identical run.
|
||||||
|
if (!id) return res.send({ code: 200, data: { counted: false, reason: 'missing_execution_identity' } });
|
||||||
|
const exitCode = typeof code === 'string' && /^-?\d+$/.test(code) ? Number(code) : code;
|
||||||
|
const elapsedMs = Number(elapsed ?? 0) * 1000;
|
||||||
|
if (!Number.isInteger(exitCode) || !Number.isSafeInteger(elapsedMs) || elapsedMs < 0) {
|
||||||
|
return res.send({ code: 400, message: 'invalid exit code or elapsed time' });
|
||||||
|
}
|
||||||
|
await new TriggerHistory(CrontabStatModel.sequelize!).recordShellResult(id, Number(ref_id), exitCode, elapsedMs, Date.now(), receiptFinishTime(req.get('X-QL-Finished-At')));
|
||||||
res.send({ code: 200 });
|
res.send({ code: 200 });
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
res.send({ code: 500 });
|
res.send({ code: 500 });
|
||||||
@@ -66,7 +58,7 @@ export default (app: Router) => {
|
|||||||
try {
|
try {
|
||||||
const today = dayjs().format('YYYY-MM-DD');
|
const today = dayjs().format('YYYY-MM-DD');
|
||||||
|
|
||||||
const [total, enabled, disabled, stats] = await Promise.all([
|
const [total, enabled, disabled, stats, triggers] = await Promise.all([
|
||||||
CrontabModel.count(),
|
CrontabModel.count(),
|
||||||
CrontabModel.count({ where: { isDisabled: 0 } }),
|
CrontabModel.count({ where: { isDisabled: 0 } }),
|
||||||
CrontabModel.count({ where: { isDisabled: 1 } }),
|
CrontabModel.count({ where: { isDisabled: 1 } }),
|
||||||
@@ -75,11 +67,13 @@ export default (app: Router) => {
|
|||||||
[fn('SUM', col('run_count')), 'total_runs'],
|
[fn('SUM', col('run_count')), 'total_runs'],
|
||||||
[fn('SUM', col('success_count')), 'total_success'],
|
[fn('SUM', col('success_count')), 'total_success'],
|
||||||
[fn('SUM', col('fail_count')), 'total_fail'],
|
[fn('SUM', col('fail_count')), 'total_fail'],
|
||||||
|
[fn('SUM', col('untimed_count')), 'untimed_count'],
|
||||||
[fn('SUM', col('total_time')), 'total_time'],
|
[fn('SUM', col('total_time')), 'total_time'],
|
||||||
],
|
],
|
||||||
where: { date: today },
|
where: { date: today },
|
||||||
raw: true,
|
raw: true,
|
||||||
}),
|
}),
|
||||||
|
new TriggerHistory(CrontabStatModel.sequelize!).metrics(dayjs().startOf('day').valueOf()),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const row = stats as any;
|
const row = stats as any;
|
||||||
@@ -94,11 +88,13 @@ export default (app: Router) => {
|
|||||||
total,
|
total,
|
||||||
enabled,
|
enabled,
|
||||||
disabled,
|
disabled,
|
||||||
|
triggers,
|
||||||
todayRuns: totalRuns,
|
todayRuns: totalRuns,
|
||||||
todaySuccess: totalSuccess,
|
todaySuccess: totalSuccess,
|
||||||
todayFail: totalFail,
|
todayFail: totalFail,
|
||||||
|
todayStopped: totalRuns - totalSuccess - totalFail,
|
||||||
successRate: totalRuns > 0 ? ((totalSuccess / totalRuns) * 100).toFixed(1) : '0',
|
successRate: totalRuns > 0 ? ((totalSuccess / totalRuns) * 100).toFixed(1) : '0',
|
||||||
avgTime: totalRuns > 0 ? Math.round(totalTime / totalRuns) : 0,
|
avgTime: averageElapsed(totalTime,totalRuns,row?.untimed_count),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
@@ -202,6 +198,7 @@ export default (app: Router) => {
|
|||||||
const rows = (await CrontabStatModel.findAll({
|
const rows = (await CrontabStatModel.findAll({
|
||||||
attributes: [
|
attributes: [
|
||||||
'ref_id',
|
'ref_id',
|
||||||
|
[fn('SUM', col('untimed_count')), 'untimed_count'],
|
||||||
[fn('SUM', col('total_time')), 'total_time'],
|
[fn('SUM', col('total_time')), 'total_time'],
|
||||||
[fn('SUM', col('run_count')), 'run_count'],
|
[fn('SUM', col('run_count')), 'run_count'],
|
||||||
[fn('MAX', col('max_time')), 'max_time'],
|
[fn('MAX', col('max_time')), 'max_time'],
|
||||||
@@ -224,7 +221,7 @@ export default (app: Router) => {
|
|||||||
const data = rows.map((r: any, i) => ({
|
const data = rows.map((r: any, i) => ({
|
||||||
rank: i + 1,
|
rank: i + 1,
|
||||||
name: nameMap[Number(r.ref_id)] || tf('任务#%s', r.ref_id),
|
name: nameMap[Number(r.ref_id)] || tf('任务#%s', r.ref_id),
|
||||||
avgTime: Math.round(Number(r.total_time) / Number(r.run_count)),
|
avgTime: averageElapsed(r.total_time,r.run_count,r.untimed_count),
|
||||||
maxTime: Number(r.max_time),
|
maxTime: Number(r.max_time),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
@@ -245,6 +242,7 @@ export default (app: Router) => {
|
|||||||
attributes: [
|
attributes: [
|
||||||
'ref_id',
|
'ref_id',
|
||||||
[fn('SUM', col('run_count')), 'run_count'],
|
[fn('SUM', col('run_count')), 'run_count'],
|
||||||
|
[fn('SUM', col('untimed_count')), 'untimed_count'],
|
||||||
[fn('SUM', col('total_time')), 'total_time'],
|
[fn('SUM', col('total_time')), 'total_time'],
|
||||||
[fn('SUM', col('success_count')), 'success_count'],
|
[fn('SUM', col('success_count')), 'success_count'],
|
||||||
],
|
],
|
||||||
@@ -267,7 +265,7 @@ export default (app: Router) => {
|
|||||||
rank: i + 1,
|
rank: i + 1,
|
||||||
name: nameMap[Number(r.ref_id)] || tf('任务#%s', r.ref_id),
|
name: nameMap[Number(r.ref_id)] || tf('任务#%s', r.ref_id),
|
||||||
runCount: Number(r.run_count),
|
runCount: Number(r.run_count),
|
||||||
avgTime: Math.round(Number(r.total_time) / Number(r.run_count)),
|
avgTime: averageElapsed(r.total_time,r.run_count,r.untimed_count),
|
||||||
successRate:
|
successRate:
|
||||||
Number(r.run_count) > 0
|
Number(r.run_count) > 0
|
||||||
? ((Number(r.success_count) / Number(r.run_count)) * 100).toFixed(1)
|
? ((Number(r.success_count) / Number(r.run_count)) * 100).toFixed(1)
|
||||||
@@ -285,6 +283,7 @@ export default (app: Router) => {
|
|||||||
'/runtime',
|
'/runtime',
|
||||||
async (req: Request, res: Response, next: NextFunction) => {
|
async (req: Request, res: Response, next: NextFunction) => {
|
||||||
try {
|
try {
|
||||||
|
const queued = await new TriggerHistory(CrontabModel.sequelize!).queueCounts();
|
||||||
const runningInstances = await RunningInstanceModel.findAll({
|
const runningInstances = await RunningInstanceModel.findAll({
|
||||||
where: {
|
where: {
|
||||||
status: InstanceStatus.running,
|
status: InstanceStatus.running,
|
||||||
@@ -292,13 +291,6 @@ export default (app: Router) => {
|
|||||||
raw: true,
|
raw: true,
|
||||||
});
|
});
|
||||||
|
|
||||||
const queuedCrons = await CrontabModel.findAll({
|
|
||||||
where: {
|
|
||||||
status: 3, // queued
|
|
||||||
},
|
|
||||||
raw: true,
|
|
||||||
});
|
|
||||||
|
|
||||||
// Fetch cron names for running instances
|
// Fetch cron names for running instances
|
||||||
const cronIds = [
|
const cronIds = [
|
||||||
...new Set(runningInstances.map((i: any) => i.cron_id)),
|
...new Set(runningInstances.map((i: any) => i.cron_id)),
|
||||||
@@ -341,7 +333,9 @@ export default (app: Router) => {
|
|||||||
code: 200,
|
code: 200,
|
||||||
data: {
|
data: {
|
||||||
runningCount: running.length,
|
runningCount: running.length,
|
||||||
queuedCount: queuedCrons.length,
|
queuedCount: queued.triggers,
|
||||||
|
queuedTasks: queued.tasks,
|
||||||
|
oldestQueued: queued.triggers ? await oldestQueued(CrontabModel.sequelize!) : [],
|
||||||
running,
|
running,
|
||||||
idleTasks: idleTasks.map((c: any) => ({
|
idleTasks: idleTasks.map((c: any) => ({
|
||||||
id: c.id,
|
id: c.id,
|
||||||
@@ -371,7 +365,7 @@ export default (app: Router) => {
|
|||||||
const statMap: Record<number, any> = {};
|
const statMap: Record<number, any> = {};
|
||||||
stats.forEach((s: any) => { statMap[s.ref_id] = s; });
|
stats.forEach((s: any) => { statMap[s.ref_id] = s; });
|
||||||
|
|
||||||
const labelMap: Record<string, { count: number; runs: number; success: number; totalTime: number }> = {};
|
const labelMap: Record<string, { count: number; runs: number; success: number; untimed: number; totalTime: number }> = {};
|
||||||
crons.forEach((c) => {
|
crons.forEach((c) => {
|
||||||
let rawLabels = c.labels;
|
let rawLabels = c.labels;
|
||||||
if (typeof rawLabels === 'string') rawLabels = JSON.parse(rawLabels);
|
if (typeof rawLabels === 'string') rawLabels = JSON.parse(rawLabels);
|
||||||
@@ -383,11 +377,12 @@ export default (app: Router) => {
|
|||||||
}
|
}
|
||||||
const st = statMap[c.id!];
|
const st = statMap[c.id!];
|
||||||
labels.forEach((label: string) => {
|
labels.forEach((label: string) => {
|
||||||
if (!labelMap[label]) labelMap[label] = { count: 0, runs: 0, success: 0, totalTime: 0 };
|
if (!labelMap[label]) labelMap[label] = { count: 0, runs: 0, success: 0, untimed: 0, totalTime: 0 };
|
||||||
labelMap[label].count += 1;
|
labelMap[label].count += 1;
|
||||||
if (st) {
|
if (st) {
|
||||||
labelMap[label].runs += Number(st.run_count) || 0;
|
labelMap[label].runs += Number(st.run_count) || 0;
|
||||||
labelMap[label].success += Number(st.success_count) || 0;
|
labelMap[label].success += Number(st.success_count) || 0;
|
||||||
|
labelMap[label].untimed += Number(st.untimed_count) || 0;
|
||||||
labelMap[label].totalTime += Number(st.total_time) || 0;
|
labelMap[label].totalTime += Number(st.total_time) || 0;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -399,7 +394,7 @@ export default (app: Router) => {
|
|||||||
count: v.count,
|
count: v.count,
|
||||||
todayRuns: v.runs,
|
todayRuns: v.runs,
|
||||||
successRate: v.runs > 0 ? ((v.success / v.runs) * 100).toFixed(1) : '0',
|
successRate: v.runs > 0 ? ((v.success / v.runs) * 100).toFixed(1) : '0',
|
||||||
avgTime: v.runs > 0 ? Math.round(v.totalTime / v.runs) : 0,
|
avgTime: averageElapsed(v.totalTime,v.runs,v.untimed),
|
||||||
}))
|
}))
|
||||||
.sort((a, b) => b.todayRuns - a.todayRuns);
|
.sort((a, b) => b.todayRuns - a.todayRuns);
|
||||||
|
|
||||||
@@ -420,13 +415,10 @@ export default (app: Router) => {
|
|||||||
data: {
|
data: {
|
||||||
platform: os.platform(),
|
platform: os.platform(),
|
||||||
uptime: Math.floor(process.uptime()),
|
uptime: Math.floor(process.uptime()),
|
||||||
memTotal: os.totalmem(),
|
...containerResources(),
|
||||||
memFree: os.freemem(),
|
|
||||||
memUsagePercent: ((1 - os.freemem() / os.totalmem()) * 100).toFixed(1),
|
|
||||||
heapUsed: Math.round(memUsage.heapUsed / 1024 / 1024),
|
heapUsed: Math.round(memUsage.heapUsed / 1024 / 1024),
|
||||||
heapTotal: Math.round(memUsage.heapTotal / 1024 / 1024),
|
heapTotal: Math.round(memUsage.heapTotal / 1024 / 1024),
|
||||||
loadAvg: os.loadavg().map((v) => Number(v.toFixed(2))),
|
loadAvg: os.loadavg().map((v) => Number(v.toFixed(2))),
|
||||||
cpus: os.cpus().length,
|
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
|||||||
+64
-2
@@ -11,8 +11,7 @@ export default (app: Router) => {
|
|||||||
route.get(
|
route.get(
|
||||||
'/',
|
'/',
|
||||||
celebrate({
|
celebrate({
|
||||||
query:
|
query: Joi.object({
|
||||||
Joi.object({
|
|
||||||
searchValue: Joi.string().optional().allow(''),
|
searchValue: Joi.string().optional().allow(''),
|
||||||
type: Joi.string().optional().allow(''),
|
type: Joi.string().optional().allow(''),
|
||||||
status: Joi.string().optional().allow(''),
|
status: Joi.string().optional().allow(''),
|
||||||
@@ -31,6 +30,69 @@ export default (app: Router) => {
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
|
route.post(
|
||||||
|
'/refresh',
|
||||||
|
celebrate({
|
||||||
|
body: Joi.object({
|
||||||
|
type: Joi.string().valid('nodejs', 'python3').required(),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
async (req: Request, res: Response, next: NextFunction) => {
|
||||||
|
try {
|
||||||
|
const dependenceService = Container.get(DependenceService);
|
||||||
|
const { job, accepted } = dependenceService.startRefresh(req.body.type);
|
||||||
|
if (!accepted) {
|
||||||
|
return res.status(409).send({
|
||||||
|
code: 409,
|
||||||
|
message: 'Another dependency refresh is already running',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return res.send({ code: 200, data: job });
|
||||||
|
} catch (e) {
|
||||||
|
return next(e);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
route.get(
|
||||||
|
'/refresh/active',
|
||||||
|
async (_req: Request, res: Response, next: NextFunction) => {
|
||||||
|
try {
|
||||||
|
const dependenceService = Container.get(DependenceService);
|
||||||
|
res.set('Cache-Control', 'no-store');
|
||||||
|
return res.send({
|
||||||
|
code: 200,
|
||||||
|
data: dependenceService.getActiveRefreshJob() || null,
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
return next(e);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
route.get(
|
||||||
|
'/refresh/:id',
|
||||||
|
celebrate({
|
||||||
|
params: Joi.object({ id: Joi.string().uuid().required() }),
|
||||||
|
}),
|
||||||
|
async (req: Request, res: Response, next: NextFunction) => {
|
||||||
|
try {
|
||||||
|
const dependenceService = Container.get(DependenceService);
|
||||||
|
const job = dependenceService.getRefreshJob(req.params.id);
|
||||||
|
if (!job) {
|
||||||
|
return res.status(404).send({
|
||||||
|
code: 404,
|
||||||
|
message: 'Dependency refresh job not found',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
res.set('Cache-Control', 'no-store');
|
||||||
|
return res.send({ code: 200, data: job });
|
||||||
|
} catch (e) {
|
||||||
|
return next(e);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
route.post(
|
route.post(
|
||||||
'/',
|
'/',
|
||||||
celebrate({
|
celebrate({
|
||||||
|
|||||||
+17
-14
@@ -1,24 +1,16 @@
|
|||||||
import { Joi, celebrate } from 'celebrate';
|
import { Joi, celebrate } from 'celebrate';
|
||||||
import { NextFunction, Request, Response, Router } from 'express';
|
import { NextFunction, Request, Response, Router } from 'express';
|
||||||
import fs from 'fs';
|
|
||||||
import multer from 'multer';
|
import multer from 'multer';
|
||||||
import { Container } from 'typedi';
|
import { Container } from 'typedi';
|
||||||
import { Logger } from 'winston';
|
import { Logger } from 'winston';
|
||||||
import config from '../config';
|
|
||||||
import { safeJSONParse } from '../config/util';
|
|
||||||
import { t } from '../shared/i18n';
|
import { t } from '../shared/i18n';
|
||||||
import EnvService from '../services/env';
|
import EnvService from '../services/env';
|
||||||
const route = Router();
|
const route = Router();
|
||||||
|
|
||||||
const storage = multer.diskStorage({
|
const upload = multer({
|
||||||
destination: function (req, file, cb) {
|
storage: multer.memoryStorage(),
|
||||||
cb(null, config.scriptPath);
|
limits: { fileSize: 5 * 1024 * 1024, files: 1 },
|
||||||
},
|
|
||||||
filename: function (req, file, cb) {
|
|
||||||
cb(null, file.originalname);
|
|
||||||
},
|
|
||||||
});
|
});
|
||||||
const upload = multer({ storage: storage });
|
|
||||||
const labelSchema = Joi.array()
|
const labelSchema = Joi.array()
|
||||||
.items(Joi.string().trim().required())
|
.items(Joi.string().trim().required())
|
||||||
.min(1)
|
.min(1)
|
||||||
@@ -282,12 +274,23 @@ export default (app: Router) => {
|
|||||||
const logger: Logger = Container.get('logger');
|
const logger: Logger = Container.get('logger');
|
||||||
try {
|
try {
|
||||||
const envService = Container.get(EnvService);
|
const envService = Container.get(EnvService);
|
||||||
const fileContent = await fs.promises.readFile(req!.file!.path, 'utf8');
|
let parseContent: unknown;
|
||||||
const parseContent = safeJSONParse(fileContent);
|
try {
|
||||||
|
parseContent = JSON.parse(req.file?.buffer.toString('utf8') || '');
|
||||||
|
} catch {
|
||||||
|
return res.status(400).send({ code: 400, message: t('参数错误') });
|
||||||
|
}
|
||||||
const data = Array.isArray(parseContent)
|
const data = Array.isArray(parseContent)
|
||||||
? parseContent
|
? parseContent
|
||||||
: [parseContent];
|
: [parseContent];
|
||||||
if (data.every((x) => x.name && x.value)) {
|
if (
|
||||||
|
data.length > 0 &&
|
||||||
|
data.every((x) =>
|
||||||
|
x && typeof x.name === 'string' &&
|
||||||
|
/^[a-zA-Z_][0-9a-zA-Z_]*$/.test(x.name) &&
|
||||||
|
typeof x.value === 'string' && x.value.length > 0,
|
||||||
|
)
|
||||||
|
) {
|
||||||
const result = await envService.create(
|
const result = await envService.create(
|
||||||
data.map((x) => ({
|
data.map((x) => ({
|
||||||
name: x.name,
|
name: x.name,
|
||||||
|
|||||||
@@ -20,7 +20,6 @@ export default (app: Router) => {
|
|||||||
res.status(500).send({
|
res.status(500).send({
|
||||||
code: 500,
|
code: 500,
|
||||||
message: 'Health check failed',
|
message: 'Health check failed',
|
||||||
error: err.message,
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|||||||
+10
-1
@@ -1,3 +1,4 @@
|
|||||||
|
import { directoryCheckHandler, directoryPageHandler } from '../shared/directoryPageRoute';
|
||||||
import { celebrate, Joi } from 'celebrate';
|
import { celebrate, Joi } from 'celebrate';
|
||||||
import { NextFunction, Request, Response, Router } from 'express';
|
import { NextFunction, Request, Response, Router } from 'express';
|
||||||
import { Container } from 'typedi';
|
import { Container } from 'typedi';
|
||||||
@@ -17,6 +18,8 @@ const blacklist = ['.tmp'];
|
|||||||
|
|
||||||
export default (app: Router) => {
|
export default (app: Router) => {
|
||||||
app.use('/logs', route);
|
app.use('/logs', route);
|
||||||
|
route.get('/entries', ...directoryPageHandler(() => config.logPath, blacklist, true));
|
||||||
|
route.post('/entries/check', ...directoryCheckHandler(() => config.logPath, blacklist));
|
||||||
|
|
||||||
route.get('/', async (req: Request, res: Response, next: NextFunction) => {
|
route.get('/', async (req: Request, res: Response, next: NextFunction) => {
|
||||||
const logger: Logger = Container.get('logger');
|
const logger: Logger = Container.get('logger');
|
||||||
@@ -38,6 +41,8 @@ export default (app: Router) => {
|
|||||||
query: Joi.object({
|
query: Joi.object({
|
||||||
path: Joi.string().allow('').optional(),
|
path: Joi.string().allow('').optional(),
|
||||||
file: Joi.string().required(),
|
file: Joi.string().required(),
|
||||||
|
follow: Joi.boolean().optional(),
|
||||||
|
cursor: Joi.string().pattern(/^\d{1,16}\.[a-f0-9]{64}$/).optional(),
|
||||||
offset: Joi.number().integer().min(0).optional(),
|
offset: Joi.number().integer().min(0).optional(),
|
||||||
limit: Joi.number()
|
limit: Joi.number()
|
||||||
.integer()
|
.integer()
|
||||||
@@ -67,14 +72,18 @@ export default (app: Router) => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
const chunk = await readLogChunk(finalPath, {
|
const chunk = await readLogChunk(finalPath, {
|
||||||
|
follow: req.query.follow as unknown as boolean,
|
||||||
|
cursor: req.query.cursor as string | undefined,
|
||||||
offset: req.query.offset as unknown as number,
|
offset: req.query.offset as unknown as number,
|
||||||
limit: req.query.limit as unknown as number,
|
limit: req.query.limit as unknown as number,
|
||||||
tail: req.query.tail as unknown as boolean,
|
tail: req.query.tail as unknown as boolean,
|
||||||
});
|
});
|
||||||
res.send({
|
res.send({
|
||||||
code: 200,
|
code: 200,
|
||||||
data: removeAnsi(chunk.content),
|
data: req.query.follow ? chunk.content : removeAnsi(chunk.content),
|
||||||
logStatus: runningInstance ? 'running' : undefined,
|
logStatus: runningInstance ? 'running' : undefined,
|
||||||
|
cursor: chunk.cursor,
|
||||||
|
reset: chunk.reset,
|
||||||
offset: chunk.offset,
|
offset: chunk.offset,
|
||||||
nextOffset: chunk.nextOffset,
|
nextOffset: chunk.nextOffset,
|
||||||
total: chunk.total,
|
total: chunk.total,
|
||||||
|
|||||||
+4
-2
@@ -3,6 +3,7 @@ import { Container } from 'typedi';
|
|||||||
import OpenService from '../services/open';
|
import OpenService from '../services/open';
|
||||||
import { Logger } from 'winston';
|
import { Logger } from 'winston';
|
||||||
import { celebrate, Joi } from 'celebrate';
|
import { celebrate, Joi } from 'celebrate';
|
||||||
|
import { rateLimit } from 'express-rate-limit';
|
||||||
const route = Router();
|
const route = Router();
|
||||||
|
|
||||||
export default (app: Router) => {
|
export default (app: Router) => {
|
||||||
@@ -26,7 +27,7 @@ export default (app: Router) => {
|
|||||||
celebrate({
|
celebrate({
|
||||||
body: Joi.object({
|
body: Joi.object({
|
||||||
name: Joi.string().optional().allow('').disallow('system'),
|
name: Joi.string().optional().allow('').disallow('system'),
|
||||||
scopes: Joi.array().items(Joi.string().required()),
|
scopes: Joi.array().items(Joi.string().required().invalid('apps')),
|
||||||
}),
|
}),
|
||||||
}),
|
}),
|
||||||
async (req: Request, res: Response, next: NextFunction) => {
|
async (req: Request, res: Response, next: NextFunction) => {
|
||||||
@@ -46,7 +47,7 @@ export default (app: Router) => {
|
|||||||
celebrate({
|
celebrate({
|
||||||
body: Joi.object({
|
body: Joi.object({
|
||||||
name: Joi.string().optional().allow(''),
|
name: Joi.string().optional().allow(''),
|
||||||
scopes: Joi.array().items(Joi.string()),
|
scopes: Joi.array().items(Joi.string().invalid('apps')),
|
||||||
id: Joi.number().required(),
|
id: Joi.number().required(),
|
||||||
}),
|
}),
|
||||||
}),
|
}),
|
||||||
@@ -100,6 +101,7 @@ export default (app: Router) => {
|
|||||||
|
|
||||||
route.get(
|
route.get(
|
||||||
'/auth/token',
|
'/auth/token',
|
||||||
|
rateLimit({ windowMs: 15 * 60 * 1000, limit: 100 }),
|
||||||
celebrate({
|
celebrate({
|
||||||
query: {
|
query: {
|
||||||
client_id: Joi.string().required(),
|
client_id: Joi.string().required(),
|
||||||
|
|||||||
+211
-22
@@ -1,3 +1,4 @@
|
|||||||
|
import { directoryCheckHandler, directoryPageHandler } from '../shared/directoryPageRoute';
|
||||||
import { randomUUID } from 'crypto';
|
import { randomUUID } from 'crypto';
|
||||||
import { resolveFileAccess } from '../shared/fileAccess';
|
import { resolveFileAccess } from '../shared/fileAccess';
|
||||||
import { fileExist, readDirs, readDir, rmPath, IFile } from '../config/util';
|
import { fileExist, readDirs, readDir, rmPath, IFile } from '../config/util';
|
||||||
@@ -7,20 +8,70 @@ import { Logger } from 'winston';
|
|||||||
import config from '../config';
|
import config from '../config';
|
||||||
import * as fs from 'fs/promises';
|
import * as fs from 'fs/promises';
|
||||||
import { celebrate, Joi } from 'celebrate';
|
import { celebrate, Joi } from 'celebrate';
|
||||||
import path, { join, parse } from 'path';
|
import path, { dirname, join, parse, resolve, sep } from 'path';
|
||||||
import ScriptService from '../services/script';
|
import ScriptService from '../services/script';
|
||||||
|
import ScriptHistoryService from '../services/scriptHistory';
|
||||||
import { t } from '../shared/i18n';
|
import { t } from '../shared/i18n';
|
||||||
import multer from 'multer';
|
import multer from 'multer';
|
||||||
import { writeFileWithLock } from '../shared/utils';
|
import { writeFileWithLock } from '../shared/utils';
|
||||||
|
import {
|
||||||
|
ScriptHistoryError,
|
||||||
|
HistoryUnavailableError,
|
||||||
|
} from '../shared/scriptHistory';
|
||||||
const route = Router();
|
const route = Router();
|
||||||
|
const blacklist = [
|
||||||
|
'node_modules',
|
||||||
|
'.git',
|
||||||
|
'.pnpm',
|
||||||
|
'pnpm-lock.yaml',
|
||||||
|
'yarn.lock',
|
||||||
|
'package-lock.json',
|
||||||
|
];
|
||||||
|
|
||||||
function isPathAllowed(targetPath: string): boolean {
|
function isPathAllowed(targetPath: string): boolean {
|
||||||
const resolved = path.resolve(targetPath);
|
const resolved = path.resolve(targetPath);
|
||||||
return config.writePathList.some((x) =>
|
return config.writePathList.some((x) =>
|
||||||
Boolean(resolveFileAccess(x, [resolved], config.blackFileList)),
|
Boolean(
|
||||||
|
resolveFileAccess(
|
||||||
|
x,
|
||||||
|
[resolved],
|
||||||
|
// Panel configuration secrets must not restrict user script filenames.
|
||||||
|
path.resolve(x) === path.resolve(config.configPath)
|
||||||
|
? config.blackFileList
|
||||||
|
: [],
|
||||||
|
),
|
||||||
|
),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function existingWritableRealPath(targetPath: string): Promise<string> {
|
||||||
|
if (!isPathAllowed(targetPath)) {
|
||||||
|
throw new ScriptHistoryError(403, '暂无权限');
|
||||||
|
}
|
||||||
|
const resolved = resolve(targetPath);
|
||||||
|
for (const writableRoot of config.writePathList) {
|
||||||
|
const root = resolve(writableRoot);
|
||||||
|
const rootPrefix = root.endsWith(sep) ? root : root + sep;
|
||||||
|
// Keep an explicit normalized boundary check at the filesystem operation.
|
||||||
|
if (!resolved.startsWith(rootPrefix)) continue;
|
||||||
|
const realPath = await fs.realpath(resolved);
|
||||||
|
const realRoot = await fs.realpath(root);
|
||||||
|
const realRootPrefix = realRoot.endsWith(sep) ? realRoot : realRoot + sep;
|
||||||
|
if (
|
||||||
|
!realPath.startsWith(realRootPrefix) ||
|
||||||
|
!resolveFileAccess(
|
||||||
|
realRoot,
|
||||||
|
[realPath],
|
||||||
|
root === resolve(config.configPath) ? config.blackFileList : [],
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
throw new ScriptHistoryError(403, '暂无权限');
|
||||||
|
}
|
||||||
|
return realPath;
|
||||||
|
}
|
||||||
|
throw new ScriptHistoryError(403, '暂无权限');
|
||||||
|
}
|
||||||
|
|
||||||
const storage = multer.diskStorage({
|
const storage = multer.diskStorage({
|
||||||
destination: function (req, file, cb) {
|
destination: function (req, file, cb) {
|
||||||
cb(null, config.tmpPath);
|
cb(null, config.tmpPath);
|
||||||
@@ -33,6 +84,8 @@ const upload = multer({ storage: storage });
|
|||||||
|
|
||||||
export default (app: Router) => {
|
export default (app: Router) => {
|
||||||
app.use('/scripts', route);
|
app.use('/scripts', route);
|
||||||
|
route.get('/entries', ...directoryPageHandler(() => config.scriptPath, blacklist, false));
|
||||||
|
route.post('/entries/check', ...directoryCheckHandler(() => config.scriptPath, blacklist));
|
||||||
|
|
||||||
route.get(
|
route.get(
|
||||||
'/',
|
'/',
|
||||||
@@ -45,21 +98,10 @@ export default (app: Router) => {
|
|||||||
const logger: Logger = Container.get('logger');
|
const logger: Logger = Container.get('logger');
|
||||||
try {
|
try {
|
||||||
let result: IFile[] = [];
|
let result: IFile[] = [];
|
||||||
const blacklist = [
|
|
||||||
'node_modules',
|
|
||||||
'.git',
|
|
||||||
'.pnpm',
|
|
||||||
'pnpm-lock.yaml',
|
|
||||||
'yarn.lock',
|
|
||||||
'package-lock.json',
|
|
||||||
];
|
|
||||||
if (req.query.path) {
|
if (req.query.path) {
|
||||||
if (
|
if (
|
||||||
!resolveFileAccess(
|
!resolveFileAccess(config.scriptPath, [req.query.path as string])
|
||||||
config.scriptPath,
|
|
||||||
[req.query.path as string],
|
|
||||||
config.blackFileList,
|
|
||||||
)
|
|
||||||
) {
|
) {
|
||||||
return res.send({ code: 403, message: t('暂无权限') });
|
return res.send({ code: 403, message: t('暂无权限') });
|
||||||
}
|
}
|
||||||
@@ -115,6 +157,82 @@ export default (app: Router) => {
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const historyError = (error: unknown, res: Response, next: NextFunction) => {
|
||||||
|
if (error instanceof ScriptHistoryError) {
|
||||||
|
return res.status(error.status).send({
|
||||||
|
code: error.status,
|
||||||
|
message: t(error.message),
|
||||||
|
...(error instanceof HistoryUnavailableError
|
||||||
|
? { historyUnavailable: true, currentHash: error.currentHash }
|
||||||
|
: {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return next(error);
|
||||||
|
};
|
||||||
|
const historyQuery = {
|
||||||
|
filename: Joi.string().required(),
|
||||||
|
path: Joi.string().optional().allow(''),
|
||||||
|
};
|
||||||
|
|
||||||
|
route.get(
|
||||||
|
'/history',
|
||||||
|
celebrate({ query: Joi.object(historyQuery).unknown(true) }),
|
||||||
|
async (req: Request, res: Response, next: NextFunction) => {
|
||||||
|
try {
|
||||||
|
const service = Container.get(ScriptHistoryService);
|
||||||
|
const data = await service.list(
|
||||||
|
(req.query.path as string) || '',
|
||||||
|
req.query.filename as string,
|
||||||
|
);
|
||||||
|
res.send({ code: 200, data });
|
||||||
|
} catch (error) {
|
||||||
|
historyError(error, res, next);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
route.get(
|
||||||
|
'/history/detail',
|
||||||
|
celebrate({
|
||||||
|
query: Joi.object({
|
||||||
|
...historyQuery,
|
||||||
|
id: Joi.string().uuid().required(),
|
||||||
|
}).unknown(true),
|
||||||
|
}),
|
||||||
|
async (req: Request, res: Response, next: NextFunction) => {
|
||||||
|
try {
|
||||||
|
const service = Container.get(ScriptHistoryService);
|
||||||
|
const data = await service.detail(
|
||||||
|
(req.query.path as string) || '',
|
||||||
|
req.query.filename as string,
|
||||||
|
req.query.id as string,
|
||||||
|
);
|
||||||
|
res.send({ code: 200, data });
|
||||||
|
} catch (error) {
|
||||||
|
historyError(error, res, next);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
route.put(
|
||||||
|
'/history/restore',
|
||||||
|
celebrate({
|
||||||
|
body: Joi.object({
|
||||||
|
...historyQuery,
|
||||||
|
id: Joi.string().uuid().required(),
|
||||||
|
expectedHash: Joi.string().hex().length(64).required(),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
async (req: Request, res: Response, next: NextFunction) => {
|
||||||
|
try {
|
||||||
|
const { path = '', filename, id, expectedHash } = req.body;
|
||||||
|
const service = Container.get(ScriptHistoryService);
|
||||||
|
const data = await service.restore(path, filename, id, expectedHash);
|
||||||
|
res.send({ code: 200, data });
|
||||||
|
} catch (error) {
|
||||||
|
historyError(error, res, next);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
route.get('/:file', (req: Request, res: Response) => {
|
route.get('/:file', (req: Request, res: Response) => {
|
||||||
return res.send({
|
return res.send({
|
||||||
code: 410,
|
code: 410,
|
||||||
@@ -136,6 +254,11 @@ export default (app: Router) => {
|
|||||||
filename: Joi.string().required(),
|
filename: Joi.string().required(),
|
||||||
path: Joi.string().optional().allow(''),
|
path: Joi.string().optional().allow(''),
|
||||||
content: Joi.string().optional().allow(''),
|
content: Joi.string().optional().allow(''),
|
||||||
|
skipHistory: Joi.boolean().optional(),
|
||||||
|
expectedHash: Joi.string()
|
||||||
|
.hex()
|
||||||
|
.length(64)
|
||||||
|
.when('skipHistory', { is: true, then: Joi.required() }),
|
||||||
originFilename: Joi.string().optional().allow(''),
|
originFilename: Joi.string().optional().allow(''),
|
||||||
directory: Joi.string().optional().allow(''),
|
directory: Joi.string().optional().allow(''),
|
||||||
file: Joi.string().optional().allow(''),
|
file: Joi.string().optional().allow(''),
|
||||||
@@ -195,8 +318,56 @@ export default (app: Router) => {
|
|||||||
if (!isPathAllowed(filePath) || !isPathAllowed(originFilePath)) {
|
if (!isPathAllowed(filePath) || !isPathAllowed(originFilePath)) {
|
||||||
return res.send({ code: 403, message: t('暂无权限') });
|
return res.send({ code: 403, message: t('暂无权限') });
|
||||||
}
|
}
|
||||||
await fs.mkdir(path, { recursive: true });
|
// Check the canonical parent directly at the directory creation boundary.
|
||||||
|
const parentPath = resolve(dirname(filePath));
|
||||||
|
let parentCreated = false;
|
||||||
|
for (const writableRoot of config.writePathList) {
|
||||||
|
const root = resolve(writableRoot);
|
||||||
|
const rootPrefix = root.endsWith(sep) ? root : root + sep;
|
||||||
|
if (parentPath === root) {
|
||||||
|
await fs.mkdir(root, { recursive: true });
|
||||||
|
} else if (parentPath.startsWith(rootPrefix)) {
|
||||||
|
await fs.mkdir(parentPath, { recursive: true });
|
||||||
|
} else {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
parentCreated = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (!parentCreated) {
|
||||||
|
return res.send({ code: 403, message: t('暂无权限') });
|
||||||
|
}
|
||||||
const fileExists = await fileExist(filePath);
|
const fileExists = await fileExist(filePath);
|
||||||
|
if (fileExists && resolveFileAccess(config.scriptPath, [filePath])) {
|
||||||
|
let removeSource = filename !== originFilename;
|
||||||
|
let backupSource = '';
|
||||||
|
if (removeSource) {
|
||||||
|
const [originRealPath, targetRealPath] = await Promise.all([
|
||||||
|
existingWritableRealPath(originFilePath),
|
||||||
|
existingWritableRealPath(filePath),
|
||||||
|
]);
|
||||||
|
// Aliases of one script are an in-place save. Removing the source
|
||||||
|
// would also remove the destination behind a target symlink.
|
||||||
|
removeSource = originRealPath !== targetRealPath;
|
||||||
|
backupSource = originRealPath;
|
||||||
|
}
|
||||||
|
// Save-as removes the source after committing the destination. Keep
|
||||||
|
// its original content too: destination history only protects the
|
||||||
|
// file being overwritten, not the source being deleted.
|
||||||
|
if (removeSource) {
|
||||||
|
await fs.copyFile(
|
||||||
|
backupSource,
|
||||||
|
join(config.bakPath, originFilename.replace(/\//g, '')),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const service = Container.get(ScriptHistoryService);
|
||||||
|
const data = await service.save(path, filename, content, {
|
||||||
|
skipHistory: req.body.skipHistory,
|
||||||
|
expectedHash: req.body.expectedHash,
|
||||||
|
});
|
||||||
|
if (removeSource) await rmPath(originFilePath);
|
||||||
|
return res.send({ code: 200, data });
|
||||||
|
}
|
||||||
if (fileExists) {
|
if (fileExists) {
|
||||||
await fs.copyFile(
|
await fs.copyFile(
|
||||||
originFilePath,
|
originFilePath,
|
||||||
@@ -209,7 +380,7 @@ export default (app: Router) => {
|
|||||||
await writeFileWithLock(filePath, content);
|
await writeFileWithLock(filePath, content);
|
||||||
return res.send({ code: 200 });
|
return res.send({ code: 200 });
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
return next(e);
|
return historyError(e, res, next);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
@@ -221,6 +392,11 @@ export default (app: Router) => {
|
|||||||
filename: Joi.string().required(),
|
filename: Joi.string().required(),
|
||||||
path: Joi.string().optional().allow(''),
|
path: Joi.string().optional().allow(''),
|
||||||
content: Joi.string().required().allow(''),
|
content: Joi.string().required().allow(''),
|
||||||
|
skipHistory: Joi.boolean().optional(),
|
||||||
|
expectedHash: Joi.string()
|
||||||
|
.hex()
|
||||||
|
.length(64)
|
||||||
|
.when('skipHistory', { is: true, then: Joi.required() }),
|
||||||
}),
|
}),
|
||||||
}),
|
}),
|
||||||
async (req: Request, res: Response, next: NextFunction) => {
|
async (req: Request, res: Response, next: NextFunction) => {
|
||||||
@@ -238,10 +414,14 @@ export default (app: Router) => {
|
|||||||
message: t('暂无权限'),
|
message: t('暂无权限'),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
await writeFileWithLock(filePath, content);
|
const service = Container.get(ScriptHistoryService);
|
||||||
return res.send({ code: 200 });
|
const data = await service.save(path || '', filename, content, {
|
||||||
|
skipHistory: req.body.skipHistory,
|
||||||
|
expectedHash: req.body.expectedHash,
|
||||||
|
});
|
||||||
|
return res.send({ code: 200, data });
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
return next(e);
|
return historyError(e, res, next);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
@@ -368,12 +548,21 @@ export default (app: Router) => {
|
|||||||
if (!isPathAllowed(filePath)) {
|
if (!isPathAllowed(filePath)) {
|
||||||
return res.send({ code: 403, message: t('暂无权限') });
|
return res.send({ code: 403, message: t('暂无权限') });
|
||||||
}
|
}
|
||||||
const logPath = join(config.logPath, path, `${name}.swap`);
|
const logPath = resolveFileAccess(config.logPath, [
|
||||||
|
path,
|
||||||
|
`${name}.swap`,
|
||||||
|
]);
|
||||||
|
if (!logPath) {
|
||||||
|
return res.send({ code: 403, message: t('暂无权限') });
|
||||||
|
}
|
||||||
|
|
||||||
const scriptService = Container.get(ScriptService);
|
const scriptService = Container.get(ScriptService);
|
||||||
const result = await scriptService.stopScript(filePath, pid);
|
const result = await scriptService.stopScript(filePath, pid);
|
||||||
setTimeout(() => {
|
setTimeout(() => {
|
||||||
rmPath(logPath);
|
const cleanupPath = resolveFileAccess(config.logPath, [logPath]);
|
||||||
|
if (cleanupPath) {
|
||||||
|
void rmPath(cleanupPath);
|
||||||
|
}
|
||||||
}, 3000);
|
}, 3000);
|
||||||
res.send(result);
|
res.send(result);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import { Container } from 'typedi';
|
|||||||
import { Logger } from 'winston';
|
import { Logger } from 'winston';
|
||||||
import SubscriptionService from '../services/subscription';
|
import SubscriptionService from '../services/subscription';
|
||||||
import { celebrate, Joi } from 'celebrate';
|
import { celebrate, Joi } from 'celebrate';
|
||||||
import CronExpressionParser from 'cron-parser';
|
import { isValidCronSchedule } from '../shared/cronSchedule';
|
||||||
const route = Router();
|
const route = Router();
|
||||||
|
|
||||||
export default (app: Router) => {
|
export default (app: Router) => {
|
||||||
@@ -58,10 +58,7 @@ export default (app: Router) => {
|
|||||||
async (req: Request, res: Response, next: NextFunction) => {
|
async (req: Request, res: Response, next: NextFunction) => {
|
||||||
const logger: Logger = Container.get('logger');
|
const logger: Logger = Container.get('logger');
|
||||||
try {
|
try {
|
||||||
if (
|
if (!req.body.schedule || isValidCronSchedule(req.body.schedule)) {
|
||||||
!req.body.schedule ||
|
|
||||||
CronExpressionParser.parse(req.body.schedule).hasNext()
|
|
||||||
) {
|
|
||||||
const subscriptionService = Container.get(SubscriptionService);
|
const subscriptionService = Container.get(SubscriptionService);
|
||||||
const data = await subscriptionService.create(req.body);
|
const data = await subscriptionService.create(req.body);
|
||||||
return res.send({ code: 200, data });
|
return res.send({ code: 200, data });
|
||||||
@@ -204,11 +201,7 @@ export default (app: Router) => {
|
|||||||
async (req: Request, res: Response, next: NextFunction) => {
|
async (req: Request, res: Response, next: NextFunction) => {
|
||||||
const logger: Logger = Container.get('logger');
|
const logger: Logger = Container.get('logger');
|
||||||
try {
|
try {
|
||||||
if (
|
if (!req.body.schedule || isValidCronSchedule(req.body.schedule)) {
|
||||||
!req.body.schedule ||
|
|
||||||
typeof req.body.schedule === 'object' ||
|
|
||||||
CronExpressionParser.parse(req.body.schedule).hasNext()
|
|
||||||
) {
|
|
||||||
const subscriptionService = Container.get(SubscriptionService);
|
const subscriptionService = Container.get(SubscriptionService);
|
||||||
const data = await subscriptionService.update(req.body);
|
const data = await subscriptionService.update(req.body);
|
||||||
return res.send({ code: 200, data });
|
return res.send({ code: 200, data });
|
||||||
|
|||||||
+41
-10
@@ -4,19 +4,17 @@ import { Logger } from 'winston';
|
|||||||
import * as fs from 'fs/promises';
|
import * as fs from 'fs/promises';
|
||||||
import config from '../config';
|
import config from '../config';
|
||||||
import SystemService from '../services/system';
|
import SystemService from '../services/system';
|
||||||
|
import { containerResources } from '../shared/containerResources';
|
||||||
import { celebrate, Joi } from 'celebrate';
|
import { celebrate, Joi } from 'celebrate';
|
||||||
import UserService from '../services/user';
|
import UserService from '../services/user';
|
||||||
import { t } from '../shared/i18n';
|
import { t } from '../shared/i18n';
|
||||||
import { isDefaultAuthInfo } from '../shared/auth';
|
import { isDefaultAuthInfo } from '../shared/auth';
|
||||||
import {
|
import { getUniqPath, handleLogPath, parseVersion } from '../config/util';
|
||||||
getUniqPath,
|
import { findCronId } from '../shared/cronCommand';
|
||||||
handleLogPath,
|
|
||||||
parseVersion,
|
|
||||||
promiseExec,
|
|
||||||
} from '../config/util';
|
|
||||||
import dayjs from 'dayjs';
|
import dayjs from 'dayjs';
|
||||||
import multer from 'multer';
|
import multer from 'multer';
|
||||||
import { logStreamManager } from '../shared/logStreamManager';
|
import { logStreamManager } from '../shared/logStreamManager';
|
||||||
|
import { rateLimit } from 'express-rate-limit';
|
||||||
|
|
||||||
const route = Router();
|
const route = Router();
|
||||||
const storage = multer.diskStorage({
|
const storage = multer.diskStorage({
|
||||||
@@ -65,7 +63,8 @@ export default (app: Router) => {
|
|||||||
try {
|
try {
|
||||||
const systemService = Container.get(SystemService);
|
const systemService = Container.get(SystemService);
|
||||||
const data = await systemService.getSystemConfig();
|
const data = await systemService.getSystemConfig();
|
||||||
res.send({ code: 200, data });
|
res.send({ code: 200, data: { ...data, effectiveCronConcurrency: systemService.cronConcurrency,
|
||||||
|
availableCpus: containerResources().cpus } });
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
return next(e);
|
return next(e);
|
||||||
}
|
}
|
||||||
@@ -208,6 +207,17 @@ export default (app: Router) => {
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
|
route.get(
|
||||||
|
'/update-status',
|
||||||
|
async (req: Request, res: Response, next: NextFunction) => {
|
||||||
|
try {
|
||||||
|
res.send(await Container.get(SystemService).getUpgradeStatus());
|
||||||
|
} catch (e) {
|
||||||
|
return next(e);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
route.put(
|
route.put(
|
||||||
'/reload',
|
'/reload',
|
||||||
celebrate({
|
celebrate({
|
||||||
@@ -249,6 +259,7 @@ export default (app: Router) => {
|
|||||||
|
|
||||||
route.put(
|
route.put(
|
||||||
'/command-run',
|
'/command-run',
|
||||||
|
rateLimit({ windowMs: 60 * 1000, limit: 60 }),
|
||||||
celebrate({
|
celebrate({
|
||||||
body: Joi.object({
|
body: Joi.object({
|
||||||
command: Joi.string().required(),
|
command: Joi.string().required(),
|
||||||
@@ -258,8 +269,13 @@ export default (app: Router) => {
|
|||||||
try {
|
try {
|
||||||
const systemService = Container.get(SystemService);
|
const systemService = Container.get(SystemService);
|
||||||
const command = req.body.command;
|
const command = req.body.command;
|
||||||
const idStr = `cat ${config.crontabFile} | grep -E "${command}" | perl -pe "s|.*ID=(.*) ${command}.*|\\1|" | head -1 | awk -F " " '{print $1}' | xargs echo -n`;
|
const crontab = await fs
|
||||||
let id = await promiseExec(idStr);
|
.readFile(config.crontabFile, 'utf8')
|
||||||
|
.catch((e) => {
|
||||||
|
if (e.code === 'ENOENT') return '';
|
||||||
|
throw e;
|
||||||
|
});
|
||||||
|
const id = findCronId(crontab, command);
|
||||||
const uniqPath = await getUniqPath(command, id);
|
const uniqPath = await getUniqPath(command, id);
|
||||||
const logTime = dayjs().format('YYYY-MM-DD-HH-mm-ss-SSS');
|
const logTime = dayjs().format('YYYY-MM-DD-HH-mm-ss-SSS');
|
||||||
const logPath = `${uniqPath}/${logTime}.log`;
|
const logPath = `${uniqPath}/${logTime}.log`;
|
||||||
@@ -274,7 +290,9 @@ export default (app: Router) => {
|
|||||||
onEnd: async (cp, endTime, diff) => {
|
onEnd: async (cp, endTime, diff) => {
|
||||||
// Close the stream after task completion
|
// Close the stream after task completion
|
||||||
try {
|
try {
|
||||||
await logStreamManager.closeStream(await handleLogPath(logPath));
|
await logStreamManager.closeStream(
|
||||||
|
await handleLogPath(logPath),
|
||||||
|
);
|
||||||
} finally {
|
} finally {
|
||||||
res.end();
|
res.end();
|
||||||
}
|
}
|
||||||
@@ -378,6 +396,19 @@ export default (app: Router) => {
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
|
route.get(
|
||||||
|
'/notify-log',
|
||||||
|
async (req: Request, res: Response, next: NextFunction) => {
|
||||||
|
try {
|
||||||
|
const systemService = Container.get(SystemService);
|
||||||
|
const data = await systemService.getNotifyLog();
|
||||||
|
res.send({ code: 200, data });
|
||||||
|
} catch (e) {
|
||||||
|
return next(e);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
route.delete(
|
route.delete(
|
||||||
'/log',
|
'/log',
|
||||||
async (req: Request, res: Response, next: NextFunction) => {
|
async (req: Request, res: Response, next: NextFunction) => {
|
||||||
|
|||||||
@@ -310,20 +310,6 @@ export default (app: Router) => {
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
route.put(
|
|
||||||
'/notification/init',
|
|
||||||
async (req: Request, res: Response, next: NextFunction) => {
|
|
||||||
const logger: Logger = Container.get('logger');
|
|
||||||
try {
|
|
||||||
const userService = Container.get(UserService);
|
|
||||||
const result = await userService.updateNotificationMode(req.body);
|
|
||||||
res.send(result);
|
|
||||||
} catch (e) {
|
|
||||||
return next(e);
|
|
||||||
}
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
route.put(
|
route.put(
|
||||||
'/avatar',
|
'/avatar',
|
||||||
upload.single('avatar'),
|
upload.single('avatar'),
|
||||||
|
|||||||
+44
-28
@@ -5,6 +5,7 @@ import { Container } from 'typedi';
|
|||||||
import config from './config';
|
import config from './config';
|
||||||
import Logger from './loaders/logger';
|
import Logger from './loaders/logger';
|
||||||
import { errStack } from './shared/errors';
|
import { errStack } from './shared/errors';
|
||||||
|
import { initializeLanguageSync } from './shared/languageSync';
|
||||||
import { type GrpcServerService } from './services/grpc';
|
import { type GrpcServerService } from './services/grpc';
|
||||||
import { type HttpServerService } from './services/http';
|
import { type HttpServerService } from './services/http';
|
||||||
|
|
||||||
@@ -27,6 +28,7 @@ class Application {
|
|||||||
if (cluster.isPrimary) {
|
if (cluster.isPrimary) {
|
||||||
await this.initializeDatabase();
|
await this.initializeDatabase();
|
||||||
}
|
}
|
||||||
|
await initializeLanguageSync();
|
||||||
if (cluster.isPrimary) {
|
if (cluster.isPrimary) {
|
||||||
this.startMasterProcess();
|
this.startMasterProcess();
|
||||||
} else {
|
} else {
|
||||||
@@ -39,11 +41,18 @@ class Application {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private startMasterProcess() {
|
private startMasterProcess() {
|
||||||
|
const readyTimeout =
|
||||||
|
process.arch === 'arm' &&
|
||||||
|
Number(
|
||||||
|
(process.config.variables as { arm_version?: string | number }).arm_version,
|
||||||
|
) === 6
|
||||||
|
? 120000
|
||||||
|
: 30000;
|
||||||
// Fork gRPC worker first and wait for it to be ready
|
// Fork gRPC worker first and wait for it to be ready
|
||||||
const grpcWorker = this.forkWorker('grpc');
|
const grpcWorker = this.forkWorker('grpc');
|
||||||
|
|
||||||
// Wait for gRPC worker to signal it's ready before starting HTTP worker
|
// Wait for gRPC worker to signal it's ready before starting HTTP worker
|
||||||
this.waitForWorkerReady(grpcWorker, 30000)
|
this.waitForWorkerReady(grpcWorker, readyTimeout)
|
||||||
.then(() => {
|
.then(() => {
|
||||||
Logger.info('[boot] gRPC worker is ready, starting HTTP worker');
|
Logger.info('[boot] gRPC worker is ready, starting HTTP worker');
|
||||||
this.httpWorker = this.forkWorker('http');
|
this.httpWorker = this.forkWorker('http');
|
||||||
@@ -69,7 +78,7 @@ class Application {
|
|||||||
Logger.warn('Unable to notify HTTP worker of scheduler exit');
|
Logger.warn('Unable to notify HTTP worker of scheduler exit');
|
||||||
}
|
}
|
||||||
const newGrpcWorker = this.forkWorker('grpc');
|
const newGrpcWorker = this.forkWorker('grpc');
|
||||||
this.waitForWorkerReady(newGrpcWorker, 30000)
|
this.waitForWorkerReady(newGrpcWorker, readyTimeout)
|
||||||
.then(() => {
|
.then(() => {
|
||||||
Logger.info('gRPC worker restarted and ready');
|
Logger.info('gRPC worker restarted and ready');
|
||||||
// Re-register cron jobs by notifying the HTTP worker
|
// Re-register cron jobs by notifying the HTTP worker
|
||||||
@@ -129,7 +138,19 @@ class Application {
|
|||||||
workerEnv.pmx = 'false';
|
workerEnv.pmx = 'false';
|
||||||
}
|
}
|
||||||
const worker = cluster.fork(workerEnv);
|
const worker = cluster.fork(workerEnv);
|
||||||
|
if (serviceType === 'grpc') worker.on('message', (msg) => {
|
||||||
|
if (msg === 'crontab-install-failed' && this.httpWorker?.isConnected()) {
|
||||||
|
this.httpWorker.send('reregister-crons');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
if (serviceType === 'http') worker.on('message', (msg) => {
|
||||||
|
if (msg?.type !== 'cancel-queued-triggers') return;
|
||||||
|
for (const [id, meta] of this.workerMetadataMap) {
|
||||||
|
const target = cluster.workers?.[id];
|
||||||
|
if (meta.serviceType === 'grpc' && target?.isConnected()) target.send(msg);
|
||||||
|
}
|
||||||
|
});
|
||||||
this.workerMetadataMap.set(worker.id, {
|
this.workerMetadataMap.set(worker.id, {
|
||||||
id: worker.id,
|
id: worker.id,
|
||||||
pid: worker.process.pid!,
|
pid: worker.process.pid!,
|
||||||
@@ -146,32 +167,8 @@ class Application {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async setupMiddlewares(): Promise<express.Application> {
|
private async setupMiddlewares(): Promise<express.Application> {
|
||||||
// Only the HTTP worker needs an Express application and its middleware.
|
const { createHttpApplication } = await import('./loaders/httpApplication');
|
||||||
const [
|
return createHttpApplication();
|
||||||
{ default: createExpress },
|
|
||||||
{ default: helmet },
|
|
||||||
{ default: cors },
|
|
||||||
{ default: compression },
|
|
||||||
{ monitoringMiddleware },
|
|
||||||
] = await Promise.all([
|
|
||||||
import('express'),
|
|
||||||
import('helmet'),
|
|
||||||
import('cors'),
|
|
||||||
import('compression'),
|
|
||||||
import('./middlewares/monitoring'),
|
|
||||||
]);
|
|
||||||
const app = createExpress();
|
|
||||||
app.use((req, res, next) => {
|
|
||||||
if (req.query.t) delete req.query.t;
|
|
||||||
next();
|
|
||||||
});
|
|
||||||
app.use(helmet({
|
|
||||||
contentSecurityPolicy: false,
|
|
||||||
}));
|
|
||||||
app.use(cors(config.cors));
|
|
||||||
app.use(compression());
|
|
||||||
app.use(monitoringMiddleware);
|
|
||||||
return app;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private setupMasterShutdown() {
|
private setupMasterShutdown() {
|
||||||
@@ -232,8 +229,21 @@ class Application {
|
|||||||
Logger.info(`[boot] ${serviceType} worker started (PID: ${process.pid})`);
|
Logger.info(`[boot] ${serviceType} worker started (PID: ${process.pid})`);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
const { triggerHistory } = await import('./shared/triggerHistory');
|
||||||
|
const history = triggerHistory();
|
||||||
|
await history.interrupt(serviceType);
|
||||||
if (serviceType === 'http') {
|
if (serviceType === 'http') {
|
||||||
await this.startHttpService();
|
await this.startHttpService();
|
||||||
|
const retryExits = async () => {
|
||||||
|
try {
|
||||||
|
await history.retryPendingExits();
|
||||||
|
} catch (error) {
|
||||||
|
Logger.error(`[scheduler] Exit receipt retry failed: ${errStack(error)}`);
|
||||||
|
} finally {
|
||||||
|
if (!this.isShuttingDown) setTimeout(retryExits, 15000).unref();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
setTimeout(retryExits, 15000).unref();
|
||||||
} else {
|
} else {
|
||||||
await this.startGrpcService();
|
await this.startGrpcService();
|
||||||
}
|
}
|
||||||
@@ -280,6 +290,10 @@ class Application {
|
|||||||
process.on('message', async (msg) => {
|
process.on('message', async (msg) => {
|
||||||
if (msg === 'shutdown') {
|
if (msg === 'shutdown') {
|
||||||
this.gracefulShutdown(serviceType);
|
this.gracefulShutdown(serviceType);
|
||||||
|
} else if (serviceType === 'grpc' && typeof msg === 'object' && msg &&
|
||||||
|
'type' in msg && msg.type === 'cancel-queued-triggers' && 'ids' in msg && Array.isArray(msg.ids)) {
|
||||||
|
const { cancelWaitingTriggers } = await import('./shared/runCron');
|
||||||
|
cancelWaitingTriggers(msg.ids);
|
||||||
} else if (serviceType === 'http' &&
|
} else if (serviceType === 'http' &&
|
||||||
(msg === 'reregister-crons' || msg === 'scheduler-unavailable')) {
|
(msg === 'reregister-crons' || msg === 'scheduler-unavailable')) {
|
||||||
const { default: cronClient } = await import('./schedule/client');
|
const { default: cronClient } = await import('./schedule/client');
|
||||||
@@ -297,6 +311,8 @@ class Application {
|
|||||||
this.isShuttingDown = true;
|
this.isShuttingDown = true;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
const { triggerHistory } = await import('./shared/triggerHistory');
|
||||||
|
await triggerHistory().interrupt(serviceType);
|
||||||
if (serviceType === 'http') {
|
if (serviceType === 'http') {
|
||||||
await this.httpServerService?.shutdown();
|
await this.httpServerService?.shutdown();
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -0,0 +1,61 @@
|
|||||||
|
import { storageFull, writeExit } from '../shared/managedExit.cjs';
|
||||||
|
import { QueryTypes } from 'sequelize';
|
||||||
|
import { randomUUID } from 'crypto';
|
||||||
|
import { sequelize } from '../data';
|
||||||
|
import { systemOwner, TriggerHistory } from '../shared/triggerHistory';
|
||||||
|
|
||||||
|
async function receipt() {
|
||||||
|
const [action, task, pid, id, status, end] = process.argv.slice(2);
|
||||||
|
const history = new TriggerHistory(sequelize);
|
||||||
|
if (action === 'start') {
|
||||||
|
await history.recoverSystemOwners(Number(task));
|
||||||
|
const triggerId = id || `system-${task}-${randomUUID()}`;
|
||||||
|
const owner = systemOwner(Number(pid));
|
||||||
|
const command = status ? systemOwner(Number(status)) : owner;
|
||||||
|
const at = Number(process.env.QL_SYSTEM_ENTERED);
|
||||||
|
const rule = process.env.QL_SYSTEM_RULE;
|
||||||
|
const dispatched = Number(process.env.QL_SYSTEM_DISPATCHED) * 1000;
|
||||||
|
const dispatch = Number.isSafeInteger(dispatched) && dispatched > 0 && dispatched <= at ? dispatched : undefined;
|
||||||
|
const wrapper = rule && Number.isSafeInteger(at) && at > 0 && at <= Date.now() ? { rule, at, dispatch } : undefined;
|
||||||
|
const acquired = await history.startSystem(triggerId, Number(task), owner, command, wrapper);
|
||||||
|
if (!acquired) process.exitCode = 75;
|
||||||
|
else if (!id) process.stdout.write(triggerId);
|
||||||
|
} else if (action === 'finish') {
|
||||||
|
// The live wrapper inherited both IDs from the start it just claimed. A
|
||||||
|
// standalone retry must match the registered system task before writing.
|
||||||
|
const fromWrapper = process.env.QL_TRIGGER_ID === id && process.env.ID === task;
|
||||||
|
if (!fromWrapper || !id?.startsWith(`system-${Number(task)}-`)) {
|
||||||
|
const [registered] = await sequelize.query(
|
||||||
|
`SELECT 1 FROM CronTriggers WHERE id=$id AND cron_id=$task AND source='system' AND owner LIKE 'crond:%'
|
||||||
|
UNION ALL SELECT 1 FROM CronLegacyExitBookings WHERE trigger_id=$id AND cron_id=$task AND source='system' LIMIT 1`,
|
||||||
|
{ bind: { id, task: Number(task) }, type: QueryTypes.SELECT });
|
||||||
|
if (!registered) return;
|
||||||
|
}
|
||||||
|
const code = Number(status);
|
||||||
|
const finishedAt = end && /^\d{13}$/.test(end) && Number(end) <= Date.now() ? Number(end) : undefined;
|
||||||
|
const bound = fromWrapper && finishedAt !== undefined;
|
||||||
|
let journaled = false;
|
||||||
|
if (bound) {
|
||||||
|
try { writeExit(id, code, null, finishedAt); journaled = true; }
|
||||||
|
catch (error) {
|
||||||
|
if (!storageFull(error)) throw error;
|
||||||
|
console.error('[crond] Receipt storage full; settling the observed exit in SQLite:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await history.complete(id, code === 0 ? 'finished' : 'failed', null, code, finishedAt,
|
||||||
|
bound ? { version: 1, code, signal: null, time: finishedAt } : finishedAt === undefined ? undefined : 'legacy');
|
||||||
|
} catch (error) {
|
||||||
|
if (!journaled || !storageFull(error)) throw error;
|
||||||
|
// The regular recovery pass will settle the receipt after space returns.
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
throw new Error('Unknown crond receipt action');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
receipt().catch((error) => {
|
||||||
|
console.error('[crond] Failed to persist execution receipt:', error);
|
||||||
|
// At start, 75 means that another execution already owns the slot.
|
||||||
|
process.exitCode = process.argv[2] === 'finish' && storageFull(error) ? 75 : 1;
|
||||||
|
}).finally(() => sequelize.close());
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
// A single, disposable backend process. Resolve startup dependencies, but never
|
||||||
|
// run appLoader: its initData/initTask hooks install dependencies and start user work.
|
||||||
|
import 'reflect-metadata';
|
||||||
|
import dns from 'dns';
|
||||||
|
import http from 'http';
|
||||||
|
|
||||||
|
// The first scheduler connection can outlive its short RPC deadline. Let the
|
||||||
|
// normal single-flight recovery finish within the supervised startup budget.
|
||||||
|
async function waitForUpgradeScheduler(
|
||||||
|
readiness: { recover(): Promise<boolean> },
|
||||||
|
timeoutMs: number,
|
||||||
|
retryMs = 250,
|
||||||
|
) {
|
||||||
|
const deadline = Date.now() + timeoutMs;
|
||||||
|
do {
|
||||||
|
if (await readiness.recover()) return;
|
||||||
|
const remaining = deadline - Date.now();
|
||||||
|
if (remaining <= 0) break;
|
||||||
|
await new Promise((resolve) =>
|
||||||
|
setTimeout(resolve, Math.min(retryMs, remaining)),
|
||||||
|
);
|
||||||
|
} while (Date.now() < deadline);
|
||||||
|
throw new Error('Upgrade scheduler readiness failed');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function probeUpgrade() {
|
||||||
|
if (!process.send || process.env.QL_UPGRADE_PROBE !== '1')
|
||||||
|
throw new Error('Upgrade probe requires its isolated supervisor');
|
||||||
|
dns.setDefaultResultOrder('ipv4first');
|
||||||
|
const { default: config } = await import('../config');
|
||||||
|
const { sequelize } = await import('../data');
|
||||||
|
const { default: loadDatabase } = await import('../loaders/db');
|
||||||
|
const { Container } = await import('typedi');
|
||||||
|
const { GrpcServerService } = await import('../services/grpc');
|
||||||
|
const { HttpServerService } = await import('../services/http');
|
||||||
|
const { createHttpApplication } = await import('../loaders/httpApplication');
|
||||||
|
const { default: loadServer } = await import('../loaders/server');
|
||||||
|
const { default: loadExpress } = await import('../loaders/express');
|
||||||
|
await import('../loaders/app'); // Resolve normal startup hooks without running them.
|
||||||
|
const { default: inject } = await import('../loaders/depInjector');
|
||||||
|
const grpc = Container.get(GrpcServerService);
|
||||||
|
const server = Container.get(HttpServerService);
|
||||||
|
try {
|
||||||
|
await loadDatabase();
|
||||||
|
// Exercise an actual on-disk transaction as well as schema initialization.
|
||||||
|
await sequelize.transaction(async (transaction) => {
|
||||||
|
await sequelize.query('CREATE TABLE upgrade_probe (value INTEGER)', {
|
||||||
|
transaction,
|
||||||
|
});
|
||||||
|
await sequelize.query('INSERT INTO upgrade_probe VALUES (42)', {
|
||||||
|
transaction,
|
||||||
|
});
|
||||||
|
const [rows] = await sequelize.query('SELECT value FROM upgrade_probe', {
|
||||||
|
transaction,
|
||||||
|
});
|
||||||
|
if ((rows[0] as { value: number })?.value !== 42)
|
||||||
|
throw new Error('Upgrade database readback failed');
|
||||||
|
});
|
||||||
|
inject();
|
||||||
|
config.grpcPort = await grpc.initialize();
|
||||||
|
const { default: cronClient } = await import('../schedule/client');
|
||||||
|
cronClient.readiness.configure(async () => {});
|
||||||
|
await waitForUpgradeScheduler(
|
||||||
|
cronClient.readiness,
|
||||||
|
process.arch === 'arm' &&
|
||||||
|
Number(
|
||||||
|
(process.config.variables as { arm_version?: string | number })
|
||||||
|
.arm_version,
|
||||||
|
) === 6
|
||||||
|
? 60000
|
||||||
|
: 10000,
|
||||||
|
);
|
||||||
|
const app = await createHttpApplication();
|
||||||
|
loadExpress({ app });
|
||||||
|
const listener = await server.initialize(app, 0);
|
||||||
|
await loadServer({ server: listener });
|
||||||
|
const address = listener.address();
|
||||||
|
if (!address || typeof address === 'string')
|
||||||
|
throw new Error('No probe HTTP port');
|
||||||
|
const health = await new Promise<any>((resolve, reject) => {
|
||||||
|
const request = http.get(
|
||||||
|
`http://127.0.0.1:${address.port}/api/health`,
|
||||||
|
(response) => {
|
||||||
|
let body = '';
|
||||||
|
response.on('data', (chunk) => {
|
||||||
|
body += chunk;
|
||||||
|
});
|
||||||
|
response.on('error', reject);
|
||||||
|
response.on('end', () => {
|
||||||
|
try {
|
||||||
|
const result = JSON.parse(body);
|
||||||
|
if (
|
||||||
|
response.statusCode !== 200 ||
|
||||||
|
result.code !== 200 ||
|
||||||
|
result.data?.status !== 'ok' ||
|
||||||
|
!result.data.services?.grpc ||
|
||||||
|
!result.data.services?.http
|
||||||
|
)
|
||||||
|
throw new Error('Upgrade HTTP/gRPC health check failed');
|
||||||
|
resolve(result.data);
|
||||||
|
} catch (error) {
|
||||||
|
reject(error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
},
|
||||||
|
);
|
||||||
|
request.setTimeout(10000, () =>
|
||||||
|
request.destroy(new Error('Upgrade health request timed out')),
|
||||||
|
);
|
||||||
|
request.on('error', reject);
|
||||||
|
});
|
||||||
|
process.send!({
|
||||||
|
type: 'upgrade-probe',
|
||||||
|
ok: true,
|
||||||
|
build: health.build,
|
||||||
|
memory: process.memoryUsage(),
|
||||||
|
pid: process.pid,
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
await server.shutdown();
|
||||||
|
grpc.getServer().forceShutdown();
|
||||||
|
await sequelize.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
process.once('disconnect', () => process.exit(1));
|
||||||
|
probeUpgrade()
|
||||||
|
.then(() => process.exit(0))
|
||||||
|
.catch((error) => {
|
||||||
|
process.send?.({
|
||||||
|
type: 'upgrade-probe',
|
||||||
|
ok: false,
|
||||||
|
code: error.code,
|
||||||
|
message: error.message,
|
||||||
|
stack: error.stack,
|
||||||
|
});
|
||||||
|
console.error(error);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
@@ -27,6 +27,7 @@ export const SAMPLE_FILES = [
|
|||||||
];
|
];
|
||||||
|
|
||||||
export const PYTHON_INSTALL_DIR = process.env.PYTHON_HOME;
|
export const PYTHON_INSTALL_DIR = process.env.PYTHON_HOME;
|
||||||
|
export const PYTHON_VENV_DIR = process.env.PYTHON_VENV_DIR;
|
||||||
|
|
||||||
export const NotificationModeStringMap = {
|
export const NotificationModeStringMap = {
|
||||||
0: 'gotify',
|
0: 'gotify',
|
||||||
|
|||||||
+93
-45
@@ -1,6 +1,5 @@
|
|||||||
import { execSync } from 'child_process';
|
import { execFileSync } from 'child_process';
|
||||||
import * as fs from 'fs/promises';
|
import * as fs from 'fs/promises';
|
||||||
import * as os from 'os';
|
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
import config from './index';
|
import config from './index';
|
||||||
import { fileExist } from './util';
|
import { fileExist } from './util';
|
||||||
@@ -24,71 +23,120 @@ const clientCertPath = path.join(certDir, 'client.crt');
|
|||||||
|
|
||||||
let cachedConfig: GrpcTlsConfig | null = null;
|
let cachedConfig: GrpcTlsConfig | null = null;
|
||||||
|
|
||||||
function run(cmd: string, execOpts?: Record<string, unknown>): string {
|
function run(args: string[]): string {
|
||||||
const opts = { stdio: 'pipe', timeout: 30000, encoding: 'utf-8', ...execOpts } as any;
|
return execFileSync('openssl', args, {
|
||||||
return (execSync(cmd, opts) as string).trim();
|
stdio: 'pipe',
|
||||||
}
|
timeout: 30000,
|
||||||
|
encoding: 'utf-8',
|
||||||
async function tmpFile(prefix: string): Promise<string> {
|
}).trim();
|
||||||
const dir = (await fileExist(certDir)) ? certDir : os.tmpdir();
|
|
||||||
await fs.mkdir(dir, { recursive: true });
|
|
||||||
return path.join(dir, `.${prefix}_${Date.now()}_${Math.random().toString(36).slice(2)}.pem`);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function generateAllCerts(): Promise<GrpcTlsConfig> {
|
async function generateAllCerts(): Promise<GrpcTlsConfig> {
|
||||||
Logger.info('[boot] Generating gRPC mTLS certificates...');
|
Logger.info('[boot] Generating gRPC mTLS certificates...');
|
||||||
|
await fs.mkdir(certDir, { recursive: true, mode: 0o700 });
|
||||||
const caKeyTmp = await tmpFile('ca_key');
|
// Atomically reserve a private workspace for keys, CSRs and OpenSSL side files.
|
||||||
const caCertTmp = await tmpFile('ca_cert');
|
const workspace = await fs.mkdtemp(path.join(certDir, '.generate-'));
|
||||||
const serverKeyTmp = await tmpFile('server_key');
|
const caKeyTmp = path.join(workspace, 'ca.key');
|
||||||
const serverCsrTmp = await tmpFile('server_csr');
|
const caCertTmp = path.join(workspace, 'ca.crt');
|
||||||
const serverExtTmp = await tmpFile('server_ext');
|
const serverKeyTmp = path.join(workspace, 'server.key');
|
||||||
const clientKeyTmp = await tmpFile('client_key');
|
const serverCsrTmp = path.join(workspace, 'server.csr');
|
||||||
const clientCsrTmp = await tmpFile('client_csr');
|
const serverExtTmp = path.join(workspace, 'server.ext');
|
||||||
const clientExtTmp = await tmpFile('client_ext');
|
const clientKeyTmp = path.join(workspace, 'client.key');
|
||||||
const srlTmp = path.join(path.dirname(caKeyTmp), '.grpc_ca.srl');
|
const clientCsrTmp = path.join(workspace, 'client.csr');
|
||||||
|
const clientExtTmp = path.join(workspace, 'client.ext');
|
||||||
|
|
||||||
const cleanup = async () => {
|
const cleanup = async () => {
|
||||||
for (const f of [caKeyTmp, caCertTmp, serverKeyTmp, serverCsrTmp, serverExtTmp,
|
await fs.rm(workspace, { recursive: true, force: true });
|
||||||
clientKeyTmp, clientCsrTmp, clientExtTmp, srlTmp]) {
|
|
||||||
try { await fs.unlink(f); } catch {}
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// 1. CA(私钥直接存盘,证书写入临时文件供签发使用)
|
run(['genrsa', '-out', caKeyTmp, '2048']);
|
||||||
run(`openssl genrsa -out '${caKeyTmp}' 2048 2>/dev/null`);
|
run([
|
||||||
run(`openssl req -new -x509 -days 3650 -key '${caKeyTmp}' -out '${caCertTmp}' -subj '/CN=qinglong-ca/O=qinglong/C=CN' 2>/dev/null`);
|
'req',
|
||||||
|
'-new',
|
||||||
|
'-x509',
|
||||||
|
'-days',
|
||||||
|
'3650',
|
||||||
|
'-key',
|
||||||
|
caKeyTmp,
|
||||||
|
'-out',
|
||||||
|
caCertTmp,
|
||||||
|
'-subj',
|
||||||
|
'/CN=qinglong-ca/O=qinglong/C=CN',
|
||||||
|
]);
|
||||||
const caKey = await fs.readFile(caKeyTmp, 'utf-8');
|
const caKey = await fs.readFile(caKeyTmp, 'utf-8');
|
||||||
const caCert = await fs.readFile(caCertTmp, 'utf-8');
|
const caCert = await fs.readFile(caCertTmp, 'utf-8');
|
||||||
await fs.mkdir(certDir, { recursive: true });
|
|
||||||
await fs.writeFile(caKeyPath, caKey, { mode: 0o600 });
|
await fs.writeFile(caKeyPath, caKey, { mode: 0o600 });
|
||||||
|
|
||||||
// 2. 服务端
|
run(['genrsa', '-out', serverKeyTmp, '2048']);
|
||||||
run(`openssl genrsa -out '${serverKeyTmp}' 2048 2>/dev/null`);
|
run([
|
||||||
run(`openssl req -new -key '${serverKeyTmp}' -out '${serverCsrTmp}' -subj '/CN=grpc-server' 2>/dev/null`);
|
'req',
|
||||||
await fs.writeFile(serverExtTmp, 'subjectAltName=DNS:localhost,IP:127.0.0.1,IP:::1\n');
|
'-new',
|
||||||
const serverCert = run(
|
'-key',
|
||||||
`openssl x509 -req -days 3650 -in '${serverCsrTmp}' -CA '${caCertTmp}' -CAkey '${caKeyTmp}' -CAcreateserial -extfile '${serverExtTmp}' 2>/dev/null`,
|
serverKeyTmp,
|
||||||
|
'-out',
|
||||||
|
serverCsrTmp,
|
||||||
|
'-subj',
|
||||||
|
'/CN=grpc-server',
|
||||||
|
]);
|
||||||
|
await fs.writeFile(
|
||||||
|
serverExtTmp,
|
||||||
|
'subjectAltName=DNS:localhost,IP:127.0.0.1,IP:::1\n',
|
||||||
);
|
);
|
||||||
|
const serverCert = run([
|
||||||
|
'x509',
|
||||||
|
'-req',
|
||||||
|
'-days',
|
||||||
|
'3650',
|
||||||
|
'-in',
|
||||||
|
serverCsrTmp,
|
||||||
|
'-CA',
|
||||||
|
caCertTmp,
|
||||||
|
'-CAkey',
|
||||||
|
caKeyTmp,
|
||||||
|
'-CAserial',
|
||||||
|
path.join(workspace, 'ca.srl'),
|
||||||
|
'-CAcreateserial',
|
||||||
|
'-extfile',
|
||||||
|
serverExtTmp,
|
||||||
|
]);
|
||||||
const serverKey = await fs.readFile(serverKeyTmp, 'utf-8');
|
const serverKey = await fs.readFile(serverKeyTmp, 'utf-8');
|
||||||
|
|
||||||
// 3. 客户端
|
run(['genrsa', '-out', clientKeyTmp, '2048']);
|
||||||
run(`openssl genrsa -out '${clientKeyTmp}' 2048 2>/dev/null`);
|
run([
|
||||||
run(`openssl req -new -key '${clientKeyTmp}' -out '${clientCsrTmp}' -subj '/CN=grpc-client' 2>/dev/null`);
|
'req',
|
||||||
|
'-new',
|
||||||
|
'-key',
|
||||||
|
clientKeyTmp,
|
||||||
|
'-out',
|
||||||
|
clientCsrTmp,
|
||||||
|
'-subj',
|
||||||
|
'/CN=grpc-client',
|
||||||
|
]);
|
||||||
await fs.writeFile(clientExtTmp, 'extendedKeyUsage=clientAuth\n');
|
await fs.writeFile(clientExtTmp, 'extendedKeyUsage=clientAuth\n');
|
||||||
const clientCert = run(
|
const clientCert = run([
|
||||||
`openssl x509 -req -days 3650 -in '${clientCsrTmp}' -CA '${caCertTmp}' -CAkey '${caKeyTmp}' -CAcreateserial -extfile '${clientExtTmp}' 2>/dev/null`,
|
'x509',
|
||||||
);
|
'-req',
|
||||||
|
'-days',
|
||||||
|
'3650',
|
||||||
|
'-in',
|
||||||
|
clientCsrTmp,
|
||||||
|
'-CA',
|
||||||
|
caCertTmp,
|
||||||
|
'-CAkey',
|
||||||
|
caKeyTmp,
|
||||||
|
'-CAserial',
|
||||||
|
path.join(workspace, 'ca.srl'),
|
||||||
|
'-CAcreateserial',
|
||||||
|
'-extfile',
|
||||||
|
clientExtTmp,
|
||||||
|
]);
|
||||||
const clientKey = await fs.readFile(clientKeyTmp, 'utf-8');
|
const clientKey = await fs.readFile(clientKeyTmp, 'utf-8');
|
||||||
|
|
||||||
await cleanup();
|
|
||||||
Logger.info('[boot] gRPC mTLS certificates generated successfully');
|
Logger.info('[boot] gRPC mTLS certificates generated successfully');
|
||||||
|
|
||||||
return { caCert, serverCert, serverKey, clientCert, clientKey };
|
return { caCert, serverCert, serverKey, clientCert, clientKey };
|
||||||
} catch (e) {
|
} finally {
|
||||||
await cleanup();
|
await cleanup();
|
||||||
throw e;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -180,12 +180,10 @@ export default {
|
|||||||
'/api/user/two-factor/login',
|
'/api/user/two-factor/login',
|
||||||
'/api/system',
|
'/api/system',
|
||||||
'/api/user/init',
|
'/api/user/init',
|
||||||
'/api/user/notification/init',
|
|
||||||
'/open/user/login',
|
'/open/user/login',
|
||||||
'/open/user/two-factor/login',
|
'/open/user/two-factor/login',
|
||||||
'/open/system',
|
'/open/system',
|
||||||
'/open/user/init',
|
'/open/user/init',
|
||||||
'/open/user/notification/init',
|
|
||||||
],
|
],
|
||||||
versionFile,
|
versionFile,
|
||||||
lastVersionFile,
|
lastVersionFile,
|
||||||
|
|||||||
+23
-26
@@ -1,5 +1,7 @@
|
|||||||
import { Subscription } from '../data/subscription';
|
import { Subscription } from '../data/subscription';
|
||||||
import isNil from 'lodash/isNil';
|
import isNil from 'lodash/isNil';
|
||||||
|
import { shellQuote } from '../shared/shellQuote';
|
||||||
|
import { getSubscriptionSshAlias } from '../shared/subscriptionPath';
|
||||||
|
|
||||||
export function formatUrl(doc: Subscription) {
|
export function formatUrl(doc: Subscription) {
|
||||||
let url = doc.url;
|
let url = doc.url;
|
||||||
@@ -7,7 +9,7 @@ export function formatUrl(doc: Subscription) {
|
|||||||
if (doc.type === 'private-repo') {
|
if (doc.type === 'private-repo') {
|
||||||
if (doc.pull_type === 'ssh-key') {
|
if (doc.pull_type === 'ssh-key') {
|
||||||
host = doc.url!.replace(/.*\@([^\:]+)\:.*/, '$1');
|
host = doc.url!.replace(/.*\@([^\:]+)\:.*/, '$1');
|
||||||
url = doc.url!.replace(host, doc.alias);
|
url = doc.url!.replace(host, getSubscriptionSshAlias(doc.alias));
|
||||||
} else {
|
} else {
|
||||||
host = doc.url!.replace(/.*\:\/\/([^\/]+)\/.*/, '$1');
|
host = doc.url!.replace(/.*\:\/\/([^\/]+)\/.*/, '$1');
|
||||||
const { username, password } = doc.pull_option as any;
|
const { username, password } = doc.pull_option as any;
|
||||||
@@ -18,29 +20,24 @@ export function formatUrl(doc: Subscription) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function formatCommand(doc: Subscription, url?: string) {
|
export function formatCommand(doc: Subscription, url?: string) {
|
||||||
let command = `SUB_ID=${doc.id} ql `;
|
const args =
|
||||||
let _url = url || formatUrl(doc).url;
|
doc.type === 'file'
|
||||||
const {
|
? ['raw', url || formatUrl(doc).url, doc.proxy || '']
|
||||||
type,
|
: [
|
||||||
whitelist,
|
'repo',
|
||||||
blacklist,
|
url || formatUrl(doc).url,
|
||||||
dependences,
|
doc.whitelist || '',
|
||||||
branch,
|
doc.blacklist || '',
|
||||||
extensions,
|
doc.dependences || '',
|
||||||
proxy,
|
doc.branch || '',
|
||||||
autoAddCron,
|
doc.extensions || '',
|
||||||
autoDelCron,
|
doc.proxy || '',
|
||||||
} = doc;
|
];
|
||||||
if (type === 'file') {
|
args.push(
|
||||||
command += `raw "${_url}" "${proxy || ''}" "${
|
String(isNil(doc.autoAddCron) ? true : Boolean(doc.autoAddCron)),
|
||||||
isNil(autoAddCron) ? true : Boolean(autoAddCron)
|
String(isNil(doc.autoDelCron) ? true : Boolean(doc.autoDelCron)),
|
||||||
}" "${isNil(autoDelCron) ? true : Boolean(autoDelCron)}"`;
|
);
|
||||||
} else {
|
return `SUB_ID=${shellQuote(String(doc.id))} ql ${args
|
||||||
command += `repo "${_url}" "${whitelist || ''}" "${blacklist || ''}" "${
|
.map((arg) => shellQuote(arg!))
|
||||||
dependences || ''
|
.join(' ')}`;
|
||||||
}" "${branch || ''}" "${extensions || ''}" "${proxy || ''}" "${
|
|
||||||
isNil(autoAddCron) ? true : Boolean(autoAddCron)
|
|
||||||
}" "${isNil(autoDelCron) ? true : Boolean(autoDelCron)}"`;
|
|
||||||
}
|
|
||||||
return command;
|
|
||||||
}
|
}
|
||||||
|
|||||||
+279
-112
@@ -1,18 +1,27 @@
|
|||||||
|
import { cronOperand } from '../shared/cronOperand';
|
||||||
|
import { finishStoppedTree, isProcessDescendant, systemOwner } from '../shared/cronProcess';
|
||||||
import * as fs from 'fs/promises';
|
import * as fs from 'fs/promises';
|
||||||
|
import { statSync } from 'fs';
|
||||||
import * as path from 'path';
|
import * as path from 'path';
|
||||||
import { exec, execSync } from 'child_process';
|
import { exec, execSync } from 'child_process';
|
||||||
import psTreeFun from 'ps-tree';
|
import psTreeFun from 'ps-tree';
|
||||||
import { promisify } from 'util';
|
import { promisify } from 'util';
|
||||||
import { load } from 'js-yaml';
|
import { load } from 'js-yaml';
|
||||||
import config from './index';
|
import config from './index';
|
||||||
import { PYTHON_INSTALL_DIR, TASK_COMMAND } from './const';
|
import { PYTHON_INSTALL_DIR, PYTHON_VENV_DIR } from './const';
|
||||||
import Logger from '../loaders/logger';
|
import Logger from '../loaders/logger';
|
||||||
import { writeFileWithLock } from '../shared/utils';
|
import { writeFileWithLock } from '../shared/utils';
|
||||||
import { DependenceTypes } from '../data/dependence';
|
import { DependenceTypes } from '../data/dependence';
|
||||||
import { FormData } from 'undici';
|
import { FormData } from 'undici';
|
||||||
import os from 'os';
|
import os from 'os';
|
||||||
import { maybeSudo, isInContainer } from './container';
|
import { maybeSudo, isInContainer } from './container';
|
||||||
|
import { shellQuote } from '../shared/shellQuote';
|
||||||
import { resolveFileAccess } from '../shared/fileAccess';
|
import { resolveFileAccess } from '../shared/fileAccess';
|
||||||
|
import {
|
||||||
|
mirrorWriteCommand,
|
||||||
|
normalizeMirror,
|
||||||
|
rewriteMirrorSources,
|
||||||
|
} from '../shared/linuxMirror';
|
||||||
|
|
||||||
export * from './share';
|
export * from './share';
|
||||||
|
|
||||||
@@ -489,13 +498,17 @@ export function psTree(pid: number): Promise<number[]> {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function killTask(pid: number, waitForExit = false) {
|
export async function killTask(pid: number, waitForExit = false, verifyOwner?: () => void,
|
||||||
|
withStopRequest?: (signalTree: () => Error | void, targets: string[]) => Promise<void | { targets: string[]; signalledAt: number | null }>) {
|
||||||
const descendants = await psTree(pid);
|
const descendants = await psTree(pid);
|
||||||
|
verifyOwner?.();
|
||||||
const signal = (target: number, sig: NodeJS.Signals) => {
|
const signal = (target: number, sig: NodeJS.Signals) => {
|
||||||
try {
|
try {
|
||||||
process.kill(target, sig);
|
process.kill(target, sig);
|
||||||
|
return true;
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
if (error.code !== 'ESRCH') throw error;
|
if (error.code !== 'ESRCH') throw error;
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
if (!waitForExit) {
|
if (!waitForExit) {
|
||||||
@@ -508,8 +521,53 @@ export async function killTask(pid: number, waitForExit = false) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const pids = [...descendants.reverse(), pid];
|
const pids = [...descendants.reverse(), pid];
|
||||||
for (const target of pids) signal(target, 'SIGTERM');
|
const births = new Map<number, string>();
|
||||||
|
if (verifyOwner) {
|
||||||
|
for (const target of pids) {
|
||||||
|
try {
|
||||||
|
const birth = systemOwner(target);
|
||||||
|
if (target === pid || isProcessDescendant(target, pid)) births.set(target, birth);
|
||||||
|
} catch (error) {
|
||||||
|
if (!['ENOENT', 'ESRCH'].includes((error as NodeJS.ErrnoException).code!)) throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
verifyOwner();
|
||||||
|
}
|
||||||
|
const sameProcess = (target: number) => {
|
||||||
|
if (!verifyOwner) return true;
|
||||||
|
if (!births.has(target)) return false;
|
||||||
|
try { return births.get(target) === systemOwner(target); }
|
||||||
|
catch (error) {
|
||||||
|
if (['ENOENT', 'ESRCH'].includes((error as NodeJS.ErrnoException).code!)) return false;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const signalTree = () => {
|
||||||
|
verifyOwner?.();
|
||||||
|
// The root owns the command's result. An ESRCH here must roll back the
|
||||||
|
// stop claim even if a descendant is still draining the output pipe.
|
||||||
|
if (verifyOwner && (!sameProcess(pid) || !signal(pid, 'SIGTERM'))) {
|
||||||
|
throw Object.assign(new Error('Command already exited'), { code: 'ESRCH' });
|
||||||
|
}
|
||||||
|
let deliveryError: Error | undefined;
|
||||||
|
for (const target of pids) {
|
||||||
|
if (verifyOwner && target === pid) continue;
|
||||||
|
try {
|
||||||
|
if (sameProcess(target)) signal(target, 'SIGTERM');
|
||||||
|
} catch (error) {
|
||||||
|
if (!withStopRequest || !verifyOwner) throw error;
|
||||||
|
// Root TERM succeeded. Commit that fact even if a child needs a retry.
|
||||||
|
deliveryError ??= error as Error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return deliveryError;
|
||||||
|
};
|
||||||
|
if (withStopRequest) {
|
||||||
|
const claim = await withStopRequest(signalTree, [...births.values()]);
|
||||||
|
if (claim) return finishStoppedTree(claim.targets, claim.signalledAt);
|
||||||
|
} else signalTree();
|
||||||
const alive = async (target: number) => {
|
const alive = async (target: number) => {
|
||||||
|
if (!sameProcess(target)) return false;
|
||||||
try {
|
try {
|
||||||
process.kill(target, 0);
|
process.kill(target, 0);
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
@@ -548,7 +606,9 @@ export async function killTask(pid: number, waitForExit = false) {
|
|||||||
};
|
};
|
||||||
let remaining = await wait();
|
let remaining = await wait();
|
||||||
if (!remaining.length) return;
|
if (!remaining.length) return;
|
||||||
for (const target of remaining) signal(target, 'SIGKILL');
|
for (const target of remaining) {
|
||||||
|
if (sameProcess(target)) signal(target, 'SIGKILL');
|
||||||
|
}
|
||||||
remaining = await wait();
|
remaining = await wait();
|
||||||
if (remaining.length)
|
if (remaining.length)
|
||||||
throw new Error(`Task processes did not exit: ${remaining.join(', ')}`);
|
throw new Error(`Task processes did not exit: ${remaining.join(', ')}`);
|
||||||
@@ -605,17 +665,7 @@ export async function getUniqPath(
|
|||||||
suffix = `_${id}`;
|
suffix = `_${id}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
let items = command.split(/ +/);
|
let str = cronOperand(command) || 'task';
|
||||||
|
|
||||||
const maxTimeCommandIndex = items.findIndex((x) => x === '-m');
|
|
||||||
if (maxTimeCommandIndex !== -1) {
|
|
||||||
items = items.slice(maxTimeCommandIndex + 2);
|
|
||||||
}
|
|
||||||
|
|
||||||
let str = items[0];
|
|
||||||
if (items[0] === TASK_COMMAND) {
|
|
||||||
str = items[1];
|
|
||||||
}
|
|
||||||
|
|
||||||
const dotIndex = str.lastIndexOf('.');
|
const dotIndex = str.lastIndexOf('.');
|
||||||
|
|
||||||
@@ -635,7 +685,8 @@ export async function getUniqPath(
|
|||||||
str = `${tempStr}_${str.slice(slashIndex + 1)}`;
|
str = `${tempStr}_${str.slice(slashIndex + 1)}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
return `${str}${suffix}`;
|
const base = str.replace(/[^a-zA-Z0-9_\u4e00-\u9fa5.-]+/g, '_').slice(0, 100 - suffix.length);
|
||||||
|
return `${base && !/^\.+$/.test(base) ? base : 'task'}${suffix}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function safeJSONParse(value?: string) {
|
export function safeJSONParse(value?: string) {
|
||||||
@@ -670,7 +721,9 @@ export async function setSystemTimezone(timezone: string): Promise<boolean> {
|
|||||||
throw new Error('Invalid timezone');
|
throw new Error('Invalid timezone');
|
||||||
}
|
}
|
||||||
|
|
||||||
await promiseExec(maybeSudo(`ln -sf /usr/share/zoneinfo/${timezone} /etc/localtime`));
|
await promiseExec(
|
||||||
|
maybeSudo(`ln -sf /usr/share/zoneinfo/${timezone} /etc/localtime`),
|
||||||
|
);
|
||||||
await promiseExec(`echo "${timezone}" | ${maybeSudo('tee /etc/timezone')}`);
|
await promiseExec(`echo "${timezone}" | ${maybeSudo('tee /etc/timezone')}`);
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
@@ -680,61 +733,177 @@ export async function setSystemTimezone(timezone: string): Promise<boolean> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Helper function to check if a name is a GitHub URL
|
||||||
|
function getPythonExecutable(): string {
|
||||||
|
return PYTHON_VENV_DIR
|
||||||
|
? shellQuote(path.resolve(config.rootPath, PYTHON_VENV_DIR, 'bin/python3'))
|
||||||
|
: 'python3';
|
||||||
|
}
|
||||||
|
|
||||||
|
function getPythonPip(): string {
|
||||||
|
return PYTHON_VENV_DIR ? `${getPythonExecutable()} -m pip` : 'pip3';
|
||||||
|
}
|
||||||
|
|
||||||
|
function isGitHubUrl(name: string): boolean {
|
||||||
|
// Support git+https://, git+http://, https://, and http:// URLs
|
||||||
|
// This covers GitHub URLs and other git-compatible repositories
|
||||||
|
return !!name.match(/^(git\+https?:\/\/|https?:\/\/)/i);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper function to check if a name is a requirements file
|
||||||
|
function isRequirementsFile(name: string): boolean {
|
||||||
|
return !!name.match(/requirements.*\.(txt|in)$/i);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper function to check if a name is a pyproject.toml file
|
||||||
|
function isPyprojectToml(name: string): boolean {
|
||||||
|
return name.endsWith('pyproject.toml');
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isPythonSourceDependency(name: string): boolean {
|
||||||
|
const source = name.trim();
|
||||||
|
return (
|
||||||
|
isGitHubUrl(source) || isRequirementsFile(source) || isPyprojectToml(source)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export function getGetCommand(type: DependenceTypes, name: string): string {
|
export function getGetCommand(type: DependenceTypes, name: string): string {
|
||||||
|
const trimmedName = name.trim();
|
||||||
|
|
||||||
|
// For Python dependencies installed from GitHub or requirements files,
|
||||||
|
// we can't reliably check if they're installed, so skip the check
|
||||||
|
if (type === DependenceTypes.python3) {
|
||||||
|
if (isPythonSourceDependency(trimmedName)) {
|
||||||
|
// Return a command that will always indicate not installed
|
||||||
|
// This ensures GitHub URLs and requirements files are always installed
|
||||||
|
return 'echo ""';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const baseCommands = {
|
const baseCommands = {
|
||||||
[DependenceTypes.nodejs]: `pnpm ls -g | grep "${name}" | head -1`,
|
[DependenceTypes.nodejs]: `pnpm ls -g | grep "${trimmedName}" | head -1`,
|
||||||
[DependenceTypes.python3]: `
|
[DependenceTypes.python3]: `${getPythonExecutable()} -c ${shellQuote(
|
||||||
python3 -c "exec('''
|
`from importlib.metadata import version, PackageNotFoundError
|
||||||
name='${name}'
|
import importlib.util
|
||||||
|
import sys
|
||||||
try:
|
try:
|
||||||
from importlib.metadata import version
|
try:
|
||||||
print(version(name))
|
from packaging.requirements import Requirement
|
||||||
except:
|
except ImportError:
|
||||||
import importlib.util as u
|
from pip._vendor.packaging.requirements import Requirement
|
||||||
import importlib.metadata as m
|
requirement = Requirement(sys.argv[1])
|
||||||
spec=u.find_spec(name)
|
if requirement.marker and not requirement.marker.evaluate():
|
||||||
print(name if spec else '')
|
print('not applicable')
|
||||||
''')"`,
|
else:
|
||||||
[DependenceTypes.linux]: getOsTypeSync() === 'Alpine'
|
try:
|
||||||
? `apk info -es ${name}`
|
installed = version(requirement.name)
|
||||||
: maybeSudo(`dpkg-query -s ${name}`),
|
except PackageNotFoundError:
|
||||||
|
if not requirement.specifier and not requirement.extras and not requirement.url:
|
||||||
|
if importlib.util.find_spec(requirement.name):
|
||||||
|
print(requirement.name)
|
||||||
|
else:
|
||||||
|
if requirement.specifier.contains(installed, prereleases=True):
|
||||||
|
print(installed)
|
||||||
|
except Exception:
|
||||||
|
pass`,
|
||||||
|
)} ${shellQuote(trimmedName)}`,
|
||||||
|
[DependenceTypes.linux]:
|
||||||
|
getOsTypeSync() === 'Alpine'
|
||||||
|
? `apk info -es ${trimmedName}`
|
||||||
|
: maybeSudo(`dpkg-query -s ${trimmedName}`),
|
||||||
};
|
};
|
||||||
|
|
||||||
return baseCommands[type];
|
return baseCommands[type];
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getInstallCommand(type: DependenceTypes, name: string): string {
|
export function getInstallCommand(type: DependenceTypes, name: string): string {
|
||||||
|
if (type === DependenceTypes.python3) {
|
||||||
|
const prefix =
|
||||||
|
PYTHON_INSTALL_DIR && !PYTHON_VENV_DIR
|
||||||
|
? ` --prefix=${shellQuote(PYTHON_INSTALL_DIR)}`
|
||||||
|
: '';
|
||||||
|
const pip = `${getPythonPip()} install --disable-pip-version-check${
|
||||||
|
PYTHON_VENV_DIR ? '' : ' --root-user-action=ignore'
|
||||||
|
}${prefix}`;
|
||||||
|
name = name.trim();
|
||||||
|
if (isRequirementsFile(name) || isPyprojectToml(name)) {
|
||||||
|
const file = [config.scriptPath, config.repoPath]
|
||||||
|
.filter(Boolean)
|
||||||
|
.map((root) => resolveFileAccess(root, [name]))
|
||||||
|
.find((file) => {
|
||||||
|
if (!file) return false;
|
||||||
|
try {
|
||||||
|
return statSync(file).isFile();
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
if (!file)
|
||||||
|
throw new Error(
|
||||||
|
'Python dependency file must be inside scripts or repo',
|
||||||
|
);
|
||||||
|
return isRequirementsFile(name)
|
||||||
|
? `${pip} -r ${shellQuote(file)}`
|
||||||
|
: `${pip} ${shellQuote(path.dirname(file))}`;
|
||||||
|
}
|
||||||
|
return `${pip} -- ${shellQuote(name)}`;
|
||||||
|
}
|
||||||
const baseCommands = {
|
const baseCommands = {
|
||||||
[DependenceTypes.nodejs]: 'pnpm add -g',
|
[DependenceTypes.nodejs]: 'pnpm add -g',
|
||||||
[DependenceTypes.python3]:
|
[DependenceTypes.linux]:
|
||||||
'pip3 install --disable-pip-version-check --root-user-action=ignore',
|
getOsTypeSync() === 'Alpine'
|
||||||
[DependenceTypes.linux]: getOsTypeSync() === 'Alpine'
|
|
||||||
? 'apk add --no-check-certificate'
|
? 'apk add --no-check-certificate'
|
||||||
: maybeSudo('apt-get install -y'),
|
: maybeSudo('apt-get install -y'),
|
||||||
};
|
};
|
||||||
|
return `${baseCommands[type]} ${name.trim()}`;
|
||||||
let command = baseCommands[type];
|
|
||||||
|
|
||||||
if (type === DependenceTypes.python3 && PYTHON_INSTALL_DIR) {
|
|
||||||
command = `${command} --prefix=${PYTHON_INSTALL_DIR}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
return `${command} ${name.trim()}`;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getUninstallCommand(
|
export function getUninstallCommand(
|
||||||
type: DependenceTypes,
|
type: DependenceTypes,
|
||||||
name: string,
|
name: string,
|
||||||
): string {
|
): string {
|
||||||
|
if (type === DependenceTypes.python3) {
|
||||||
|
name = name.trim();
|
||||||
|
if (isRequirementsFile(name)) {
|
||||||
|
const file = [config.scriptPath, config.repoPath]
|
||||||
|
.filter(Boolean)
|
||||||
|
.map((root) => resolveFileAccess(root, [name]))
|
||||||
|
.find((file) => {
|
||||||
|
if (!file) return false;
|
||||||
|
try {
|
||||||
|
return statSync(file).isFile();
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
if (!file)
|
||||||
|
throw new Error(
|
||||||
|
'Python dependency file must be inside scripts or repo',
|
||||||
|
);
|
||||||
|
return `${getPythonPip()} uninstall --disable-pip-version-check${
|
||||||
|
PYTHON_VENV_DIR ? '' : ' --root-user-action=ignore'
|
||||||
|
} -y -r ${shellQuote(file)}`;
|
||||||
|
}
|
||||||
|
if (isPyprojectToml(name) || isGitHubUrl(name)) {
|
||||||
|
// A source URL/project can install several distributions and has no safe
|
||||||
|
// inverse. Removing its panel entry leaves installed packages intact.
|
||||||
|
return `printf '%s\\n' ${shellQuote(
|
||||||
|
'Remove installed Python distributions separately by package name.',
|
||||||
|
)}`;
|
||||||
|
}
|
||||||
|
const packageName =
|
||||||
|
name.match(
|
||||||
|
/^([A-Za-z0-9][A-Za-z0-9._-]*)(?:\[[^\]]+\])?(?=\s*(?:[<>=!~;@]|$))/,
|
||||||
|
)?.[1] || name;
|
||||||
|
return `${getPythonPip()} uninstall --disable-pip-version-check${
|
||||||
|
PYTHON_VENV_DIR ? '' : ' --root-user-action=ignore'
|
||||||
|
} -y -- ${shellQuote(packageName)}`;
|
||||||
|
}
|
||||||
const baseCommands = {
|
const baseCommands = {
|
||||||
[DependenceTypes.nodejs]: 'pnpm remove -g',
|
[DependenceTypes.nodejs]: 'pnpm remove -g',
|
||||||
[DependenceTypes.python3]:
|
[DependenceTypes.linux]:
|
||||||
'pip3 uninstall --disable-pip-version-check --root-user-action=ignore -y',
|
getOsTypeSync() === 'Alpine' ? 'apk del' : maybeSudo('apt-get remove -y'),
|
||||||
[DependenceTypes.linux]: getOsTypeSync() === 'Alpine'
|
|
||||||
? 'apk del'
|
|
||||||
: maybeSudo('apt-get remove -y'),
|
|
||||||
};
|
};
|
||||||
|
|
||||||
return `${baseCommands[type]} ${name.trim()}`;
|
return `${baseCommands[type]} ${name.trim()}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -806,75 +975,73 @@ export async function detectOS(): Promise<
|
|||||||
return osType;
|
return osType;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function getCurrentMirrorDomain(
|
|
||||||
filePath: string,
|
|
||||||
): Promise<string | null> {
|
|
||||||
const fileContent = await fs.readFile(filePath, 'utf8');
|
|
||||||
const lines = fileContent.split('\n');
|
|
||||||
for (const line of lines) {
|
|
||||||
if (line.trim().startsWith('#')) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const match = line.match(/https?:\/\/[^\/]+/);
|
|
||||||
if (match) {
|
|
||||||
return match[0];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function replaceDomainInFile(
|
|
||||||
filePath: string,
|
|
||||||
oldDomainWithScheme: string,
|
|
||||||
newDomainWithScheme: string,
|
|
||||||
): Promise<void> {
|
|
||||||
let fileContent = await fs.readFile(filePath, 'utf8');
|
|
||||||
let updatedContent = fileContent.replace(
|
|
||||||
new RegExp(oldDomainWithScheme, 'g'),
|
|
||||||
newDomainWithScheme,
|
|
||||||
);
|
|
||||||
|
|
||||||
if (!newDomainWithScheme.endsWith('/')) {
|
|
||||||
newDomainWithScheme += '/';
|
|
||||||
}
|
|
||||||
|
|
||||||
await writeFileWithLock(filePath, updatedContent);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function _updateLinuxMirror(
|
async function _updateLinuxMirror(
|
||||||
osType: string,
|
osType: string,
|
||||||
mirrorDomainWithScheme: string,
|
mirror: string,
|
||||||
): Promise<string> {
|
): Promise<string> {
|
||||||
const S = isInContainer() ? 'sudo ' : '';
|
const defaults: Record<string, string> = {
|
||||||
let filePath: string, currentDomainWithScheme: string | null;
|
Debian: 'http://deb.debian.org/debian',
|
||||||
switch (osType) {
|
Ubuntu: 'http://archive.ubuntu.com/ubuntu',
|
||||||
case 'Debian':
|
Alpine: 'http://dl-cdn.alpinelinux.org/alpine',
|
||||||
filePath = '/etc/apt/sources.list.d/debian.sources';
|
};
|
||||||
currentDomainWithScheme = await getCurrentMirrorDomain(filePath);
|
if (!defaults[osType])
|
||||||
if (currentDomainWithScheme) {
|
throw new Error('Unsupported OS type for updating mirrors.');
|
||||||
return `${S}sed -i 's|${currentDomainWithScheme}|${mirrorDomainWithScheme || 'http://deb.debian.org'}|g' ${filePath} || (${S}mkdir -p /etc/apt/sources.list.d && echo -e "Types: deb\\nURIs: ${mirrorDomainWithScheme || 'http://deb.debian.org'}\\nSuites: \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-updates\\nComponents: main\\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg" | ${S}tee ${filePath}) && ${S}apt-get update`;
|
const target = mirror || defaults[osType];
|
||||||
|
normalizeMirror(target);
|
||||||
|
const S = osType !== 'Alpine' && isInContainer() ? 'sudo ' : '';
|
||||||
|
const update =
|
||||||
|
osType === 'Alpine' ? 'apk update' : `${S}apt-get update --error-on=any`;
|
||||||
|
let file =
|
||||||
|
osType === 'Alpine'
|
||||||
|
? '/etc/apk/repositories'
|
||||||
|
: `/etc/apt/sources.list.d/${osType.toLowerCase()}.sources`;
|
||||||
|
let content = '';
|
||||||
|
try {
|
||||||
|
content = await fs.readFile(file, 'utf8');
|
||||||
|
} catch (error: any) {
|
||||||
|
if (error.code !== 'ENOENT') throw error;
|
||||||
|
if (osType !== 'Alpine') {
|
||||||
|
try {
|
||||||
|
file = '/etc/apt/sources.list';
|
||||||
|
content = await fs.readFile(file, 'utf8');
|
||||||
|
} catch (legacyError: any) {
|
||||||
|
if (legacyError.code !== 'ENOENT') throw legacyError;
|
||||||
|
file = `/etc/apt/sources.list.d/${osType.toLowerCase()}.sources`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!content.trim()) {
|
||||||
|
const release = await fs.readFile('/etc/os-release', 'utf8');
|
||||||
|
if (osType === 'Alpine') {
|
||||||
|
const version = release.match(/^VERSION_ID=["']?(\d+\.\d+)/m)?.[1];
|
||||||
|
if (!version)
|
||||||
|
throw new Error('Cannot determine Alpine repository version');
|
||||||
|
content = `${defaults.Alpine}/v${version}/main\n${defaults.Alpine}/v${version}/community\n`;
|
||||||
} else {
|
} else {
|
||||||
return `${S}mkdir -p /etc/apt/sources.list.d && echo -e "Types: deb\\nURIs: ${mirrorDomainWithScheme || 'http://deb.debian.org'}\\nSuites: \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-updates\\nComponents: main\\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg" | ${S}tee ${filePath} && ${S}apt-get update`;
|
// The fallback may have read an empty legacy .list. Deb822 stanzas must
|
||||||
|
// be written to a .sources file, leaving the legacy file untouched.
|
||||||
|
file = `/etc/apt/sources.list.d/${osType.toLowerCase()}.sources`;
|
||||||
|
const codename = release.match(/^VERSION_CODENAME=["']?([\w-]+)/m)?.[1];
|
||||||
|
if (!codename)
|
||||||
|
throw new Error('Cannot determine APT repository codename');
|
||||||
|
const ubuntu = osType === 'Ubuntu';
|
||||||
|
content = `Types: deb\nURIs: ${
|
||||||
|
defaults[osType]
|
||||||
|
}\nSuites: ${codename} ${codename}-updates${
|
||||||
|
ubuntu ? ` ${codename}-backports` : ''
|
||||||
|
}\nComponents: ${
|
||||||
|
ubuntu ? 'main restricted universe multiverse' : 'main'
|
||||||
|
}\nSigned-By: /usr/share/keyrings/${osType.toLowerCase()}-archive-keyring.gpg\n`;
|
||||||
|
if (!ubuntu)
|
||||||
|
content += `\nTypes: deb\nURIs: http://security.debian.org/debian-security\nSuites: ${codename}-security\nComponents: main\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg\n`;
|
||||||
}
|
}
|
||||||
case 'Ubuntu':
|
|
||||||
filePath = '/etc/apt/sources.list.d/ubuntu.sources';
|
|
||||||
currentDomainWithScheme = await getCurrentMirrorDomain(filePath);
|
|
||||||
if (currentDomainWithScheme) {
|
|
||||||
return `${S}sed -i 's|${currentDomainWithScheme}|${mirrorDomainWithScheme || 'http://archive.ubuntu.com'}|g' ${filePath} || (${S}mkdir -p /etc/apt/sources.list.d && echo -e "Types: deb\\nURIs: ${mirrorDomainWithScheme || 'http://archive.ubuntu.com'}\\nSuites: \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-updates \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-backports\\nComponents: main restricted universe multiverse\\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg" | ${S}tee ${filePath}) && ${S}apt-get update`;
|
|
||||||
} else {
|
|
||||||
return `${S}mkdir -p /etc/apt/sources.list.d && echo -e "Types: deb\\nURIs: ${mirrorDomainWithScheme || 'http://archive.ubuntu.com'}\\nSuites: \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-updates \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-backports\\nComponents: main restricted universe multiverse\\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg" | ${S}tee ${filePath} && ${S}apt-get update`;
|
|
||||||
}
|
|
||||||
case 'Alpine':
|
|
||||||
filePath = '/etc/apk/repositories';
|
|
||||||
currentDomainWithScheme = await getCurrentMirrorDomain(filePath);
|
|
||||||
if (currentDomainWithScheme) {
|
|
||||||
return `sed -i 's|${currentDomainWithScheme}|${mirrorDomainWithScheme || 'http://dl-cdn.alpinelinux.org'}|g' ${filePath} || (mkdir -p /etc/apk && echo -e "\\$(grep VERSION_ID /etc/os-release | cut -d= -f2 | cut -d. -f1,2)/main\\n\\$(grep VERSION_ID /etc/os-release | cut -d= -f2 | cut -d. -f1,2)/community" | sed "s|^|${mirrorDomainWithScheme || 'http://dl-cdn.alpinelinux.org'}/alpine/v|" | tee ${filePath}) && apk update`;
|
|
||||||
} else {
|
|
||||||
return `mkdir -p /etc/apk && echo -e "\\$(grep VERSION_ID /etc/os-release | cut -d= -f2 | cut -d. -f1,2)/main\\n\\$(grep VERSION_ID /etc/os-release | cut -d= -f2 | cut -d. -f1,2)/community" | sed "s|^|${mirrorDomainWithScheme || 'http://dl-cdn.alpinelinux.org'}/alpine/v|" | tee ${filePath} && apk update`;
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
throw Error('Unsupported OS type for updating mirrors.');
|
|
||||||
}
|
}
|
||||||
|
return mirrorWriteCommand(
|
||||||
|
file,
|
||||||
|
rewriteMirrorSources(content, target),
|
||||||
|
S,
|
||||||
|
update,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function updateLinuxMirrorFile(mirror: string): Promise<string> {
|
export async function updateLinuxMirrorFile(mirror: string): Promise<string> {
|
||||||
|
|||||||
+15
-1
@@ -16,7 +16,8 @@ export class Crontab {
|
|||||||
queued_token?: string | null;
|
queued_token?: string | null;
|
||||||
isPinned?: 1 | 0;
|
isPinned?: 1 | 0;
|
||||||
labels?: string[];
|
labels?: string[];
|
||||||
last_running_time?: number;
|
last_instance_id?: number;
|
||||||
|
last_running_time?: number | null;
|
||||||
last_execution_time?: number;
|
last_execution_time?: number;
|
||||||
sub_id?: number;
|
sub_id?: number;
|
||||||
extra_schedules?: Array<{ schedule: string }>;
|
extra_schedules?: Array<{ schedule: string }>;
|
||||||
@@ -24,6 +25,10 @@ export class Crontab {
|
|||||||
task_after?: string;
|
task_after?: string;
|
||||||
log_name?: string;
|
log_name?: string;
|
||||||
allow_multiple_instances?: 1 | 0;
|
allow_multiple_instances?: 1 | 0;
|
||||||
|
skip_if_pending?: boolean;
|
||||||
|
max_queue_wait?: number;
|
||||||
|
catchup_seconds?: number;
|
||||||
|
recovery_from?: number;
|
||||||
work_dir?: string;
|
work_dir?: string;
|
||||||
|
|
||||||
constructor(options: Crontab) {
|
constructor(options: Crontab) {
|
||||||
@@ -51,6 +56,10 @@ export class Crontab {
|
|||||||
this.task_after = options.task_after;
|
this.task_after = options.task_after;
|
||||||
this.log_name = options.log_name;
|
this.log_name = options.log_name;
|
||||||
this.allow_multiple_instances = options.allow_multiple_instances || 0;
|
this.allow_multiple_instances = options.allow_multiple_instances || 0;
|
||||||
|
this.skip_if_pending = Boolean(options.skip_if_pending);
|
||||||
|
this.max_queue_wait = options.max_queue_wait || 0;
|
||||||
|
this.catchup_seconds = options.catchup_seconds || 0;
|
||||||
|
this.recovery_from = options.recovery_from;
|
||||||
this.work_dir = options.work_dir;
|
this.work_dir = options.work_dir;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -86,6 +95,7 @@ export const CrontabModel = sequelize.define<CronInstance>('Crontab', {
|
|||||||
log_path: DataTypes.STRING,
|
log_path: DataTypes.STRING,
|
||||||
queued_token: DataTypes.STRING,
|
queued_token: DataTypes.STRING,
|
||||||
labels: DataTypes.JSON,
|
labels: DataTypes.JSON,
|
||||||
|
last_instance_id: DataTypes.INTEGER,
|
||||||
last_running_time: DataTypes.NUMBER,
|
last_running_time: DataTypes.NUMBER,
|
||||||
last_execution_time: DataTypes.NUMBER,
|
last_execution_time: DataTypes.NUMBER,
|
||||||
sub_id: { type: DataTypes.NUMBER, allowNull: true },
|
sub_id: { type: DataTypes.NUMBER, allowNull: true },
|
||||||
@@ -94,5 +104,9 @@ export const CrontabModel = sequelize.define<CronInstance>('Crontab', {
|
|||||||
task_after: DataTypes.STRING,
|
task_after: DataTypes.STRING,
|
||||||
log_name: DataTypes.STRING,
|
log_name: DataTypes.STRING,
|
||||||
allow_multiple_instances: DataTypes.NUMBER,
|
allow_multiple_instances: DataTypes.NUMBER,
|
||||||
|
skip_if_pending: DataTypes.BOOLEAN,
|
||||||
|
max_queue_wait: DataTypes.INTEGER,
|
||||||
|
catchup_seconds: DataTypes.INTEGER,
|
||||||
|
recovery_from: DataTypes.BIGINT,
|
||||||
work_dir: DataTypes.STRING,
|
work_dir: DataTypes.STRING,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -8,8 +8,10 @@ export class CrontabStat {
|
|||||||
run_count?: number;
|
run_count?: number;
|
||||||
success_count?: number;
|
success_count?: number;
|
||||||
fail_count?: number;
|
fail_count?: number;
|
||||||
|
untimed_count?: number;
|
||||||
total_time?: number;
|
total_time?: number;
|
||||||
max_time?: number;
|
max_time?: number;
|
||||||
|
fixed_max_time?: number | null;
|
||||||
|
|
||||||
constructor(options: CrontabStat) {
|
constructor(options: CrontabStat) {
|
||||||
this.id = options.id;
|
this.id = options.id;
|
||||||
@@ -18,6 +20,7 @@ export class CrontabStat {
|
|||||||
this.run_count = options.run_count || 0;
|
this.run_count = options.run_count || 0;
|
||||||
this.success_count = options.success_count || 0;
|
this.success_count = options.success_count || 0;
|
||||||
this.fail_count = options.fail_count || 0;
|
this.fail_count = options.fail_count || 0;
|
||||||
|
this.untimed_count = options.untimed_count || 0;
|
||||||
this.total_time = options.total_time || 0;
|
this.total_time = options.total_time || 0;
|
||||||
this.max_time = options.max_time || 0;
|
this.max_time = options.max_time || 0;
|
||||||
}
|
}
|
||||||
@@ -53,10 +56,16 @@ export const CrontabStatModel = sequelize.define<CrontabStatInstance>(
|
|||||||
type: DataTypes.NUMBER,
|
type: DataTypes.NUMBER,
|
||||||
defaultValue: 0,
|
defaultValue: 0,
|
||||||
},
|
},
|
||||||
|
untimed_count: {
|
||||||
|
type: DataTypes.INTEGER,
|
||||||
|
allowNull: false,
|
||||||
|
defaultValue: 0,
|
||||||
|
},
|
||||||
total_time: {
|
total_time: {
|
||||||
type: DataTypes.NUMBER,
|
type: DataTypes.NUMBER,
|
||||||
defaultValue: 0,
|
defaultValue: 0,
|
||||||
},
|
},
|
||||||
|
fixed_max_time: { type: DataTypes.NUMBER, allowNull: true },
|
||||||
max_time: {
|
max_time: {
|
||||||
type: DataTypes.NUMBER,
|
type: DataTypes.NUMBER,
|
||||||
defaultValue: 0,
|
defaultValue: 0,
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import { Sequelize, Transaction } from 'sequelize';
|
import { Sequelize, Transaction } from 'sequelize';
|
||||||
import config from '../config/index';
|
import config from '../config/index';
|
||||||
|
import type { Database } from 'sqlite3';
|
||||||
import { join } from 'path';
|
import { join } from 'path';
|
||||||
|
|
||||||
export const sequelize = new Sequelize({
|
export const sequelize = new Sequelize({
|
||||||
@@ -20,4 +21,29 @@ export const sequelize = new Sequelize({
|
|||||||
transactionType: Transaction.TYPES.IMMEDIATE,
|
transactionType: Transaction.TYPES.IMMEDIATE,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// In DELETE journal mode, EXTRA flushes the unlink that makes a transaction
|
||||||
|
// final. Sequelize opens a separate SQLite handle for each transaction.
|
||||||
|
const initialized = new WeakMap<Database, Promise<void>>();
|
||||||
|
sequelize.addHook('beforeQuery', async (_options, query) => {
|
||||||
|
const connection = query.connection as Database;
|
||||||
|
let ready = initialized.get(connection);
|
||||||
|
if (!ready) {
|
||||||
|
ready = new Promise<void>((resolve, reject) => {
|
||||||
|
connection.run('PRAGMA synchronous=EXTRA', error => error ? reject(error) : resolve());
|
||||||
|
}).catch(error => {
|
||||||
|
initialized.delete(connection);
|
||||||
|
throw error;
|
||||||
|
});
|
||||||
|
initialized.set(connection, ready);
|
||||||
|
}
|
||||||
|
await ready;
|
||||||
|
});
|
||||||
|
|
||||||
|
// SQLite transactions use dedicated file handles outside Sequelize 6's pool.
|
||||||
|
// Its default destroyConnection targets the pool on COMMIT/ROLLBACK errors,
|
||||||
|
// leaving the native writer open. The SQLite release path closes that handle.
|
||||||
|
sequelize.connectionManager.destroyConnection = async (connection) => {
|
||||||
|
sequelize.connectionManager.releaseConnection(connection);
|
||||||
|
};
|
||||||
|
|
||||||
export type ResponseType<T> = { code: number; data?: T; message?: string };
|
export type ResponseType<T> = { code: number; data?: T; message?: string };
|
||||||
|
|||||||
+19
-1
@@ -24,7 +24,25 @@ export interface AppToken {
|
|||||||
expiration: number;
|
expiration: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type AppScope = 'envs' | 'crons' | 'configs' | 'scripts' | 'logs' | 'system' | 'dashboard';
|
export type AppScope =
|
||||||
|
| 'crons'
|
||||||
|
| 'crons:read'
|
||||||
|
| 'subscriptions'
|
||||||
|
| 'subscriptions:read'
|
||||||
|
| 'envs'
|
||||||
|
| 'envs:read'
|
||||||
|
| 'configs'
|
||||||
|
| 'configs:read'
|
||||||
|
| 'scripts'
|
||||||
|
| 'scripts:read'
|
||||||
|
| 'logs'
|
||||||
|
| 'logs:read'
|
||||||
|
| 'dependencies'
|
||||||
|
| 'dependencies:read'
|
||||||
|
| 'dashboard'
|
||||||
|
| 'dashboard:read'
|
||||||
|
| 'system'
|
||||||
|
| 'apps';
|
||||||
|
|
||||||
export interface AppInstance extends Model<App, App>, App {}
|
export interface AppInstance extends Model<App, App>, App {}
|
||||||
export const AppModel = sequelize.define<AppInstance>('App', {
|
export const AppModel = sequelize.define<AppInstance>('App', {
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ export enum InstanceStatus {
|
|||||||
'finished' = 1,
|
'finished' = 1,
|
||||||
'stopped' = 2,
|
'stopped' = 2,
|
||||||
'error' = 3,
|
'error' = 3,
|
||||||
|
'unknown' = 4,
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RunningInstanceAttributes {
|
export interface RunningInstanceAttributes {
|
||||||
@@ -78,4 +79,5 @@ export const RunningInstanceModel = sequelize.define<RunningInstanceModel>(
|
|||||||
allowNull: true,
|
allowNull: true,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{ indexes: [{ fields: ['cron_id', 'pid', 'started_at'] }, { fields: ['cron_id', 'id'] }] },
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -28,8 +28,12 @@ export enum AuthDataType {
|
|||||||
'removeLogFrequency' = 'removeLogFrequency',
|
'removeLogFrequency' = 'removeLogFrequency',
|
||||||
'systemConfig' = 'systemConfig',
|
'systemConfig' = 'systemConfig',
|
||||||
'authConfig' = 'authConfig',
|
'authConfig' = 'authConfig',
|
||||||
|
'notifyLog' = 'notifyLog',
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export const NotifyStatus = LoginStatus;
|
||||||
|
export type NotifyStatus = LoginStatus;
|
||||||
|
|
||||||
export interface SystemConfigInfo {
|
export interface SystemConfigInfo {
|
||||||
lang?: string;
|
lang?: string;
|
||||||
panelTitle?: string;
|
panelTitle?: string;
|
||||||
@@ -53,6 +57,17 @@ export interface LoginLogInfo {
|
|||||||
status?: LoginStatus;
|
status?: LoginStatus;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface NotifyLogInfo {
|
||||||
|
timestamp?: number;
|
||||||
|
title?: string;
|
||||||
|
content?: string;
|
||||||
|
status?: NotifyStatus;
|
||||||
|
notifyType?: string;
|
||||||
|
source?: 'system' | 'script';
|
||||||
|
scriptName?: string;
|
||||||
|
taskId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
export interface TokenInfo {
|
export interface TokenInfo {
|
||||||
value: string;
|
value: string;
|
||||||
timestamp: number;
|
timestamp: number;
|
||||||
@@ -88,6 +103,7 @@ export interface AuthInfo {
|
|||||||
export type SystemModelInfo = SystemConfigInfo &
|
export type SystemModelInfo = SystemConfigInfo &
|
||||||
Partial<NotificationInfo> &
|
Partial<NotificationInfo> &
|
||||||
LoginLogInfo &
|
LoginLogInfo &
|
||||||
|
Partial<NotifyLogInfo> &
|
||||||
Partial<AuthInfo>;
|
Partial<AuthInfo>;
|
||||||
|
|
||||||
export interface SystemInstance
|
export interface SystemInstance
|
||||||
|
|||||||
+84
-40
@@ -6,7 +6,7 @@ import config from '../config';
|
|||||||
import { UnauthorizedError, expressjwt } from 'express-jwt';
|
import { UnauthorizedError, expressjwt } from 'express-jwt';
|
||||||
import { getPlatform, getToken } from '../config/util';
|
import { getPlatform, getToken } from '../config/util';
|
||||||
import rewrite from 'express-urlrewrite';
|
import rewrite from 'express-urlrewrite';
|
||||||
import { errors } from 'celebrate';
|
import validationErrors from '../middlewares/validationErrors';
|
||||||
import { serveEnv } from '../config/serverEnv';
|
import { serveEnv } from '../config/serverEnv';
|
||||||
import { shareStore } from '../shared/store';
|
import { shareStore } from '../shared/store';
|
||||||
import { isValidToken, isDefaultAuthInfo } from '../shared/auth';
|
import { isValidToken, isDefaultAuthInfo } from '../shared/auth';
|
||||||
@@ -14,7 +14,9 @@ import { AuthInfo } from '../data/system';
|
|||||||
import path from 'path';
|
import path from 'path';
|
||||||
import { t } from '../shared/i18n';
|
import { t } from '../shared/i18n';
|
||||||
import { AppScope } from '../data/open';
|
import { AppScope } from '../data/open';
|
||||||
|
import uniqueConstraintError from '../middlewares/uniqueConstraintError';
|
||||||
import protectedPathCase from '../middlewares/protectedPathCase';
|
import protectedPathCase from '../middlewares/protectedPathCase';
|
||||||
|
import Logger from './logger';
|
||||||
|
|
||||||
function resolveTrustProxy(value = process.env.QL_TRUST_PROXY) {
|
function resolveTrustProxy(value = process.env.QL_TRUST_PROXY) {
|
||||||
const setting = value?.trim();
|
const setting = value?.trim();
|
||||||
@@ -111,7 +113,43 @@ export default ({ app }: { app: Application }) => {
|
|||||||
const keyMatch = pathLower.match(/\/open\/([a-z]+)\/*/);
|
const keyMatch = pathLower.match(/\/open\/([a-z]+)\/*/);
|
||||||
const key = keyMatch && keyMatch[1];
|
const key = keyMatch && keyMatch[1];
|
||||||
|
|
||||||
if (!doc.scopes.includes(key as AppScope)) {
|
const isReadRequest =
|
||||||
|
(['GET', 'HEAD'].includes(req.method) &&
|
||||||
|
!(
|
||||||
|
key === 'crons' && /^\/open\/crons\/import\/?$/.test(pathLower)
|
||||||
|
)) ||
|
||||||
|
(['scripts', 'logs'].includes(key || '') &&
|
||||||
|
req.method === 'POST' &&
|
||||||
|
/^\/open\/(?:scripts|logs)\/(?:download|entries\/check)\/?$/.test(pathLower));
|
||||||
|
const hasNamespaceScope =
|
||||||
|
doc.scopes.includes(key as AppScope) ||
|
||||||
|
([
|
||||||
|
'crons',
|
||||||
|
'subscriptions',
|
||||||
|
'envs',
|
||||||
|
'configs',
|
||||||
|
'scripts',
|
||||||
|
'logs',
|
||||||
|
'dependencies',
|
||||||
|
'dashboard',
|
||||||
|
].includes(key || '') &&
|
||||||
|
doc.scopes.includes(`${key}:read` as AppScope) &&
|
||||||
|
isReadRequest);
|
||||||
|
if (!hasNamespaceScope) {
|
||||||
|
const err = new UnauthorizedError('credentials_bad_scheme', {
|
||||||
|
message: t('暂无权限'),
|
||||||
|
});
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
// The built-in system app still needs its local CLI maintenance API.
|
||||||
|
const appRouteAllowed =
|
||||||
|
key !== 'system' ||
|
||||||
|
doc.name === 'system' ||
|
||||||
|
((req.method === 'GET' || req.method === 'HEAD') &&
|
||||||
|
(pathLower === '/open/system' || pathLower === '/open/system/')) ||
|
||||||
|
(req.method === 'PUT' && pathLower === '/open/system/notify');
|
||||||
|
if (!appRouteAllowed) {
|
||||||
const err = new UnauthorizedError('credentials_bad_scheme', {
|
const err = new UnauthorizedError('credentials_bad_scheme', {
|
||||||
message: t('暂无权限'),
|
message: t('暂无权限'),
|
||||||
});
|
});
|
||||||
@@ -154,14 +192,7 @@ export default ({ app }: { app: Application }) => {
|
|||||||
|
|
||||||
app.use(async (req, res, next) => {
|
app.use(async (req, res, next) => {
|
||||||
const pathLower = req.path.toLowerCase();
|
const pathLower = req.path.toLowerCase();
|
||||||
if (
|
if (!['/api/user/init', '/open/user/init'].includes(pathLower)) {
|
||||||
![
|
|
||||||
'/api/user/init',
|
|
||||||
'/api/user/notification/init',
|
|
||||||
'/open/user/init',
|
|
||||||
'/open/user/notification/init',
|
|
||||||
].includes(pathLower)
|
|
||||||
) {
|
|
||||||
return next();
|
return next();
|
||||||
}
|
}
|
||||||
const authInfo = (await shareStore.getAuthInfo()) || ({} as AuthInfo);
|
const authInfo = (await shareStore.getAuthInfo()) || ({} as AuthInfo);
|
||||||
@@ -178,7 +209,14 @@ export default ({ app }: { app: Application }) => {
|
|||||||
app.use(rewrite('/open/*', '/api/$1'));
|
app.use(rewrite('/open/*', '/api/$1'));
|
||||||
app.use(config.api.prefix, routes());
|
app.use(config.api.prefix, routes());
|
||||||
|
|
||||||
app.get('*', (_, res, next) => {
|
app.get('*', (req, res, next) => {
|
||||||
|
// Don't serve index.html for API routes
|
||||||
|
if (
|
||||||
|
req.path === config.api.prefix ||
|
||||||
|
req.path.startsWith(`${config.api.prefix}/`)
|
||||||
|
) {
|
||||||
|
return res.status(404).json({ code: 404, message: 'Not Found' });
|
||||||
|
}
|
||||||
const indexPath = path.join(frontendPath, 'index.html');
|
const indexPath = path.join(frontendPath, 'index.html');
|
||||||
res.sendFile(indexPath, (err) => {
|
res.sendFile(indexPath, (err) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
@@ -189,41 +227,28 @@ export default ({ app }: { app: Application }) => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
app.use(errors());
|
app.use(validationErrors);
|
||||||
|
app.use(uniqueConstraintError);
|
||||||
|
|
||||||
app.use(
|
app.use(
|
||||||
(
|
(
|
||||||
err: Error & { status: number },
|
err: Error & { status: number; code?: string },
|
||||||
req: Request,
|
req: Request,
|
||||||
res: Response,
|
res: Response,
|
||||||
next: NextFunction,
|
next: NextFunction,
|
||||||
) => {
|
) => {
|
||||||
if (err.name === 'UnauthorizedError') {
|
if (err.name === 'UnauthorizedError') {
|
||||||
return res
|
let message = err.message;
|
||||||
.status(err.status)
|
if (!config.isDevelopment) {
|
||||||
.send({ code: 401, message: err.message })
|
if (err.code === 'credentials_bad_scheme') {
|
||||||
.end();
|
message = t('暂无权限');
|
||||||
|
} else if (err.code === 'credentials_required') {
|
||||||
|
message = t('请先登录');
|
||||||
|
} else {
|
||||||
|
message = t('Token 已失效');
|
||||||
}
|
}
|
||||||
return next(err);
|
}
|
||||||
},
|
return res.status(401).send({ code: 401, message }).end();
|
||||||
);
|
|
||||||
|
|
||||||
app.use(
|
|
||||||
(
|
|
||||||
err: Error & { errors: any[] },
|
|
||||||
req: Request,
|
|
||||||
res: Response,
|
|
||||||
next: NextFunction,
|
|
||||||
) => {
|
|
||||||
if (err.name.includes('Sequelize')) {
|
|
||||||
return res
|
|
||||||
.status(500)
|
|
||||||
.send({
|
|
||||||
code: 400,
|
|
||||||
message: `${err.message}`,
|
|
||||||
errors: err.errors,
|
|
||||||
})
|
|
||||||
.end();
|
|
||||||
}
|
}
|
||||||
return next(err);
|
return next(err);
|
||||||
},
|
},
|
||||||
@@ -236,10 +261,29 @@ export default ({ app }: { app: Application }) => {
|
|||||||
res: Response,
|
res: Response,
|
||||||
next: NextFunction,
|
next: NextFunction,
|
||||||
) => {
|
) => {
|
||||||
res.status(err.status || 500);
|
const status =
|
||||||
|
Number.isInteger(err.status) && err.status >= 400 && err.status < 600
|
||||||
|
? err.status
|
||||||
|
: 500;
|
||||||
|
if (status >= 500) {
|
||||||
|
const route =
|
||||||
|
typeof req.route?.path === 'string' ? req.route.path : '?';
|
||||||
|
const errorName = /^[A-Za-z][A-Za-z0-9]{0,63}$/.test(err.name)
|
||||||
|
? err.name
|
||||||
|
: 'Error';
|
||||||
|
Logger.error(
|
||||||
|
'HTTP request failed: status=%d method=%s route=%s type=%s',
|
||||||
|
status,
|
||||||
|
req.method,
|
||||||
|
route,
|
||||||
|
errorName,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
res.status(status);
|
||||||
res.json({
|
res.json({
|
||||||
code: err.status || 500,
|
code: status,
|
||||||
message: err.message,
|
message:
|
||||||
|
!config.isDevelopment && status >= 500 ? t('未知错误') : err.message,
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
import type express from 'express';
|
||||||
|
import config from '../config';
|
||||||
|
|
||||||
|
export async function createHttpApplication(): Promise<express.Application> {
|
||||||
|
// Only the HTTP worker needs an Express application and its middleware.
|
||||||
|
const [
|
||||||
|
{ default: createExpress },
|
||||||
|
{ default: helmet },
|
||||||
|
{ default: cors },
|
||||||
|
{ default: compression },
|
||||||
|
{ monitoringMiddleware },
|
||||||
|
] = await Promise.all([
|
||||||
|
import('express'),
|
||||||
|
import('helmet'),
|
||||||
|
import('cors'),
|
||||||
|
import('compression'),
|
||||||
|
import('../middlewares/monitoring'),
|
||||||
|
]);
|
||||||
|
const app = createExpress();
|
||||||
|
app.use((req, res, next) => {
|
||||||
|
if (req.query.t) delete req.query.t;
|
||||||
|
next();
|
||||||
|
});
|
||||||
|
app.use(
|
||||||
|
helmet({
|
||||||
|
contentSecurityPolicy: false,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
app.use(cors(config.cors));
|
||||||
|
app.use(compression());
|
||||||
|
app.use(monitoringMiddleware);
|
||||||
|
return app;
|
||||||
|
}
|
||||||
+18
-38
@@ -1,7 +1,7 @@
|
|||||||
import DependenceService from '../services/dependence';
|
import DependenceService from '../services/dependence';
|
||||||
import { exec } from 'child_process';
|
import { exec } from 'child_process';
|
||||||
import { Container } from 'typedi';
|
import { Container } from 'typedi';
|
||||||
import { Crontab, CrontabModel, CrontabStatus } from '../data/cron';
|
import { Crontab, CrontabModel } from '../data/cron';
|
||||||
import CronService from '../services/cron';
|
import CronService from '../services/cron';
|
||||||
import EnvService from '../services/env';
|
import EnvService from '../services/env';
|
||||||
import { DependenceModel, DependenceStatus } from '../data/dependence';
|
import { DependenceModel, DependenceStatus } from '../data/dependence';
|
||||||
@@ -13,13 +13,12 @@ import { AuthDataType, SystemModel } from '../data/system';
|
|||||||
import SystemService from '../services/system';
|
import SystemService from '../services/system';
|
||||||
import UserService from '../services/user';
|
import UserService from '../services/user';
|
||||||
import { writeFile, readFile } from 'fs/promises';
|
import { writeFile, readFile } from 'fs/promises';
|
||||||
import { createRandomString, fileExist, isDemoEnv, safeJSONParse } from '../config/util';
|
import { fileExist, isDemoEnv, safeJSONParse } from '../config/util';
|
||||||
import OpenService from '../services/open';
|
import OpenService from '../services/open';
|
||||||
import { shareStore } from '../shared/store';
|
import { shareStore } from '../shared/store';
|
||||||
import Logger from './logger';
|
import Logger from './logger';
|
||||||
import cronClient from '../schedule/client';
|
import cronClient from '../schedule/client';
|
||||||
import { AppModel } from '../data/open';
|
import { TriggerHistory } from '../shared/triggerHistory';
|
||||||
import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
|
|
||||||
import { setLang, systemLang } from '../shared/i18n';
|
import { setLang, systemLang } from '../shared/i18n';
|
||||||
|
|
||||||
export default async () => {
|
export default async () => {
|
||||||
@@ -31,24 +30,7 @@ export default async () => {
|
|||||||
const openService = Container.get(OpenService);
|
const openService = Container.get(OpenService);
|
||||||
|
|
||||||
// 初始化增加系统配置
|
// 初始化增加系统配置
|
||||||
let systemApp = (
|
await openService.initializeSystemApp();
|
||||||
await AppModel.findOne({
|
|
||||||
where: { name: 'system' },
|
|
||||||
})
|
|
||||||
)?.get({ plain: true });
|
|
||||||
if (!systemApp) {
|
|
||||||
systemApp = await AppModel.create({
|
|
||||||
name: 'system',
|
|
||||||
scopes: ['crons', 'system', 'dashboard'],
|
|
||||||
client_id: createRandomString(12, 12),
|
|
||||||
client_secret: createRandomString(24, 24),
|
|
||||||
});
|
|
||||||
} else if (!systemApp.scopes.includes('dashboard')) {
|
|
||||||
await AppModel.update(
|
|
||||||
{ scopes: [...systemApp.scopes, 'dashboard'] },
|
|
||||||
{ where: { name: 'system' } },
|
|
||||||
);
|
|
||||||
}
|
|
||||||
const [systemConfig] = await SystemModel.findOrCreate({
|
const [systemConfig] = await SystemModel.findOrCreate({
|
||||||
where: { type: AuthDataType.systemConfig },
|
where: { type: AuthDataType.systemConfig },
|
||||||
});
|
});
|
||||||
@@ -139,14 +121,11 @@ export default async () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// 初始化更新所有任务状态为空闲
|
const history = new TriggerHistory(CrontabModel.sequelize!);
|
||||||
await CrontabModel.update({ status: CrontabStatus.idle }, { where: {} });
|
const pendingStops = await history.pendingStops();
|
||||||
|
const pendingBatches = await history.pendingStopBatches();
|
||||||
// 清空所有运行中的实例记录(服务重启后进程已不存在)
|
if (pendingStops.length || pendingBatches.length) await cronService.resumePendingStops(history, pendingStops, pendingBatches);
|
||||||
await RunningInstanceModel.update(
|
await history.reconcileInstances();
|
||||||
{ status: InstanceStatus.stopped },
|
|
||||||
{ where: { status: InstanceStatus.running } },
|
|
||||||
);
|
|
||||||
|
|
||||||
// 初始化时执行一次所有的 ql repo 任务
|
// 初始化时执行一次所有的 ql repo 任务
|
||||||
CrontabModel.findAll({
|
CrontabModel.findAll({
|
||||||
@@ -236,16 +215,17 @@ export default async () => {
|
|||||||
}
|
}
|
||||||
} catch { }
|
} catch { }
|
||||||
|
|
||||||
// 初始化保存一次ck和定时任务数据
|
// Replace legacy task environment files before recovered tasks can run.
|
||||||
cronClient.readiness.configure(() => cronService.autosave_crontab(true));
|
|
||||||
await cronClient.readiness.recover();
|
|
||||||
|
|
||||||
await envService.set_envs();
|
await envService.set_envs();
|
||||||
|
|
||||||
|
// 初始化保存一次ck和定时任务数据
|
||||||
|
cronClient.readiness.configure(async () => {
|
||||||
|
await cronService.autosave_crontab(true);
|
||||||
|
await systemService.restoreCronConcurrency();
|
||||||
|
});
|
||||||
|
await cronClient.readiness.recover();
|
||||||
|
|
||||||
const authInfo = await userService.getAuthInfo();
|
const authInfo = await userService.getAuthInfo();
|
||||||
const apps = await openService.findApps();
|
|
||||||
await shareStore.updateAuthInfo(authInfo);
|
await shareStore.updateAuthInfo(authInfo);
|
||||||
if (apps?.length) {
|
await openService.refreshApps();
|
||||||
await shareStore.updateApps(apps);
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -7,6 +7,8 @@ import config from '../config';
|
|||||||
import { fileExist } from '../config/util';
|
import { fileExist } from '../config/util';
|
||||||
import { join } from 'path';
|
import { join } from 'path';
|
||||||
import { t } from '../shared/i18n';
|
import { t } from '../shared/i18n';
|
||||||
|
import Logger from './logger';
|
||||||
|
import { assertSubscriptionAlias } from '../shared/subscriptionPath';
|
||||||
|
|
||||||
export default async () => {
|
export default async () => {
|
||||||
const systemService = Container.get(SystemService);
|
const systemService = Container.get(SystemService);
|
||||||
@@ -24,6 +26,7 @@ export default async () => {
|
|||||||
if (await fileExist(tokenFile)) {
|
if (await fileExist(tokenFile)) {
|
||||||
tokenCommand = `node ${tokenFile}`;
|
tokenCommand = `node ${tokenFile}`;
|
||||||
}
|
}
|
||||||
|
tokenCommand += ' --renew';
|
||||||
const cron = {
|
const cron = {
|
||||||
id: NaN,
|
id: NaN,
|
||||||
name: t('生成token'),
|
name: t('生成token'),
|
||||||
@@ -70,6 +73,12 @@ export default async () => {
|
|||||||
await subscriptionService.setSshConfig();
|
await subscriptionService.setSshConfig();
|
||||||
const subs = await subscriptionService.list();
|
const subs = await subscriptionService.list();
|
||||||
for (const sub of subs) {
|
for (const sub of subs) {
|
||||||
subscriptionService.handleTask(sub.get({ plain: true }), !sub.is_disabled);
|
try {
|
||||||
|
const doc = sub.get({ plain: true });
|
||||||
|
assertSubscriptionAlias(doc.alias);
|
||||||
|
await subscriptionService.handleTask(doc, !sub.is_disabled);
|
||||||
|
} catch (error) {
|
||||||
|
Logger.warn(t('跳过订阅 %s 的无效定时配置: %s'), sub.id, error);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -0,0 +1,14 @@
|
|||||||
|
import { ErrorRequestHandler } from 'express';
|
||||||
|
import { UniqueConstraintError } from 'sequelize';
|
||||||
|
import { t } from '../shared/i18n';
|
||||||
|
|
||||||
|
// Handle the database constraint itself so concurrent writes are covered too.
|
||||||
|
const uniqueConstraintError: ErrorRequestHandler = (err, req, res, next) => {
|
||||||
|
if (!(err instanceof UniqueConstraintError)) return next(err);
|
||||||
|
// Constraint details can contain environment values or application credentials.
|
||||||
|
res
|
||||||
|
.status(409)
|
||||||
|
.json({ code: 409, message: t('资源已存在,请检查重复的名称或值') });
|
||||||
|
};
|
||||||
|
|
||||||
|
export default uniqueConstraintError;
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
import { errors, isCelebrateError } from 'celebrate';
|
||||||
|
import { ErrorRequestHandler } from 'express';
|
||||||
|
import { t } from '../shared/i18n';
|
||||||
|
|
||||||
|
const respond = errors();
|
||||||
|
|
||||||
|
// Schemas are compiled once at startup; resolve their messages per request so
|
||||||
|
// changing the system language also changes errors from those existing schemas.
|
||||||
|
const validationErrors: ErrorRequestHandler = (error, req, res, next) => {
|
||||||
|
if (isCelebrateError(error)) {
|
||||||
|
for (const detail of error.details.values()) {
|
||||||
|
detail.message = detail.details.map((item) => t(item.message)).join('. ');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return respond(error, req, res, next);
|
||||||
|
};
|
||||||
|
|
||||||
|
export default validationErrors;
|
||||||
@@ -260,6 +260,15 @@ message SystemNotifyRequest {
|
|||||||
optional NotificationInfo notificationInfo = 3;
|
optional NotificationInfo notificationInfo = 3;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
message RecordNotifyLogRequest {
|
||||||
|
string title = 1;
|
||||||
|
string content = 2;
|
||||||
|
bool success = 3;
|
||||||
|
string notifyType = 4;
|
||||||
|
string scriptName = 5;
|
||||||
|
string taskId = 6;
|
||||||
|
}
|
||||||
|
|
||||||
service Api {
|
service Api {
|
||||||
rpc GetEnvs(GetEnvsRequest) returns (EnvsResponse) {}
|
rpc GetEnvs(GetEnvsRequest) returns (EnvsResponse) {}
|
||||||
rpc CreateEnv(CreateEnvRequest) returns (EnvsResponse) {}
|
rpc CreateEnv(CreateEnvRequest) returns (EnvsResponse) {}
|
||||||
@@ -271,6 +280,7 @@ service Api {
|
|||||||
rpc UpdateEnvNames(UpdateEnvNamesRequest) returns (Response) {}
|
rpc UpdateEnvNames(UpdateEnvNamesRequest) returns (Response) {}
|
||||||
rpc GetEnvById(GetEnvByIdRequest) returns (EnvResponse) {}
|
rpc GetEnvById(GetEnvByIdRequest) returns (EnvResponse) {}
|
||||||
rpc SystemNotify(SystemNotifyRequest) returns (Response) {}
|
rpc SystemNotify(SystemNotifyRequest) returns (Response) {}
|
||||||
|
rpc RecordNotifyLog(RecordNotifyLogRequest) returns (Response) {}
|
||||||
rpc GetCronDetail(CronDetailRequest) returns (CronDetailResponse) {}
|
rpc GetCronDetail(CronDetailRequest) returns (CronDetailResponse) {}
|
||||||
rpc CreateCron(CreateCronRequest) returns (CronResponse) {}
|
rpc CreateCron(CreateCronRequest) returns (CronResponse) {}
|
||||||
rpc UpdateCron(UpdateCronRequest) returns (CronResponse) {}
|
rpc UpdateCron(UpdateCronRequest) returns (CronResponse) {}
|
||||||
|
|||||||
+175
-1
@@ -1,7 +1,7 @@
|
|||||||
// Code generated by protoc-gen-ts_proto. DO NOT EDIT.
|
// Code generated by protoc-gen-ts_proto. DO NOT EDIT.
|
||||||
// versions:
|
// versions:
|
||||||
// protoc-gen-ts_proto v2.6.1
|
// protoc-gen-ts_proto v2.6.1
|
||||||
// protoc v3.17.3
|
// protoc v6.31.1
|
||||||
// source: back/protos/api.proto
|
// source: back/protos/api.proto
|
||||||
|
|
||||||
/* eslint-disable */
|
/* eslint-disable */
|
||||||
@@ -417,6 +417,15 @@ export interface SystemNotifyRequest {
|
|||||||
notificationInfo?: NotificationInfo | undefined;
|
notificationInfo?: NotificationInfo | undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface RecordNotifyLogRequest {
|
||||||
|
title: string;
|
||||||
|
content: string;
|
||||||
|
success: boolean;
|
||||||
|
notifyType: string;
|
||||||
|
scriptName: string;
|
||||||
|
taskId: string;
|
||||||
|
}
|
||||||
|
|
||||||
function createBaseEnvItem(): EnvItem {
|
function createBaseEnvItem(): EnvItem {
|
||||||
return {
|
return {
|
||||||
id: undefined,
|
id: undefined,
|
||||||
@@ -4296,6 +4305,146 @@ export const SystemNotifyRequest: MessageFns<SystemNotifyRequest> = {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
function createBaseRecordNotifyLogRequest(): RecordNotifyLogRequest {
|
||||||
|
return { title: "", content: "", success: false, notifyType: "", scriptName: "", taskId: "" };
|
||||||
|
}
|
||||||
|
|
||||||
|
export const RecordNotifyLogRequest: MessageFns<RecordNotifyLogRequest> = {
|
||||||
|
encode(message: RecordNotifyLogRequest, writer: BinaryWriter = new BinaryWriter()): BinaryWriter {
|
||||||
|
if (message.title !== "") {
|
||||||
|
writer.uint32(10).string(message.title);
|
||||||
|
}
|
||||||
|
if (message.content !== "") {
|
||||||
|
writer.uint32(18).string(message.content);
|
||||||
|
}
|
||||||
|
if (message.success !== false) {
|
||||||
|
writer.uint32(24).bool(message.success);
|
||||||
|
}
|
||||||
|
if (message.notifyType !== "") {
|
||||||
|
writer.uint32(34).string(message.notifyType);
|
||||||
|
}
|
||||||
|
if (message.scriptName !== "") {
|
||||||
|
writer.uint32(42).string(message.scriptName);
|
||||||
|
}
|
||||||
|
if (message.taskId !== "") {
|
||||||
|
writer.uint32(50).string(message.taskId);
|
||||||
|
}
|
||||||
|
return writer;
|
||||||
|
},
|
||||||
|
|
||||||
|
decode(input: BinaryReader | Uint8Array, length?: number): RecordNotifyLogRequest {
|
||||||
|
const reader = input instanceof BinaryReader ? input : new BinaryReader(input);
|
||||||
|
let end = length === undefined ? reader.len : reader.pos + length;
|
||||||
|
const message = createBaseRecordNotifyLogRequest();
|
||||||
|
while (reader.pos < end) {
|
||||||
|
const tag = reader.uint32();
|
||||||
|
switch (tag >>> 3) {
|
||||||
|
case 1: {
|
||||||
|
if (tag !== 10) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
message.title = reader.string();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
case 2: {
|
||||||
|
if (tag !== 18) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
message.content = reader.string();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
case 3: {
|
||||||
|
if (tag !== 24) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
message.success = reader.bool();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
case 4: {
|
||||||
|
if (tag !== 34) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
message.notifyType = reader.string();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
case 5: {
|
||||||
|
if (tag !== 42) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
message.scriptName = reader.string();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
case 6: {
|
||||||
|
if (tag !== 50) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
message.taskId = reader.string();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ((tag & 7) === 4 || tag === 0) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
reader.skip(tag & 7);
|
||||||
|
}
|
||||||
|
return message;
|
||||||
|
},
|
||||||
|
|
||||||
|
fromJSON(object: any): RecordNotifyLogRequest {
|
||||||
|
return {
|
||||||
|
title: isSet(object.title) ? globalThis.String(object.title) : "",
|
||||||
|
content: isSet(object.content) ? globalThis.String(object.content) : "",
|
||||||
|
success: isSet(object.success) ? globalThis.Boolean(object.success) : false,
|
||||||
|
notifyType: isSet(object.notifyType) ? globalThis.String(object.notifyType) : "",
|
||||||
|
scriptName: isSet(object.scriptName) ? globalThis.String(object.scriptName) : "",
|
||||||
|
taskId: isSet(object.taskId) ? globalThis.String(object.taskId) : "",
|
||||||
|
};
|
||||||
|
},
|
||||||
|
|
||||||
|
toJSON(message: RecordNotifyLogRequest): unknown {
|
||||||
|
const obj: any = {};
|
||||||
|
if (message.title !== "") {
|
||||||
|
obj.title = message.title;
|
||||||
|
}
|
||||||
|
if (message.content !== "") {
|
||||||
|
obj.content = message.content;
|
||||||
|
}
|
||||||
|
if (message.success !== false) {
|
||||||
|
obj.success = message.success;
|
||||||
|
}
|
||||||
|
if (message.notifyType !== "") {
|
||||||
|
obj.notifyType = message.notifyType;
|
||||||
|
}
|
||||||
|
if (message.scriptName !== "") {
|
||||||
|
obj.scriptName = message.scriptName;
|
||||||
|
}
|
||||||
|
if (message.taskId !== "") {
|
||||||
|
obj.taskId = message.taskId;
|
||||||
|
}
|
||||||
|
return obj;
|
||||||
|
},
|
||||||
|
|
||||||
|
create<I extends Exact<DeepPartial<RecordNotifyLogRequest>, I>>(base?: I): RecordNotifyLogRequest {
|
||||||
|
return RecordNotifyLogRequest.fromPartial(base ?? ({} as any));
|
||||||
|
},
|
||||||
|
fromPartial<I extends Exact<DeepPartial<RecordNotifyLogRequest>, I>>(object: I): RecordNotifyLogRequest {
|
||||||
|
const message = createBaseRecordNotifyLogRequest();
|
||||||
|
message.title = object.title ?? "";
|
||||||
|
message.content = object.content ?? "";
|
||||||
|
message.success = object.success ?? false;
|
||||||
|
message.notifyType = object.notifyType ?? "";
|
||||||
|
message.scriptName = object.scriptName ?? "";
|
||||||
|
message.taskId = object.taskId ?? "";
|
||||||
|
return message;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
export type ApiService = typeof ApiService;
|
export type ApiService = typeof ApiService;
|
||||||
export const ApiService = {
|
export const ApiService = {
|
||||||
getEnvs: {
|
getEnvs: {
|
||||||
@@ -4388,6 +4537,15 @@ export const ApiService = {
|
|||||||
responseSerialize: (value: Response) => Buffer.from(Response.encode(value).finish()),
|
responseSerialize: (value: Response) => Buffer.from(Response.encode(value).finish()),
|
||||||
responseDeserialize: (value: Buffer) => Response.decode(value),
|
responseDeserialize: (value: Buffer) => Response.decode(value),
|
||||||
},
|
},
|
||||||
|
recordNotifyLog: {
|
||||||
|
path: "/com.ql.api.Api/RecordNotifyLog",
|
||||||
|
requestStream: false,
|
||||||
|
responseStream: false,
|
||||||
|
requestSerialize: (value: RecordNotifyLogRequest) => Buffer.from(RecordNotifyLogRequest.encode(value).finish()),
|
||||||
|
requestDeserialize: (value: Buffer) => RecordNotifyLogRequest.decode(value),
|
||||||
|
responseSerialize: (value: Response) => Buffer.from(Response.encode(value).finish()),
|
||||||
|
responseDeserialize: (value: Buffer) => Response.decode(value),
|
||||||
|
},
|
||||||
getCronDetail: {
|
getCronDetail: {
|
||||||
path: "/com.ql.api.Api/GetCronDetail",
|
path: "/com.ql.api.Api/GetCronDetail",
|
||||||
requestStream: false,
|
requestStream: false,
|
||||||
@@ -4482,6 +4640,7 @@ export interface ApiServer extends UntypedServiceImplementation {
|
|||||||
updateEnvNames: handleUnaryCall<UpdateEnvNamesRequest, Response>;
|
updateEnvNames: handleUnaryCall<UpdateEnvNamesRequest, Response>;
|
||||||
getEnvById: handleUnaryCall<GetEnvByIdRequest, EnvResponse>;
|
getEnvById: handleUnaryCall<GetEnvByIdRequest, EnvResponse>;
|
||||||
systemNotify: handleUnaryCall<SystemNotifyRequest, Response>;
|
systemNotify: handleUnaryCall<SystemNotifyRequest, Response>;
|
||||||
|
recordNotifyLog: handleUnaryCall<RecordNotifyLogRequest, Response>;
|
||||||
getCronDetail: handleUnaryCall<CronDetailRequest, CronDetailResponse>;
|
getCronDetail: handleUnaryCall<CronDetailRequest, CronDetailResponse>;
|
||||||
createCron: handleUnaryCall<CreateCronRequest, CronResponse>;
|
createCron: handleUnaryCall<CreateCronRequest, CronResponse>;
|
||||||
updateCron: handleUnaryCall<UpdateCronRequest, CronResponse>;
|
updateCron: handleUnaryCall<UpdateCronRequest, CronResponse>;
|
||||||
@@ -4644,6 +4803,21 @@ export interface ApiClient extends Client {
|
|||||||
options: Partial<CallOptions>,
|
options: Partial<CallOptions>,
|
||||||
callback: (error: ServiceError | null, response: Response) => void,
|
callback: (error: ServiceError | null, response: Response) => void,
|
||||||
): ClientUnaryCall;
|
): ClientUnaryCall;
|
||||||
|
recordNotifyLog(
|
||||||
|
request: RecordNotifyLogRequest,
|
||||||
|
callback: (error: ServiceError | null, response: Response) => void,
|
||||||
|
): ClientUnaryCall;
|
||||||
|
recordNotifyLog(
|
||||||
|
request: RecordNotifyLogRequest,
|
||||||
|
metadata: Metadata,
|
||||||
|
callback: (error: ServiceError | null, response: Response) => void,
|
||||||
|
): ClientUnaryCall;
|
||||||
|
recordNotifyLog(
|
||||||
|
request: RecordNotifyLogRequest,
|
||||||
|
metadata: Metadata,
|
||||||
|
options: Partial<CallOptions>,
|
||||||
|
callback: (error: ServiceError | null, response: Response) => void,
|
||||||
|
): ClientUnaryCall;
|
||||||
getCronDetail(
|
getCronDetail(
|
||||||
request: CronDetailRequest,
|
request: CronDetailRequest,
|
||||||
callback: (error: ServiceError | null, response: CronDetailResponse) => void,
|
callback: (error: ServiceError | null, response: CronDetailResponse) => void,
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ package com.ql.cron;
|
|||||||
service Cron {
|
service Cron {
|
||||||
rpc addCron(AddCronRequest) returns (AddCronResponse);
|
rpc addCron(AddCronRequest) returns (AddCronResponse);
|
||||||
rpc delCron(DeleteCronRequest) returns (DeleteCronResponse);
|
rpc delCron(DeleteCronRequest) returns (DeleteCronResponse);
|
||||||
|
rpc setConcurrency(SetConcurrencyRequest) returns (AddCronResponse);
|
||||||
}
|
}
|
||||||
|
|
||||||
message ISchedule { string schedule = 1; }
|
message ISchedule { string schedule = 1; }
|
||||||
@@ -15,6 +16,10 @@ message ICron {
|
|||||||
string command = 3;
|
string command = 3;
|
||||||
repeated ISchedule extra_schedules = 4;
|
repeated ISchedule extra_schedules = 4;
|
||||||
string name = 5;
|
string name = 5;
|
||||||
|
optional uint32 max_queue_wait = 6;
|
||||||
|
optional uint32 catchup_seconds = 7;
|
||||||
|
optional int64 recovery_from = 8;
|
||||||
|
optional bool skip_if_pending = 9;
|
||||||
}
|
}
|
||||||
|
|
||||||
message AddCronRequest {
|
message AddCronRequest {
|
||||||
@@ -27,3 +32,6 @@ message AddCronResponse {}
|
|||||||
message DeleteCronRequest { repeated string ids = 1; }
|
message DeleteCronRequest { repeated string ids = 1; }
|
||||||
|
|
||||||
message DeleteCronResponse {}
|
message DeleteCronResponse {}
|
||||||
|
|
||||||
|
// Zero restores the default limit.
|
||||||
|
message SetConcurrencyRequest { int32 concurrency = 1; }
|
||||||
|
|||||||
+179
-2
@@ -1,7 +1,7 @@
|
|||||||
// Code generated by protoc-gen-ts_proto. DO NOT EDIT.
|
// Code generated by protoc-gen-ts_proto. DO NOT EDIT.
|
||||||
// versions:
|
// versions:
|
||||||
// protoc-gen-ts_proto v2.6.1
|
// protoc-gen-ts_proto v2.6.1
|
||||||
// protoc v3.17.3
|
// protoc v5.29.0
|
||||||
// source: back/protos/cron.proto
|
// source: back/protos/cron.proto
|
||||||
|
|
||||||
/* eslint-disable */
|
/* eslint-disable */
|
||||||
@@ -31,6 +31,10 @@ export interface ICron {
|
|||||||
command: string;
|
command: string;
|
||||||
extra_schedules: ISchedule[];
|
extra_schedules: ISchedule[];
|
||||||
name: string;
|
name: string;
|
||||||
|
max_queue_wait?: number | undefined;
|
||||||
|
catchup_seconds?: number | undefined;
|
||||||
|
recovery_from?: number | undefined;
|
||||||
|
skip_if_pending?: boolean | undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface AddCronRequest {
|
export interface AddCronRequest {
|
||||||
@@ -48,6 +52,11 @@ export interface DeleteCronRequest {
|
|||||||
export interface DeleteCronResponse {
|
export interface DeleteCronResponse {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Zero restores the default limit. */
|
||||||
|
export interface SetConcurrencyRequest {
|
||||||
|
concurrency: number;
|
||||||
|
}
|
||||||
|
|
||||||
function createBaseISchedule(): ISchedule {
|
function createBaseISchedule(): ISchedule {
|
||||||
return { schedule: "" };
|
return { schedule: "" };
|
||||||
}
|
}
|
||||||
@@ -107,7 +116,17 @@ export const ISchedule: MessageFns<ISchedule> = {
|
|||||||
};
|
};
|
||||||
|
|
||||||
function createBaseICron(): ICron {
|
function createBaseICron(): ICron {
|
||||||
return { id: "", schedule: "", command: "", extra_schedules: [], name: "" };
|
return {
|
||||||
|
id: "",
|
||||||
|
schedule: "",
|
||||||
|
command: "",
|
||||||
|
extra_schedules: [],
|
||||||
|
name: "",
|
||||||
|
max_queue_wait: undefined,
|
||||||
|
catchup_seconds: undefined,
|
||||||
|
recovery_from: undefined,
|
||||||
|
skip_if_pending: undefined,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export const ICron: MessageFns<ICron> = {
|
export const ICron: MessageFns<ICron> = {
|
||||||
@@ -127,6 +146,18 @@ export const ICron: MessageFns<ICron> = {
|
|||||||
if (message.name !== "") {
|
if (message.name !== "") {
|
||||||
writer.uint32(42).string(message.name);
|
writer.uint32(42).string(message.name);
|
||||||
}
|
}
|
||||||
|
if (message.max_queue_wait !== undefined) {
|
||||||
|
writer.uint32(48).uint32(message.max_queue_wait);
|
||||||
|
}
|
||||||
|
if (message.catchup_seconds !== undefined) {
|
||||||
|
writer.uint32(56).uint32(message.catchup_seconds);
|
||||||
|
}
|
||||||
|
if (message.recovery_from !== undefined) {
|
||||||
|
writer.uint32(64).int64(message.recovery_from);
|
||||||
|
}
|
||||||
|
if (message.skip_if_pending !== undefined) {
|
||||||
|
writer.uint32(72).bool(message.skip_if_pending);
|
||||||
|
}
|
||||||
return writer;
|
return writer;
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -177,6 +208,38 @@ export const ICron: MessageFns<ICron> = {
|
|||||||
message.name = reader.string();
|
message.name = reader.string();
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
case 6: {
|
||||||
|
if (tag !== 48) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
message.max_queue_wait = reader.uint32();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
case 7: {
|
||||||
|
if (tag !== 56) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
message.catchup_seconds = reader.uint32();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
case 8: {
|
||||||
|
if (tag !== 64) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
message.recovery_from = longToNumber(reader.int64());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
case 9: {
|
||||||
|
if (tag !== 72) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
message.skip_if_pending = reader.bool();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if ((tag & 7) === 4 || tag === 0) {
|
if ((tag & 7) === 4 || tag === 0) {
|
||||||
break;
|
break;
|
||||||
@@ -195,6 +258,10 @@ export const ICron: MessageFns<ICron> = {
|
|||||||
? object.extra_schedules.map((e: any) => ISchedule.fromJSON(e))
|
? object.extra_schedules.map((e: any) => ISchedule.fromJSON(e))
|
||||||
: [],
|
: [],
|
||||||
name: isSet(object.name) ? globalThis.String(object.name) : "",
|
name: isSet(object.name) ? globalThis.String(object.name) : "",
|
||||||
|
max_queue_wait: isSet(object.max_queue_wait) ? globalThis.Number(object.max_queue_wait) : undefined,
|
||||||
|
catchup_seconds: isSet(object.catchup_seconds) ? globalThis.Number(object.catchup_seconds) : undefined,
|
||||||
|
recovery_from: isSet(object.recovery_from) ? globalThis.Number(object.recovery_from) : undefined,
|
||||||
|
skip_if_pending: isSet(object.skip_if_pending) ? globalThis.Boolean(object.skip_if_pending) : undefined,
|
||||||
};
|
};
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -215,6 +282,18 @@ export const ICron: MessageFns<ICron> = {
|
|||||||
if (message.name !== "") {
|
if (message.name !== "") {
|
||||||
obj.name = message.name;
|
obj.name = message.name;
|
||||||
}
|
}
|
||||||
|
if (message.max_queue_wait !== undefined) {
|
||||||
|
obj.max_queue_wait = Math.round(message.max_queue_wait);
|
||||||
|
}
|
||||||
|
if (message.catchup_seconds !== undefined) {
|
||||||
|
obj.catchup_seconds = Math.round(message.catchup_seconds);
|
||||||
|
}
|
||||||
|
if (message.recovery_from !== undefined) {
|
||||||
|
obj.recovery_from = Math.round(message.recovery_from);
|
||||||
|
}
|
||||||
|
if (message.skip_if_pending !== undefined) {
|
||||||
|
obj.skip_if_pending = message.skip_if_pending;
|
||||||
|
}
|
||||||
return obj;
|
return obj;
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -228,6 +307,10 @@ export const ICron: MessageFns<ICron> = {
|
|||||||
message.command = object.command ?? "";
|
message.command = object.command ?? "";
|
||||||
message.extra_schedules = object.extra_schedules?.map((e) => ISchedule.fromPartial(e)) || [];
|
message.extra_schedules = object.extra_schedules?.map((e) => ISchedule.fromPartial(e)) || [];
|
||||||
message.name = object.name ?? "";
|
message.name = object.name ?? "";
|
||||||
|
message.max_queue_wait = object.max_queue_wait ?? undefined;
|
||||||
|
message.catchup_seconds = object.catchup_seconds ?? undefined;
|
||||||
|
message.recovery_from = object.recovery_from ?? undefined;
|
||||||
|
message.skip_if_pending = object.skip_if_pending ?? undefined;
|
||||||
return message;
|
return message;
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
@@ -452,6 +535,64 @@ export const DeleteCronResponse: MessageFns<DeleteCronResponse> = {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
function createBaseSetConcurrencyRequest(): SetConcurrencyRequest {
|
||||||
|
return { concurrency: 0 };
|
||||||
|
}
|
||||||
|
|
||||||
|
export const SetConcurrencyRequest: MessageFns<SetConcurrencyRequest> = {
|
||||||
|
encode(message: SetConcurrencyRequest, writer: BinaryWriter = new BinaryWriter()): BinaryWriter {
|
||||||
|
if (message.concurrency !== 0) {
|
||||||
|
writer.uint32(8).int32(message.concurrency);
|
||||||
|
}
|
||||||
|
return writer;
|
||||||
|
},
|
||||||
|
|
||||||
|
decode(input: BinaryReader | Uint8Array, length?: number): SetConcurrencyRequest {
|
||||||
|
const reader = input instanceof BinaryReader ? input : new BinaryReader(input);
|
||||||
|
let end = length === undefined ? reader.len : reader.pos + length;
|
||||||
|
const message = createBaseSetConcurrencyRequest();
|
||||||
|
while (reader.pos < end) {
|
||||||
|
const tag = reader.uint32();
|
||||||
|
switch (tag >>> 3) {
|
||||||
|
case 1: {
|
||||||
|
if (tag !== 8) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
message.concurrency = reader.int32();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ((tag & 7) === 4 || tag === 0) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
reader.skip(tag & 7);
|
||||||
|
}
|
||||||
|
return message;
|
||||||
|
},
|
||||||
|
|
||||||
|
fromJSON(object: any): SetConcurrencyRequest {
|
||||||
|
return { concurrency: isSet(object.concurrency) ? globalThis.Number(object.concurrency) : 0 };
|
||||||
|
},
|
||||||
|
|
||||||
|
toJSON(message: SetConcurrencyRequest): unknown {
|
||||||
|
const obj: any = {};
|
||||||
|
if (message.concurrency !== 0) {
|
||||||
|
obj.concurrency = Math.round(message.concurrency);
|
||||||
|
}
|
||||||
|
return obj;
|
||||||
|
},
|
||||||
|
|
||||||
|
create<I extends Exact<DeepPartial<SetConcurrencyRequest>, I>>(base?: I): SetConcurrencyRequest {
|
||||||
|
return SetConcurrencyRequest.fromPartial(base ?? ({} as any));
|
||||||
|
},
|
||||||
|
fromPartial<I extends Exact<DeepPartial<SetConcurrencyRequest>, I>>(object: I): SetConcurrencyRequest {
|
||||||
|
const message = createBaseSetConcurrencyRequest();
|
||||||
|
message.concurrency = object.concurrency ?? 0;
|
||||||
|
return message;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
export type CronService = typeof CronService;
|
export type CronService = typeof CronService;
|
||||||
export const CronService = {
|
export const CronService = {
|
||||||
addCron: {
|
addCron: {
|
||||||
@@ -472,11 +613,21 @@ export const CronService = {
|
|||||||
responseSerialize: (value: DeleteCronResponse) => Buffer.from(DeleteCronResponse.encode(value).finish()),
|
responseSerialize: (value: DeleteCronResponse) => Buffer.from(DeleteCronResponse.encode(value).finish()),
|
||||||
responseDeserialize: (value: Buffer) => DeleteCronResponse.decode(value),
|
responseDeserialize: (value: Buffer) => DeleteCronResponse.decode(value),
|
||||||
},
|
},
|
||||||
|
setConcurrency: {
|
||||||
|
path: "/com.ql.cron.Cron/setConcurrency",
|
||||||
|
requestStream: false,
|
||||||
|
responseStream: false,
|
||||||
|
requestSerialize: (value: SetConcurrencyRequest) => Buffer.from(SetConcurrencyRequest.encode(value).finish()),
|
||||||
|
requestDeserialize: (value: Buffer) => SetConcurrencyRequest.decode(value),
|
||||||
|
responseSerialize: (value: AddCronResponse) => Buffer.from(AddCronResponse.encode(value).finish()),
|
||||||
|
responseDeserialize: (value: Buffer) => AddCronResponse.decode(value),
|
||||||
|
},
|
||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
export interface CronServer extends UntypedServiceImplementation {
|
export interface CronServer extends UntypedServiceImplementation {
|
||||||
addCron: handleUnaryCall<AddCronRequest, AddCronResponse>;
|
addCron: handleUnaryCall<AddCronRequest, AddCronResponse>;
|
||||||
delCron: handleUnaryCall<DeleteCronRequest, DeleteCronResponse>;
|
delCron: handleUnaryCall<DeleteCronRequest, DeleteCronResponse>;
|
||||||
|
setConcurrency: handleUnaryCall<SetConcurrencyRequest, AddCronResponse>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface CronClient extends Client {
|
export interface CronClient extends Client {
|
||||||
@@ -510,6 +661,21 @@ export interface CronClient extends Client {
|
|||||||
options: Partial<CallOptions>,
|
options: Partial<CallOptions>,
|
||||||
callback: (error: ServiceError | null, response: DeleteCronResponse) => void,
|
callback: (error: ServiceError | null, response: DeleteCronResponse) => void,
|
||||||
): ClientUnaryCall;
|
): ClientUnaryCall;
|
||||||
|
setConcurrency(
|
||||||
|
request: SetConcurrencyRequest,
|
||||||
|
callback: (error: ServiceError | null, response: AddCronResponse) => void,
|
||||||
|
): ClientUnaryCall;
|
||||||
|
setConcurrency(
|
||||||
|
request: SetConcurrencyRequest,
|
||||||
|
metadata: Metadata,
|
||||||
|
callback: (error: ServiceError | null, response: AddCronResponse) => void,
|
||||||
|
): ClientUnaryCall;
|
||||||
|
setConcurrency(
|
||||||
|
request: SetConcurrencyRequest,
|
||||||
|
metadata: Metadata,
|
||||||
|
options: Partial<CallOptions>,
|
||||||
|
callback: (error: ServiceError | null, response: AddCronResponse) => void,
|
||||||
|
): ClientUnaryCall;
|
||||||
}
|
}
|
||||||
|
|
||||||
export const CronClient = makeGenericClientConstructor(CronService, "com.ql.cron.Cron") as unknown as {
|
export const CronClient = makeGenericClientConstructor(CronService, "com.ql.cron.Cron") as unknown as {
|
||||||
@@ -530,6 +696,17 @@ type KeysOfUnion<T> = T extends T ? keyof T : never;
|
|||||||
export type Exact<P, I extends P> = P extends Builtin ? P
|
export type Exact<P, I extends P> = P extends Builtin ? P
|
||||||
: P & { [K in keyof P]: Exact<P[K], I[K]> } & { [K in Exclude<keyof I, KeysOfUnion<P>>]: never };
|
: P & { [K in keyof P]: Exact<P[K], I[K]> } & { [K in Exclude<keyof I, KeysOfUnion<P>>]: never };
|
||||||
|
|
||||||
|
function longToNumber(int64: { toString(): string }): number {
|
||||||
|
const num = globalThis.Number(int64.toString());
|
||||||
|
if (num > globalThis.Number.MAX_SAFE_INTEGER) {
|
||||||
|
throw new globalThis.Error("Value is larger than Number.MAX_SAFE_INTEGER");
|
||||||
|
}
|
||||||
|
if (num < globalThis.Number.MIN_SAFE_INTEGER) {
|
||||||
|
throw new globalThis.Error("Value is smaller than Number.MIN_SAFE_INTEGER");
|
||||||
|
}
|
||||||
|
return num;
|
||||||
|
}
|
||||||
|
|
||||||
function isSet(value: any): boolean {
|
function isSet(value: any): boolean {
|
||||||
return value !== null && value !== undefined;
|
return value !== null && value !== undefined;
|
||||||
}
|
}
|
||||||
|
|||||||
+86
-76
@@ -1,20 +1,25 @@
|
|||||||
import { ServerUnaryCall, sendUnaryData, status } from '@grpc/grpc-js';
|
import { ServerUnaryCall, sendUnaryData, status } from '@grpc/grpc-js';
|
||||||
import { AddCronRequest, AddCronResponse } from '../protos/cron';
|
import { AddCronRequest, AddCronResponse } from '../protos/cron';
|
||||||
import nodeSchedule from 'node-schedule';
|
import { createCronJob, CronJob } from '../shared/cronScheduler';
|
||||||
import { isValidCronSchedule } from '../shared/cronSchedule';
|
import { isValidCronSchedule } from '../shared/cronSchedule';
|
||||||
import { scheduleStacks } from './data';
|
import { scheduleStacks } from './data';
|
||||||
|
import { latestMissedSlot } from '../shared/cronRecovery';
|
||||||
|
import { slotId } from '../shared/triggerHistory';
|
||||||
import { runCron } from '../shared/runCron';
|
import { runCron } from '../shared/runCron';
|
||||||
import Logger from '../loaders/logger';
|
import Logger from '../loaders/logger';
|
||||||
import { tf } from '../shared/i18n';
|
import { t, tf } from '../shared/i18n';
|
||||||
|
|
||||||
// Validate the entire batch before replacing any existing jobs.
|
// Validate the entire batch before replacing any existing jobs.
|
||||||
const isValidCronField = (cron: string): boolean => {
|
const isValidCronField = (cron: string): boolean => {
|
||||||
return isValidCronSchedule(cron);
|
return isValidCronSchedule(cron);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const definitions = new WeakMap<CronJob[], string>();
|
||||||
|
|
||||||
const addCron = (
|
const addCron = (
|
||||||
call: ServerUnaryCall<AddCronRequest, AddCronResponse>,
|
call: ServerUnaryCall<AddCronRequest, AddCronResponse>,
|
||||||
callback: sendUnaryData<AddCronResponse>,
|
callback: sendUnaryData<AddCronResponse>,
|
||||||
|
savePlan?: () => Promise<void>,
|
||||||
) => {
|
) => {
|
||||||
// ===== 第一遍:预校验所有 cron 表达式 =====
|
// ===== 第一遍:预校验所有 cron 表达式 =====
|
||||||
const validationErrors: string[] = [];
|
const validationErrors: string[] = [];
|
||||||
@@ -24,11 +29,7 @@ const addCron = (
|
|||||||
|
|
||||||
if (!isValidCronField(schedule)) {
|
if (!isValidCronField(schedule)) {
|
||||||
validationErrors.push(
|
validationErrors.push(
|
||||||
tf(
|
tf('任务ID %s: 无效的 cron 表达式 "%s"', String(id), schedule),
|
||||||
'任务ID %s: 无效的 cron 表达式 "%s"',
|
|
||||||
String(id),
|
|
||||||
schedule,
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -56,82 +57,91 @@ const addCron = (
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Recovery replaces the whole snapshot, including deletions and disabled jobs.
|
// Prepare stopped jobs first: construction errors must preserve the old snapshot.
|
||||||
// Validation above must finish before touching the previous schedule.
|
const prepared = new Map<string, CronJob[]>();
|
||||||
|
const desired = new Map(call.request.crons.map((item) => [item.id, item]));
|
||||||
|
const recoveries: (() => Promise<void>)[] = [];
|
||||||
|
try {
|
||||||
|
for (const item of desired.values()) {
|
||||||
|
const signature = JSON.stringify([item.schedule, item.command,
|
||||||
|
(item.extra_schedules || []).map((s) => s.schedule), item.max_queue_wait || 0,
|
||||||
|
!!item.skip_if_pending, item.catchup_seconds || 0, item.catchup_seconds || item.skip_if_pending ? item.recovery_from : 0]);
|
||||||
|
const current = scheduleStacks.get(item.id);
|
||||||
|
if (current && definitions.get(current) === signature) continue;
|
||||||
|
const jobs: CronJob[] = [];
|
||||||
|
definitions.set(jobs, signature);
|
||||||
|
prepared.set(item.id, jobs);
|
||||||
|
for (const [ruleIndex, schedule] of [
|
||||||
|
item.schedule,
|
||||||
|
...(item.extra_schedules || []).map((x) => x.schedule),
|
||||||
|
].entries()) {
|
||||||
|
const fire = async (date: Date, source: 'node' | 'recovery' = 'node') => {
|
||||||
|
const isCurrent = () => scheduleStacks.get(item.id) === jobs;
|
||||||
|
if (!isCurrent()) return;
|
||||||
|
await runCron(item.command, item, isCurrent, date && {
|
||||||
|
id: slotId(item.id, ruleIndex, schedule, +date), time: +date, source,
|
||||||
|
});
|
||||||
|
};
|
||||||
|
if (call.request.replace && item.catchup_seconds && item.recovery_from) {
|
||||||
|
recoveries.push(async () => {
|
||||||
|
const now = Date.now();
|
||||||
|
const since = Math.max(item.recovery_from!, now - item.catchup_seconds! * 1000);
|
||||||
|
const time = latestMissedSlot(schedule, since, now);
|
||||||
|
if (time !== null) await fire(new Date(time), 'recovery');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
jobs.push(
|
||||||
|
createCronJob(
|
||||||
|
schedule,
|
||||||
|
fire,
|
||||||
|
{
|
||||||
|
name: `${item.id}: ${item.name || ''}`,
|
||||||
|
logger: Logger,
|
||||||
|
start: false,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
for (const jobs of prepared.values()) jobs.forEach((job) => job.cancel());
|
||||||
|
const err: any = new Error(
|
||||||
|
error instanceof Error ? error.message : String(error),
|
||||||
|
);
|
||||||
|
err.code = status.INVALID_ARGUMENT;
|
||||||
|
err.details = err.message;
|
||||||
|
callback(err, null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const activate = () => {
|
||||||
if (call.request.replace) {
|
if (call.request.replace) {
|
||||||
for (const jobs of scheduleStacks.values()) {
|
for (const [id, jobs] of scheduleStacks) {
|
||||||
for (const job of jobs) job?.cancel();
|
if (desired.has(id)) continue;
|
||||||
|
jobs.forEach((job) => job?.cancel());
|
||||||
|
scheduleStacks.delete(id);
|
||||||
}
|
}
|
||||||
scheduleStacks.clear();
|
|
||||||
}
|
}
|
||||||
|
for (const [id, jobs] of prepared) {
|
||||||
// ===== 第二遍:注册所有任务 =====
|
scheduleStacks.get(id)?.forEach((job) => job.cancel());
|
||||||
for (const item of call.request.crons) {
|
|
||||||
const { id, schedule, command, extra_schedules, name } = item;
|
|
||||||
|
|
||||||
// 取消该 id 已有的旧任务
|
|
||||||
if (scheduleStacks.has(id)) {
|
|
||||||
scheduleStacks.get(id)?.forEach((x) => x.cancel());
|
|
||||||
}
|
|
||||||
|
|
||||||
Logger.info(
|
|
||||||
'[schedule][创建定时任务] 任务ID: %s, 名称: %s, cron: %s, 执行命令: %s',
|
|
||||||
id,
|
|
||||||
name,
|
|
||||||
schedule,
|
|
||||||
command,
|
|
||||||
);
|
|
||||||
|
|
||||||
if (extra_schedules?.length) {
|
|
||||||
extra_schedules.forEach((x) => {
|
|
||||||
Logger.info(
|
|
||||||
'[schedule][创建定时任务] 任务ID: %s, 名称: %s, cron: %s, 执行命令: %s',
|
|
||||||
id,
|
|
||||||
name,
|
|
||||||
x.schedule,
|
|
||||||
command,
|
|
||||||
);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
const mainJob = nodeSchedule.scheduleJob(id, schedule, async () => {
|
|
||||||
Logger.info(`[schedule][准备运行任务] 命令: ${command}`);
|
|
||||||
runCron(command, item);
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!mainJob) {
|
|
||||||
Logger.warn(
|
|
||||||
'[schedule][创建定时任务] scheduleJob 返回 null(不符合预期,已通过预校验): 任务ID: %s, cron: %s',
|
|
||||||
id,
|
|
||||||
schedule,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const extraJobs = extra_schedules?.length
|
|
||||||
? extra_schedules.map((x) => {
|
|
||||||
const job = nodeSchedule.scheduleJob(id, x.schedule, async () => {
|
|
||||||
Logger.info(`[schedule][准备运行任务] 命令: ${command}`);
|
|
||||||
runCron(command, item);
|
|
||||||
});
|
|
||||||
if (!job) {
|
|
||||||
Logger.warn(
|
|
||||||
'[schedule][创建定时任务] scheduleJob 返回 null(不符合预期,已通过预校验): 任务ID: %s, cron: %s',
|
|
||||||
id,
|
|
||||||
x.schedule,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return job;
|
|
||||||
})
|
|
||||||
: [];
|
|
||||||
|
|
||||||
// 过滤 null(兜底保护,正常情况下预校验已拦截)
|
|
||||||
const jobs = [mainJob, ...extraJobs].filter((x) => x != null);
|
|
||||||
if (jobs.length > 0) {
|
|
||||||
scheduleStacks.set(id, jobs);
|
scheduleStacks.set(id, jobs);
|
||||||
}
|
jobs.forEach((job) => job.start());
|
||||||
|
Logger.info(
|
||||||
|
t('[schedule][创建定时任务] 任务ID: %s, 规则数: %s'),
|
||||||
|
id,
|
||||||
|
jobs.length,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
callback(null, null);
|
callback(null, null);
|
||||||
|
for (const recover of recoveries) setImmediate(() => recover().catch((err) => Logger.error(String(err))));
|
||||||
|
};
|
||||||
|
if (savePlan) {
|
||||||
|
savePlan().then(activate).catch(error => {
|
||||||
|
for (const jobs of prepared.values()) jobs.forEach(job => job.cancel());
|
||||||
|
callback(error);
|
||||||
|
});
|
||||||
|
} else activate();
|
||||||
};
|
};
|
||||||
|
|
||||||
export { addCron };
|
export { addCron };
|
||||||
|
|||||||
+25
-2
@@ -16,6 +16,7 @@ import {
|
|||||||
MoveEnvRequest,
|
MoveEnvRequest,
|
||||||
Response,
|
Response,
|
||||||
SystemNotifyRequest,
|
SystemNotifyRequest,
|
||||||
|
RecordNotifyLogRequest,
|
||||||
UpdateEnvNamesRequest,
|
UpdateEnvNamesRequest,
|
||||||
UpdateEnvRequest,
|
UpdateEnvRequest,
|
||||||
} from '../protos/api';
|
} from '../protos/api';
|
||||||
@@ -238,7 +239,8 @@ export const systemNotify = async (
|
|||||||
const data = await systemService.notify({
|
const data = await systemService.notify({
|
||||||
title: call.request.title,
|
title: call.request.title,
|
||||||
content: call.request.content,
|
content: call.request.content,
|
||||||
notificationInfo: call.request.notificationInfo as unknown as NotificationInfo,
|
notificationInfo: call.request
|
||||||
|
.notificationInfo as unknown as NotificationInfo,
|
||||||
});
|
});
|
||||||
callback(null, data);
|
callback(null, data);
|
||||||
} catch (e: any) {
|
} catch (e: any) {
|
||||||
@@ -246,10 +248,31 @@ export const systemNotify = async (
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const recordNotifyLog = async (
|
||||||
|
call: ServerUnaryCall<RecordNotifyLogRequest, Response>,
|
||||||
|
callback: sendUnaryData<Response>,
|
||||||
|
) => {
|
||||||
|
try {
|
||||||
|
await Container.get(SystemService).recordNotifyLog({
|
||||||
|
title: call.request.title,
|
||||||
|
content: call.request.content,
|
||||||
|
status: call.request.success ? 0 : 1,
|
||||||
|
notifyType: call.request.notifyType,
|
||||||
|
source: 'script',
|
||||||
|
scriptName: call.request.scriptName,
|
||||||
|
taskId: call.request.taskId,
|
||||||
|
});
|
||||||
|
callback(null, { code: 200 });
|
||||||
|
} catch (error: any) {
|
||||||
|
callback(error);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const normalizeCronData = (data: CronItem | null): CronItem | undefined => {
|
const normalizeCronData = (data: CronItem | null): CronItem | undefined => {
|
||||||
if (!data) return undefined;
|
if (!data) return undefined;
|
||||||
// create() returns a Sequelize instance; spreading it omits attribute getters.
|
// create() returns a Sequelize instance; spreading it omits attribute getters.
|
||||||
const cron = data instanceof Model ? (data.get({ plain: true }) as CronItem) : data;
|
const cron =
|
||||||
|
data instanceof Model ? (data.get({ plain: true }) as CronItem) : data;
|
||||||
return {
|
return {
|
||||||
...cron,
|
...cron,
|
||||||
labels: cron.labels ?? [],
|
labels: cron.labels ?? [],
|
||||||
|
|||||||
+27
-2
@@ -62,16 +62,41 @@ class Client {
|
|||||||
return this._client;
|
return this._client;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async setConcurrency(concurrency: number): Promise<void> {
|
||||||
|
await this.waitForReady(2000);
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
this.client.setConcurrency(
|
||||||
|
{ concurrency }, new Metadata(), { deadline: Date.now() + 5000 },
|
||||||
|
(err) => {
|
||||||
|
if (err) {
|
||||||
|
if (err.code === status.UNAVAILABLE || err.code === status.DEADLINE_EXCEEDED) {
|
||||||
|
this.readiness.invalidate();
|
||||||
|
Object.assign(err, { status: 503 });
|
||||||
|
}
|
||||||
|
return reject(err);
|
||||||
|
}
|
||||||
|
resolve();
|
||||||
|
},
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
async addCron(
|
async addCron(
|
||||||
request: AddCronRequest['crons'],
|
request: AddCronRequest['crons'],
|
||||||
replace = false
|
replace = false
|
||||||
): Promise<AddCronResponse> {
|
): Promise<AddCronResponse> {
|
||||||
await this.waitForReady(2000);
|
await this.waitForReady(2000);
|
||||||
|
// Include every rule in a recovery snapshot, but bound a stuck worker.
|
||||||
|
const ruleCount = request.reduce(
|
||||||
|
(count, item) => count + 1 + (item.extra_schedules?.length || 0),
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
const registrationTimeoutMs = Math.min(120000, 5000 + ruleCount * 5);
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
this.client.addCron(
|
this.client.addCron(
|
||||||
{ crons: request, replace },
|
AddCronRequest.fromPartial({ crons: request, replace }),
|
||||||
new Metadata(),
|
new Metadata(),
|
||||||
{ deadline: Date.now() + 5000 },
|
{ deadline: Date.now() + registrationTimeoutMs },
|
||||||
(err, res) => {
|
(err, res) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
if (err.code === status.UNAVAILABLE || err.code === status.DEADLINE_EXCEEDED) {
|
if (err.code === status.UNAVAILABLE || err.code === status.DEADLINE_EXCEEDED) {
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import nodeSchedule from 'node-schedule';
|
import { CronJob } from '../shared/cronScheduler';
|
||||||
import { ToadScheduler } from 'toad-scheduler';
|
import { ToadScheduler } from 'toad-scheduler';
|
||||||
|
|
||||||
export const scheduleStacks = new Map<string, nodeSchedule.Job[]>();
|
export const scheduleStacks = new Map<string, CronJob[]>();
|
||||||
|
|
||||||
export const intervalSchedule = new ToadScheduler();
|
export const intervalSchedule = new ToadScheduler();
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { t } from '../shared/i18n';
|
||||||
import { ServerUnaryCall, sendUnaryData } from '@grpc/grpc-js';
|
import { ServerUnaryCall, sendUnaryData } from '@grpc/grpc-js';
|
||||||
import { DeleteCronRequest, DeleteCronResponse } from '../protos/cron';
|
import { DeleteCronRequest, DeleteCronResponse } from '../protos/cron';
|
||||||
import { scheduleStacks } from './data';
|
import { scheduleStacks } from './data';
|
||||||
@@ -9,19 +10,19 @@ const delCron = (
|
|||||||
) => {
|
) => {
|
||||||
for (const id of call.request.ids) {
|
for (const id of call.request.ids) {
|
||||||
if (scheduleStacks.has(id)) {
|
if (scheduleStacks.has(id)) {
|
||||||
Logger.info(
|
Logger.info(t('[schedule][取消定时任务] 任务ID: %s'), id);
|
||||||
'[schedule][取消定时任务] 任务ID: %s',
|
// 防御性过滤历史调度栈中的空任务,
|
||||||
id,
|
|
||||||
);
|
|
||||||
// 过滤掉 nodeSchedule.scheduleJob() 对无效表达式返回的 null,
|
|
||||||
// 否则对 null 调 cancel() 会让整个取消流程抛出 UNKNOWN 错误,
|
// 否则对 null 调 cancel() 会让整个取消流程抛出 UNKNOWN 错误,
|
||||||
// 进而导致 HTTP 端的 remove() 跳过 setCrontab(),造成 crontab.list 残留。
|
// 进而导致 HTTP 端的 remove() 跳过 setCrontab(),造成 crontab.list 残留。
|
||||||
scheduleStacks.get(id)?.filter((x) => x != null).forEach((x) => {
|
scheduleStacks
|
||||||
|
.get(id)
|
||||||
|
?.filter((x) => x != null)
|
||||||
|
.forEach((x) => {
|
||||||
try {
|
try {
|
||||||
x.cancel();
|
x.cancel();
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
Logger.warn(
|
Logger.warn(
|
||||||
'[schedule][取消任务失败] 任务ID: %s, 错误: %s',
|
t('[schedule][取消任务失败] 任务ID: %s, 错误: %s'),
|
||||||
id,
|
id,
|
||||||
error?.message || error,
|
error?.message || error,
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -0,0 +1,59 @@
|
|||||||
|
import { ServerUnaryCall, sendUnaryData } from '@grpc/grpc-js';
|
||||||
|
import {
|
||||||
|
AddCronRequest,
|
||||||
|
AddCronResponse,
|
||||||
|
DeleteCronRequest,
|
||||||
|
DeleteCronResponse,
|
||||||
|
} from '../protos/cron';
|
||||||
|
import { addCron } from './addCron';
|
||||||
|
import { delCron } from './delCron';
|
||||||
|
import { randomUUID } from 'crypto';
|
||||||
|
import { saveSchedulePlan, confirmSchedulePlan } from '../shared/scheduleAudit';
|
||||||
|
|
||||||
|
// Keep acknowledgement, publication and timer replacement in request order,
|
||||||
|
// even if the client times out and sends a replacement snapshot.
|
||||||
|
let pending = Promise.resolve();
|
||||||
|
function serialize(action: () => Promise<void>) {
|
||||||
|
pending = pending.then(action, action);
|
||||||
|
return pending;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function registeredAdd(
|
||||||
|
call: ServerUnaryCall<AddCronRequest, AddCronResponse>,
|
||||||
|
callback: sendUnaryData<AddCronResponse>,
|
||||||
|
) {
|
||||||
|
return serialize(async () => {
|
||||||
|
const db = require('../data').sequelize;
|
||||||
|
const token = randomUUID();
|
||||||
|
let first = 0;
|
||||||
|
const last = await new Promise<number>((resolve, reject) => {
|
||||||
|
addCron(
|
||||||
|
call,
|
||||||
|
(error) => (error ? reject(error) : resolve(Date.now())),
|
||||||
|
async () => {
|
||||||
|
await saveSchedulePlan(db, call.request, undefined, token);
|
||||||
|
first = Date.now();
|
||||||
|
},
|
||||||
|
);
|
||||||
|
});
|
||||||
|
await confirmSchedulePlan(db, token, first, last, call.request.replace);
|
||||||
|
callback(null, null);
|
||||||
|
}).catch((error) => callback(error));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function registeredDelete(
|
||||||
|
call: ServerUnaryCall<DeleteCronRequest, DeleteCronResponse>,
|
||||||
|
callback: sendUnaryData<DeleteCronResponse>,
|
||||||
|
) {
|
||||||
|
return serialize(async () => {
|
||||||
|
const db = require('../data').sequelize;
|
||||||
|
const token = randomUUID();
|
||||||
|
await saveSchedulePlan(db, call.request, undefined, token);
|
||||||
|
const first = Date.now();
|
||||||
|
delCron(call, (error) => {
|
||||||
|
if (error) throw error;
|
||||||
|
});
|
||||||
|
await confirmSchedulePlan(db, token, first, Date.now(), false);
|
||||||
|
callback(null, null);
|
||||||
|
}).catch((error) => callback(error));
|
||||||
|
}
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
import { ServerUnaryCall, sendUnaryData, status } from '@grpc/grpc-js';
|
||||||
|
import os from 'os';
|
||||||
|
import { AddCronResponse, SetConcurrencyRequest } from '../protos/cron';
|
||||||
|
import taskLimit from '../shared/pLimit';
|
||||||
|
|
||||||
|
export async function setConcurrency(
|
||||||
|
call: ServerUnaryCall<SetConcurrencyRequest, AddCronResponse>,
|
||||||
|
callback: sendUnaryData<AddCronResponse>,
|
||||||
|
) {
|
||||||
|
const limit = call.request.concurrency;
|
||||||
|
if (!Number.isInteger(limit) || limit < 0 || limit > 2147483647) {
|
||||||
|
callback(
|
||||||
|
Object.assign(new Error('Invalid concurrency'), {
|
||||||
|
code: status.INVALID_ARGUMENT,
|
||||||
|
}),
|
||||||
|
null,
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await taskLimit.setCustomLimit(limit || Math.max(os.cpus().length, 4));
|
||||||
|
callback(null, {});
|
||||||
|
} catch (error) {
|
||||||
|
callback(error as Error, null);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -12,13 +12,12 @@ export default class ConfigService {
|
|||||||
|
|
||||||
public async getFile(filePath: string, res: Response) {
|
public async getFile(filePath: string, res: Response) {
|
||||||
let content = '';
|
let content = '';
|
||||||
if (!filePath) {
|
if (!filePath || filePath.startsWith('data/scripts/')) {
|
||||||
return res.send({ code: 403, message: t('文件无法访问') });
|
return res.send({ code: 403, message: t('文件无法访问') });
|
||||||
}
|
}
|
||||||
const scriptFile = filePath.startsWith('data/scripts/');
|
|
||||||
const resolved = resolveFileAccess(
|
const resolved = resolveFileAccess(
|
||||||
scriptFile ? config.scriptPath : config.configPath,
|
config.configPath,
|
||||||
[scriptFile ? filePath.slice('data/scripts/'.length) : filePath],
|
[filePath],
|
||||||
config.blackFileList,
|
config.blackFileList,
|
||||||
);
|
);
|
||||||
if (!resolved) {
|
if (!resolved) {
|
||||||
|
|||||||
+618
-279
File diff suppressed because it is too large
Load Diff
+131
-29
@@ -20,22 +20,35 @@ import {
|
|||||||
getInstallCommand,
|
getInstallCommand,
|
||||||
getUninstallCommand,
|
getUninstallCommand,
|
||||||
getGetCommand,
|
getGetCommand,
|
||||||
|
isPythonSourceDependency,
|
||||||
} from '../config/util';
|
} from '../config/util';
|
||||||
import dayjs from 'dayjs';
|
import dayjs from 'dayjs';
|
||||||
import taskLimit from '../shared/pLimit';
|
import taskLimit from '../shared/pLimit';
|
||||||
import { detectOS } from '../config/util';
|
import { detectOS } from '../config/util';
|
||||||
import { LINUX_DEPENDENCE_COMMAND } from '../config/const';
|
import { LINUX_DEPENDENCE_COMMAND } from '../config/const';
|
||||||
import { t, tf } from '../shared/i18n';
|
import { t, tf } from '../shared/i18n';
|
||||||
|
import { randomUUID } from 'crypto';
|
||||||
|
|
||||||
|
type RefreshType = 'nodejs' | 'python3';
|
||||||
|
type RefreshJob = {
|
||||||
|
id: string;
|
||||||
|
type: RefreshType;
|
||||||
|
state: 'running' | 'completed' | 'failed';
|
||||||
|
};
|
||||||
|
|
||||||
@Service()
|
@Service()
|
||||||
export default class DependenceService {
|
export default class DependenceService {
|
||||||
|
private activeRefreshJob?: RefreshJob;
|
||||||
|
private refreshJobs = new Map<string, RefreshJob>();
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
@Inject('logger') private logger: winston.Logger,
|
@Inject('logger') private logger: winston.Logger,
|
||||||
private sockService: SockService,
|
private sockService: SockService,
|
||||||
) { }
|
) {}
|
||||||
|
|
||||||
public async create(payloads: Dependence[]): Promise<Dependence[]> {
|
public async create(payloads: Dependence[]): Promise<Dependence[]> {
|
||||||
const tabs = payloads.map((x) => {
|
const tabs = payloads.map((x) => {
|
||||||
|
getInstallCommand(x.type, x.name);
|
||||||
const tab = new Dependence({ ...x, status: DependenceStatus.queued });
|
const tab = new Dependence({ ...x, status: DependenceStatus.queued });
|
||||||
return tab;
|
return tab;
|
||||||
});
|
});
|
||||||
@@ -59,6 +72,7 @@ export default class DependenceService {
|
|||||||
...other,
|
...other,
|
||||||
status: DependenceStatus.queued,
|
status: DependenceStatus.queued,
|
||||||
});
|
});
|
||||||
|
getInstallCommand(tab.type, tab.name);
|
||||||
const newDoc = await this.updateDb(tab);
|
const newDoc = await this.updateDb(tab);
|
||||||
this.installDependenceOneByOne([newDoc]);
|
this.installDependenceOneByOne([newDoc]);
|
||||||
return newDoc;
|
return newDoc;
|
||||||
@@ -74,21 +88,25 @@ export default class DependenceService {
|
|||||||
for (const doc of docs) {
|
for (const doc of docs) {
|
||||||
taskLimit.removeQueuedDependency(doc);
|
taskLimit.removeQueuedDependency(doc);
|
||||||
}
|
}
|
||||||
const unInstalledDeps = docs.filter(
|
|
||||||
(x) => x.status !== DependenceStatus.installed,
|
|
||||||
);
|
|
||||||
const installedDeps = docs.filter(
|
const installedDeps = docs.filter(
|
||||||
(x) => x.status === DependenceStatus.installed,
|
(x) =>
|
||||||
|
x.status === DependenceStatus.installed ||
|
||||||
|
// Older list refreshes marked unprobeable sources as failed. They may
|
||||||
|
// still have installed distributions (including partial pip installs).
|
||||||
|
(x.status === DependenceStatus.installFailed &&
|
||||||
|
x.type === DependenceTypes.python3 &&
|
||||||
|
isPythonSourceDependency(x.name)),
|
||||||
);
|
);
|
||||||
|
const unInstalledDeps = docs.filter((x) => !installedDeps.includes(x));
|
||||||
await this.removeDb(unInstalledDeps.map((x) => x.id!));
|
await this.removeDb(unInstalledDeps.map((x) => x.id!));
|
||||||
|
|
||||||
if (installedDeps.length) {
|
if (installedDeps.length) {
|
||||||
await DependenceModel.update(
|
await DependenceModel.update(
|
||||||
{ status: DependenceStatus.queued, log: [] },
|
{ status: DependenceStatus.queued, log: [] },
|
||||||
{ where: { id: ids } },
|
{ where: { id: installedDeps.map((x) => x.id!) } },
|
||||||
);
|
);
|
||||||
|
|
||||||
this.installDependenceOneByOne(docs, false, force);
|
this.installDependenceOneByOne(installedDeps, false, force);
|
||||||
}
|
}
|
||||||
return docs;
|
return docs;
|
||||||
}
|
}
|
||||||
@@ -111,11 +129,6 @@ export default class DependenceService {
|
|||||||
query: any = {},
|
query: any = {},
|
||||||
): Promise<Dependence[]> {
|
): Promise<Dependence[]> {
|
||||||
let condition = query;
|
let condition = query;
|
||||||
const dependenceType =
|
|
||||||
type && DependenceTypes[type] !== undefined
|
|
||||||
? DependenceTypes[type]
|
|
||||||
: undefined;
|
|
||||||
await this.refreshInstalledStatuses(dependenceType);
|
|
||||||
if (type && DependenceTypes[type] !== undefined) {
|
if (type && DependenceTypes[type] !== undefined) {
|
||||||
condition.type = DependenceTypes[type];
|
condition.type = DependenceTypes[type];
|
||||||
}
|
}
|
||||||
@@ -138,6 +151,52 @@ export default class DependenceService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public startRefresh(type: RefreshType): {
|
||||||
|
job: RefreshJob;
|
||||||
|
accepted: boolean;
|
||||||
|
} {
|
||||||
|
if (this.activeRefreshJob) {
|
||||||
|
return {
|
||||||
|
job: this.activeRefreshJob,
|
||||||
|
accepted: this.activeRefreshJob.type === type,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const job: RefreshJob = { id: randomUUID(), type, state: 'running' };
|
||||||
|
this.activeRefreshJob = job;
|
||||||
|
this.refreshJobs.set(job.id, job);
|
||||||
|
while (this.refreshJobs.size > 20) {
|
||||||
|
const oldestId = this.refreshJobs.keys().next().value;
|
||||||
|
this.refreshJobs.delete(oldestId);
|
||||||
|
}
|
||||||
|
|
||||||
|
void this.refreshInstalledStatuses(DependenceTypes[type])
|
||||||
|
.then(() => {
|
||||||
|
job.state = 'completed';
|
||||||
|
})
|
||||||
|
.catch((error) => {
|
||||||
|
this.logger.error('Dependency refresh failed: %o', error);
|
||||||
|
job.state = 'failed';
|
||||||
|
})
|
||||||
|
.finally(() => {
|
||||||
|
if (this.activeRefreshJob === job) {
|
||||||
|
this.activeRefreshJob = undefined;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
return { job, accepted: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
public getRefreshJob(id: string): RefreshJob | undefined {
|
||||||
|
return this.refreshJobs.get(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
public getActiveRefreshJob(): RefreshJob | undefined {
|
||||||
|
return this.activeRefreshJob?.state === 'running'
|
||||||
|
? this.activeRefreshJob
|
||||||
|
: undefined;
|
||||||
|
}
|
||||||
|
|
||||||
private async refreshInstalledStatuses(type?: DependenceTypes) {
|
private async refreshInstalledStatuses(type?: DependenceTypes) {
|
||||||
const cacheDependenceTypes = [
|
const cacheDependenceTypes = [
|
||||||
DependenceTypes.nodejs,
|
DependenceTypes.nodejs,
|
||||||
@@ -153,20 +212,31 @@ export default class DependenceService {
|
|||||||
type: type === undefined ? { [Op.in]: cacheDependenceTypes } : type,
|
type: type === undefined ? { [Op.in]: cacheDependenceTypes } : type,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
const checkableDocs = docs.filter(
|
||||||
|
(doc) =>
|
||||||
|
doc.type !== DependenceTypes.python3 ||
|
||||||
|
!isPythonSourceDependency(doc.name),
|
||||||
|
);
|
||||||
const checks = await concurrentRun(
|
const checks = await concurrentRun(
|
||||||
docs.map((doc) => async () => {
|
checkableDocs.map((doc) => async () => this.isDependenceInstalled(doc)),
|
||||||
return (await this.isDependenceInstalled(doc)) ? undefined : doc.id;
|
|
||||||
}),
|
|
||||||
5,
|
5,
|
||||||
);
|
);
|
||||||
const missingIds = (checks || []).filter(
|
|
||||||
(id): id is number => id !== undefined,
|
|
||||||
);
|
|
||||||
|
|
||||||
if (missingIds.length) {
|
for (const [index, installed] of (checks || []).entries()) {
|
||||||
|
if (installed) continue;
|
||||||
|
const doc = checkableDocs[index];
|
||||||
await DependenceModel.update(
|
await DependenceModel.update(
|
||||||
{ status: DependenceStatus.installFailed },
|
{ status: DependenceStatus.installFailed },
|
||||||
{ where: { id: missingIds } },
|
{
|
||||||
|
where: {
|
||||||
|
id: doc.id,
|
||||||
|
status: DependenceStatus.installed,
|
||||||
|
name: doc.name,
|
||||||
|
type: doc.type,
|
||||||
|
updatedAt: (doc as typeof doc & { updatedAt: Date }).updatedAt,
|
||||||
|
// Sequelize's automatic timestamp is absent from Dependence's attribute type.
|
||||||
|
} as any,
|
||||||
|
},
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -175,7 +245,10 @@ export default class DependenceService {
|
|||||||
let depName = dependency.name.trim();
|
let depName = dependency.name.trim();
|
||||||
const depVersionStr = versionDependenceCommandTypes[dependency.type];
|
const depVersionStr = versionDependenceCommandTypes[dependency.type];
|
||||||
let depVersion = '';
|
let depVersion = '';
|
||||||
if (depName.includes(depVersionStr)) {
|
if (
|
||||||
|
dependency.type !== DependenceTypes.python3 &&
|
||||||
|
depName.includes(depVersionStr)
|
||||||
|
) {
|
||||||
const symbolRegx = new RegExp(
|
const symbolRegx = new RegExp(
|
||||||
`(.*)${depVersionStr}([0-9\\.\\-\\+a-zA-Z]*)`,
|
`(.*)${depVersionStr}([0-9\\.\\-\\+a-zA-Z]*)`,
|
||||||
);
|
);
|
||||||
@@ -296,6 +369,26 @@ export default class DependenceService {
|
|||||||
force: boolean = false,
|
force: boolean = false,
|
||||||
) {
|
) {
|
||||||
return taskLimit.runDependeny(dependency, () => {
|
return taskLimit.runDependeny(dependency, () => {
|
||||||
|
let validatedCommand: string;
|
||||||
|
try {
|
||||||
|
validatedCommand = isInstall
|
||||||
|
? getInstallCommand(dependency.type, dependency.name.trim())
|
||||||
|
: getUninstallCommand(dependency.type, dependency.name.trim());
|
||||||
|
} catch (error) {
|
||||||
|
taskLimit.removeQueuedDependency(dependency);
|
||||||
|
if (!isInstall && force) {
|
||||||
|
return this.removeDb([dependency.id!]).then(() => null);
|
||||||
|
}
|
||||||
|
return DependenceModel.update(
|
||||||
|
{
|
||||||
|
status: isInstall
|
||||||
|
? DependenceStatus.installFailed
|
||||||
|
: DependenceStatus.removeFailed,
|
||||||
|
log: [String(error)],
|
||||||
|
},
|
||||||
|
{ where: { id: dependency.id } },
|
||||||
|
).then(() => null);
|
||||||
|
}
|
||||||
return new Promise(async (resolve) => {
|
return new Promise(async (resolve) => {
|
||||||
if (taskLimit.firstDependencyId !== dependency.id) {
|
if (taskLimit.firstDependencyId !== dependency.id) {
|
||||||
return resolve(null);
|
return resolve(null);
|
||||||
@@ -345,9 +438,7 @@ export default class DependenceService {
|
|||||||
: DependenceStatus.removing;
|
: DependenceStatus.removing;
|
||||||
await DependenceModel.update({ status }, { where: { id: depIds } });
|
await DependenceModel.update({ status }, { where: { id: depIds } });
|
||||||
|
|
||||||
let command = isInstall
|
let command = validatedCommand;
|
||||||
? getInstallCommand(dependency.type, depName)
|
|
||||||
: getUninstallCommand(dependency.type, depName);
|
|
||||||
if (isLinuxDependence) {
|
if (isLinuxDependence) {
|
||||||
command = isInstall
|
command = isInstall
|
||||||
? `${linuxCommand.install} ${depName.trim()}`
|
? `${linuxCommand.install} ${depName.trim()}`
|
||||||
@@ -370,7 +461,13 @@ export default class DependenceService {
|
|||||||
this.updateLog(depIds, message);
|
this.updateLog(depIds, message);
|
||||||
|
|
||||||
// 判断是否已经安装过依赖
|
// 判断是否已经安装过依赖
|
||||||
if (isInstall && !force) {
|
// pip must resolve optional dependencies even if the base distribution
|
||||||
|
// is already present; its version alone cannot prove extras are ready.
|
||||||
|
if (
|
||||||
|
isInstall &&
|
||||||
|
!force &&
|
||||||
|
!(isPythonDependence && depName.includes('['))
|
||||||
|
) {
|
||||||
let getCommand = getGetCommand(dependency.type, depName);
|
let getCommand = getGetCommand(dependency.type, depName);
|
||||||
const depVersionStr = versionDependenceCommandTypes[dependency.type];
|
const depVersionStr = versionDependenceCommandTypes[dependency.type];
|
||||||
if (isLinuxDependence) {
|
if (isLinuxDependence) {
|
||||||
@@ -378,7 +475,7 @@ export default class DependenceService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let depVersion = '';
|
let depVersion = '';
|
||||||
if (depName.includes(depVersionStr)) {
|
if (!isPythonDependence && depName.includes(depVersionStr)) {
|
||||||
const symbolRegx = new RegExp(
|
const symbolRegx = new RegExp(
|
||||||
`(.*)${depVersionStr}([0-9\\.\\-\\+a-zA-Z]*)`,
|
`(.*)${depVersionStr}([0-9\\.\\-\\+a-zA-Z]*)`,
|
||||||
);
|
);
|
||||||
@@ -470,15 +567,20 @@ export default class DependenceService {
|
|||||||
|
|
||||||
const message =
|
const message =
|
||||||
'\n' +
|
'\n' +
|
||||||
tf('依赖%s%s,结束时间 %s,耗时 %s 秒',
|
tf(
|
||||||
|
'依赖%s%s,结束时间 %s,耗时 %s 秒',
|
||||||
actionText,
|
actionText,
|
||||||
resultText,
|
resultText,
|
||||||
endTime.format('YYYY-MM-DD HH:mm:ss'),
|
endTime.format('YYYY-MM-DD HH:mm:ss'),
|
||||||
String(endTime.diff(startTime, 'second')),
|
String(endTime.diff(startTime, 'second')),
|
||||||
);
|
);
|
||||||
const exitStatus = isSucceed
|
const exitStatus = isSucceed
|
||||||
? (isInstall ? DependenceStatus.installed : DependenceStatus.removed)
|
? isInstall
|
||||||
: (isInstall ? DependenceStatus.installFailed : DependenceStatus.removeFailed);
|
? DependenceStatus.installed
|
||||||
|
: DependenceStatus.removed
|
||||||
|
: isInstall
|
||||||
|
? DependenceStatus.installFailed
|
||||||
|
: DependenceStatus.removeFailed;
|
||||||
this.sockService.sendMessage({
|
this.sockService.sendMessage({
|
||||||
type: socketMessageType,
|
type: socketMessageType,
|
||||||
message,
|
message,
|
||||||
|
|||||||
+25
-7
@@ -14,12 +14,17 @@ import {
|
|||||||
} from '../data/env';
|
} from '../data/env';
|
||||||
import { writeFileWithLock } from '../shared/utils';
|
import { writeFileWithLock } from '../shared/utils';
|
||||||
import { sequelize } from '../data';
|
import { sequelize } from '../data';
|
||||||
|
import {
|
||||||
|
assertAllowedTaskEnvName,
|
||||||
|
isAllowedTaskEnvName,
|
||||||
|
} from '../shared/taskEnvName';
|
||||||
|
|
||||||
@Service()
|
@Service()
|
||||||
export default class EnvService {
|
export default class EnvService {
|
||||||
constructor(@Inject('logger') private logger: winston.Logger) { }
|
constructor(@Inject('logger') private logger: winston.Logger) { }
|
||||||
|
|
||||||
public async create(payloads: Env[]): Promise<Env[]> {
|
public async create(payloads: Env[]): Promise<Env[]> {
|
||||||
|
payloads.forEach((payload) => assertAllowedTaskEnvName(payload.name));
|
||||||
const envs = await this.envs();
|
const envs = await this.envs();
|
||||||
let position = initPosition;
|
let position = initPosition;
|
||||||
if (
|
if (
|
||||||
@@ -52,6 +57,7 @@ export default class EnvService {
|
|||||||
public async update(payload: Env): Promise<Env> {
|
public async update(payload: Env): Promise<Env> {
|
||||||
const doc = await this.getDb({ id: payload.id });
|
const doc = await this.getDb({ id: payload.id });
|
||||||
const tab = new Env({ ...doc, ...payload });
|
const tab = new Env({ ...doc, ...payload });
|
||||||
|
assertAllowedTaskEnvName(tab.name);
|
||||||
const newDoc = await this.updateDb(tab);
|
const newDoc = await this.updateDb(tab);
|
||||||
await this.set_envs();
|
await this.set_envs();
|
||||||
return newDoc;
|
return newDoc;
|
||||||
@@ -151,7 +157,10 @@ export default class EnvService {
|
|||||||
['position', 'DESC'],
|
['position', 'DESC'],
|
||||||
['createdAt', 'ASC'],
|
['createdAt', 'ASC'],
|
||||||
]);
|
]);
|
||||||
return result;
|
return result.map((env) => ({
|
||||||
|
...env,
|
||||||
|
securityBlocked: !isAllowedTaskEnvName(env.name),
|
||||||
|
}));
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
@@ -182,11 +191,14 @@ export default class EnvService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async enabled(ids: number[]) {
|
public async enabled(ids: number[]) {
|
||||||
|
const docs = await EnvModel.findAll({ where: { id: ids } });
|
||||||
|
docs.forEach((doc) => assertAllowedTaskEnvName(doc.name));
|
||||||
await EnvModel.update({ status: EnvStatus.normal }, { where: { id: ids } });
|
await EnvModel.update({ status: EnvStatus.normal }, { where: { id: ids } });
|
||||||
await this.set_envs();
|
await this.set_envs();
|
||||||
}
|
}
|
||||||
|
|
||||||
public async updateNames({ ids, name }: { ids: number[]; name: string }) {
|
public async updateNames({ ids, name }: { ids: number[]; name: string }) {
|
||||||
|
assertAllowedTaskEnvName(name);
|
||||||
await EnvModel.update({ name }, { where: { id: ids } });
|
await EnvModel.update({ name }, { where: { id: ids } });
|
||||||
await this.set_envs();
|
await this.set_envs();
|
||||||
}
|
}
|
||||||
@@ -242,6 +254,13 @@ export default class EnvService {
|
|||||||
name: { [Op.not]: null },
|
name: { [Op.not]: null },
|
||||||
status: EnvStatus.normal,
|
status: EnvStatus.normal,
|
||||||
});
|
});
|
||||||
|
const invalid = envs.filter((env) => !isAllowedTaskEnvName(env.name));
|
||||||
|
if (invalid.length > 0) {
|
||||||
|
this.logger.warn(
|
||||||
|
'Skipped reserved environment variables in task files: %s',
|
||||||
|
JSON.stringify(invalid.map((env) => ({ id: env.id, name: env.name }))),
|
||||||
|
);
|
||||||
|
}
|
||||||
const groups = groupBy(envs, 'name');
|
const groups = groupBy(envs, 'name');
|
||||||
let env_string = '';
|
let env_string = '';
|
||||||
let js_env_string = '';
|
let js_env_string = '';
|
||||||
@@ -250,8 +269,8 @@ export default class EnvService {
|
|||||||
if (Object.prototype.hasOwnProperty.call(groups, key)) {
|
if (Object.prototype.hasOwnProperty.call(groups, key)) {
|
||||||
const group = groups[key];
|
const group = groups[key];
|
||||||
|
|
||||||
// 忽略不符合bash要求的环境变量名称
|
// Ignore invalid and reserved names, including legacy database rows.
|
||||||
if (/^[a-zA-Z_][0-9a-zA-Z_]*$/.test(key)) {
|
if (isAllowedTaskEnvName(key)) {
|
||||||
let value = group
|
let value = group
|
||||||
.map((x) => x.value)
|
.map((x) => x.value)
|
||||||
.join('&')
|
.join('&')
|
||||||
@@ -262,10 +281,9 @@ export default class EnvService {
|
|||||||
.map((x) => x.value)
|
.map((x) => x.value)
|
||||||
.join('&')
|
.join('&')
|
||||||
.replace(/\\/g, '\\\\')}`;
|
.replace(/\\/g, '\\\\')}`;
|
||||||
js_env_string += `process.env.${key}=\`${_env_value.replace(
|
js_env_string += `process.env[${JSON.stringify(key)}]=${JSON.stringify(
|
||||||
/\`/g,
|
group.map((x) => x.value).join('&'),
|
||||||
'\\`',
|
)};\n`;
|
||||||
)}\`;\n`;
|
|
||||||
py_env_string += `os.environ['${key}']='''${_env_value.replace(
|
py_env_string += `os.environ['${key}']='''${_env_value.replace(
|
||||||
/\'/g,
|
/\'/g,
|
||||||
"\\'",
|
"\\'",
|
||||||
|
|||||||
@@ -2,8 +2,8 @@ import { Server, ServerCredentials } from '@grpc/grpc-js';
|
|||||||
import { CronService } from '../protos/cron';
|
import { CronService } from '../protos/cron';
|
||||||
import { HealthService } from '../protos/health';
|
import { HealthService } from '../protos/health';
|
||||||
import { ApiService } from '../protos/api';
|
import { ApiService } from '../protos/api';
|
||||||
import { addCron } from '../schedule/addCron';
|
import { registeredAdd, registeredDelete } from '../schedule/registeredPlans';
|
||||||
import { delCron } from '../schedule/delCron';
|
import { setConcurrency } from '../schedule/setConcurrency';
|
||||||
import { check } from '../schedule/health';
|
import { check } from '../schedule/health';
|
||||||
import * as Api from '../schedule/api';
|
import * as Api from '../schedule/api';
|
||||||
import Logger from '../loaders/logger';
|
import Logger from '../loaders/logger';
|
||||||
@@ -27,7 +27,7 @@ export class GrpcServerService {
|
|||||||
async initialize() {
|
async initialize() {
|
||||||
try {
|
try {
|
||||||
this.server.addService(HealthService, { check });
|
this.server.addService(HealthService, { check });
|
||||||
this.server.addService(CronService, { addCron, delCron });
|
this.server.addService(CronService, { addCron: registeredAdd, delCron: registeredDelete, setConcurrency });
|
||||||
this.server.addService(ApiService, Api);
|
this.server.addService(ApiService, Api);
|
||||||
|
|
||||||
const tlsConfig = await initGrpcCerts();
|
const tlsConfig = await initGrpcCerts();
|
||||||
@@ -49,13 +49,13 @@ export class GrpcServerService {
|
|||||||
for (const host of hostsToTry) {
|
for (const host of hostsToTry) {
|
||||||
try {
|
try {
|
||||||
const address = this.formatGrpcAddress(host, grpcPort);
|
const address = this.formatGrpcAddress(host, grpcPort);
|
||||||
await bindAsync(address, credentials);
|
const boundPort = await bindAsync(address, credentials);
|
||||||
Logger.debug(`[boot] gRPC service started successfully on ${address}`);
|
Logger.debug(`[boot] gRPC service started successfully on ${address}`);
|
||||||
metricsService.record('grpc_service_start', 1, {
|
metricsService.record('grpc_service_start', 1, {
|
||||||
port: grpcPort.toString(),
|
port: grpcPort.toString(),
|
||||||
host
|
host
|
||||||
});
|
});
|
||||||
return grpcPort;
|
return boundPort;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
lastError = err as Error;
|
lastError = err as Error;
|
||||||
Logger.warn(`Failed to bind gRPC on ${host}:${grpcPort}, trying next...`, err);
|
Logger.warn(`Failed to bind gRPC on ${host}:${grpcPort}, trying next...`, err);
|
||||||
|
|||||||
@@ -2,8 +2,10 @@ import { Service } from 'typedi';
|
|||||||
import Logger from '../loaders/logger';
|
import Logger from '../loaders/logger';
|
||||||
import cronClient from '../schedule/client';
|
import cronClient from '../schedule/client';
|
||||||
import { HttpServerService } from './http';
|
import { HttpServerService } from './http';
|
||||||
|
import { runtimeBuild } from '../shared/runtimeBuild';
|
||||||
|
|
||||||
interface HealthStatus {
|
interface HealthStatus {
|
||||||
|
build: typeof runtimeBuild;
|
||||||
status: 'ok' | 'error';
|
status: 'ok' | 'error';
|
||||||
services: {
|
services: {
|
||||||
http: boolean;
|
http: boolean;
|
||||||
@@ -22,12 +24,11 @@ interface HealthStatus {
|
|||||||
export class HealthService {
|
export class HealthService {
|
||||||
private startTime = Date.now();
|
private startTime = Date.now();
|
||||||
|
|
||||||
constructor(
|
constructor(private httpServerService: HttpServerService) {}
|
||||||
private httpServerService: HttpServerService,
|
|
||||||
) {}
|
|
||||||
|
|
||||||
async check(): Promise<HealthStatus> {
|
async check(): Promise<HealthStatus> {
|
||||||
const status: HealthStatus = {
|
const status: HealthStatus = {
|
||||||
|
build: runtimeBuild,
|
||||||
status: 'ok',
|
status: 'ok',
|
||||||
services: {
|
services: {
|
||||||
http: true,
|
http: true,
|
||||||
|
|||||||
@@ -1,5 +1,4 @@
|
|||||||
import { resolveFileAccess } from '../shared/fileAccess';
|
import { resolveFileAccess } from '../shared/fileAccess';
|
||||||
import path from 'path';
|
|
||||||
import { Inject, Service } from 'typedi';
|
import { Inject, Service } from 'typedi';
|
||||||
import winston from 'winston';
|
import winston from 'winston';
|
||||||
import config from '../config';
|
import config from '../config';
|
||||||
@@ -9,10 +8,6 @@ export default class LogService {
|
|||||||
constructor(@Inject('logger') private logger: winston.Logger) {}
|
constructor(@Inject('logger') private logger: winston.Logger) {}
|
||||||
|
|
||||||
public checkFilePath(filePath: string, fileName: string) {
|
public checkFilePath(filePath: string, fileName: string) {
|
||||||
return resolveFileAccess(
|
return resolveFileAccess(config.logPath, [filePath || '', fileName], ['.tmp']);
|
||||||
config.logPath,
|
|
||||||
[filePath || '', fileName],
|
|
||||||
config.blackFileList,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { t } from '../shared/i18n';
|
||||||
import { performance } from 'perf_hooks';
|
import { performance } from 'perf_hooks';
|
||||||
import Logger from '../loaders/logger';
|
import Logger from '../loaders/logger';
|
||||||
|
|
||||||
@@ -126,7 +127,7 @@ export class MetricsService {
|
|||||||
timestamp: Date.now(),
|
timestamp: Date.now(),
|
||||||
metrics: this.getMetrics(),
|
metrics: this.getMetrics(),
|
||||||
};
|
};
|
||||||
Logger.info('性能指标报告:', report);
|
Logger.info(t('性能指标报告:'), report);
|
||||||
return report;
|
return report;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+36
-20
@@ -1,3 +1,4 @@
|
|||||||
|
import { persistNotificationAudit } from '../shared/notificationAudit';
|
||||||
import crypto from 'crypto';
|
import crypto from 'crypto';
|
||||||
import nodemailer from 'nodemailer';
|
import nodemailer from 'nodemailer';
|
||||||
import { Inject, Service } from 'typedi';
|
import { Inject, Service } from 'typedi';
|
||||||
@@ -55,27 +56,33 @@ export default class NotificationService {
|
|||||||
content: string,
|
content: string,
|
||||||
notificationInfo?: NotificationInfo,
|
notificationInfo?: NotificationInfo,
|
||||||
): Promise<boolean | undefined> {
|
): Promise<boolean | undefined> {
|
||||||
let { type, ...rest } = await this.userService.getNotificationMode();
|
let type: string | undefined;
|
||||||
if (notificationInfo?.type) {
|
let success = false;
|
||||||
type = notificationInfo?.type;
|
try {
|
||||||
}
|
const saved = await this.userService.getNotificationMode();
|
||||||
|
type = notificationInfo?.type || saved.type;
|
||||||
if (type) {
|
if (type) {
|
||||||
this.title = title;
|
this.title = title;
|
||||||
this.content = content;
|
this.content = content;
|
||||||
let params = rest;
|
const { type: _, ...overrides } = notificationInfo || {};
|
||||||
if (notificationInfo) {
|
const { type: _saved, ...rest } = saved;
|
||||||
const { type: _, ...others } = notificationInfo;
|
this.params = { ...rest, ...overrides };
|
||||||
params = { ...rest, ...others };
|
|
||||||
}
|
|
||||||
this.params = params;
|
|
||||||
const notificationModeAction = this.modeMap.get(type);
|
|
||||||
try {
|
try {
|
||||||
return await notificationModeAction?.call(this);
|
success = !!(await this.modeMap.get(type)?.call(this));
|
||||||
} catch (error: any) {
|
} catch (error) {
|
||||||
console.error(error);
|
console.error(error);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return false;
|
return success;
|
||||||
|
} finally {
|
||||||
|
await persistNotificationAudit({
|
||||||
|
title,
|
||||||
|
content,
|
||||||
|
notifyType: type,
|
||||||
|
source: 'system',
|
||||||
|
status: success ? 0 : 1,
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public async testNotify(
|
public async testNotify(
|
||||||
@@ -84,14 +91,23 @@ export default class NotificationService {
|
|||||||
content: string,
|
content: string,
|
||||||
) {
|
) {
|
||||||
const { type, ...rest } = info;
|
const { type, ...rest } = info;
|
||||||
if (type) {
|
if (!type) return true;
|
||||||
|
let success = false;
|
||||||
|
try {
|
||||||
this.title = title;
|
this.title = title;
|
||||||
this.content = content;
|
this.content = content;
|
||||||
this.params = rest;
|
this.params = rest;
|
||||||
const notificationModeAction = this.modeMap.get(type);
|
success = !!(await this.modeMap.get(type)?.call(this));
|
||||||
return await notificationModeAction?.call(this);
|
return success;
|
||||||
|
} finally {
|
||||||
|
await persistNotificationAudit({
|
||||||
|
title,
|
||||||
|
content,
|
||||||
|
notifyType: type,
|
||||||
|
source: 'system',
|
||||||
|
status: success ? 0 : 1,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private parseMailRecipients(value?: string) {
|
private parseMailRecipients(value?: string) {
|
||||||
@@ -921,7 +937,8 @@ export default class NotificationService {
|
|||||||
private async openiLink() {
|
private async openiLink() {
|
||||||
const { openiLinkAppToken, openiLinkHubUrl, openiLinkContextToken } =
|
const { openiLinkAppToken, openiLinkHubUrl, openiLinkContextToken } =
|
||||||
this.params;
|
this.params;
|
||||||
const baseUrl = openiLinkHubUrl?.replace(/\/$/, '') || 'https://hub.openilink.com';
|
const baseUrl =
|
||||||
|
openiLinkHubUrl?.replace(/\/$/, '') || 'https://hub.openilink.com';
|
||||||
const url = `${baseUrl}/bot/v1/message/send`;
|
const url = `${baseUrl}/bot/v1/message/send`;
|
||||||
const body: Record<string, string> = {
|
const body: Record<string, string> = {
|
||||||
type: 'text',
|
type: 'text',
|
||||||
@@ -974,5 +991,4 @@ export default class NotificationService {
|
|||||||
throw new Error(error.response ? error.response.body : error);
|
throw new Error(error.response ? error.response.body : error);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
+109
-17
@@ -6,17 +6,66 @@ import { v4 as uuidV4 } from 'uuid';
|
|||||||
import sequelize, { Op } from 'sequelize';
|
import sequelize, { Op } from 'sequelize';
|
||||||
import { shareStore } from '../shared/store';
|
import { shareStore } from '../shared/store';
|
||||||
import { t } from '../shared/i18n';
|
import { t } from '../shared/i18n';
|
||||||
|
import { lock } from 'proper-lockfile';
|
||||||
|
import path from 'path';
|
||||||
|
import os from 'os';
|
||||||
|
import { writeFile, rename, rm, stat, chmod } from 'fs/promises';
|
||||||
|
import config from '../config';
|
||||||
|
|
||||||
@Service()
|
@Service()
|
||||||
export default class OpenService {
|
export default class OpenService {
|
||||||
constructor(@Inject('logger') private logger: winston.Logger) {}
|
constructor(@Inject('logger') private logger: winston.Logger) {}
|
||||||
|
|
||||||
|
// Every database mutation and full authentication-cache publication shares
|
||||||
|
// this cross-process lock. Lock a separate marker, not token.json itself.
|
||||||
|
private async withAppsLock<T>(operation: () => Promise<T>): Promise<T> {
|
||||||
|
const target = path.join(config.configPath, '.apps-state');
|
||||||
|
const release = await lock(target, {
|
||||||
|
realpath: false,
|
||||||
|
stale: 10_000,
|
||||||
|
retries: { retries: 10, factor: 2, minTimeout: 100, maxTimeout: 3000 },
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
return await operation();
|
||||||
|
} finally {
|
||||||
|
await release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async initializeSystemApp(): Promise<void> {
|
||||||
|
await this.withAppsLock(async () => {
|
||||||
|
const doc = await AppModel.findOne({ where: { name: 'system' } });
|
||||||
|
const app = doc?.get({ plain: true });
|
||||||
|
if (!app) {
|
||||||
|
await this.insert({
|
||||||
|
name: 'system',
|
||||||
|
scopes: ['crons', 'system', 'dashboard'],
|
||||||
|
client_id: createRandomString(12, 12),
|
||||||
|
client_secret: createRandomString(24, 24),
|
||||||
|
});
|
||||||
|
} else if (!app.scopes.includes('dashboard')) {
|
||||||
|
await AppModel.update(
|
||||||
|
{ scopes: [...app.scopes, 'dashboard'] },
|
||||||
|
{ where: { id: app.id } },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await shareStore.updateApps(await this.find({}));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public async refreshApps(): Promise<void> {
|
||||||
|
await this.withAppsLock(async () => {
|
||||||
|
await shareStore.updateApps(await this.find({}));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
public async findApps(): Promise<App[] | null> {
|
public async findApps(): Promise<App[] | null> {
|
||||||
const docs = await this.find({});
|
const docs = await this.find({});
|
||||||
return docs;
|
return docs;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async create(payload: App): Promise<App> {
|
public async create(payload: App): Promise<App> {
|
||||||
|
return this.withAppsLock(async () => {
|
||||||
const tab = { ...payload };
|
const tab = { ...payload };
|
||||||
tab.client_id = createRandomString(12, 12);
|
tab.client_id = createRandomString(12, 12);
|
||||||
tab.client_secret = createRandomString(24, 24);
|
tab.client_secret = createRandomString(24, 24);
|
||||||
@@ -24,9 +73,10 @@ export default class OpenService {
|
|||||||
const apps = await this.find({});
|
const apps = await this.find({});
|
||||||
await shareStore.updateApps(apps);
|
await shareStore.updateApps(apps);
|
||||||
return { ...doc, tokens: [] };
|
return { ...doc, tokens: [] };
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
public async insert(payload: App): Promise<App> {
|
private async insert(payload: App): Promise<App> {
|
||||||
const doc = await AppModel.create(payload, { returning: true });
|
const doc = await AppModel.create(payload, { returning: true });
|
||||||
return doc.get({ plain: true });
|
return doc.get({ plain: true });
|
||||||
}
|
}
|
||||||
@@ -41,10 +91,16 @@ export default class OpenService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async updateDb(payload: Partial<App>): Promise<App> {
|
private async updateDb(payload: Partial<App>): Promise<App> {
|
||||||
|
return this.withAppsLock(async () => {
|
||||||
|
const target = await AppModel.findOne({ where: { id: payload.id } });
|
||||||
|
if (target?.name === 'system' || payload.name === 'system') {
|
||||||
|
throw Object.assign(new Error(t('暂无权限')), { status: 403 });
|
||||||
|
}
|
||||||
await AppModel.update(payload, { where: { id: payload.id } });
|
await AppModel.update(payload, { where: { id: payload.id } });
|
||||||
const apps = await this.find({});
|
const apps = await this.find({});
|
||||||
await shareStore.updateApps(apps);
|
await shareStore.updateApps(apps);
|
||||||
return apps?.find((x) => x.id === payload.id) as App;
|
return apps?.find((x) => x.id === payload.id) as App;
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
public async getDb(query: Record<string, any>): Promise<App> {
|
public async getDb(query: Record<string, any>): Promise<App> {
|
||||||
@@ -56,9 +112,17 @@ export default class OpenService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async remove(ids: number[]) {
|
public async remove(ids: number[]) {
|
||||||
|
return this.withAppsLock(async () => {
|
||||||
|
const systemApp = await AppModel.findOne({
|
||||||
|
where: { id: ids, name: 'system' },
|
||||||
|
});
|
||||||
|
if (systemApp) {
|
||||||
|
throw Object.assign(new Error(t('暂无权限')), { status: 403 });
|
||||||
|
}
|
||||||
await AppModel.destroy({ where: { id: ids } });
|
await AppModel.destroy({ where: { id: ids } });
|
||||||
const apps = await this.find({});
|
const apps = await this.find({});
|
||||||
await shareStore.updateApps(apps);
|
await shareStore.updateApps(apps);
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
public async resetSecret(id: number): Promise<App> {
|
public async resetSecret(id: number): Promise<App> {
|
||||||
@@ -112,7 +176,15 @@ export default class OpenService {
|
|||||||
return docs.map((x) => x.get({ plain: true }));
|
return docs.map((x) => x.get({ plain: true }));
|
||||||
}
|
}
|
||||||
|
|
||||||
public async authToken({
|
public async authToken(credentials: {
|
||||||
|
client_id: string;
|
||||||
|
client_secret: string;
|
||||||
|
}): Promise<any> {
|
||||||
|
return this.withAppsLock(() => this.issueToken(credentials));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Caller holds withAppsLock through database update and cache publication.
|
||||||
|
private async issueToken({
|
||||||
client_id,
|
client_id,
|
||||||
client_secret,
|
client_secret,
|
||||||
}: {
|
}: {
|
||||||
@@ -156,27 +228,47 @@ export default class OpenService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public async generateSystemToken(): Promise<{
|
public async generateSystemToken(renew = false): Promise<{
|
||||||
value: string;
|
value: string;
|
||||||
expiration: number;
|
expiration: number;
|
||||||
}> {
|
}> {
|
||||||
const apps = await shareStore.getApps();
|
return this.withAppsLock(async () => {
|
||||||
const systemApp = apps?.find((x) => x.name === 'system');
|
// Read after acquiring the lock; a concurrent issuer may have renewed.
|
||||||
if (!systemApp) {
|
const systemApp = await this.getDb({ name: 'system' });
|
||||||
throw new Error('system app not found');
|
|
||||||
}
|
|
||||||
const now = Math.round(Date.now() / 1000);
|
const now = Math.round(Date.now() / 1000);
|
||||||
const currentToken = systemApp.tokens?.find((x) => x.expiration > now);
|
let appToken = systemApp.tokens
|
||||||
if (currentToken) {
|
?.filter((token) => token.expiration > now)
|
||||||
return currentToken;
|
.sort((left, right) => right.expiration - left.expiration)[0];
|
||||||
}
|
if (renew || !appToken) {
|
||||||
const { data } = await this.authToken({
|
const { data } = await this.issueToken({
|
||||||
client_id: systemApp.client_id,
|
client_id: systemApp.client_id,
|
||||||
client_secret: systemApp.client_secret,
|
client_secret: systemApp.client_secret,
|
||||||
});
|
});
|
||||||
return {
|
appToken = { value: data.token, expiration: data.expiration };
|
||||||
...data,
|
} else {
|
||||||
value: data.token,
|
// Recover a prior database commit whose cache publication failed.
|
||||||
};
|
await shareStore.updateApps(await this.find({}));
|
||||||
|
}
|
||||||
|
const tokenFile = path.join(config.configPath, 'token.json');
|
||||||
|
const temporaryFile = `${tokenFile}.${process.pid}.tmp`;
|
||||||
|
const mode = await stat(tokenFile).then(
|
||||||
|
(file) => file.mode & 0o777,
|
||||||
|
(error: NodeJS.ErrnoException) => {
|
||||||
|
if (error.code === 'ENOENT') return 0o600;
|
||||||
|
throw error;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
await writeFile(temporaryFile, `${JSON.stringify(appToken)}${os.EOL}`, {
|
||||||
|
mode,
|
||||||
|
});
|
||||||
|
await chmod(temporaryFile, mode);
|
||||||
|
// Shell readers see a complete old or new document, never a partial one.
|
||||||
|
await rename(temporaryFile, tokenFile);
|
||||||
|
} finally {
|
||||||
|
await rm(temporaryFile, { force: true });
|
||||||
|
}
|
||||||
|
return appToken;
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+53
-22
@@ -1,11 +1,13 @@
|
|||||||
import fs from 'fs/promises';
|
import fs from 'fs/promises';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
import dayjs from 'dayjs';
|
import dayjs from 'dayjs';
|
||||||
import { Op } from 'sequelize';
|
import { literal, Op } from 'sequelize';
|
||||||
import { Service } from 'typedi';
|
import { Service } from 'typedi';
|
||||||
import config from '../config';
|
import config from '../config';
|
||||||
import { CrontabStatModel } from '../data/cronStats';
|
import { CrontabStatModel } from '../data/cronStats';
|
||||||
import { sequelize } from '../data';
|
import { sequelize } from '../data';
|
||||||
|
import { TriggerHistory } from '../shared/triggerHistory';
|
||||||
|
import { prunedExitMigrationId } from '../shared/schemaMigrations';
|
||||||
import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
|
import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
|
||||||
import { AuthDataType, SystemModel } from '../data/system';
|
import { AuthDataType, SystemModel } from '../data/system';
|
||||||
import {
|
import {
|
||||||
@@ -21,10 +23,12 @@ export interface StorageCleanupRequest extends RetentionPolicy {
|
|||||||
compactDatabase?: boolean;
|
compactDatabase?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
function runningInstanceWhere(days: number) {
|
function runningInstanceWhere(cutoffMs: number) {
|
||||||
const cutoff = dayjs().subtract(days, 'day').unix();
|
const cutoff = cutoffMs / 1000;
|
||||||
return {
|
return {
|
||||||
status: { [Op.ne]: InstanceStatus.running },
|
status: { [Op.ne]: InstanceStatus.running },
|
||||||
|
id: { [Op.notIn]: literal(`(SELECT instance_id FROM CronTriggers WHERE instance_id IS NOT NULL
|
||||||
|
AND (exit_pending=1 OR state IN ('queued','running') OR finished_at IS NULL OR observed_at >= ${cutoffMs} OR finished_at >= ${cutoffMs}))`) },
|
||||||
[Op.or]: [
|
[Op.or]: [
|
||||||
{ finished_at: { [Op.lt]: cutoff } },
|
{ finished_at: { [Op.lt]: cutoff } },
|
||||||
{
|
{
|
||||||
@@ -35,9 +39,11 @@ function runningInstanceWhere(days: number) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function cronStatWhere(days: number) {
|
function retentionCutoffs(policy: RetentionPolicy) {
|
||||||
|
const now = dayjs();
|
||||||
return {
|
return {
|
||||||
date: { [Op.lt]: dayjs().subtract(days, 'day').format('YYYY-MM-DD') },
|
history: now.subtract(policy.runningInstanceRetentionDays, 'day').startOf('second').valueOf(),
|
||||||
|
firstDay: now.subtract(policy.cronStatRetentionDays, 'day').format('YYYY-MM-DD'),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -63,20 +69,20 @@ export default class RetentionService {
|
|||||||
return normalized;
|
return normalized;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async preview(request: StorageCleanupRequest) {
|
public async preview(request: StorageCleanupRequest, cutoffs = retentionCutoffs(normalizeRetentionPolicy(request))) {
|
||||||
const policy = normalizeRetentionPolicy(request);
|
const policy = normalizeRetentionPolicy(request);
|
||||||
const dependenceCacheTypes = (request.dependenceCacheTypes || []).filter(
|
const dependenceCacheTypes = (request.dependenceCacheTypes || []).filter(
|
||||||
isDependenceCacheType,
|
isDependenceCacheType,
|
||||||
);
|
);
|
||||||
const [runningInstances, cronStats, dependenceCaches] = await Promise.all([
|
const [runningInstances, cronStats, dependenceCaches, triggers] = await Promise.all([
|
||||||
policy.runningInstanceRetentionDays > 0
|
policy.runningInstanceRetentionDays > 0
|
||||||
? RunningInstanceModel.count({
|
? RunningInstanceModel.count({
|
||||||
where: runningInstanceWhere(policy.runningInstanceRetentionDays),
|
where: runningInstanceWhere(cutoffs.history),
|
||||||
})
|
})
|
||||||
: 0,
|
: 0,
|
||||||
policy.cronStatRetentionDays > 0
|
policy.cronStatRetentionDays > 0
|
||||||
? CrontabStatModel.count({
|
? CrontabStatModel.count({
|
||||||
where: cronStatWhere(policy.cronStatRetentionDays),
|
where: { date: { [Op.lt]: cutoffs.firstDay } },
|
||||||
})
|
})
|
||||||
: 0,
|
: 0,
|
||||||
Promise.all(
|
Promise.all(
|
||||||
@@ -87,11 +93,15 @@ export default class RetentionService {
|
|||||||
),
|
),
|
||||||
})),
|
})),
|
||||||
),
|
),
|
||||||
|
policy.runningInstanceRetentionDays > 0
|
||||||
|
? new TriggerHistory(sequelize).countExpired(cutoffs.history)
|
||||||
|
: 0,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
policy,
|
policy,
|
||||||
runningInstances,
|
runningInstances,
|
||||||
|
triggers,
|
||||||
cronStats,
|
cronStats,
|
||||||
dependenceCaches,
|
dependenceCaches,
|
||||||
dependenceCacheBytes: dependenceCaches.reduce(
|
dependenceCacheBytes: dependenceCaches.reduce(
|
||||||
@@ -103,25 +113,46 @@ export default class RetentionService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async cleanup(request: StorageCleanupRequest) {
|
public async cleanup(request: StorageCleanupRequest) {
|
||||||
const preview = await this.preview(request);
|
const cutoffs = retentionCutoffs(normalizeRetentionPolicy(request));
|
||||||
|
const preview = await this.preview(request, cutoffs);
|
||||||
const deleted = await sequelize.transaction(async (transaction) => {
|
const deleted = await sequelize.transaction(async (transaction) => {
|
||||||
|
const history = new TriggerHistory(sequelize);
|
||||||
|
if (preview.policy.runningInstanceRetentionDays || preview.policy.cronStatRetentionDays) {
|
||||||
|
await history.replayPublishedExits(transaction);
|
||||||
|
}
|
||||||
const runningInstances =
|
const runningInstances =
|
||||||
preview.policy.runningInstanceRetentionDays > 0
|
preview.policy.runningInstanceRetentionDays > 0
|
||||||
? await RunningInstanceModel.destroy({
|
? await RunningInstanceModel.destroy({
|
||||||
where: runningInstanceWhere(
|
where: runningInstanceWhere(cutoffs.history),
|
||||||
preview.policy.runningInstanceRetentionDays,
|
|
||||||
),
|
|
||||||
transaction,
|
transaction,
|
||||||
})
|
})
|
||||||
: 0;
|
: 0;
|
||||||
const cronStats =
|
let cronStats = 0;
|
||||||
preview.policy.cronStatRetentionDays > 0
|
if (preview.policy.cronStatRetentionDays > 0) {
|
||||||
? await CrontabStatModel.destroy({
|
const firstDay = cutoffs.firstDay;
|
||||||
where: cronStatWhere(preview.policy.cronStatRetentionDays),
|
cronStats = await CrontabStatModel.destroy({ where: { date: { [Op.lt]: firstDay } }, transaction });
|
||||||
transaction,
|
await sequelize.query(`INSERT INTO CronStatRetention(id,first_day) VALUES (1,$firstDay)
|
||||||
})
|
ON CONFLICT(id) DO UPDATE SET first_day=MAX(first_day,excluded.first_day)`,
|
||||||
|
{ bind: { firstDay }, transaction });
|
||||||
|
await sequelize.query(`DELETE FROM CronPrunedUnknownRuns WHERE last_possible_day <
|
||||||
|
(SELECT first_day FROM CronStatRetention WHERE id=1)`, { transaction });
|
||||||
|
}
|
||||||
|
const triggers = preview.policy.runningInstanceRetentionDays > 0
|
||||||
|
? await history.deleteExpired(cutoffs.history, transaction)
|
||||||
: 0;
|
: 0;
|
||||||
return { runningInstances, cronStats };
|
if (preview.policy.runningInstanceRetentionDays || preview.policy.cronStatRetentionDays) {
|
||||||
|
// An unbound receipt's day can be wrong. A later bound receipt still
|
||||||
|
// needs the execution-to-task link even after both visible histories expire.
|
||||||
|
// A writer can also win the lock before an interrupted upgrade retries.
|
||||||
|
// Keep its bound bookings until the proof-copy transaction has committed.
|
||||||
|
await sequelize.query(`DELETE FROM CronLegacyExitBookings WHERE day <
|
||||||
|
(SELECT first_day FROM CronStatRetention WHERE id=1)
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM CronTriggers WHERE id=CronLegacyExitBookings.trigger_id)
|
||||||
|
AND ((bound=1 AND EXISTS (SELECT 1 FROM SchemaMigrations WHERE id=$proofMigration))
|
||||||
|
OR (bound=0 AND NOT EXISTS (SELECT 1 FROM Crontabs WHERE id=CronLegacyExitBookings.cron_id)))`,
|
||||||
|
{ bind: { proofMigration: prunedExitMigrationId }, transaction });
|
||||||
|
}
|
||||||
|
return { runningInstances, cronStats, triggers };
|
||||||
});
|
});
|
||||||
|
|
||||||
const dependenceCaches = [];
|
const dependenceCaches = [];
|
||||||
@@ -135,7 +166,7 @@ export default class RetentionService {
|
|||||||
|
|
||||||
if (
|
if (
|
||||||
request.compactDatabase &&
|
request.compactDatabase &&
|
||||||
(deleted.runningInstances || deleted.cronStats)
|
(deleted.runningInstances || deleted.cronStats || deleted.triggers)
|
||||||
) {
|
) {
|
||||||
await sequelize.query('VACUUM');
|
await sequelize.query('VACUUM');
|
||||||
}
|
}
|
||||||
@@ -148,7 +179,7 @@ export default class RetentionService {
|
|||||||
},
|
},
|
||||||
compactedDatabase: Boolean(
|
compactedDatabase: Boolean(
|
||||||
request.compactDatabase &&
|
request.compactDatabase &&
|
||||||
(deleted.runningInstances || deleted.cronStats),
|
(deleted.runningInstances || deleted.cronStats || deleted.triggers),
|
||||||
),
|
),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
+38
-18
@@ -1,6 +1,7 @@
|
|||||||
|
import { t } from '../shared/i18n';
|
||||||
import { Service, Inject } from 'typedi';
|
import { Service, Inject } from 'typedi';
|
||||||
import winston from 'winston';
|
import winston from 'winston';
|
||||||
import nodeSchedule from 'node-schedule';
|
import { createCronJob, CronJob } from '../shared/cronScheduler';
|
||||||
import { ChildProcessWithoutNullStreams } from 'child_process';
|
import { ChildProcessWithoutNullStreams } from 'child_process';
|
||||||
import {
|
import {
|
||||||
ToadScheduler,
|
ToadScheduler,
|
||||||
@@ -11,7 +12,11 @@ import {
|
|||||||
import dayjs from 'dayjs';
|
import dayjs from 'dayjs';
|
||||||
import taskLimit from '../shared/pLimit';
|
import taskLimit from '../shared/pLimit';
|
||||||
import { spawn } from 'cross-spawn';
|
import { spawn } from 'cross-spawn';
|
||||||
import { observeChildProcess, asError, ProcessResult } from '../shared/childProcess';
|
import {
|
||||||
|
observeChildProcess,
|
||||||
|
asError,
|
||||||
|
ProcessResult,
|
||||||
|
} from '../shared/childProcess';
|
||||||
|
|
||||||
export interface ScheduleTaskType {
|
export interface ScheduleTaskType {
|
||||||
id?: number;
|
id?: number;
|
||||||
@@ -38,7 +43,7 @@ export interface TaskCallbacks {
|
|||||||
|
|
||||||
@Service()
|
@Service()
|
||||||
export default class ScheduleService {
|
export default class ScheduleService {
|
||||||
private scheduleStacks = new Map<string, nodeSchedule.Job>();
|
private scheduleStacks = new Map<string, CronJob>();
|
||||||
|
|
||||||
private intervalSchedule = new ToadScheduler();
|
private intervalSchedule = new ToadScheduler();
|
||||||
|
|
||||||
@@ -79,7 +84,7 @@ export default class ScheduleService {
|
|||||||
let cp: ChildProcessWithoutNullStreams | undefined;
|
let cp: ChildProcessWithoutNullStreams | undefined;
|
||||||
let result: ProcessResult = { code: null, signal: null };
|
let result: ProcessResult = { code: null, signal: null };
|
||||||
try {
|
try {
|
||||||
this.logger.info('[panel][开始执行任务] 任务ID: %s', others.id);
|
this.logger.info(t('[panel][开始执行任务] 任务ID: %s'), others.id);
|
||||||
await callbacks.onBefore?.(startTime);
|
await callbacks.onBefore?.(startTime);
|
||||||
cp = spawn(command, { shell: '/bin/bash' });
|
cp = spawn(command, { shell: '/bin/bash' });
|
||||||
const child = cp;
|
const child = cp;
|
||||||
@@ -99,7 +104,7 @@ export default class ScheduleService {
|
|||||||
|
|
||||||
if (result.error) {
|
if (result.error) {
|
||||||
this.logger.error(
|
this.logger.error(
|
||||||
'[panel][执行任务失败] 任务ID: %s, 错误: %s',
|
t('[panel][执行任务失败] 任务ID: %s, 错误: %s'),
|
||||||
others.id,
|
others.id,
|
||||||
result.error.message,
|
result.error.message,
|
||||||
);
|
);
|
||||||
@@ -107,7 +112,7 @@ export default class ScheduleService {
|
|||||||
await callbacks.onError?.(result.error.message);
|
await callbacks.onError?.(result.error.message);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.logger.error(
|
this.logger.error(
|
||||||
'[panel][任务错误回调失败] %s',
|
t('[panel][任务错误回调失败] %s'),
|
||||||
asError(error).message,
|
asError(error).message,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -123,12 +128,12 @@ export default class ScheduleService {
|
|||||||
} catch (error) {
|
} catch (error) {
|
||||||
result.error ??= asError(error);
|
result.error ??= asError(error);
|
||||||
this.logger.error(
|
this.logger.error(
|
||||||
'[panel][任务结束回调失败] %s',
|
t('[panel][任务结束回调失败] %s'),
|
||||||
asError(error).message,
|
asError(error).message,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
this.logger.info(
|
this.logger.info(
|
||||||
'[panel][执行任务结束] 任务ID: %s, 退出码: %j',
|
t('[panel][执行任务结束] 任务ID: %s, 退出码: %j'),
|
||||||
others.id,
|
others.id,
|
||||||
result.code,
|
result.code,
|
||||||
);
|
);
|
||||||
@@ -137,7 +142,7 @@ export default class ScheduleService {
|
|||||||
).catch((error) => {
|
).catch((error) => {
|
||||||
// Queue/setup failures must not become unhandled rejections in detached callers.
|
// Queue/setup failures must not become unhandled rejections in detached callers.
|
||||||
rejectStart(asError(error));
|
rejectStart(asError(error));
|
||||||
this.logger.error('[panel][任务队列失败] %s', asError(error).message);
|
this.logger.error(t('[panel][任务队列失败] %s'), asError(error).message);
|
||||||
return { ...others, code: null, signal: null, error: asError(error) };
|
return { ...others, code: null, signal: null, error: asError(error) };
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -152,25 +157,40 @@ export default class ScheduleService {
|
|||||||
) {
|
) {
|
||||||
const _id = this.formatId(id);
|
const _id = this.formatId(id);
|
||||||
this.logger.info(
|
this.logger.info(
|
||||||
'[panel][创建cron任务] 任务ID: %s, cron: %s, 任务名: %s, 执行命令: %s',
|
t('[panel][创建cron任务] 任务ID: %s, cron: %s, 任务名: %s, 执行命令: %s'),
|
||||||
_id,
|
_id,
|
||||||
schedule,
|
schedule,
|
||||||
name,
|
name,
|
||||||
command,
|
command,
|
||||||
);
|
);
|
||||||
|
|
||||||
this.scheduleStacks.set(
|
if (schedule) {
|
||||||
_id,
|
try {
|
||||||
nodeSchedule.scheduleJob(_id, schedule, async () => {
|
const job = createCronJob(
|
||||||
|
schedule,
|
||||||
|
() =>
|
||||||
this.runTask(command, callbacks, {
|
this.runTask(command, callbacks, {
|
||||||
name,
|
name,
|
||||||
schedule,
|
schedule,
|
||||||
command,
|
command,
|
||||||
id: _id,
|
id: _id,
|
||||||
runOrigin,
|
runOrigin,
|
||||||
});
|
|
||||||
}),
|
}),
|
||||||
|
{ name: `${_id}: ${name || ''}`, logger: this.logger, start: false },
|
||||||
);
|
);
|
||||||
|
this.scheduleStacks.get(_id)?.cancel();
|
||||||
|
this.scheduleStacks.set(_id, job);
|
||||||
|
job.start();
|
||||||
|
} catch (error) {
|
||||||
|
// A persisted invalid subscription must not become an unhandled startup rejection.
|
||||||
|
this.logger.warn(
|
||||||
|
t('[panel][跳过无效定时任务] 任务ID: %s, cron: %s, 错误: %s'),
|
||||||
|
_id,
|
||||||
|
schedule,
|
||||||
|
asError(error).message,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (runImmediately) {
|
if (runImmediately) {
|
||||||
this.runTask(command, callbacks, {
|
this.runTask(command, callbacks, {
|
||||||
@@ -185,7 +205,7 @@ export default class ScheduleService {
|
|||||||
|
|
||||||
async cancelCronTask({ id = 0, name }: ScheduleTaskType) {
|
async cancelCronTask({ id = 0, name }: ScheduleTaskType) {
|
||||||
const _id = this.formatId(id);
|
const _id = this.formatId(id);
|
||||||
this.logger.info('[panel][取消定时任务] 任务名: %s', name);
|
this.logger.info(t('[panel][取消定时任务] 任务名: %s'), name);
|
||||||
if (this.scheduleStacks.has(_id)) {
|
if (this.scheduleStacks.has(_id)) {
|
||||||
this.scheduleStacks.get(_id)?.cancel();
|
this.scheduleStacks.get(_id)?.cancel();
|
||||||
this.scheduleStacks.delete(_id);
|
this.scheduleStacks.delete(_id);
|
||||||
@@ -200,7 +220,7 @@ export default class ScheduleService {
|
|||||||
) {
|
) {
|
||||||
const _id = this.formatId(id);
|
const _id = this.formatId(id);
|
||||||
this.logger.info(
|
this.logger.info(
|
||||||
'[panel][创建interval任务] 任务ID: %s, 任务名: %s, 执行命令: %s',
|
t('[panel][创建interval任务] 任务ID: %s, 任务名: %s, 执行命令: %s'),
|
||||||
_id,
|
_id,
|
||||||
name,
|
name,
|
||||||
command,
|
command,
|
||||||
@@ -217,7 +237,7 @@ export default class ScheduleService {
|
|||||||
},
|
},
|
||||||
(err) => {
|
(err) => {
|
||||||
this.logger.error(
|
this.logger.error(
|
||||||
'[panel][执行任务失败] 命令: %s, 错误信息: %j',
|
t('[panel][执行任务失败] 命令: %s, 错误信息: %j'),
|
||||||
command,
|
command,
|
||||||
err,
|
err,
|
||||||
);
|
);
|
||||||
@@ -245,7 +265,7 @@ export default class ScheduleService {
|
|||||||
async cancelIntervalTask({ id = 0, name }: ScheduleTaskType) {
|
async cancelIntervalTask({ id = 0, name }: ScheduleTaskType) {
|
||||||
const _id = this.formatId(id);
|
const _id = this.formatId(id);
|
||||||
this.logger.info(
|
this.logger.info(
|
||||||
'[panel][取消interval任务] 任务ID: %s, 任务名: %s',
|
t('[panel][取消interval任务] 任务ID: %s, 任务名: %s'),
|
||||||
_id,
|
_id,
|
||||||
name,
|
name,
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -66,11 +66,7 @@ export default class ScriptService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public checkFilePath(filePath: string, fileName: string) {
|
public checkFilePath(filePath: string, fileName: string) {
|
||||||
return resolveFileAccess(
|
return resolveFileAccess(config.scriptPath, [filePath || '', fileName]);
|
||||||
config.scriptPath,
|
|
||||||
[filePath || '', fileName],
|
|
||||||
config.blackFileList,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public async getFile(filePath: string, fileName: string) {
|
public async getFile(filePath: string, fileName: string) {
|
||||||
|
|||||||
@@ -0,0 +1,12 @@
|
|||||||
|
import path from 'path';
|
||||||
|
import { Service } from 'typedi';
|
||||||
|
import config from '../config';
|
||||||
|
import { ScriptHistory } from '../shared/scriptHistory';
|
||||||
|
|
||||||
|
/** Panel configuration and lifecycle; the storage core stays independently testable. */
|
||||||
|
@Service()
|
||||||
|
export default class ScriptHistoryService extends ScriptHistory {
|
||||||
|
constructor() {
|
||||||
|
super(config.scriptPath, path.join(config.dataPath, 'script-history'));
|
||||||
|
}
|
||||||
|
}
|
||||||
+45
-14
@@ -1,3 +1,4 @@
|
|||||||
|
import { t } from '../shared/i18n';
|
||||||
import { Service, Inject } from 'typedi';
|
import { Service, Inject } from 'typedi';
|
||||||
import winston from 'winston';
|
import winston from 'winston';
|
||||||
import fs from 'fs/promises';
|
import fs from 'fs/promises';
|
||||||
@@ -8,10 +9,22 @@ import { formatUrl } from '../config/subscription';
|
|||||||
import config from '../config';
|
import config from '../config';
|
||||||
import { fileExist, rmPath } from '../config/util';
|
import { fileExist, rmPath } from '../config/util';
|
||||||
import { writeFileWithLock } from '../shared/utils';
|
import { writeFileWithLock } from '../shared/utils';
|
||||||
|
import {
|
||||||
|
getSubscriptionSshAlias,
|
||||||
|
resolveSubscriptionPath,
|
||||||
|
} from '../shared/subscriptionPath';
|
||||||
|
|
||||||
@Service()
|
@Service()
|
||||||
export default class SshKeyService {
|
export default class SshKeyService {
|
||||||
private homedir = os.homedir();
|
// OpenSSH uses the effective user's passwd entry, not the HOME override.
|
||||||
|
private get homedir(): string {
|
||||||
|
try {
|
||||||
|
return os.userInfo().homedir;
|
||||||
|
} catch {
|
||||||
|
// Custom container UIDs may have no passwd entry. Keep startup working.
|
||||||
|
return os.homedir();
|
||||||
|
}
|
||||||
|
}
|
||||||
private sshPath = config.sshdPath;
|
private sshPath = config.sshdPath;
|
||||||
private sshConfigFilePath = path.resolve(this.homedir, '.ssh', 'config');
|
private sshConfigFilePath = path.resolve(this.homedir, '.ssh', 'config');
|
||||||
private sshConfigHeader = `Include ${path.join(this.sshPath, '*.config')}`;
|
private sshConfigHeader = `Include ${path.join(this.sshPath, '*.config')}`;
|
||||||
@@ -21,6 +34,10 @@ export default class SshKeyService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async initSshConfigFile() {
|
private async initSshConfigFile() {
|
||||||
|
await fs.mkdir(path.dirname(this.sshConfigFilePath), {
|
||||||
|
recursive: true,
|
||||||
|
mode: 0o700,
|
||||||
|
});
|
||||||
let config = '';
|
let config = '';
|
||||||
const _exist = await fileExist(this.sshConfigFilePath);
|
const _exist = await fileExist(this.sshConfigFilePath);
|
||||||
if (_exist) {
|
if (_exist) {
|
||||||
@@ -42,22 +59,22 @@ export default class SshKeyService {
|
|||||||
key: string,
|
key: string,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
try {
|
try {
|
||||||
const filePath = path.join(this.sshPath, alias);
|
const filePath = resolveSubscriptionPath(this.sshPath, alias);
|
||||||
try {
|
try {
|
||||||
await rmPath(filePath);
|
await rmPath(filePath);
|
||||||
} catch { }
|
} catch {}
|
||||||
await writeFileWithLock(filePath, `${key}${os.EOL}`, { mode: '400' });
|
await writeFileWithLock(filePath, `${key}${os.EOL}`, { mode: '400' });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.logger.error('生成私钥文件失败', error);
|
this.logger.error(t('生成私钥文件失败'), error);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async removePrivateKeyFile(alias: string): Promise<void> {
|
private async removePrivateKeyFile(alias: string): Promise<void> {
|
||||||
try {
|
try {
|
||||||
const filePath = path.join(this.sshPath, alias);
|
const filePath = resolveSubscriptionPath(this.sshPath, alias);
|
||||||
await rmPath(filePath);
|
await rmPath(filePath);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.logger.error('删除私钥文件失败', error);
|
this.logger.error(t('删除私钥文件失败'), error);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -66,18 +83,26 @@ export default class SshKeyService {
|
|||||||
host: string,
|
host: string,
|
||||||
proxy?: string,
|
proxy?: string,
|
||||||
) {
|
) {
|
||||||
|
if (
|
||||||
|
!/^[a-zA-Z0-9.-]+$/.test(host) ||
|
||||||
|
(proxy && !/^[a-zA-Z0-9.:[\]-]+$/.test(proxy))
|
||||||
|
) {
|
||||||
|
throw Object.assign(new Error('Invalid SSH host or proxy'), {
|
||||||
|
status: 400,
|
||||||
|
});
|
||||||
|
}
|
||||||
if (host === 'github.com') {
|
if (host === 'github.com') {
|
||||||
host = `ssh.github.com\n Port 443\n HostkeyAlgorithms +ssh-rsa`;
|
host = `ssh.github.com\n Port 443\n HostkeyAlgorithms +ssh-rsa`;
|
||||||
}
|
}
|
||||||
const proxyStr = proxy
|
const proxyStr = proxy
|
||||||
? ` ProxyCommand nc -v -x ${proxy} %h %p 2>/dev/null\n`
|
? ` ProxyCommand nc -v -x ${proxy} %h %p 2>/dev/null\n`
|
||||||
: '';
|
: '';
|
||||||
const config = `Host ${alias}\n Hostname ${host}\n IdentityFile ${path.join(
|
const config = `Host ${alias}\n Hostname ${host}\n IdentityFile ${resolveSubscriptionPath(
|
||||||
this.sshPath,
|
this.sshPath,
|
||||||
alias,
|
alias,
|
||||||
)}\n StrictHostKeyChecking no\n${proxyStr}`;
|
)}\n StrictHostKeyChecking no\n${proxyStr}`;
|
||||||
await writeFileWithLock(
|
await writeFileWithLock(
|
||||||
`${path.join(this.sshPath, `${alias}.config`)}`,
|
`${resolveSubscriptionPath(this.sshPath, alias, '.config')}`,
|
||||||
config,
|
config,
|
||||||
{
|
{
|
||||||
encoding: 'utf8',
|
encoding: 'utf8',
|
||||||
@@ -88,10 +113,10 @@ export default class SshKeyService {
|
|||||||
|
|
||||||
private async removeSshConfig(alias: string) {
|
private async removeSshConfig(alias: string) {
|
||||||
try {
|
try {
|
||||||
const filePath = path.join(this.sshPath, `${alias}.config`);
|
const filePath = resolveSubscriptionPath(this.sshPath, alias, '.config');
|
||||||
await rmPath(filePath);
|
await rmPath(filePath);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.logger.error(`删除ssh配置文件${alias}失败`, error);
|
this.logger.error(t('删除ssh配置文件%s失败'), alias, error);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -101,6 +126,7 @@ export default class SshKeyService {
|
|||||||
host: string,
|
host: string,
|
||||||
proxy?: string,
|
proxy?: string,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
|
alias = getSubscriptionSshAlias(alias);
|
||||||
await this.generatePrivateKeyFile(alias, key);
|
await this.generatePrivateKeyFile(alias, key);
|
||||||
await this.generateSingleSshConfig(alias, host, proxy);
|
await this.generateSingleSshConfig(alias, host, proxy);
|
||||||
}
|
}
|
||||||
@@ -110,6 +136,7 @@ export default class SshKeyService {
|
|||||||
host: string,
|
host: string,
|
||||||
proxy?: string,
|
proxy?: string,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
|
alias = getSubscriptionSshAlias(alias);
|
||||||
await this.removePrivateKeyFile(alias);
|
await this.removePrivateKeyFile(alias);
|
||||||
await this.removeSshConfig(alias);
|
await this.removeSshConfig(alias);
|
||||||
}
|
}
|
||||||
@@ -117,7 +144,8 @@ export default class SshKeyService {
|
|||||||
public async setSshConfig(docs: Subscription[]) {
|
public async setSshConfig(docs: Subscription[]) {
|
||||||
for (const doc of docs) {
|
for (const doc of docs) {
|
||||||
if (doc.type === 'private-repo' && doc.pull_type === 'ssh-key') {
|
if (doc.type === 'private-repo' && doc.pull_type === 'ssh-key') {
|
||||||
const { alias, proxy } = doc;
|
try {
|
||||||
|
const alias = getSubscriptionSshAlias(doc.alias);
|
||||||
const { host } = formatUrl(doc);
|
const { host } = formatUrl(doc);
|
||||||
await this.removePrivateKeyFile(alias);
|
await this.removePrivateKeyFile(alias);
|
||||||
await this.removeSshConfig(alias);
|
await this.removeSshConfig(alias);
|
||||||
@@ -125,7 +153,10 @@ export default class SshKeyService {
|
|||||||
alias,
|
alias,
|
||||||
(doc.pull_option as any).private_key,
|
(doc.pull_option as any).private_key,
|
||||||
);
|
);
|
||||||
await this.generateSingleSshConfig(alias, host, proxy);
|
await this.generateSingleSshConfig(alias, host, doc.proxy);
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.warn(t('跳过订阅 %s 的无效 SSH 配置: %s'), doc.id, error);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -144,12 +175,12 @@ export default class SshKeyService {
|
|||||||
|
|
||||||
private async generateGlobalSshConfig(alias: string) {
|
private async generateGlobalSshConfig(alias: string) {
|
||||||
// Create a config that matches all hosts, making this key globally available
|
// Create a config that matches all hosts, making this key globally available
|
||||||
const config = `Host *\n IdentityFile ${path.join(
|
const config = `Host *\n IdentityFile ${resolveSubscriptionPath(
|
||||||
this.sshPath,
|
this.sshPath,
|
||||||
alias,
|
alias,
|
||||||
)}\n StrictHostKeyChecking no\n`;
|
)}\n StrictHostKeyChecking no\n`;
|
||||||
await writeFileWithLock(
|
await writeFileWithLock(
|
||||||
`${path.join(this.sshPath, `${alias}.config`)}`,
|
`${resolveSubscriptionPath(this.sshPath, alias, '.config')}`,
|
||||||
config,
|
config,
|
||||||
{
|
{
|
||||||
encoding: 'utf8',
|
encoding: 'utf8',
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ import {
|
|||||||
} from '../config/util';
|
} from '../config/util';
|
||||||
import fs from 'fs/promises';
|
import fs from 'fs/promises';
|
||||||
import { FindOptions, Op } from 'sequelize';
|
import { FindOptions, Op } from 'sequelize';
|
||||||
import path, { join } from 'path';
|
import path from 'path';
|
||||||
import ScheduleService, { TaskCallbacks } from './schedule';
|
import ScheduleService, { TaskCallbacks } from './schedule';
|
||||||
import { SimpleIntervalSchedule } from 'toad-scheduler';
|
import { SimpleIntervalSchedule } from 'toad-scheduler';
|
||||||
import SockService from './sock';
|
import SockService from './sock';
|
||||||
@@ -30,6 +30,14 @@ import dayjs from 'dayjs';
|
|||||||
import { LOG_END_SYMBOL } from '../config/const';
|
import { LOG_END_SYMBOL } from '../config/const';
|
||||||
import { formatCommand, formatUrl } from '../config/subscription';
|
import { formatCommand, formatUrl } from '../config/subscription';
|
||||||
import { CrontabModel } from '../data/cron';
|
import { CrontabModel } from '../data/cron';
|
||||||
|
import { assertSubscriptionAlias } from '../shared/subscriptionPath';
|
||||||
|
import { resolveLogPath } from '../shared/logPath';
|
||||||
|
import {
|
||||||
|
getSubscriptionStorageCandidates,
|
||||||
|
isSubscriptionStorageName,
|
||||||
|
resolveSubscriptionStoragePath,
|
||||||
|
} from '../shared/subscriptionStorage';
|
||||||
|
import { withSubscriptionMutation } from '../shared/subscriptionMutationLock';
|
||||||
import CrontabService from './cron';
|
import CrontabService from './cron';
|
||||||
import taskLimit from '../shared/pLimit';
|
import taskLimit from '../shared/pLimit';
|
||||||
import { logStreamManager } from '../shared/logStreamManager';
|
import { logStreamManager } from '../shared/logStreamManager';
|
||||||
@@ -88,9 +96,11 @@ export default class SubscriptionService {
|
|||||||
needCreate = true,
|
needCreate = true,
|
||||||
runImmediately = false,
|
runImmediately = false,
|
||||||
) {
|
) {
|
||||||
|
// Cancelling by ID must also work for invalid historical subscriptions.
|
||||||
|
if (needCreate) {
|
||||||
const { url } = formatUrl(doc);
|
const { url } = formatUrl(doc);
|
||||||
|
|
||||||
doc.command = formatCommand(doc, url as string);
|
doc.command = formatCommand(doc, url as string);
|
||||||
|
}
|
||||||
|
|
||||||
if (doc.schedule_type === 'crontab') {
|
if (doc.schedule_type === 'crontab') {
|
||||||
this.scheduleService.cancelCronTask(doc as any);
|
this.scheduleService.cancelCronTask(doc as any);
|
||||||
@@ -222,11 +232,14 @@ export default class SubscriptionService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async create(payload: Subscription): Promise<Subscription> {
|
public async create(payload: Subscription): Promise<Subscription> {
|
||||||
|
assertSubscriptionAlias(payload.alias);
|
||||||
|
return withSubscriptionMutation(async () => {
|
||||||
const tab = new Subscription(payload);
|
const tab = new Subscription(payload);
|
||||||
const doc = await this.insert(tab);
|
const doc = await this.insert(tab);
|
||||||
await this.handleTask(doc.get({ plain: true }));
|
await this.handleTask(doc.get({ plain: true }));
|
||||||
await this.setSshConfig();
|
await this.setSshConfig();
|
||||||
return doc;
|
return doc;
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
public async insert(payload: Subscription): Promise<SubscriptionInstance> {
|
public async insert(payload: Subscription): Promise<SubscriptionInstance> {
|
||||||
@@ -234,12 +247,27 @@ export default class SubscriptionService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async update(payload: Subscription): Promise<Subscription> {
|
public async update(payload: Subscription): Promise<Subscription> {
|
||||||
|
assertSubscriptionAlias(payload.alias);
|
||||||
|
return withSubscriptionMutation(async () => {
|
||||||
const doc = await this.getDb({ id: payload.id });
|
const doc = await this.getDb({ id: payload.id });
|
||||||
const tab = new Subscription({ ...doc, ...payload });
|
const tab = new Subscription({ ...doc, ...payload });
|
||||||
const newDoc = await this.updateDb(tab);
|
const newDoc = await this.updateDb(tab);
|
||||||
await this.handleTask(newDoc, !newDoc.is_disabled);
|
await this.handleTask(newDoc, !newDoc.is_disabled);
|
||||||
|
await this.removeSshConfigForSubscription(doc);
|
||||||
await this.setSshConfig();
|
await this.setSshConfig();
|
||||||
return newDoc;
|
return newDoc;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private async removeSshConfigForSubscription(doc: Subscription) {
|
||||||
|
if (doc.type !== 'private-repo' || doc.pull_type !== 'ssh-key') return;
|
||||||
|
// Invalid historical rows must remain deletable without deriving unsafe paths.
|
||||||
|
try {
|
||||||
|
assertSubscriptionAlias(doc.alias);
|
||||||
|
} catch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await this.sshKeyService.removeSSHKey(doc.alias, '');
|
||||||
}
|
}
|
||||||
|
|
||||||
public async updateDb(payload: Subscription): Promise<Subscription> {
|
public async updateDb(payload: Subscription): Promise<Subscription> {
|
||||||
@@ -262,6 +290,7 @@ export default class SubscriptionService {
|
|||||||
last_running_time: number;
|
last_running_time: number;
|
||||||
last_execution_time: number;
|
last_execution_time: number;
|
||||||
}) {
|
}) {
|
||||||
|
if (log_path) resolveLogPath(config.logPath, log_path);
|
||||||
const options: any = {
|
const options: any = {
|
||||||
status,
|
status,
|
||||||
pid,
|
pid,
|
||||||
@@ -279,11 +308,52 @@ export default class SubscriptionService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async remove(ids: number[], query: { force?: boolean }) {
|
public async remove(ids: number[], query: { force?: boolean }) {
|
||||||
|
return withSubscriptionMutation(async () => {
|
||||||
const docs = await SubscriptionModel.findAll({ where: { id: ids } });
|
const docs = await SubscriptionModel.findAll({ where: { id: ids } });
|
||||||
|
const storagePaths: string[] = [];
|
||||||
|
const retainedPaths: string[] = [];
|
||||||
|
const collectPaths = (
|
||||||
|
doc: Subscription,
|
||||||
|
targets: string[],
|
||||||
|
deleting = false,
|
||||||
|
) => {
|
||||||
|
for (const names of getSubscriptionStorageCandidates(doc)) {
|
||||||
|
// A URL accepted by one engine can produce an invalid name in another.
|
||||||
|
// Validate each candidate independently, retaining the safe candidate.
|
||||||
|
if (!Object.values(names).every(isSubscriptionStorageName)) continue;
|
||||||
|
const resolve = deleting ? resolveSubscriptionStoragePath : path.resolve;
|
||||||
|
const scriptPath = resolve(config.scriptPath, names.script);
|
||||||
|
if (scriptPath) targets.push(scriptPath);
|
||||||
|
if (names.repo) {
|
||||||
|
const repoPath = resolve(config.repoPath, names.repo);
|
||||||
|
if (repoPath) targets.push(repoPath);
|
||||||
|
}
|
||||||
|
if (names.raw) {
|
||||||
|
const rawPath = resolve(path.join(config.dataPath, 'raw'), names.raw);
|
||||||
|
if (rawPath) targets.push(rawPath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
// Validate all deletion targets before changing rows or removing any files.
|
||||||
|
if (query?.force === true) {
|
||||||
|
for (const doc of docs) {
|
||||||
|
assertSubscriptionAlias(doc.alias);
|
||||||
|
resolveSubscriptionStoragePath(config.scriptPath, doc.alias);
|
||||||
|
resolveSubscriptionStoragePath(config.repoPath, doc.alias);
|
||||||
|
collectPaths(doc, storagePaths, true);
|
||||||
|
}
|
||||||
|
const remaining = await SubscriptionModel.findAll({ where: {} });
|
||||||
|
for (const doc of remaining) {
|
||||||
|
if (!ids.includes(doc.id!)) collectPaths(doc, retainedPaths);
|
||||||
|
}
|
||||||
|
}
|
||||||
for (const doc of docs) {
|
for (const doc of docs) {
|
||||||
await this.handleTask(doc.get({ plain: true }), false);
|
await this.handleTask(doc.get({ plain: true }), false);
|
||||||
}
|
}
|
||||||
await SubscriptionModel.destroy({ where: { id: ids } });
|
await SubscriptionModel.destroy({ where: { id: ids } });
|
||||||
|
for (const doc of docs) {
|
||||||
|
await this.removeSshConfigForSubscription(doc);
|
||||||
|
}
|
||||||
await this.setSshConfig();
|
await this.setSshConfig();
|
||||||
|
|
||||||
if (query?.force === true) {
|
if (query?.force === true) {
|
||||||
@@ -291,13 +361,18 @@ export default class SubscriptionService {
|
|||||||
if (crons?.length) {
|
if (crons?.length) {
|
||||||
await this.crontabService.remove(crons.map((x) => x.id!));
|
await this.crontabService.remove(crons.map((x) => x.id!));
|
||||||
}
|
}
|
||||||
for (const doc of docs) {
|
for (const storagePath of new Set(storagePaths)) {
|
||||||
const filePath = join(config.scriptPath, doc.alias);
|
// rm is recursive: an ancestor must also be retained if a surviving
|
||||||
const repoPath = join(config.repoPath, doc.alias);
|
// subscription uses a nested branch, and vice versa.
|
||||||
await rmPath(filePath);
|
const shared = retainedPaths.some((retained) =>
|
||||||
await rmPath(repoPath);
|
retained === storagePath ||
|
||||||
|
retained.startsWith(`${storagePath}${path.sep}`) ||
|
||||||
|
storagePath.startsWith(`${retained}${path.sep}`),
|
||||||
|
);
|
||||||
|
if (!shared) await rmPath(storagePath);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
public async getDb(
|
public async getDb(
|
||||||
@@ -399,7 +474,7 @@ export default class SubscriptionService {
|
|||||||
|
|
||||||
if (doc.log_path) {
|
if (doc.log_path) {
|
||||||
const relativeDir = path.dirname(`${doc.log_path}`);
|
const relativeDir = path.dirname(`${doc.log_path}`);
|
||||||
const dir = path.resolve(config.logPath, relativeDir);
|
const dir = path.dirname(resolveLogPath(config.logPath, doc.log_path));
|
||||||
const _exist = await fileExist(dir);
|
const _exist = await fileExist(dir);
|
||||||
if (_exist) {
|
if (_exist) {
|
||||||
let files = await fs.readdir(dir);
|
let files = await fs.readdir(dir);
|
||||||
|
|||||||
+162
-69
@@ -1,7 +1,11 @@
|
|||||||
|
import { persistNotificationAudit } from '../shared/notificationAudit';
|
||||||
import { spawn } from 'cross-spawn';
|
import { spawn } from 'cross-spawn';
|
||||||
|
import { execFile } from 'child_process';
|
||||||
|
import { promisify } from 'util';
|
||||||
|
import { shellQuote } from '../shared/shellQuote';
|
||||||
import { Response } from 'express';
|
import { Response } from 'express';
|
||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
import { Agent, request } from 'undici';
|
import { request } from 'undici';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
import { Inject, Service } from 'typedi';
|
import { Inject, Service } from 'typedi';
|
||||||
import winston from 'winston';
|
import winston from 'winston';
|
||||||
@@ -29,8 +33,12 @@ import {
|
|||||||
SystemInstance,
|
SystemInstance,
|
||||||
SystemModel,
|
SystemModel,
|
||||||
SystemModelInfo,
|
SystemModelInfo,
|
||||||
|
NotifyStatus,
|
||||||
|
NotifyLogInfo,
|
||||||
} from '../data/system';
|
} from '../data/system';
|
||||||
import taskLimit from '../shared/pLimit';
|
import taskLimit from '../shared/pLimit';
|
||||||
|
import cronClient from '../schedule/client';
|
||||||
|
import { withSchedulerMutation } from '../shared/schedulerMutationLock';
|
||||||
import NotificationService from './notify';
|
import NotificationService from './notify';
|
||||||
import ScheduleService, { TaskCallbacks } from './schedule';
|
import ScheduleService, { TaskCallbacks } from './schedule';
|
||||||
import SockService from './sock';
|
import SockService from './sock';
|
||||||
@@ -48,7 +56,11 @@ export default class SystemService {
|
|||||||
@Inject('logger') private logger: winston.Logger,
|
@Inject('logger') private logger: winston.Logger,
|
||||||
private scheduleService: ScheduleService,
|
private scheduleService: ScheduleService,
|
||||||
private sockService: SockService,
|
private sockService: SockService,
|
||||||
) { }
|
) {}
|
||||||
|
|
||||||
|
public get cronConcurrency() {
|
||||||
|
return taskLimit.cronConcurrency;
|
||||||
|
}
|
||||||
|
|
||||||
public async getSystemConfig() {
|
public async getSystemConfig() {
|
||||||
const doc = await this.getDb({ type: AuthDataType.systemConfig });
|
const doc = await this.getDb({ type: AuthDataType.systemConfig });
|
||||||
@@ -119,15 +131,29 @@ export default class SystemService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async updateCronConcurrency(info: SystemModelInfo) {
|
public async updateCronConcurrency(info: SystemModelInfo) {
|
||||||
|
const limit = info.cronConcurrency ?? 0;
|
||||||
|
if (!Number.isInteger(limit) || limit < 0 || limit > 2147483647) {
|
||||||
|
throw Object.assign(new Error(t('参数错误')), { status: 400 });
|
||||||
|
}
|
||||||
|
return withSchedulerMutation(async () => {
|
||||||
const oDoc = await this.getSystemConfig();
|
const oDoc = await this.getSystemConfig();
|
||||||
await this.updateAuthDb({
|
await this.updateAuthDb({
|
||||||
...oDoc,
|
...oDoc,
|
||||||
info: { ...oDoc.info, ...info },
|
info: { ...oDoc.info, ...info, cronConcurrency: limit },
|
||||||
});
|
});
|
||||||
if (info.cronConcurrency) {
|
await taskLimit.setCustomLimit(limit || Math.max(os.cpus().length, 4));
|
||||||
await taskLimit.setCustomLimit(info.cronConcurrency);
|
await cronClient.setConcurrency(limit);
|
||||||
}
|
|
||||||
return { code: 200, data: info };
|
return { code: 200, data: info };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public async restoreCronConcurrency() {
|
||||||
|
await withSchedulerMutation(async () => {
|
||||||
|
const doc = await this.getSystemConfig();
|
||||||
|
const limit = doc.info?.cronConcurrency ?? 0;
|
||||||
|
await taskLimit.setCustomLimit(limit || Math.max(os.cpus().length, 4));
|
||||||
|
await cronClient.setConcurrency(limit);
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
public async updateDependenceProxy(info: SystemModelInfo) {
|
public async updateDependenceProxy(info: SystemModelInfo) {
|
||||||
@@ -139,7 +165,9 @@ export default class SystemService {
|
|||||||
if (info.dependenceProxy) {
|
if (info.dependenceProxy) {
|
||||||
await fs.promises.writeFile(
|
await fs.promises.writeFile(
|
||||||
config.dependenceProxyFile,
|
config.dependenceProxyFile,
|
||||||
`export http_proxy="${info.dependenceProxy}"\nexport https_proxy="${info.dependenceProxy}"`,
|
`export http_proxy=${shellQuote(
|
||||||
|
info.dependenceProxy,
|
||||||
|
)}\nexport https_proxy=${shellQuote(info.dependenceProxy)}`,
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
await fs.promises.rm(config.dependenceProxyFile);
|
await fs.promises.rm(config.dependenceProxyFile);
|
||||||
@@ -155,7 +183,7 @@ export default class SystemService {
|
|||||||
});
|
});
|
||||||
let cmd = 'pnpm config delete registry';
|
let cmd = 'pnpm config delete registry';
|
||||||
if (info.nodeMirror) {
|
if (info.nodeMirror) {
|
||||||
cmd = `pnpm config set registry ${info.nodeMirror}`;
|
cmd = `pnpm config set registry ${shellQuote(info.nodeMirror)}`;
|
||||||
}
|
}
|
||||||
let command = `cd && ${cmd}`;
|
let command = `cd && ${cmd}`;
|
||||||
const docs = await DependenceModel.findAll({
|
const docs = await DependenceModel.findAll({
|
||||||
@@ -204,7 +232,7 @@ export default class SystemService {
|
|||||||
});
|
});
|
||||||
let cmd = 'pip config unset global.index-url';
|
let cmd = 'pip config unset global.index-url';
|
||||||
if (info.pythonMirror) {
|
if (info.pythonMirror) {
|
||||||
cmd = `pip3 config set global.index-url ${info.pythonMirror}`;
|
cmd = `pip3 config set global.index-url ${shellQuote(info.pythonMirror)}`;
|
||||||
}
|
}
|
||||||
await promiseExec(cmd);
|
await promiseExec(cmd);
|
||||||
return { code: 200, data: info };
|
return { code: 200, data: info };
|
||||||
@@ -220,7 +248,6 @@ export default class SystemService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const command = await updateLinuxMirrorFile(info.linuxMirror || '');
|
const command = await updateLinuxMirrorFile(info.linuxMirror || '');
|
||||||
let hasError = false;
|
|
||||||
this.scheduleService.runTask(
|
this.scheduleService.runTask(
|
||||||
command,
|
command,
|
||||||
{
|
{
|
||||||
@@ -228,14 +255,15 @@ export default class SystemService {
|
|||||||
res?.setHeader('QL-Task-Pid', `${cp.pid}`);
|
res?.setHeader('QL-Task-Pid', `${cp.pid}`);
|
||||||
res?.end();
|
res?.end();
|
||||||
},
|
},
|
||||||
onEnd: async () => {
|
onEnd: async (cp) => {
|
||||||
this.sockService.sendMessage({
|
this.sockService.sendMessage({
|
||||||
type: 'updateLinuxMirror',
|
type: 'updateLinuxMirror',
|
||||||
message: 'update linux mirror end',
|
message: 'update linux mirror end',
|
||||||
status: 'completed',
|
status: 'completed',
|
||||||
});
|
});
|
||||||
onEnd?.();
|
onEnd?.();
|
||||||
if (!hasError) {
|
// Tools such as sudo and apt may write warnings to stderr on success.
|
||||||
|
if (cp?.exitCode === 0 && cp.signalCode === null) {
|
||||||
await this.updateAuthDb({
|
await this.updateAuthDb({
|
||||||
...oDoc,
|
...oDoc,
|
||||||
info: { ...oDoc.info, ...info },
|
info: { ...oDoc.info, ...info },
|
||||||
@@ -243,7 +271,6 @@ export default class SystemService {
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
onError: async (message: string) => {
|
onError: async (message: string) => {
|
||||||
hasError = true;
|
|
||||||
this.sockService.sendMessage({ type: 'updateLinuxMirror', message });
|
this.sockService.sendMessage({ type: 'updateLinuxMirror', message });
|
||||||
},
|
},
|
||||||
onLog: async (message: string) => {
|
onLog: async (message: string) => {
|
||||||
@@ -259,26 +286,24 @@ export default class SystemService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async checkUpdate() {
|
public async checkUpdate() {
|
||||||
|
const controller = new AbortController();
|
||||||
|
const timeout = setTimeout(() => controller.abort(), 15000);
|
||||||
try {
|
try {
|
||||||
const currentVersionContent = await parseVersion(config.versionFile);
|
const currentVersionContent = await parseVersion(config.versionFile);
|
||||||
|
|
||||||
let lastVersionContent;
|
const { body, statusCode } = await request(
|
||||||
try {
|
|
||||||
const { body } = await request(
|
|
||||||
`${config.lastVersionFile}?t=${Date.now()}`,
|
`${config.lastVersionFile}?t=${Date.now()}`,
|
||||||
{
|
{ signal: controller.signal },
|
||||||
dispatcher: new Agent({
|
|
||||||
keepAliveTimeout: 30000,
|
|
||||||
keepAliveMaxTimeout: 30000,
|
|
||||||
}),
|
|
||||||
},
|
|
||||||
);
|
);
|
||||||
const text = await body.text();
|
const text = await body.text();
|
||||||
lastVersionContent = parseContentVersion(text);
|
if (statusCode !== 200) throw new Error('Version request failed');
|
||||||
} catch (error) { }
|
const lastVersionContent = parseContentVersion(text);
|
||||||
|
if (
|
||||||
if (!lastVersionContent) {
|
!lastVersionContent ||
|
||||||
lastVersionContent = currentVersionContent;
|
typeof lastVersionContent.version !== 'string' ||
|
||||||
|
!/^\d+\.\d+\.\d+$/.test(lastVersionContent.version)
|
||||||
|
) {
|
||||||
|
throw new Error('Invalid version response');
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -296,8 +321,10 @@ export default class SystemService {
|
|||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
return {
|
return {
|
||||||
code: 400,
|
code: 400,
|
||||||
message: error.message,
|
message: t('检查更新失败,请检查网络后重试'),
|
||||||
};
|
};
|
||||||
|
} finally {
|
||||||
|
clearTimeout(timeout);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -321,43 +348,52 @@ export default class SystemService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async updateSystem() {
|
public async updateSystem() {
|
||||||
const cp = spawn('real_time=true ql update false', { shell: '/bin/bash' });
|
return this.startUpgradeJob('download');
|
||||||
|
}
|
||||||
|
|
||||||
cp.stdout.on('data', (data) => {
|
private upgradeJobs() {
|
||||||
this.sockService.sendMessage({
|
return require(path.join(
|
||||||
type: 'updateSystemVersion',
|
config.rootPath,
|
||||||
message: data.toString(),
|
'cli/src/internal/maintenance/panelUpgrade.cjs',
|
||||||
});
|
));
|
||||||
});
|
}
|
||||||
|
|
||||||
cp.stderr.on('data', (data) => {
|
public async getUpgradeStatus() {
|
||||||
this.sockService.sendMessage({
|
return {
|
||||||
type: 'updateSystemVersion',
|
code: 200,
|
||||||
message: data.toString(),
|
data: await this.upgradeJobs().getStatus(config.tmpPath),
|
||||||
});
|
};
|
||||||
});
|
}
|
||||||
|
|
||||||
cp.on('error', (err) => {
|
private async startUpgradeJob(action: 'download' | 'apply') {
|
||||||
this.sockService.sendMessage({
|
try {
|
||||||
type: 'updateSystemVersion',
|
const data = await this.upgradeJobs().startJob(
|
||||||
message: JSON.stringify(err),
|
{
|
||||||
status: 'failed',
|
root: config.rootPath,
|
||||||
});
|
tmp: config.tmpPath,
|
||||||
});
|
env: {
|
||||||
|
...process.env,
|
||||||
cp.on('exit', (code) => {
|
dir_data: config.dataPath,
|
||||||
this.sockService.sendMessage({
|
dir_config: config.configPath,
|
||||||
type: 'updateSystemVersion',
|
QlPort: String(config.port),
|
||||||
message: '',
|
QlBaseUrl: config.baseUrl,
|
||||||
status: code === 0 ? 'success' : 'failed',
|
},
|
||||||
});
|
},
|
||||||
});
|
action,
|
||||||
|
);
|
||||||
return { code: 200 };
|
return { code: 200, data };
|
||||||
|
} catch (error: any) {
|
||||||
|
this.logger.error('[upgrade] %s', error.message);
|
||||||
|
return { code: 400, message: t('升级任务启动失败,请检查日志后重试') };
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public async reloadSystem(target?: 'system' | 'data') {
|
public async reloadSystem(target?: 'system' | 'data') {
|
||||||
const cmd = `real_time=true ql reload ${target || ''}`;
|
if (target && !['system', 'data'].includes(target)) {
|
||||||
|
return { code: 400, message: t('参数错误') };
|
||||||
|
}
|
||||||
|
if (target === 'system') return this.startUpgradeJob('apply');
|
||||||
|
const cmd = `real_time=true ql reload ${shellQuote(target || '')}`;
|
||||||
const cp = spawn(cmd, {
|
const cp = spawn(cmd, {
|
||||||
shell: '/bin/bash',
|
shell: '/bin/bash',
|
||||||
detached: true,
|
detached: true,
|
||||||
@@ -386,11 +422,13 @@ export default class SystemService {
|
|||||||
if (notificationInfo && typeString) {
|
if (notificationInfo && typeString) {
|
||||||
notificationInfo.type = typeString;
|
notificationInfo.type = typeString;
|
||||||
}
|
}
|
||||||
|
|
||||||
const isSuccess = await this.notificationService.notify(
|
const isSuccess = await this.notificationService.notify(
|
||||||
title,
|
title,
|
||||||
content,
|
content,
|
||||||
notificationInfo,
|
notificationInfo,
|
||||||
);
|
);
|
||||||
|
|
||||||
if (isSuccess) {
|
if (isSuccess) {
|
||||||
return { code: 200, message: t('通知发送成功') };
|
return { code: 200, message: t('通知发送成功') };
|
||||||
} else {
|
} else {
|
||||||
@@ -398,16 +436,36 @@ export default class SystemService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public async run({ command, logPath }: { command: string; logPath?: string }, callback: TaskCallbacks) {
|
public async recordNotifyLog(info: NotifyLogInfo): Promise<void> {
|
||||||
|
await persistNotificationAudit(info);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async getNotifyLog(): Promise<Array<NotifyLogInfo>> {
|
||||||
|
const docs = await SystemModel.findAll({
|
||||||
|
where: { type: AuthDataType.notifyLog },
|
||||||
|
order: [['id', 'DESC']],
|
||||||
|
limit: 200,
|
||||||
|
});
|
||||||
|
return docs.map((x) => ({ ...x.info, id: x.id }));
|
||||||
|
}
|
||||||
|
|
||||||
|
public async run(
|
||||||
|
{ command, logPath }: { command: string; logPath?: string },
|
||||||
|
callback: TaskCallbacks,
|
||||||
|
) {
|
||||||
if (!command.startsWith(TASK_COMMAND)) {
|
if (!command.startsWith(TASK_COMMAND)) {
|
||||||
command = `${TASK_COMMAND} ${command}`;
|
command = `${TASK_COMMAND} ${command}`;
|
||||||
}
|
}
|
||||||
const logPathPrefix = logPath ? `real_log_path=${logPath}` : ''
|
const logPathPrefix = logPath ? `real_log_path=${logPath}` : '';
|
||||||
this.scheduleService.runTask(`${logPathPrefix} real_time=true ${command}`, callback, {
|
this.scheduleService.runTask(
|
||||||
|
`${logPathPrefix} real_time=true ${command}`,
|
||||||
|
callback,
|
||||||
|
{
|
||||||
command,
|
command,
|
||||||
id: command.replace(/ /g, '-'),
|
id: command.replace(/ /g, '-'),
|
||||||
runOrigin: 'system',
|
runOrigin: 'system',
|
||||||
});
|
},
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async stop({ command, pid }: { command: string; pid: number }) {
|
public async stop({ command, pid }: { command: string; pid: number }) {
|
||||||
@@ -438,10 +496,35 @@ export default class SystemService {
|
|||||||
if (type && type.length) {
|
if (type && type.length) {
|
||||||
dataDirs = dataDirs.concat(type.filter((x) => x !== 'base'));
|
dataDirs = dataDirs.concat(type.filter((x) => x !== 'base'));
|
||||||
}
|
}
|
||||||
const dataPaths = dataDirs.map((dir) => `data/${dir}`);
|
const allowed = new Set([
|
||||||
await promiseExec(
|
'db',
|
||||||
`cd ${config.dataPath} && cd ../ && tar -zcvf ${config.dataTgzFile
|
'upload',
|
||||||
} ${dataPaths.join(' ')}`,
|
'config',
|
||||||
|
'scripts',
|
||||||
|
'log',
|
||||||
|
'deps',
|
||||||
|
'syslog',
|
||||||
|
'dep_cache',
|
||||||
|
'raw',
|
||||||
|
'repo',
|
||||||
|
'ssh.d',
|
||||||
|
'script-history',
|
||||||
|
]);
|
||||||
|
if (dataDirs.some((dir) => !allowed.has(dir))) {
|
||||||
|
return res.status(400).send({ code: 400, message: t('参数错误') });
|
||||||
|
}
|
||||||
|
if (dataDirs.includes('scripts')) dataDirs.push('script-history');
|
||||||
|
// History is created lazily; old/fresh installations must still export.
|
||||||
|
if (!fs.existsSync(path.join(config.dataPath, 'script-history'))) {
|
||||||
|
dataDirs = dataDirs.filter((dir) => dir !== 'script-history');
|
||||||
|
}
|
||||||
|
const dataPaths = [...new Set(dataDirs)].map((dir) => `data/${dir}`);
|
||||||
|
await promisify(execFile)(
|
||||||
|
'tar',
|
||||||
|
['-zcf', config.dataTgzFile, '--', ...dataPaths],
|
||||||
|
{
|
||||||
|
cwd: path.dirname(config.dataPath),
|
||||||
|
},
|
||||||
);
|
);
|
||||||
res.download(config.dataTgzFile);
|
res.download(config.dataTgzFile);
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
@@ -533,7 +616,17 @@ export default class SystemService {
|
|||||||
const result = await readDirs(config.systemLogPath, config.systemLogPath);
|
const result = await readDirs(config.systemLogPath, config.systemLogPath);
|
||||||
const logs = result.reverse().filter((x) => x.title.endsWith('.log'));
|
const logs = result.reverse().filter((x) => x.title.endsWith('.log'));
|
||||||
for (const log of logs) {
|
for (const log of logs) {
|
||||||
await rmPath(path.join(config.systemLogPath, log.title));
|
// All workers keep append handles open; unlinking strands future writes
|
||||||
|
// in a deleted inode until rotation. Preserve the file while clearing it.
|
||||||
|
const file = await fs.promises.open(
|
||||||
|
path.join(config.systemLogPath, log.title),
|
||||||
|
fs.constants.O_WRONLY | fs.constants.O_NOFOLLOW,
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
await file.truncate(0);
|
||||||
|
} finally {
|
||||||
|
await file.close();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -565,7 +658,7 @@ export default class SystemService {
|
|||||||
try {
|
try {
|
||||||
await fs.promises.writeFile(
|
await fs.promises.writeFile(
|
||||||
config.langEnvFile,
|
config.langEnvFile,
|
||||||
`export QL_LANG='${lang}'\n`,
|
`export QL_LANG=${shellQuote(lang)}\n`,
|
||||||
);
|
);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.logger.error(`Failed to write lang_env.sh: ${error}`);
|
this.logger.error(`Failed to write lang_env.sh: ${error}`);
|
||||||
|
|||||||
+17
-14
@@ -33,9 +33,11 @@ import {
|
|||||||
verifyPassword,
|
verifyPassword,
|
||||||
} from '../shared/password';
|
} from '../shared/password';
|
||||||
import { serializeAuthMutation } from '../shared/authMutation';
|
import { serializeAuthMutation } from '../shared/authMutation';
|
||||||
|
import { LoginThrottle } from '../shared/loginThrottle';
|
||||||
|
|
||||||
@Service()
|
@Service()
|
||||||
export default class UserService {
|
export default class UserService {
|
||||||
|
private loginThrottle = new LoginThrottle();
|
||||||
@Inject((type) => NotificationService)
|
@Inject((type) => NotificationService)
|
||||||
private notificationService!: NotificationService;
|
private notificationService!: NotificationService;
|
||||||
|
|
||||||
@@ -106,11 +108,8 @@ export default class UserService {
|
|||||||
return { code: 403, message: t('该 IP 已被列入黑名单') };
|
return { code: 403, message: t('该 IP 已被列入黑名单') };
|
||||||
}
|
}
|
||||||
|
|
||||||
const retriesTime = Math.pow(3, retries) * 1000;
|
const waitTime = this.loginThrottle.retryAfter(ip, timestamp);
|
||||||
if (retries > 2 && timestamp - lastlogon < retriesTime) {
|
if (waitTime > 0) {
|
||||||
const waitTime = Math.ceil(
|
|
||||||
(retriesTime - (timestamp - lastlogon)) / 1000,
|
|
||||||
);
|
|
||||||
return {
|
return {
|
||||||
code: 410,
|
code: 410,
|
||||||
message: tf('失败次数过多,请%s秒后重试', waitTime),
|
message: tf('失败次数过多,请%s秒后重试', waitTime),
|
||||||
@@ -133,6 +132,7 @@ export default class UserService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (passwordMatches) {
|
if (passwordMatches) {
|
||||||
|
this.loginThrottle.reset(ip);
|
||||||
const data = createRandomString(50, 100);
|
const data = createRandomString(50, 100);
|
||||||
const expiration = twoFactorActivated ? '60d' : '20d';
|
const expiration = twoFactorActivated ? '60d' : '20d';
|
||||||
let token = jwt.sign({ data }, config.jwt.secret, {
|
let token = jwt.sign({ data }, config.jwt.secret, {
|
||||||
@@ -173,7 +173,7 @@ export default class UserService {
|
|||||||
t('你于') +
|
t('你于') +
|
||||||
dayjs(timestamp).format('YYYY-MM-DD HH:mm:ss') +
|
dayjs(timestamp).format('YYYY-MM-DD HH:mm:ss') +
|
||||||
t('在') +
|
t('在') +
|
||||||
address +
|
t(address) +
|
||||||
' ' +
|
' ' +
|
||||||
req.platform +
|
req.platform +
|
||||||
t('端') +
|
t('端') +
|
||||||
@@ -206,8 +206,9 @@ export default class UserService {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
} else {
|
} else {
|
||||||
|
const failures = this.loginThrottle.fail(ip, timestamp);
|
||||||
await this.updateAuthInfo(content, {
|
await this.updateAuthInfo(content, {
|
||||||
retries: retries + 1,
|
retries: failures,
|
||||||
lastlogon: timestamp,
|
lastlogon: timestamp,
|
||||||
lastip: ip,
|
lastip: ip,
|
||||||
lastaddr: address,
|
lastaddr: address,
|
||||||
@@ -218,7 +219,7 @@ export default class UserService {
|
|||||||
t('你于') +
|
t('你于') +
|
||||||
dayjs(timestamp).format('YYYY-MM-DD HH:mm:ss') +
|
dayjs(timestamp).format('YYYY-MM-DD HH:mm:ss') +
|
||||||
t('在') +
|
t('在') +
|
||||||
address +
|
t(address) +
|
||||||
' ' +
|
' ' +
|
||||||
req.platform +
|
req.platform +
|
||||||
t('端') +
|
t('端') +
|
||||||
@@ -239,8 +240,8 @@ export default class UserService {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
this.getLoginLog();
|
this.getLoginLog();
|
||||||
if (retries > 2) {
|
if (failures > 3) {
|
||||||
const waitTime = Math.round(Math.pow(3, retries + 1));
|
const waitTime = this.loginThrottle.retryAfter(ip, timestamp);
|
||||||
return {
|
return {
|
||||||
code: 410,
|
code: 410,
|
||||||
message: tf('失败次数过多,请%s秒后重试', waitTime),
|
message: tf('失败次数过多,请%s秒后重试', waitTime),
|
||||||
@@ -439,8 +440,8 @@ export default class UserService {
|
|||||||
const authInfo = await this.getAuthInfo();
|
const authInfo = await this.getAuthInfo();
|
||||||
const { isTwoFactorChecking, twoFactorSecret } = authInfo;
|
const { isTwoFactorChecking, twoFactorSecret } = authInfo;
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
const retries = authInfo.retries || 0;
|
const source = getClientIp(req);
|
||||||
if (retries > 2 && now - authInfo.lastlogon < Math.pow(3, retries) * 1000) {
|
if (this.loginThrottle.retryAfter(source, now) > 0) {
|
||||||
return { code: 410, message: t('失败次数过多,请稍后重试') };
|
return { code: 410, message: t('失败次数过多,请稍后重试') };
|
||||||
}
|
}
|
||||||
if (
|
if (
|
||||||
@@ -472,9 +473,8 @@ export default class UserService {
|
|||||||
.join(' ');
|
.join(' ');
|
||||||
}
|
}
|
||||||
await this.updateAuthInfo(authInfo, {
|
await this.updateAuthInfo(authInfo, {
|
||||||
retries: retries + 1,
|
retries: this.loginThrottle.fail(source, now),
|
||||||
lastlogon: now,
|
lastlogon: now,
|
||||||
isTwoFactorChecking: retries + 1 < 5,
|
|
||||||
lastip: ip,
|
lastip: ip,
|
||||||
lastaddr: address,
|
lastaddr: address,
|
||||||
platform: req.platform,
|
platform: req.platform,
|
||||||
@@ -690,5 +690,8 @@ export default class UserService {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
await this.updateAuthInfo(authInfo, payload);
|
await this.updateAuthInfo(authInfo, payload);
|
||||||
|
if (retries === 0 || password !== undefined || username !== undefined) {
|
||||||
|
this.loginThrottle.clear();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,62 @@
|
|||||||
|
import os from 'os';
|
||||||
|
import { readFileSync } from 'fs';
|
||||||
|
import path from 'path';
|
||||||
|
|
||||||
|
function read(file: string): string {
|
||||||
|
try { return readFileSync(file, 'utf8').trim(); } catch { return ''; }
|
||||||
|
}
|
||||||
|
|
||||||
|
// Walk ancestors too: a nested cgroup can inherit a tighter parent quota.
|
||||||
|
export function containerResources(readValue = read, platform = process.platform) {
|
||||||
|
const hostCpus = os.cpus().length;
|
||||||
|
const hostMemory = os.totalmem();
|
||||||
|
let cpus = hostCpus;
|
||||||
|
let memTotal = hostMemory;
|
||||||
|
let memFree = os.freemem();
|
||||||
|
if (platform === 'linux') {
|
||||||
|
const groups = readValue('/proc/self/cgroup').split('\n');
|
||||||
|
for (const line of groups) {
|
||||||
|
const match = /^\d+:([^:]*):(\/.*)$/.exec(line);
|
||||||
|
if (!match) continue;
|
||||||
|
const controllers = match[1].split(',');
|
||||||
|
const unified = !match[1];
|
||||||
|
for (const kind of ['cpu', 'memory', 'cpuset']) {
|
||||||
|
if (!unified && !controllers.includes(kind)) continue;
|
||||||
|
const roots = unified ? ['/sys/fs/cgroup'] : [...new Set([kind, match[1]].map(k => '/sys/fs/cgroup/' + k))];
|
||||||
|
for (const root of roots) {
|
||||||
|
let dir = path.join(root, path.normalize(match[2]));
|
||||||
|
while (dir === root || dir.startsWith(root + '/')) {
|
||||||
|
const value = (file: string) => readValue(path.join(dir, file));
|
||||||
|
if (kind === 'cpu') {
|
||||||
|
const [quota, period] = (unified ? value('cpu.max').split(/\s+/) :
|
||||||
|
[value('cpu.cfs_quota_us'), value('cpu.cfs_period_us')]).map(Number);
|
||||||
|
if (quota > 0 && period > 0) cpus = Math.min(cpus, quota / period);
|
||||||
|
} else if (kind === 'cpuset') {
|
||||||
|
const set = value(unified ? 'cpuset.cpus.effective' : 'cpuset.cpus');
|
||||||
|
if (/^\d+(?:-\d+)?(?:,\d+(?:-\d+)?)*$/.test(set)) {
|
||||||
|
const count = set.split(',').reduce((n, range) => {
|
||||||
|
const [lo, hi] = range.split('-').map(Number);
|
||||||
|
return n + (hi ?? lo) - lo + 1;
|
||||||
|
}, 0);
|
||||||
|
if (count > 0) cpus = Math.min(cpus, count);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
const limit = Number(value(unified ? 'memory.max' : 'memory.limit_in_bytes'));
|
||||||
|
if (limit > 0 && limit <= memTotal) {
|
||||||
|
const usage = value(unified ? 'memory.current' : 'memory.usage_in_bytes');
|
||||||
|
if (usage && Number.isFinite(Number(usage))) {
|
||||||
|
memTotal = limit;
|
||||||
|
memFree = Math.max(0, limit - Number(usage));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (dir === root) break;
|
||||||
|
dir = path.dirname(dir);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { cpus, memTotal, memFree, hostCpus, hostMemory,
|
||||||
|
memUsagePercent: ((1 - memFree / memTotal) * 100).toFixed(1) };
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
// Match command text literally: looking up a task must never execute its input.
|
||||||
|
export function findCronId(crontab: string, command: string): string {
|
||||||
|
if (!command) return '';
|
||||||
|
for (const line of crontab.split('\n')) {
|
||||||
|
const id = /\bID=(\d+)\s/.exec(line);
|
||||||
|
if (id && line.includes(command, id.index + id[0].length)) {
|
||||||
|
return id[1];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return '';
|
||||||
|
}
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
import { join } from 'path';
|
||||||
|
import { existsSync } from 'fs';
|
||||||
|
import { receiptPath } from './managedExit.cjs';
|
||||||
|
import { spawn } from 'cross-spawn';
|
||||||
|
import type { ChildProcessWithoutNullStreams } from 'child_process';
|
||||||
|
import type { Readable } from 'stream';
|
||||||
|
|
||||||
|
// A managed command waits on its parent's pipe. EOF before authorization exits
|
||||||
|
// without interpreting the command, even when the worker is killed by SIGKILL.
|
||||||
|
export function spawnCronCommand(command: string, triggerId?: string, onCommandStart?: (at: number) => void) {
|
||||||
|
if (!triggerId) return spawn(command, { shell: '/bin/bash' });
|
||||||
|
const sourceScript = join(__dirname, '../../shell/cron-command.py');
|
||||||
|
const supervisor = existsSync(sourceScript) ? sourceScript : join(process.env.QL_DIR!, 'shell/cron-command.py');
|
||||||
|
const args = ['-I', '-S', supervisor, command, String(process.pid),
|
||||||
|
receiptPath(triggerId) || '', process.execPath, join(__dirname, 'managedExit.cjs'),
|
||||||
|
];
|
||||||
|
if (onCommandStart) args.push('report-start');
|
||||||
|
const child = spawn('python3', args, { env: { ...process.env, QL_TRIGGER_ID: triggerId },
|
||||||
|
stdio: onCommandStart ? ['pipe', 'pipe', 'pipe', 'pipe'] : 'pipe' }) as ChildProcessWithoutNullStreams;
|
||||||
|
if (onCommandStart && child.stdio[3]) {
|
||||||
|
const timing = child.stdio[3] as Readable;
|
||||||
|
let stamp = '';
|
||||||
|
timing.setEncoding('ascii').on('data', chunk => { stamp += chunk; });
|
||||||
|
timing.once('end', () => { if (/^\d{13}$/.test(stamp)) onCommandStart(Number(stamp)); });
|
||||||
|
timing.on('error', () => {}); // Missing telemetry must not change the command result.
|
||||||
|
}
|
||||||
|
// A concurrent stop can close the pipe before the authorization write.
|
||||||
|
child.stdin.on('error', () => {});
|
||||||
|
return child;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function releaseCronCommand(child: ChildProcessWithoutNullStreams): Promise<void> {
|
||||||
|
return new Promise((resolve, reject) =>
|
||||||
|
child.stdin.end('run\n', (error?: Error) => error ? reject(error) : resolve()));
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
// Read literal shell words only. Never expand or execute a command to derive a
|
||||||
|
// path. Adjacent quoted/unquoted fragments belong to the same argument.
|
||||||
|
function words(command: string): string[] {
|
||||||
|
const result: string[] = [];
|
||||||
|
let value = '', quote = '', active = false;
|
||||||
|
for (let i = 0; i < command.length; i++) {
|
||||||
|
const c = command[i];
|
||||||
|
if (quote && c === quote) { quote = ''; continue; }
|
||||||
|
if (!quote && (c === "'" || c === '"')) { quote = c; active = true; continue; }
|
||||||
|
if (c === '\\' && quote !== "'") {
|
||||||
|
const next = command[++i];
|
||||||
|
if (next === undefined) return [];
|
||||||
|
if (quote === '"' && !/[$`"\\\n]/.test(next)) value += '\\';
|
||||||
|
if (next !== '\n') { value += next; active = true; }
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!quote && /[;&|<>()\n]/.test(c)) break;
|
||||||
|
if (!quote && /\s/.test(c)) {
|
||||||
|
if (active) result.push(value);
|
||||||
|
value = ''; active = false; continue;
|
||||||
|
}
|
||||||
|
if (!active && !quote && c === '#') break;
|
||||||
|
value += c; active = true;
|
||||||
|
}
|
||||||
|
if (quote) return [];
|
||||||
|
if (active) result.push(value);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function cronOperand(command: string, taskOnly = false): string | undefined {
|
||||||
|
const args = words(command.trim());
|
||||||
|
if (args[0] === 'task') args.shift();
|
||||||
|
else if (taskOnly && args[0] === 'ql') return;
|
||||||
|
while (args[0] === '-l' || args[0] === '-m') {
|
||||||
|
if (args.shift() === '-m') args.shift();
|
||||||
|
}
|
||||||
|
if (args.length > 1 && /^(?:.*\/)?(?:node|python3?|bash|sh|ts-node(?:-transpile-only)?)$/.test(args[0])) args.shift();
|
||||||
|
return args[0];
|
||||||
|
}
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
import { upcomingSlots } from './cronScheduler';
|
||||||
|
|
||||||
|
type Definition = {
|
||||||
|
schedule?: string;
|
||||||
|
extra_schedules?: { schedule: string }[];
|
||||||
|
isDisabled?: number;
|
||||||
|
saved?: boolean;
|
||||||
|
};
|
||||||
|
const cache = new Map<string, { after: number; times: number[] }>();
|
||||||
|
|
||||||
|
export function previewSchedule(
|
||||||
|
schedule: string,
|
||||||
|
count: number,
|
||||||
|
now: number,
|
||||||
|
): number[] {
|
||||||
|
const key = JSON.stringify([
|
||||||
|
Intl.DateTimeFormat().resolvedOptions().timeZone,
|
||||||
|
schedule,
|
||||||
|
]);
|
||||||
|
const cached = cache.get(key);
|
||||||
|
if (cached && now >= cached.after) {
|
||||||
|
const future = cached.times.filter((time) => time > now);
|
||||||
|
if (future.length >= count) return future.slice(0, count);
|
||||||
|
}
|
||||||
|
const times = upcomingSlots(schedule, now, count);
|
||||||
|
if (cache.size >= 512) cache.delete(cache.keys().next().value!);
|
||||||
|
cache.set(key, { after: now, times });
|
||||||
|
return times;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function withNextRun<T extends Definition>(cron: T) {
|
||||||
|
let nextRunTime: number | null = null;
|
||||||
|
if (
|
||||||
|
cron.schedule &&
|
||||||
|
!cron.isDisabled &&
|
||||||
|
cron.saved !== false &&
|
||||||
|
!cron.schedule.startsWith('@once') &&
|
||||||
|
!cron.schedule.startsWith('@boot')
|
||||||
|
) {
|
||||||
|
try {
|
||||||
|
const now = Date.now();
|
||||||
|
nextRunTime = Math.min(
|
||||||
|
...[
|
||||||
|
cron.schedule,
|
||||||
|
...(cron.extra_schedules || []).map((x) => x.schedule),
|
||||||
|
].map((rule) => previewSchedule(rule, 1, now)[0]),
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
// Legacy invalid rows remain visible and editable.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { ...cron, nextRunTime };
|
||||||
|
}
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
import { readFileSync } from 'fs';
|
||||||
|
import { execFileSync, spawnSync } from 'child_process';
|
||||||
|
|
||||||
|
// Legacy runs can outlive a scheduler restart without an active trigger receipt.
|
||||||
|
export function hasLiveCronProcess(status?: number, pid?: number | null) {
|
||||||
|
if (status !== 0 || !pid || pid < 0) return false; // CrontabStatus.running
|
||||||
|
try {
|
||||||
|
process.kill(pid, 0);
|
||||||
|
return true;
|
||||||
|
} catch (error) {
|
||||||
|
const code = (error as NodeJS.ErrnoException).code;
|
||||||
|
if (code === 'ESRCH') return false;
|
||||||
|
if (code === 'EPERM') return true;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function readBsdProcessField(pid: number, field: 'lstart' | 'ppid') {
|
||||||
|
const result = spawnSync('ps', ['-p', String(pid), '-o', field + '='], {
|
||||||
|
encoding: 'utf8', env: { ...process.env, LC_ALL: 'C', TZ: 'UTC' },
|
||||||
|
});
|
||||||
|
// ENOENT from spawning ps refers to the executable, not the target PID.
|
||||||
|
if (result.error) throw new Error('Cannot run ps: ' + result.error.message, { cause: result.error });
|
||||||
|
const value = result.stdout.trim();
|
||||||
|
if (result.status === 0 && value) return value;
|
||||||
|
// BSD ps exits 1 without output when the selected PID has already exited.
|
||||||
|
if (result.status === 1 && !value && !result.stderr.trim())
|
||||||
|
throw Object.assign(new Error('Process has exited'), { code: 'ESRCH' });
|
||||||
|
throw new Error(`Cannot read ${field} for PID ${pid}: ${result.stderr.trim() || result.signal || 'ps status ' + result.status}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function systemOwner(pid: number) {
|
||||||
|
if (process.platform !== 'linux') {
|
||||||
|
// BSD crontab installations have no procfs. ps exposes the process birth time.
|
||||||
|
process.kill(pid, 0);
|
||||||
|
const birth = readBsdProcessField(pid, 'lstart');
|
||||||
|
return `crond:${pid}:${birth}`;
|
||||||
|
}
|
||||||
|
const stat = readFileSync(`/proc/${pid}/stat`, 'utf8');
|
||||||
|
const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
|
||||||
|
if (['Z', 'X', 'x'].includes(fields[0])) throw Object.assign(new Error('Process has exited'), { code: 'ESRCH' });
|
||||||
|
const startTicks = fields[19];
|
||||||
|
const boot = readFileSync('/proc/sys/kernel/random/boot_id', 'utf8').trim();
|
||||||
|
return `crond:${pid}:${startTicks}:${boot}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
let linuxClockTicks: number | undefined;
|
||||||
|
|
||||||
|
// A start packet may arrive after its PID was recycled, before the replacement reports.
|
||||||
|
export function legacyProcessOwner(pid: number, started: number): string | undefined {
|
||||||
|
try {
|
||||||
|
const identity = systemOwner(pid);
|
||||||
|
const birth = identity.slice(`crond:${pid}:`.length);
|
||||||
|
let bornAt: number;
|
||||||
|
if (process.platform === 'linux') {
|
||||||
|
linuxClockTicks ??= Number(execFileSync('getconf', ['CLK_TCK'], { encoding: 'utf8' }));
|
||||||
|
const boot = /^btime (\d+)$/m.exec(readFileSync('/proc/stat', 'utf8'));
|
||||||
|
if (!boot || !(linuxClockTicks > 0)) throw new Error('Cannot read process birth time');
|
||||||
|
bornAt = Number(boot[1]) + Number(birth.split(':')[0]) / linuxClockTicks;
|
||||||
|
} else {
|
||||||
|
bornAt = Date.parse(birth + ' UTC') / 1000;
|
||||||
|
}
|
||||||
|
// Both the callback and Linux boot time have one-second precision.
|
||||||
|
return Math.floor(bornAt) <= started + 1 ? identity : undefined;
|
||||||
|
} catch (error) {
|
||||||
|
if (['ESRCH', 'ENOENT'].includes((error as NodeJS.ErrnoException).code!)) return undefined;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// PID 1 survives worker restarts, but changes when a container or Linux host restarts.
|
||||||
|
export function pidNamespaceOwner() {
|
||||||
|
return process.platform === 'linux' ? systemOwner(1) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// psTree enumerates asynchronously. Reject a PID already reparented or recycled
|
||||||
|
// between enumeration and the birth-token snapshot.
|
||||||
|
export function isProcessDescendant(pid: number, root: number) {
|
||||||
|
const seen = new Set<number>();
|
||||||
|
while (pid > 1 && !seen.has(pid)) {
|
||||||
|
if (pid === root) return true;
|
||||||
|
seen.add(pid);
|
||||||
|
if (process.platform === 'linux') {
|
||||||
|
const stat = readFileSync(`/proc/${pid}/stat`, 'utf8');
|
||||||
|
pid = Number(stat.slice(stat.lastIndexOf(')') + 2).split(' ')[1]);
|
||||||
|
} else {
|
||||||
|
pid = Number(readBsdProcessField(pid, 'ppid'));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// For an individual legacy instance the task ID alone also matches its siblings.
|
||||||
|
export function verifyCronProcess(pid: number, cronId: number, identity?: string | null, started?: number,
|
||||||
|
instance?: { logPath?: string }) {
|
||||||
|
const conflict = () => Object.assign(new Error('Task process identity could not be verified'), { status: 409 });
|
||||||
|
if (!Number.isInteger(pid) || pid <= 1) throw conflict();
|
||||||
|
if (identity) {
|
||||||
|
// A birth token for this PID proves the original process is gone if the
|
||||||
|
// kernel now reports a different birth. Access errors remain retryable.
|
||||||
|
if (!identity.startsWith(`crond:${pid}:`)) throw conflict();
|
||||||
|
let current: string;
|
||||||
|
try { current = systemOwner(pid); }
|
||||||
|
catch (error) {
|
||||||
|
if ((error as NodeJS.ErrnoException).code === 'ENOENT')
|
||||||
|
throw Object.assign(new Error('Command already exited'), { code: 'ESRCH' });
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
if (current !== identity) throw Object.assign(new Error('Command PID was reused'), { code: 'PID_REUSED', status: 409 });
|
||||||
|
return current;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
if (process.platform === 'linux') {
|
||||||
|
const env = readFileSync(`/proc/${pid}/environ`, 'utf8').split('\0');
|
||||||
|
if (instance && (!instance.logPath || instance.logPath === '/dev/null' ||
|
||||||
|
!env.includes('real_log_path=' + instance.logPath))) throw conflict();
|
||||||
|
if (env.includes('ID=' + cronId) && started) {
|
||||||
|
const boot = /^btime (\d+)$/m.exec(readFileSync('/proc/stat', 'utf8'));
|
||||||
|
const birth = systemOwner(pid);
|
||||||
|
const ticks = Number(birth.split(':')[2]);
|
||||||
|
const hz = Number(execFileSync('getconf', ['CLK_TCK'], { encoding: 'utf8' }));
|
||||||
|
if (boot && hz > 0 && Math.floor(Number(boot[1]) + ticks / hz) <= started + 1) return birth;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
if (['ENOENT', 'ESRCH'].includes((error as NodeJS.ErrnoException).code!)) {
|
||||||
|
// Distinguish a completed child whose receipt is still being written
|
||||||
|
// from a live PID with an unverifiable identity.
|
||||||
|
try { systemOwner(pid); } catch (probe) {
|
||||||
|
if (['ENOENT', 'ESRCH'].includes((probe as NodeJS.ErrnoException).code!))
|
||||||
|
throw Object.assign(new Error('Command already exited'), { code: 'ESRCH' });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw conflict();
|
||||||
|
}
|
||||||
|
|
||||||
|
// After a stop worker exits, descendants may already have been adopted by PID 1.
|
||||||
|
// A saved birth identity still identifies them; walking the dead root no longer can.
|
||||||
|
export async function finishStoppedTree(identities: string[], signalledAt?: number | null) {
|
||||||
|
const surviving = () => identities.flatMap(identity => {
|
||||||
|
const pid = Number(identity.split(':')[1]);
|
||||||
|
if (!Number.isSafeInteger(pid) || pid <= 1) throw new Error('Invalid saved process identity');
|
||||||
|
try { return systemOwner(pid) === identity ? [pid] : []; }
|
||||||
|
catch (error) {
|
||||||
|
if (['ESRCH', 'ENOENT'].includes((error as NodeJS.ErrnoException).code!)) return [];
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
const remainingGrace = Math.min(1000, Math.max(0, (signalledAt ?? Date.now()) + 1000 - Date.now()));
|
||||||
|
const graceDeadline = Date.now() + remainingGrace;
|
||||||
|
while (surviving().length) {
|
||||||
|
const remaining = graceDeadline - Date.now();
|
||||||
|
if (remaining <= 0) break;
|
||||||
|
await new Promise(resolve => setTimeout(resolve, Math.min(20, remaining)));
|
||||||
|
}
|
||||||
|
if (!surviving().length) return;
|
||||||
|
// Revalidate each PID immediately before signalling, including after the grace period.
|
||||||
|
for (const identity of identities) {
|
||||||
|
const pid = Number(identity.split(':')[1]);
|
||||||
|
try {
|
||||||
|
if (systemOwner(pid) === identity) process.kill(pid, 'SIGKILL');
|
||||||
|
} catch (error) {
|
||||||
|
if (!['ESRCH', 'ENOENT'].includes((error as NodeJS.ErrnoException).code!)) throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const deadline = Date.now() + 1000;
|
||||||
|
while (surviving().length) {
|
||||||
|
if (Date.now() >= deadline) throw new Error('Stopped process tree did not exit');
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 20));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
import { upcomingSlots } from './cronScheduler';
|
||||||
|
|
||||||
|
// Find only the latest eligible slot even for a second-level rule after hours
|
||||||
|
// offline. Search by time, so cost is logarithmic in the configured window.
|
||||||
|
export function latestMissedSlot(
|
||||||
|
schedule: string,
|
||||||
|
since: number,
|
||||||
|
until: number,
|
||||||
|
): number | null {
|
||||||
|
let low = Math.floor(since / 1000);
|
||||||
|
let high = Math.floor(until / 1000);
|
||||||
|
let latest: number | null = null;
|
||||||
|
while (low < high) {
|
||||||
|
const middle = Math.floor((low + high) / 2);
|
||||||
|
const [next] = upcomingSlots(schedule, middle * 1000);
|
||||||
|
if (next > until) high = middle;
|
||||||
|
else {
|
||||||
|
latest = next;
|
||||||
|
low = next / 1000;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return latest;
|
||||||
|
}
|
||||||
+147
-7
@@ -1,12 +1,150 @@
|
|||||||
import cronParser from 'cron-parser-v4';
|
import CronExpressionParser from 'cron-parser';
|
||||||
|
import { validate } from 'node-cron';
|
||||||
import { ScheduleType } from '../interface/schedule';
|
import { ScheduleType } from '../interface/schedule';
|
||||||
|
|
||||||
// Keep validation aligned with node-schedule's locked cron-parser version.
|
const aliases: Record<string, string> = {
|
||||||
// cron-parser 5 accepts syntax (e.g. H) that the scheduler cannot execute.
|
'@yearly': '0 0 0 1 1 *',
|
||||||
|
'@annually': '0 0 0 1 1 *',
|
||||||
|
'@monthly': '0 0 0 1 * *',
|
||||||
|
'@weekly': '0 0 0 * * 0',
|
||||||
|
'@daily': '0 0 0 * * *',
|
||||||
|
'@midnight': '0 0 0 * * *',
|
||||||
|
'@hourly': '0 0 * * * *',
|
||||||
|
'@minutely': '0 * * * * *',
|
||||||
|
};
|
||||||
|
|
||||||
|
export interface CronSchedule {
|
||||||
|
patterns: string[];
|
||||||
|
dayCount: number;
|
||||||
|
nthDay: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validation and registration repeatedly see identical rules in large snapshots.
|
||||||
|
// Cache only immutable syntax, never dates or running tasks, and bound user input.
|
||||||
|
const scheduleCache = new Map<string, CronSchedule>();
|
||||||
|
const daysInMonth = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
|
||||||
|
|
||||||
|
// Canonicalize legacy shorthand, aliases, names and numeric-start steps before
|
||||||
|
// node-cron sees them. Extend legacy aliases explicitly; reject H and bare /N.
|
||||||
|
export function parseCronSchedule(schedule: unknown): CronSchedule {
|
||||||
|
if (typeof schedule !== 'string' || !schedule.trim()) {
|
||||||
|
throw new Error('Invalid cron schedule');
|
||||||
|
}
|
||||||
|
let source = schedule.trim();
|
||||||
|
const cacheKey = source;
|
||||||
|
const cached = scheduleCache.get(cacheKey);
|
||||||
|
if (cached) return cached;
|
||||||
|
if (source.startsWith('@')) {
|
||||||
|
if (!Object.hasOwn(aliases, source)) throw new Error('Invalid cron alias');
|
||||||
|
source = aliases[source];
|
||||||
|
}
|
||||||
|
let parts = source.split(/\s+/);
|
||||||
|
if (
|
||||||
|
parts.length > 6 ||
|
||||||
|
parts.some((part) => /(^|,)\//.test(part) || /H(?:\(|\/|$)/i.test(part))
|
||||||
|
) {
|
||||||
|
throw new Error('Unsupported cron syntax');
|
||||||
|
}
|
||||||
|
parts = [
|
||||||
|
...['0', '*', '*', '*', '*', '*'].slice(0, 6 - parts.length),
|
||||||
|
...parts,
|
||||||
|
];
|
||||||
|
if (
|
||||||
|
parts.some(
|
||||||
|
(part, index) => index !== 3 && index !== 5 && part.includes('?'),
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
throw new Error('Question mark is only valid in day fields');
|
||||||
|
}
|
||||||
|
const expression = CronExpressionParser.parse(parts.join(' '));
|
||||||
|
if (!expression.hasNext())
|
||||||
|
throw new Error('Cron schedule has no next execution');
|
||||||
|
let normalized = parts.map((part) => part.replace(/\?/g, '*'));
|
||||||
|
// cron-parser includes Sunday in stepped ranges ending in 7 even when the
|
||||||
|
// step does not land on 7. node-cron accepts those ranges with different days.
|
||||||
|
if (/(^|,)(?:\d+|[a-z]{3})-0*7\/\d+(?=,|$)/i.test(normalized[5])) {
|
||||||
|
normalized[5] = expression.fields.dayOfWeek.values.join(',');
|
||||||
|
}
|
||||||
|
// Most rules are already supported; only inspect individual fields on failure.
|
||||||
|
if (!validate(normalized.join(' '))) {
|
||||||
|
const fields = [
|
||||||
|
expression.fields.second,
|
||||||
|
expression.fields.minute,
|
||||||
|
expression.fields.hour,
|
||||||
|
expression.fields.dayOfMonth,
|
||||||
|
expression.fields.month,
|
||||||
|
expression.fields.dayOfWeek,
|
||||||
|
];
|
||||||
|
normalized = normalized.map((value, index) => {
|
||||||
|
// Preserve supported lists, ranges and names. Expand only fields node-cron
|
||||||
|
// cannot consume, such as numeric/name-start steps; never compress them.
|
||||||
|
const probe = ['0', '0', '0', '*', '*', '*'];
|
||||||
|
probe[index] = value;
|
||||||
|
if (validate(probe.join(' '))) return value;
|
||||||
|
const values = fields[index].values;
|
||||||
|
if (index === 5 && expression.fields.dayOfWeek.nthDay) {
|
||||||
|
// The legacy normalizer treated *#N / ?#N as an unrestricted weekday.
|
||||||
|
if (values.length === 8) return '*';
|
||||||
|
// # applies to a single weekday; normalize its numeric spelling only.
|
||||||
|
return values.length === 1
|
||||||
|
? `${values[0]}#${expression.fields.dayOfWeek.nthDay}`
|
||||||
|
: value;
|
||||||
|
}
|
||||||
|
return values.join(',');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const months = expression.fields.month.values;
|
||||||
|
// cron-parser 4 clipped dates outside a single selected month's maximum
|
||||||
|
// before deciding whether the day field covered the whole month.
|
||||||
|
const dayCount = expression.fields.dayOfMonth.values.filter(
|
||||||
|
(day) =>
|
||||||
|
months.length !== 1 ||
|
||||||
|
typeof day !== 'number' ||
|
||||||
|
day <= daysInMonth[months[0] - 1],
|
||||||
|
).length;
|
||||||
|
const weekCount = expression.fields.dayOfWeek.values.length;
|
||||||
|
// Preserve the legacy count-based wildcard rules, including repeated Sunday
|
||||||
|
// and L tokens: an ignored day field must not remain an AND constraint.
|
||||||
|
const weekdayPattern = [
|
||||||
|
...normalized.slice(0, 3),
|
||||||
|
'*',
|
||||||
|
...normalized.slice(4),
|
||||||
|
].join(' ');
|
||||||
|
const patterns =
|
||||||
|
weekCount === 8
|
||||||
|
? [[...normalized.slice(0, 5), '*'].join(' ')]
|
||||||
|
: dayCount < 31
|
||||||
|
? [[...normalized.slice(0, 5), '*'].join(' '), weekdayPattern]
|
||||||
|
: [weekdayPattern];
|
||||||
|
const valid = patterns.map((pattern) => validate(pattern));
|
||||||
|
if (patterns.length === 2 && !valid[0] && valid[1]) {
|
||||||
|
// A date branch can be impossible for all chosen months while the weekday
|
||||||
|
// branch still runs. Drop it only if its syntax is valid with any month.
|
||||||
|
if (validate([...normalized.slice(0, 4), '*', '*'].join(' '))) {
|
||||||
|
patterns.shift();
|
||||||
|
valid.shift();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!valid.every(Boolean)) {
|
||||||
|
throw new Error('Unsupported cron schedule');
|
||||||
|
}
|
||||||
|
const parsed = {
|
||||||
|
patterns,
|
||||||
|
dayCount,
|
||||||
|
nthDay: expression.fields.dayOfWeek.nthDay,
|
||||||
|
};
|
||||||
|
Object.freeze(patterns);
|
||||||
|
Object.freeze(parsed);
|
||||||
|
if (scheduleCache.size >= 512)
|
||||||
|
scheduleCache.delete(scheduleCache.keys().next().value);
|
||||||
|
scheduleCache.set(cacheKey, parsed);
|
||||||
|
return parsed;
|
||||||
|
}
|
||||||
|
|
||||||
export function isValidCronSchedule(schedule: unknown): schedule is string {
|
export function isValidCronSchedule(schedule: unknown): schedule is string {
|
||||||
if (typeof schedule !== 'string' || !schedule.trim()) return false;
|
|
||||||
try {
|
try {
|
||||||
return cronParser.parseExpression(schedule).hasNext();
|
parseCronSchedule(schedule);
|
||||||
|
return true;
|
||||||
} catch {
|
} catch {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -22,6 +160,8 @@ export function getInvalidCronSchedules(cron: {
|
|||||||
) {
|
) {
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
return [cron.schedule, ...(cron.extra_schedules || []).map((x) => x.schedule)]
|
return [
|
||||||
.filter((schedule) => !isValidCronSchedule(schedule));
|
cron.schedule,
|
||||||
|
...(cron.extra_schedules || []).map((x) => x.schedule),
|
||||||
|
].filter((schedule) => !isValidCronSchedule(schedule));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,142 @@
|
|||||||
|
import { t } from './i18n';
|
||||||
|
import { createTask, ScheduledTask } from 'node-cron';
|
||||||
|
import { parseCronSchedule } from './cronSchedule';
|
||||||
|
|
||||||
|
interface SchedulerLogger {
|
||||||
|
warn(message: string, ...args: unknown[]): unknown;
|
||||||
|
error(message: string, ...args: unknown[]): unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CronJob {
|
||||||
|
start(): void;
|
||||||
|
cancel(): void;
|
||||||
|
}
|
||||||
|
|
||||||
|
function acceptsSlot(parsed: ReturnType<typeof parseCronSchedule>, tasks: ScheduledTask[], index: number, date: Date) {
|
||||||
|
if (tasks.length === 1) return true;
|
||||||
|
// Preserve cron-parser 4's day/weekday OR semantics, including its
|
||||||
|
// month-dependent full-day range and the global nth-week constraint.
|
||||||
|
if (parsed.nthDay && Math.ceil(date.getDate() / 7) !== parsed.nthDay)
|
||||||
|
return false;
|
||||||
|
if (index === 1) return !tasks[0].match(date);
|
||||||
|
const monthDays = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
|
||||||
|
return parsed.dayCount < monthDays[date.getMonth()] || tasks[1].match(date);
|
||||||
|
}
|
||||||
|
|
||||||
|
// node-cron owns timers and calendar calculation; Qinglong owns execution
|
||||||
|
// concurrency and process lifetime. Missed in-process slots join that same queue.
|
||||||
|
export function createCronJob(
|
||||||
|
schedule: string,
|
||||||
|
callback: (date: Date) => unknown | Promise<unknown>,
|
||||||
|
options: { name: string; logger: SchedulerLogger; start?: boolean },
|
||||||
|
): CronJob {
|
||||||
|
const parsed = parseCronSchedule(schedule);
|
||||||
|
const tasks: ScheduledTask[] = [];
|
||||||
|
let cancelled = false;
|
||||||
|
let started = false;
|
||||||
|
const dispatch = async (index: number, date: Date, missed: boolean) => {
|
||||||
|
if (cancelled || !started || !acceptsSlot(parsed, tasks, index, date)) return;
|
||||||
|
if (missed) {
|
||||||
|
options.logger.warn(
|
||||||
|
t('[schedule][补执行迟到任务] 任务: %s, 计划时间: %s'),
|
||||||
|
options.name,
|
||||||
|
date.toISOString(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await callback(date);
|
||||||
|
} catch (error) {
|
||||||
|
options.logger.error(
|
||||||
|
t('[schedule][定时回调失败] 任务: %s, 错误: %s'),
|
||||||
|
options.name,
|
||||||
|
error instanceof Error ? error.message : String(error),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
for (const [index, pattern] of parsed.patterns.entries()) {
|
||||||
|
const task = createTask(
|
||||||
|
pattern,
|
||||||
|
(context) => dispatch(index, context.date, false),
|
||||||
|
{
|
||||||
|
noOverlap: false,
|
||||||
|
missedExecutionTolerance: 5000,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
tasks.push(task);
|
||||||
|
task.on('execution:missed', (context) =>
|
||||||
|
dispatch(index, context.date, true),
|
||||||
|
);
|
||||||
|
// Fail before an existing schedule is cancelled, including impossible dates.
|
||||||
|
task.getNextRuns(1);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
tasks.forEach((task) => task.destroy());
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
const job: CronJob = {
|
||||||
|
start() {
|
||||||
|
if (started || cancelled) return;
|
||||||
|
started = true;
|
||||||
|
tasks.forEach((task) => task.start());
|
||||||
|
},
|
||||||
|
cancel() {
|
||||||
|
if (cancelled) return;
|
||||||
|
cancelled = true;
|
||||||
|
tasks.forEach((task) => task.destroy());
|
||||||
|
},
|
||||||
|
};
|
||||||
|
if (options.start !== false) job.start();
|
||||||
|
return job;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// node-cron 4.6.0 exposes getNextRuns only from the wall clock. Its pinned
|
||||||
|
// inline matcher also accepts an explicit cursor, needed for previews and recovery.
|
||||||
|
export function upcomingSlots(schedule: string, after: number, count = 1, through?: number): number[] {
|
||||||
|
const parsed = parseCronSchedule(schedule);
|
||||||
|
const tasks: ScheduledTask[] = [];
|
||||||
|
try {
|
||||||
|
const matchers = parsed.patterns.map((pattern) => {
|
||||||
|
const task = createTask(pattern, () => {});
|
||||||
|
tasks.push(task);
|
||||||
|
return (task as ScheduledTask & {
|
||||||
|
timeMatcher: { getNextMatch(date: Date): Date; expressions: number[][] };
|
||||||
|
}).timeMatcher;
|
||||||
|
});
|
||||||
|
const next = matchers.map((matcher) => matcher.getNextMatch(new Date(after)));
|
||||||
|
// All calendar branches share their clock fields. For bounded audit ranges,
|
||||||
|
// resolve each matching minute once, including the matcher's DST choice.
|
||||||
|
const seconds = through == null ? [] : [...new Set(matchers[0].expressions[0])].sort((a,b) => a-b);
|
||||||
|
const slots: number[] = [];
|
||||||
|
let horizon = after + 366 * 100 * 86400000;
|
||||||
|
while (slots.length < count) {
|
||||||
|
const time = Math.min(...next.map(Number));
|
||||||
|
if (through != null && time > through) break;
|
||||||
|
if (!Number.isFinite(time) || time > horizon) throw new Error('No matching calendar date');
|
||||||
|
let accepted = false;
|
||||||
|
next.forEach((date, index) => {
|
||||||
|
if (+date !== time) return;
|
||||||
|
const valid = acceptsSlot(parsed, tasks, index, date);
|
||||||
|
accepted ||= valid;
|
||||||
|
// Both branches share the clock fields. Rejection applies to the
|
||||||
|
// entire local day, including schedules that fire every second.
|
||||||
|
const cursor = new Date(time);
|
||||||
|
if (!valid) cursor.setHours(23, 59, 59, 999);
|
||||||
|
else if (through != null) cursor.setTime(time - time % 60000 + 59999);
|
||||||
|
next[index] = matchers[index].getNextMatch(cursor);
|
||||||
|
});
|
||||||
|
if (accepted) {
|
||||||
|
if (through == null) slots.push(time);
|
||||||
|
else for (const second of seconds) {
|
||||||
|
const slot = time - time % 60000 + second * 1000;
|
||||||
|
if (slot >= time && slot <= through && slots.length < count) slots.push(slot);
|
||||||
|
}
|
||||||
|
horizon = time + 366 * 100 * 86400000;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return slots;
|
||||||
|
} finally {
|
||||||
|
tasks.forEach((task) => task.destroy());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
import dayjs from 'dayjs';
|
||||||
|
import { QueryTypes, Sequelize, Transaction } from 'sequelize';
|
||||||
|
|
||||||
|
// Hold the receipt claim and the daily increment in the same write transaction.
|
||||||
|
export async function recordCronResult(
|
||||||
|
database: Sequelize, cronId: number, _code: number, elapsedMs: number,
|
||||||
|
finishedAt: number, triggerId: string, transaction?: Transaction,
|
||||||
|
): Promise<void> {
|
||||||
|
if (!triggerId) throw new Error('Execution identity required');
|
||||||
|
if (!transaction) {
|
||||||
|
return database.transaction((tx) =>
|
||||||
|
recordCronResult(database, cronId, _code, elapsedMs, finishedAt, triggerId, tx));
|
||||||
|
}
|
||||||
|
const bind = { ref: cronId, id: triggerId };
|
||||||
|
const claimed = await database.query(
|
||||||
|
"UPDATE CronTriggers SET counted=1,exit_pending=NULL WHERE cron_id=$ref AND id=$id AND counted=0 AND started_at IS NOT NULL AND state IN ('finished','failed','stopped')",
|
||||||
|
{ bind, type: QueryTypes.BULKUPDATE, transaction },
|
||||||
|
);
|
||||||
|
if (!claimed) {
|
||||||
|
const rows = await database.query('SELECT 1 FROM CronTriggers WHERE cron_id=$ref AND id=$id',
|
||||||
|
{ bind, type: QueryTypes.SELECT, transaction });
|
||||||
|
if (!rows.length) throw new Error('Unknown trigger');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const [receipt] = await database.query<{ state: string; finished_at: number; reversible: number }>(
|
||||||
|
`SELECT state,finished_at,EXISTS(SELECT 1 FROM CronLegacyExitBookings WHERE trigger_id=$id) AS reversible
|
||||||
|
FROM CronTriggers WHERE cron_id=$ref AND id=$id`,
|
||||||
|
{ bind, type: QueryTypes.SELECT, transaction });
|
||||||
|
const day = dayjs(receipt.finished_at ?? finishedAt).format('YYYY-MM-DD');
|
||||||
|
await addDailyExit(database, cronId, day, receipt.state, elapsedMs, Boolean(receipt.reversible), transaction);
|
||||||
|
if (receipt.reversible) {
|
||||||
|
await database.query(`UPDATE CronLegacyExitBookings SET stopped=$stopped,
|
||||||
|
stat_id=(SELECT id FROM CrontabStats WHERE ref_id=$ref AND date=$day
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM CronStatRetention WHERE id=1 AND first_day>$day))
|
||||||
|
WHERE trigger_id=$id`, { bind: { ...bind, day, stopped: Number(receipt.state === 'stopped') }, transaction });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function addDailyExit(database: Sequelize, cronId: number, day: string, state: string,
|
||||||
|
elapsedMs: number, reversible: boolean, transaction: Transaction) {
|
||||||
|
await database.query(
|
||||||
|
`INSERT INTO "CrontabStats"
|
||||||
|
(ref_id, date, run_count, success_count, fail_count, total_time, max_time, fixed_max_time, createdAt, updatedAt)
|
||||||
|
SELECT $id, $date, 1, $success, $fail, $elapsed, $elapsed, $fixedMax, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
|
||||||
|
WHERE NOT EXISTS (SELECT 1 FROM CronStatRetention WHERE id=1 AND first_day>$date)
|
||||||
|
ON CONFLICT(ref_id, date) DO UPDATE SET
|
||||||
|
run_count = COALESCE(run_count, 0) + 1,
|
||||||
|
success_count = COALESCE(success_count, 0) + excluded.success_count,
|
||||||
|
fail_count = COALESCE(fail_count, 0) + excluded.fail_count,
|
||||||
|
total_time = COALESCE(total_time, 0) + excluded.total_time,
|
||||||
|
max_time = MAX(COALESCE(max_time, 0), excluded.max_time),
|
||||||
|
fixed_max_time = MAX(COALESCE(fixed_max_time,max_time,0),excluded.fixed_max_time),
|
||||||
|
updatedAt = excluded.updatedAt`,
|
||||||
|
{ transaction, bind: { id: cronId, date: day, fixedMax: reversible ? 0 : elapsedMs,
|
||||||
|
success: Number(state === 'finished'), fail: Number(state === 'failed'), elapsed: elapsedMs } },
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
import { EventEmitter } from 'events';
|
||||||
|
import { QueueAddOptions } from 'p-queue-cjs';
|
||||||
|
|
||||||
|
type Job = () => Promise<unknown>;
|
||||||
|
export interface CronQueueOptions extends QueueAddOptions {
|
||||||
|
waitingSignal?: EventEmitter;
|
||||||
|
onRemoved?: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
// p-queue checks its signal only after dequeue. Remove expired waits while the
|
||||||
|
// execution slots are still occupied, without aborting a running child.
|
||||||
|
export class CronWaitQueue {
|
||||||
|
private entries: { run: Job; priority: number; detach: () => void }[] = [];
|
||||||
|
|
||||||
|
get size() { return this.entries.length; }
|
||||||
|
|
||||||
|
enqueue(run: Job, options: Partial<CronQueueOptions> = {}) {
|
||||||
|
const { priority = 0, waitingSignal, onRemoved } = options;
|
||||||
|
const remove = () => {
|
||||||
|
const index = this.entries.indexOf(entry);
|
||||||
|
if (index < 0) return;
|
||||||
|
this.entries.splice(index, 1);
|
||||||
|
onRemoved?.();
|
||||||
|
};
|
||||||
|
const entry = { run, priority, detach: () => waitingSignal?.off('expired', remove) };
|
||||||
|
const last = this.entries[this.size - 1];
|
||||||
|
if (!last || last.priority >= priority) this.entries.push(entry);
|
||||||
|
else this.entries.splice(this.entries.findIndex((job) => job.priority < priority), 0, entry);
|
||||||
|
waitingSignal?.once('expired', remove);
|
||||||
|
}
|
||||||
|
|
||||||
|
dequeue() {
|
||||||
|
const next = this.entries.shift();
|
||||||
|
next?.detach();
|
||||||
|
return next?.run;
|
||||||
|
}
|
||||||
|
|
||||||
|
filter(options: Partial<CronQueueOptions>) {
|
||||||
|
return this.entries.filter((entry) => entry.priority === options.priority).map((entry) => entry.run);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,111 @@
|
|||||||
|
import { promises as fs } from 'fs';
|
||||||
|
import path from 'path';
|
||||||
|
import { resolveFileAccess } from './fileAccess';
|
||||||
|
|
||||||
|
export type DirectoryVersion = [string, string | null];
|
||||||
|
|
||||||
|
function isGone(error: unknown) {
|
||||||
|
const code = (error as NodeJS.ErrnoException).code;
|
||||||
|
return code === 'ENOENT' || code === 'ENOTDIR';
|
||||||
|
}
|
||||||
|
|
||||||
|
async function directoryVersion(root: string, parent: string, blacklist: string[]) {
|
||||||
|
const dir = parent ? resolveFileAccess(root, [parent], blacklist) : root;
|
||||||
|
const denied = () => Object.assign(new Error('Invalid directory'), { status: 403 });
|
||||||
|
if (!dir) throw denied();
|
||||||
|
try {
|
||||||
|
if (await fs.realpath(dir) !== path.join(root, parent)) throw denied();
|
||||||
|
const stat = await fs.lstat(dir, { bigint: true });
|
||||||
|
return stat.isDirectory() ? [stat.dev, stat.ino, stat.mtimeNs, stat.ctimeNs].join(':') : null;
|
||||||
|
} catch (error) {
|
||||||
|
if (isGone(error)) return null;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function checkDirectoryVersions(root: string, blacklist: string[], versions: DirectoryVersion[]) {
|
||||||
|
const realRoot = await fs.realpath(root);
|
||||||
|
const matches = await Promise.all(versions.map(async ([parent, expected]) => {
|
||||||
|
try {
|
||||||
|
return await directoryVersion(realRoot, parent, blacklist) === expected;
|
||||||
|
} catch (error) {
|
||||||
|
// A formerly accessible directory may now be a symlink outside the root.
|
||||||
|
if ((error as { status?: number }).status === 403) return false;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
return matches.every(Boolean);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function readDirectoryPage(root: string, blacklist: string[], directory: string, after: string,
|
||||||
|
search: string, foldersOnly: boolean, newestFirst: boolean, limit = 50) {
|
||||||
|
const blocked = new Set(blacklist.map(name => name.toLowerCase()));
|
||||||
|
const realRoot = await fs.realpath(root);
|
||||||
|
const versions: DirectoryVersion[] = [];
|
||||||
|
async function entries(parent: string) {
|
||||||
|
const version = await directoryVersion(realRoot, parent, blacklist);
|
||||||
|
versions.push([parent, version]);
|
||||||
|
if (version === null) return [];
|
||||||
|
try {
|
||||||
|
return (await fs.readdir(path.join(realRoot, parent), { withFileTypes: true })).filter(e =>
|
||||||
|
!blocked.has(e.name.toLowerCase()) && (e.isDirectory() || (!foldersOnly && e.isFile())));
|
||||||
|
} catch (error) {
|
||||||
|
if (isGone(error)) return [];
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const compare = (a: string, b: string) => a < b ? -1 : a > b ? 1 : 0;
|
||||||
|
const rows = [];
|
||||||
|
let next: string | null = null;
|
||||||
|
async function row(parent: string, title: string, directory: boolean) {
|
||||||
|
const key = path.join(parent, title);
|
||||||
|
if (directory) return { title, parent, key, type: 'directory', isLeaf: false };
|
||||||
|
try {
|
||||||
|
const stat = await fs.lstat(path.join(root, key));
|
||||||
|
if (stat.isFile()) return { title, parent, key, type: 'file', isLeaf: true, size: stat.size };
|
||||||
|
} catch (error) { if (!isGone(error)) throw error; }
|
||||||
|
}
|
||||||
|
if (!search) {
|
||||||
|
const sorted = (await entries(directory)).sort((a,b) =>
|
||||||
|
Number(b.isDirectory()) - Number(a.isDirectory()) ||
|
||||||
|
compare(a.name, b.name) * (newestFirst && !a.isDirectory() ? -1 : 1));
|
||||||
|
const selected = sorted.filter(e => {
|
||||||
|
if (!after) return true;
|
||||||
|
const group = e.isDirectory() ? '0' : '1';
|
||||||
|
return group > after[0] || (group === after[0] &&
|
||||||
|
compare(e.name, after.slice(1)) * (newestFirst && !e.isDirectory() ? -1 : 1) > 0);
|
||||||
|
});
|
||||||
|
const page = selected.slice(0, limit);
|
||||||
|
const data = (await Promise.all(page.map(e => row(directory, e.name, e.isDirectory())))).filter(Boolean);
|
||||||
|
if (selected.length > limit) {
|
||||||
|
const last = page[page.length - 1];
|
||||||
|
next = (last.isDirectory() ? '0' : '1') + last.name;
|
||||||
|
}
|
||||||
|
return { data, next, versions };
|
||||||
|
}
|
||||||
|
// Resume a depth-first search at the last visited entry, including on pages with zero matches.
|
||||||
|
async function* walk(parent: string, resume: string[]): AsyncGenerator<{ parent: string; name: string; dir: boolean }> {
|
||||||
|
const items = (await entries(parent)).sort((a,b) => compare(a.name,b.name));
|
||||||
|
for (const item of items) {
|
||||||
|
if (resume.length && item.name < resume[0]) continue;
|
||||||
|
const same = item.name === resume[0];
|
||||||
|
if (!same) yield { parent, name: item.name, dir: item.isDirectory() };
|
||||||
|
if (item.isDirectory()) {
|
||||||
|
try { yield* walk(path.join(parent, item.name), same ? resume.slice(1) : []); }
|
||||||
|
catch (error) {
|
||||||
|
if (!isGone(error) && (error as { status?: number }).status !== 403) throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let visited = 0;
|
||||||
|
for await (const entry of walk('', after ? after.split('/') : [])) {
|
||||||
|
const key = path.join(entry.parent, entry.name);
|
||||||
|
if (key.toLowerCase().includes(search.toLowerCase())) {
|
||||||
|
const item = await row(entry.parent, entry.name, entry.dir);
|
||||||
|
if (item) rows.push(item);
|
||||||
|
}
|
||||||
|
if (++visited >= 512 || rows.length === limit) { next = key; break; }
|
||||||
|
}
|
||||||
|
return { data: rows, next, versions };
|
||||||
|
}
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
import { celebrate, Joi } from 'celebrate';
|
||||||
|
import { Request, Response, NextFunction } from 'express';
|
||||||
|
import { checkDirectoryVersions, readDirectoryPage } from './directoryPage';
|
||||||
|
|
||||||
|
export function directoryCheckHandler(root: () => string, blacklist: string[]) {
|
||||||
|
return [
|
||||||
|
celebrate({ body: Joi.array().items(Joi.array().ordered(
|
||||||
|
Joi.string().allow('').max(4096).required(), Joi.string().max(128).allow(null).required(),
|
||||||
|
).length(2)).min(1).max(128).required() }),
|
||||||
|
async (req: Request, res: Response, next: NextFunction) => {
|
||||||
|
try {
|
||||||
|
const current = await checkDirectoryVersions(root(), blacklist, req.body);
|
||||||
|
res.send({ code: 200, current });
|
||||||
|
} catch (error) { next(error); }
|
||||||
|
},
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function directoryPageHandler(root: () => string, blacklist: string[], newestFirst: boolean) {
|
||||||
|
return [
|
||||||
|
celebrate({ query: Joi.object({
|
||||||
|
path: Joi.string().allow(''), after: Joi.string().allow(''), search: Joi.string().allow(''),
|
||||||
|
folders: Joi.boolean(), t: Joi.string(),
|
||||||
|
}) }),
|
||||||
|
async (req: Request, res: Response, next: NextFunction) => {
|
||||||
|
try {
|
||||||
|
const { path = '', after = '', search = '', folders } = req.query;
|
||||||
|
const page = await readDirectoryPage(root(), blacklist, String(path), String(after),
|
||||||
|
String(search), String(folders) === 'true', newestFirst);
|
||||||
|
res.send({ code: 200, ...page });
|
||||||
|
} catch (error) { next(error); }
|
||||||
|
},
|
||||||
|
];
|
||||||
|
}
|
||||||
@@ -2,3 +2,10 @@
|
|||||||
export function errStack(error: unknown): string {
|
export function errStack(error: unknown): string {
|
||||||
return error instanceof Error && error.stack ? error.stack : String(error);
|
return error instanceof Error && error.stack ? error.stack : String(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export class PendingCronInstall extends Error {
|
||||||
|
readonly status = 503;
|
||||||
|
constructor(readonly cronId: number, message: string) {
|
||||||
|
super(message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
// The original scheduled CLI wire format maps to its existing persisted trigger ID.
|
||||||
|
export const executionIdPattern = /^(?:client-|legacy-system:\d{1,12}:)[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/;
|
||||||
|
export function executionTriggerId(id: string) {
|
||||||
|
return id.replace('legacy-system:', 'system-cli-').replace(':', '-');
|
||||||
|
}
|
||||||
|
|
||||||
|
// A header lets upgraded clients keep the same identified body on older panels.
|
||||||
|
export function receiptFinishTime(header?: string) {
|
||||||
|
if (header && /^\d{1,13}$/.test(header)) {
|
||||||
|
const time = Number(header);
|
||||||
|
if (time > 0 && time <= Date.now()) return time;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -24,8 +24,11 @@ export function resolveFileAccess(
|
|||||||
const resolvedRoot = path.resolve(root);
|
const resolvedRoot = path.resolve(root);
|
||||||
const target = path.resolve(resolvedRoot, ...parts);
|
const target = path.resolve(resolvedRoot, ...parts);
|
||||||
if (target === resolvedRoot || !isWithin(resolvedRoot, target)) return '';
|
if (target === resolvedRoot || !isWithin(resolvedRoot, target)) return '';
|
||||||
|
// File systems such as default APFS can resolve a differently cased spelling
|
||||||
|
// of the same entry. Check every protected name without case sensitivity.
|
||||||
|
const blockedNames = new Set(blacklist.map((part) => part.toLowerCase()));
|
||||||
const isBlocked = (relative: string) =>
|
const isBlocked = (relative: string) =>
|
||||||
relative.split(path.sep).some((part) => blacklist.includes(part));
|
relative.split(path.sep).some((part) => blockedNames.has(part.toLowerCase()));
|
||||||
if (isBlocked(path.relative(resolvedRoot, target))) return '';
|
if (isBlocked(path.relative(resolvedRoot, target))) return '';
|
||||||
try {
|
try {
|
||||||
const realRoot = fs.realpathSync(resolvedRoot);
|
const realRoot = fs.realpathSync(resolvedRoot);
|
||||||
|
|||||||
+230
-91
@@ -1,111 +1,238 @@
|
|||||||
import { shareStore } from './store';
|
|
||||||
|
|
||||||
const messages: Record<string, Record<string, string>> = {
|
const messages: Record<string, Record<string, string>> = {
|
||||||
zh: {},
|
zh: {},
|
||||||
en: {
|
en: {
|
||||||
'暂无权限': 'Access denied',
|
'系统定时规则安装失败,旧规则可能仍在运行,正在重试': 'System crontab installation failed. The previous rules may still run. Retrying.',
|
||||||
'参数错误': 'Invalid parameter',
|
'Cron表达式格式有误': 'Invalid cron expression',
|
||||||
'参数不正确': 'Invalid parameter',
|
'升级任务启动失败,请检查日志后重试':
|
||||||
'文件无法访问': 'File not accessible',
|
'Failed to start the update. Check the logs and try again.',
|
||||||
'文件不存在': 'File not found',
|
'[schedule][停止已运行任务] 任务ID: %s, PID: %s':
|
||||||
'路径不存在': 'Path does not exist',
|
'[schedule][Stopping running task] Task ID: %s, PID: %s',
|
||||||
'路径不正确': 'Invalid path',
|
|
||||||
|
'接口已下线,请使用 /configs/detail 接口':
|
||||||
|
'This endpoint has been retired. Use /configs/detail instead.',
|
||||||
|
'接口已下线,请使用 /logs/detail 接口':
|
||||||
|
'This endpoint has been retired. Use /logs/detail instead.',
|
||||||
|
'接口已下线,请使用 /scripts/detail 接口':
|
||||||
|
'This endpoint has been retired. Use /scripts/detail instead.',
|
||||||
|
'仅支持 PNG、JPEG、GIF、WebP、AVIF 图片':
|
||||||
|
'Only PNG, JPEG, GIF, WebP and AVIF images are supported',
|
||||||
|
请先初始化: 'Please initialize the panel first',
|
||||||
|
请先关闭两步验证: 'Please disable two-factor authentication first',
|
||||||
|
'失败次数过多,请稍后重试':
|
||||||
|
'Too many failed attempts. Please try again later.',
|
||||||
|
配置更新失败: 'Failed to update configuration',
|
||||||
|
'trust proxy 配置格式无效': 'Invalid trust proxy configuration',
|
||||||
|
'代理层数不能超过 20': 'Proxy hop count cannot exceed 20',
|
||||||
|
'环境变量 QL_TRUST_PROXY 已生效,系统设置不可覆盖':
|
||||||
|
'QL_TRUST_PROXY is active and cannot be overridden by system settings',
|
||||||
|
环境变量名称不可使用: 'Environment variable name is reserved or invalid',
|
||||||
|
系统配置不存在: 'System configuration does not exist',
|
||||||
|
'历史版本仅支持不超过 1 MiB 的 UTF-8 文本脚本':
|
||||||
|
'Version history only supports UTF-8 text scripts up to 1 MiB',
|
||||||
|
脚本不存在: 'Script does not exist',
|
||||||
|
请选择脚本文件: 'Please select a script file',
|
||||||
|
'历史版本读取失败,请检查历史文件':
|
||||||
|
'Failed to read version history. Check the history file.',
|
||||||
|
历史版本不存在或已清理:
|
||||||
|
'This history version does not exist or has been removed',
|
||||||
|
'文件已变化,请刷新历史版本后重试':
|
||||||
|
'The file has changed. Refresh version history and try again.',
|
||||||
|
'历史版本容量不足,无法安全保存':
|
||||||
|
'Version history storage is full. The file cannot be saved safely.',
|
||||||
|
'跳过订阅 %s 的无效定时配置: %s':
|
||||||
|
'Skipping invalid schedule for subscription %s: %s',
|
||||||
|
'[schedule][准备运行任务] 命令: %s':
|
||||||
|
'[schedule][Preparing task] Command: %s',
|
||||||
|
'[schedule][创建定时任务] 任务ID: %s, 规则数: %s':
|
||||||
|
'[schedule][Creating scheduled task] Task ID: %s, schedules: %s',
|
||||||
|
'[schedule][取消定时任务] 任务ID: %s':
|
||||||
|
'[schedule][Cancelling scheduled task] Task ID: %s',
|
||||||
|
'[schedule][取消任务失败] 任务ID: %s, 错误: %s':
|
||||||
|
'[schedule][Failed to cancel task] Task ID: %s, error: %s',
|
||||||
|
'[panel][停止任务失败] PID: %s, 错误: %s':
|
||||||
|
'[panel][Failed to stop task] PID: %s, error: %s',
|
||||||
|
'[panel][停止实例失败] 实例ID: %s, PID: %s, 错误: %s':
|
||||||
|
'[panel][Failed to stop instance] Instance ID: %s, PID: %s, error: %s',
|
||||||
|
'[panel][执行任务失败] 任务ID: %s, 错误: %s':
|
||||||
|
'[panel][Task failed] Task ID: %s, error: %s',
|
||||||
|
'[panel][执行任务结束] 任务ID: %s, 退出码: %j':
|
||||||
|
'[panel][Task finished] Task ID: %s, exit code: %j',
|
||||||
|
'[panel][创建任务失败] 任务ID: %s, 错误: %s':
|
||||||
|
'[panel][Failed to create task] Task ID: %s, error: %s',
|
||||||
|
'[panel][关闭任务日志失败] %s': '[panel][Failed to close task log] %s',
|
||||||
|
'[panel][清理任务状态失败] %s': '[panel][Failed to clear task status] %s',
|
||||||
|
'性能指标报告:': 'Performance metrics:',
|
||||||
|
'[panel][开始执行任务] 任务ID: %s': '[panel][Starting task] Task ID: %s',
|
||||||
|
'[panel][任务错误回调失败] %s': '[panel][Task error callback failed] %s',
|
||||||
|
'[panel][任务结束回调失败] %s':
|
||||||
|
'[panel][Task completion callback failed] %s',
|
||||||
|
'[panel][任务队列失败] %s': '[panel][Task queue failed] %s',
|
||||||
|
'[panel][创建cron任务] 任务ID: %s, cron: %s, 任务名: %s, 执行命令: %s':
|
||||||
|
'[panel][Creating cron task] Task ID: %s, cron: %s, name: %s, command: %s',
|
||||||
|
'[panel][跳过无效定时任务] 任务ID: %s, cron: %s, 错误: %s':
|
||||||
|
'[panel][Skipping invalid schedule] Task ID: %s, cron: %s, error: %s',
|
||||||
|
'[panel][取消定时任务] 任务名: %s':
|
||||||
|
'[panel][Cancelling scheduled task] Name: %s',
|
||||||
|
'[panel][创建interval任务] 任务ID: %s, 任务名: %s, 执行命令: %s':
|
||||||
|
'[panel][Creating interval task] Task ID: %s, name: %s, command: %s',
|
||||||
|
'[panel][执行任务失败] 命令: %s, 错误信息: %j':
|
||||||
|
'[panel][Task failed] Command: %s, error: %j',
|
||||||
|
'[panel][取消interval任务] 任务ID: %s, 任务名: %s':
|
||||||
|
'[panel][Cancelling interval task] Task ID: %s, name: %s',
|
||||||
|
生成私钥文件失败: 'Failed to generate private key file',
|
||||||
|
删除私钥文件失败: 'Failed to delete private key file',
|
||||||
|
'删除ssh配置文件%s失败': 'Failed to delete SSH configuration file %s',
|
||||||
|
'跳过订阅 %s 的无效 SSH 配置: %s':
|
||||||
|
'Skipping invalid SSH configuration for subscription %s: %s',
|
||||||
|
'[schedule][补执行迟到任务] 任务: %s, 计划时间: %s':
|
||||||
|
'[schedule][Running late task] Task: %s, scheduled time: %s',
|
||||||
|
'[schedule][定时回调失败] 任务: %s, 错误: %s':
|
||||||
|
'[schedule][Scheduled callback failed] Task: %s, error: %s',
|
||||||
|
'[schedule][任务队列] 空闲中...': '[schedule][Task queue] Idle...',
|
||||||
|
'[schedule][任务加入队列] 运行中任务数: %s, 等待中任务数: %s':
|
||||||
|
'[schedule][Task queued] Running: %s, waiting: %s',
|
||||||
|
'[schedule][开始处理任务] 运行中任务数: %s, 等待中任务数: %s':
|
||||||
|
'[schedule][Processing task] Running: %s, waiting: %s',
|
||||||
|
'[schedule][任务处理成功] 参数 %s':
|
||||||
|
'[schedule][Task succeeded] Arguments: %s',
|
||||||
|
'[schedule][任务处理错误] 参数 %s': '[schedule][Task failed] Arguments: %s',
|
||||||
|
'[schedule][任务处理结束] 运行中任务数: %s, 等待中任务数: %s':
|
||||||
|
'[schedule][Task finished] Running: %s, waiting: %s',
|
||||||
|
'[schedule][任务重复运行] 通知失败 %s':
|
||||||
|
'[schedule][Duplicate task] Notification failed: %s',
|
||||||
|
'[schedule][任务重复运行] 参数 %s':
|
||||||
|
'[schedule][Duplicate task] Arguments: %s',
|
||||||
|
'[schedule][停止已运行任务] 任务ID: %s':
|
||||||
|
'[schedule][Stopping running task] Task ID: %s',
|
||||||
|
'[schedule][检查已运行任务失败] 任务ID: %s, 错误: %s':
|
||||||
|
'[schedule][Failed to check running task] Task ID: %s, error: %s',
|
||||||
|
'[schedule][开始执行任务] 参数 %s':
|
||||||
|
'[schedule][Starting task] Arguments: %s',
|
||||||
|
'[schedule][任务标准错误] 命令: %s, 信息: %s':
|
||||||
|
'[schedule][Task stderr] Command: %s, message: %s',
|
||||||
|
'[schedule][执行任务失败] 命令: %s, 错误: %s':
|
||||||
|
'[schedule][Task failed] Command: %s, error: %s',
|
||||||
|
'[schedule][执行任务结束] 任务ID: %s, 退出码: %j':
|
||||||
|
'[schedule][Task finished] Task ID: %s, exit code: %j',
|
||||||
|
'[schedule][创建任务失败] 命令: %s, 错误: %s':
|
||||||
|
'[schedule][Failed to create task] Command: %s, error: %s',
|
||||||
|
|
||||||
|
'检查更新失败,请检查网络后重试':
|
||||||
|
'Failed to check for updates. Check your network and try again.',
|
||||||
|
'资源已存在,请检查重复的名称或值':
|
||||||
|
'Resource already exists; check for duplicate names or values',
|
||||||
|
暂无权限: 'Access denied',
|
||||||
|
参数错误: 'Invalid parameter',
|
||||||
|
参数不正确: 'Invalid parameter',
|
||||||
|
文件无法访问: 'File not accessible',
|
||||||
|
文件不存在: 'File not found',
|
||||||
|
路径不存在: 'Path does not exist',
|
||||||
|
路径不正确: 'Invalid path',
|
||||||
'通知发送失败,请检查参数': 'Notification failed, check parameters',
|
'通知发送失败,请检查参数': 'Notification failed, check parameters',
|
||||||
'通知发送失败,请检查系统设置/通知配置': 'Notification failed, check system settings',
|
'通知发送失败,请检查系统设置/通知配置':
|
||||||
'通知发送成功': 'Notification sent successfully',
|
'Notification failed, check system settings',
|
||||||
'设置时区失败': 'Failed to set timezone',
|
通知发送成功: 'Notification sent successfully',
|
||||||
'任务未找到': 'Task not found',
|
设置时区失败: 'Failed to set timezone',
|
||||||
'密码不能设置为admin': 'Password cannot be admin',
|
任务未找到: 'Task not found',
|
||||||
'更新成功': 'Update successful',
|
密码不能设置为admin: 'Password cannot be admin',
|
||||||
'未知错误': 'Unknown error',
|
更新成功: 'Update successful',
|
||||||
'验证失败': 'Verification failed',
|
未知错误: 'Unknown error',
|
||||||
'实例不存在或已停止': 'Instance does not exist or stopped',
|
验证失败: 'Verification failed',
|
||||||
'实例已停止': 'Instance stopped',
|
实例不存在或已停止: 'Instance does not exist or stopped',
|
||||||
'确认停止实例': 'Confirm to stop instance',
|
实例已停止: 'Instance stopped',
|
||||||
'确认停止运行实例': 'Confirm to stop running instance',
|
实例已结束: 'Instance already ended',
|
||||||
'确认停止': 'Confirm to stop',
|
确认停止实例: 'Confirm to stop instance',
|
||||||
'确认停止定时任务': 'Confirm to stop scheduled task',
|
确认停止运行实例: 'Confirm to stop running instance',
|
||||||
'确认删除': 'Confirm to delete',
|
确认停止: 'Confirm to stop',
|
||||||
'确认删除定时任务': 'Confirm to delete scheduled task',
|
确认停止定时任务: 'Confirm to stop scheduled task',
|
||||||
'确认删除选中的定时任务吗': 'Confirm to delete selected tasks?',
|
确认删除: 'Confirm to delete',
|
||||||
'确认运行': 'Confirm to run',
|
确认删除定时任务: 'Confirm to delete scheduled task',
|
||||||
'确认运行定时任务': 'Confirm to run scheduled task',
|
确认删除选中的定时任务吗: 'Confirm to delete selected tasks?',
|
||||||
'确认保存': 'Confirm to save',
|
确认运行: 'Confirm to run',
|
||||||
'确认保存文件': 'Confirm to save file',
|
确认运行定时任务: 'Confirm to run scheduled task',
|
||||||
'确认重启': 'Confirm restart',
|
确认保存: 'Confirm to save',
|
||||||
'确认启用': 'Confirm to enable',
|
确认保存文件: 'Confirm to save file',
|
||||||
'确认禁用': 'Confirm to disable',
|
确认重启: 'Confirm restart',
|
||||||
'确认': 'Confirm',
|
确认启用: 'Confirm to enable',
|
||||||
'删除成功': 'Deleted successfully',
|
确认禁用: 'Confirm to disable',
|
||||||
'操作成功': 'Operation successful',
|
确认: 'Confirm',
|
||||||
'参数不完整': 'Incomplete parameters',
|
删除成功: 'Deleted successfully',
|
||||||
'默认路径不支持删除': 'Default path cannot be deleted',
|
操作成功: 'Operation successful',
|
||||||
'必须在日志目录下': 'Must be within log directory',
|
参数不完整: 'Incomplete parameters',
|
||||||
|
默认路径不支持删除: 'Default path cannot be deleted',
|
||||||
|
必须在日志目录下: 'Must be within log directory',
|
||||||
'备份数据上传成功,确认覆盖数据': 'Backup uploaded, confirm overwrite',
|
'备份数据上传成功,确认覆盖数据': 'Backup uploaded, confirm overwrite',
|
||||||
'如果恢复失败,可进入容器执行': 'If restore fails, run in container:',
|
'如果恢复失败,可进入容器执行': 'If restore fails, run in container:',
|
||||||
'系统将在': 'System will',
|
系统将在: 'System will',
|
||||||
'秒后自动刷新': 'refresh in seconds',
|
秒后自动刷新: 'refresh in seconds',
|
||||||
'生成数据中...': 'Generating data...',
|
'生成数据中...': 'Generating data...',
|
||||||
'每条数据 name 或者 value 字段不能为空,参考导出文件格式': 'Each entry must have name and value, see export format',
|
'每条数据 name 或者 value 字段不能为空,参考导出文件格式':
|
||||||
'不支持当前依赖类型': 'Unsupported dependency type',
|
'Each entry must have name and value, see export format',
|
||||||
'依赖已存在': 'Dependency already exists',
|
不支持当前依赖类型: 'Unsupported dependency type',
|
||||||
'依赖不存在': 'Dependency does not exist',
|
依赖已存在: 'Dependency already exists',
|
||||||
'该脚本正在运行中': 'Script is running',
|
依赖不存在: 'Dependency does not exist',
|
||||||
'该脚本未在运行中': 'Script is not running',
|
该脚本正在运行中: 'Script is running',
|
||||||
'文件内容为空': 'File content is empty',
|
该脚本未在运行中: 'Script is not running',
|
||||||
'文件名不能为空': 'File name cannot be empty',
|
文件内容为空: 'File content is empty',
|
||||||
'标签不能为空': 'Label cannot be empty',
|
文件名不能为空: 'File name cannot be empty',
|
||||||
'名称不能为空': 'Name cannot be empty',
|
标签不能为空: 'Label cannot be empty',
|
||||||
'名称不能为保留关键字': 'Name cannot be reserved keyword',
|
名称不能为空: 'Name cannot be empty',
|
||||||
'名称已存在': 'Name already exists',
|
名称不能为保留关键字: 'Name cannot be reserved keyword',
|
||||||
'密码错误': 'Incorrect password',
|
名称已存在: 'Name already exists',
|
||||||
'用户不存在': 'User does not exist',
|
密码错误: 'Incorrect password',
|
||||||
'请输入用户名和密码': 'Please enter username and password',
|
用户不存在: 'User does not exist',
|
||||||
'无权访问': 'Access denied',
|
请输入用户名和密码: 'Please enter username and password',
|
||||||
'登录成功': 'Login successful',
|
无权访问: 'Access denied',
|
||||||
'退出成功': 'Logout successful',
|
登录成功: 'Login successful',
|
||||||
|
退出成功: 'Logout successful',
|
||||||
'Token 已失效': 'Token expired',
|
'Token 已失效': 'Token expired',
|
||||||
'Token 无效': 'Invalid token',
|
'Token 无效': 'Invalid token',
|
||||||
'两步骤验证已开启': '2FA enabled',
|
两步骤验证已开启: '2FA enabled',
|
||||||
'两步骤验证已关闭': '2FA disabled',
|
两步骤验证已关闭: '2FA disabled',
|
||||||
'验证码错误': 'Invalid verification code',
|
验证码错误: 'Invalid verification code',
|
||||||
'验证码已过期': 'Verification code expired',
|
验证码已过期: 'Verification code expired',
|
||||||
'请先开启两步骤验证': 'Please enable 2FA first',
|
请先开启两步骤验证: 'Please enable 2FA first',
|
||||||
'两步骤验证密钥不能为空': '2FA secret cannot be empty',
|
两步骤验证密钥不能为空: '2FA secret cannot be empty',
|
||||||
'用户已存在': 'User already exists',
|
用户已存在: 'User already exists',
|
||||||
'用户名不能为admin': 'Username cannot be admin',
|
用户名不能为admin: 'Username cannot be admin',
|
||||||
'不能删除自己': 'Cannot delete yourself',
|
不能删除自己: 'Cannot delete yourself',
|
||||||
'不能禁用自己': 'Cannot disable yourself',
|
不能禁用自己: 'Cannot disable yourself',
|
||||||
'文件路径无效': 'Invalid file path',
|
文件路径无效: 'Invalid file path',
|
||||||
'保存成功': 'Saved successfully',
|
保存成功: 'Saved successfully',
|
||||||
'client_id 或 client_seret 有误': 'Invalid client_id or client_secret',
|
'client_id 或 client_seret 有误': 'Invalid client_id or client_secret',
|
||||||
'订阅执行完成': 'Subscription completed',
|
订阅执行完成: 'Subscription completed',
|
||||||
'wxPusher 服务的 TopicIds 和 Uids 至少配置一个才行': 'wxPusher requires at least one of TopicIds or Uids',
|
'wxPusher 服务的 TopicIds 和 Uids 至少配置一个才行':
|
||||||
|
'wxPusher requires at least one of TopicIds or Uids',
|
||||||
'wxPusher SPT 不能为空': 'wxPusher SPT cannot be empty',
|
'wxPusher SPT 不能为空': 'wxPusher SPT cannot be empty',
|
||||||
'wxPusher SPT 最多支持 10 个': 'wxPusher SPT supports at most 10 tokens',
|
'wxPusher SPT 最多支持 10 个': 'wxPusher SPT supports at most 10 tokens',
|
||||||
'Url 或者 Body 中必须包含 $title': 'Url or Body must contain $title',
|
'Url 或者 Body 中必须包含 $title': 'Url or Body must contain $title',
|
||||||
'绝对路径必须在日志目录内或使用 /dev/null':
|
'绝对路径必须在日志目录内或使用 /dev/null':
|
||||||
'Absolute path must be within log directory or use /dev/null',
|
'Absolute path must be within log directory or use /dev/null',
|
||||||
'请先登录': 'Please login first',
|
请先登录: 'Please login first',
|
||||||
'运行中...': 'Running...',
|
'运行中...': 'Running...',
|
||||||
'日志不存在...': 'Log does not exist...',
|
'日志不存在...': 'Log does not exist...',
|
||||||
'未分类': 'Uncategorized',
|
未分类: 'Uncategorized',
|
||||||
'任务重复运行': 'Duplicate task execution',
|
任务重复运行: 'Duplicate task execution',
|
||||||
'日志设置为忽略': 'Log set to ignore',
|
日志设置为忽略: 'Log set to ignore',
|
||||||
'定时规则不能为空': 'Schedule rule cannot be empty',
|
定时规则不能为空: 'Schedule rule cannot be empty',
|
||||||
'无效的定时规则': 'Invalid schedule rule',
|
无效的定时规则: 'Invalid schedule rule',
|
||||||
|
定时规则必须是字符串: 'Schedule rule must be a string',
|
||||||
|
额外定时规则必须是对象: 'Extra schedule must be an object',
|
||||||
|
额外定时规则必须是数组: 'Extra schedules must be an array',
|
||||||
'日志名称只能包含中文、字母、数字、下划线、连字符、点和路径分隔符':
|
'日志名称只能包含中文、字母、数字、下划线、连字符、点和路径分隔符':
|
||||||
'Log name can only contain Chinese characters, letters, numbers, underscores, hyphens, dots, and path separators',
|
'Log name can only contain Chinese characters, letters, numbers, underscores, hyphens, dots, and path separators',
|
||||||
'日志名称不能超过100个字符': 'Log name cannot exceed 100 characters',
|
日志名称不能超过100个字符: 'Log name cannot exceed 100 characters',
|
||||||
'错误的用户名密码,请重试': 'Incorrect username or password, please try again',
|
'错误的用户名密码,请重试':
|
||||||
|
'Incorrect username or password, please try again',
|
||||||
'该 IP 已被列入黑名单': 'This IP address has been blocked',
|
'该 IP 已被列入黑名单': 'This IP address has been blocked',
|
||||||
'已加入 IP 黑名单': 'IP address added to the blacklist',
|
'已加入 IP 黑名单': 'IP address added to the blacklist',
|
||||||
'已移出 IP 黑名单': 'IP address removed from the blacklist',
|
'已移出 IP 黑名单': 'IP address removed from the blacklist',
|
||||||
'青龙快讯': 'QingLong',
|
青龙快讯: 'QingLong',
|
||||||
'登录通知': 'Login Notification',
|
登录通知: 'Login Notification',
|
||||||
'你于': 'You at ',
|
你于: 'You at ',
|
||||||
'在': ' in ',
|
在: ' in ',
|
||||||
端: '',
|
端: ' client',
|
||||||
|
内网IP: 'Private IP',
|
||||||
登录失败: 'login failed',
|
登录失败: 'login failed',
|
||||||
',ip地址': ', IP: ',
|
',ip地址': ', IP: ',
|
||||||
'任务#%s': 'Task#%s',
|
'任务#%s': 'Task#%s',
|
||||||
@@ -130,12 +257,15 @@ const messages: Record<string, Record<string, string>> = {
|
|||||||
青龙: 'QingLong',
|
青龙: 'QingLong',
|
||||||
'【蛟龙】测试通知 https://t.me/jiao_long':
|
'【蛟龙】测试通知 https://t.me/jiao_long':
|
||||||
'[JiaoLong] Test notification https://t.me/jiao_long',
|
'[JiaoLong] Test notification https://t.me/jiao_long',
|
||||||
'生成token': 'Generate token',
|
生成token: 'Generate token',
|
||||||
'删除日志': 'Delete logs',
|
删除日志: 'Delete logs',
|
||||||
'## 开始执行... %s\n': '## Start executing... %s\n',
|
'## 开始执行... %s\n': '## Start executing... %s\n',
|
||||||
'执行before命令...': 'Execute before command...',
|
'执行before命令...': 'Execute before command...',
|
||||||
'执行after命令...': 'Execute after command...',
|
'执行after命令...': 'Execute after command...',
|
||||||
'## 执行结束... %s 耗时 %s 秒': '## Execution finished... %s elapsed %s seconds',
|
'## 执行结束... %s 耗时 %s 秒':
|
||||||
|
'## Execution finished... %s elapsed %s seconds',
|
||||||
|
'原调度未能恢复,正在重试': 'Could not restore the previous schedule. Retrying.',
|
||||||
|
'任务已修改,请刷新后重试': 'The task has changed. Refresh and try again.',
|
||||||
'调度器注册失败,任务创建已回滚':
|
'调度器注册失败,任务创建已回滚':
|
||||||
'Scheduler registration failed, task creation rolled back',
|
'Scheduler registration failed, task creation rolled back',
|
||||||
'调度器注册失败,任务更新已回滚':
|
'调度器注册失败,任务更新已回滚':
|
||||||
@@ -151,9 +281,18 @@ const messages: Record<string, Record<string, string>> = {
|
|||||||
|
|
||||||
let currentLang: string = 'zh';
|
let currentLang: string = 'zh';
|
||||||
|
|
||||||
export function setLang(lang: string) {
|
export function setLang(lang: string, persist = true) {
|
||||||
currentLang = lang || 'zh';
|
currentLang = lang || 'zh';
|
||||||
shareStore.setLang(currentLang);
|
if (!persist) return;
|
||||||
|
// Translation is also used by standalone scheduler and CLI helpers. Load
|
||||||
|
// persistent storage only when changing the language, not when reading it.
|
||||||
|
const { shareStore } = require('./store') as typeof import('./store');
|
||||||
|
const language = currentLang;
|
||||||
|
return shareStore.setLang(language).then(() => {
|
||||||
|
if (currentLang === language && process.connected && process.send) {
|
||||||
|
process.send({ type: 'language-changed', lang: language }, () => {});
|
||||||
|
}
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 系统默认语言:Intl 检测 → 'zh'(仅返回 zh/en) */
|
/** 系统默认语言:Intl 检测 → 'zh'(仅返回 zh/en) */
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ export interface IDependencyFn<T> {
|
|||||||
export interface ICronFn<T> {
|
export interface ICronFn<T> {
|
||||||
(): Promise<T>;
|
(): Promise<T>;
|
||||||
cron?: TCron;
|
cron?: TCron;
|
||||||
|
isCurrent?: () => boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ISchedule {
|
export interface ISchedule {
|
||||||
|
|||||||
@@ -0,0 +1,32 @@
|
|||||||
|
import cluster from 'cluster';
|
||||||
|
import { setLang, systemLang } from './i18n';
|
||||||
|
|
||||||
|
function languageMessage(
|
||||||
|
message: unknown,
|
||||||
|
): message is { type: string; lang: string } {
|
||||||
|
const value = message as { type?: string; lang?: string } | null;
|
||||||
|
return (
|
||||||
|
value?.type === 'language-changed' && ['zh', 'en'].includes(value.lang!)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Each worker owns a translation cache. Relay saved settings to live workers;
|
||||||
|
// a replacement worker restores the persisted language before serving requests.
|
||||||
|
export async function initializeLanguageSync() {
|
||||||
|
if (cluster.isPrimary) {
|
||||||
|
cluster.on('message', (_worker, message) => {
|
||||||
|
if (!languageMessage(message)) return;
|
||||||
|
setLang(message.lang, false);
|
||||||
|
for (const worker of Object.values(cluster.workers || {})) {
|
||||||
|
if (worker?.isConnected()) worker.send(message, () => {});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
} else {
|
||||||
|
process.on('message', (message) => {
|
||||||
|
if (languageMessage(message)) setLang(message.lang, false);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const { shareStore } = await import('./store');
|
||||||
|
setLang((await shareStore.getLang()) || systemLang(), false);
|
||||||
|
}
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
import dayjs from 'dayjs';
|
||||||
|
import { QueryTypes, Sequelize, Transaction } from 'sequelize';
|
||||||
|
|
||||||
|
type DayCounts = {
|
||||||
|
cron_id: number;
|
||||||
|
day: string;
|
||||||
|
successes: number;
|
||||||
|
failures: number;
|
||||||
|
later_successes: number;
|
||||||
|
later_failures: number;
|
||||||
|
has_bookings: number;
|
||||||
|
saved_successes: number;
|
||||||
|
saved_failures: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Process one outcome, newest day first. Same-day receipts can only cover
|
||||||
|
// eligible exits. Surplus becomes available to earlier days after this match.
|
||||||
|
function matchDailyCounts(exits: number, saved: number, laterCredits: number, afterLastBooking = 0) {
|
||||||
|
const localMatches = Math.min(saved, exits - afterLastBooking);
|
||||||
|
const deficit = exits - localMatches;
|
||||||
|
const borrowed = Math.min(deficit, laterCredits);
|
||||||
|
return { missing: deficit - borrowed, remaining: laterCredits - borrowed + saved - localMatches };
|
||||||
|
}
|
||||||
|
|
||||||
|
// A legacy terminal status needs a matching exit code. An identified supervisor
|
||||||
|
// also knows a child failed when the OS returns a signal instead of a code.
|
||||||
|
const confirmedResult = `t.started_at IS NOT NULL AND t.finished_at>=t.started_at
|
||||||
|
AND t.state IN ('finished','failed')
|
||||||
|
AND (t.source<>'legacy' OR (
|
||||||
|
((t.state='finished' AND t.exit_code=0) OR (t.state='failed' AND t.exit_code<>0))
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM RunningInstances sibling
|
||||||
|
LEFT JOIN CronTriggers linked ON linked.instance_id=sibling.id
|
||||||
|
WHERE sibling.cron_id=i.cron_id AND sibling.pid=i.pid AND sibling.finished_at=i.finished_at
|
||||||
|
AND sibling.id<>i.id AND sibling.status IN (1,3)
|
||||||
|
AND (linked.id IS NULL OR (linked.source='legacy' AND linked.owner IS NULL))
|
||||||
|
AND (sibling.started_at<>i.started_at OR COALESCE(sibling.log_path,'')<>COALESCE(i.log_path,'')
|
||||||
|
OR sibling.status<>i.status OR sibling.exit_code IS NOT i.exit_code OR linked.id IS NOT NULL))))`;
|
||||||
|
|
||||||
|
export function readDailyResults(database: Sequelize, cronId: number) {
|
||||||
|
const today = dayjs().startOf('day');
|
||||||
|
const first = today.subtract(6, 'day');
|
||||||
|
const next = today.add(1, 'day');
|
||||||
|
return database.query<{
|
||||||
|
day: string; exitSuccess: number; exitFail: number;
|
||||||
|
bookedSuccess: number; bookedFail: number; supplement: number;
|
||||||
|
}>(`WITH results AS (
|
||||||
|
SELECT date(t.finished_at/1000,'unixepoch','localtime') AS day,
|
||||||
|
SUM(t.state='finished') AS exitSuccess,SUM(t.state='failed') AS exitFail,
|
||||||
|
0 AS bookedSuccess,0 AS bookedFail,0 AS supplement
|
||||||
|
FROM CronTriggers t LEFT JOIN RunningInstances i ON i.id=t.instance_id
|
||||||
|
WHERE t.cron_id=$id AND t.finished_at >= $start AND t.finished_at < $end
|
||||||
|
AND t.reason IS NOT 'spawn' AND ${confirmedResult} GROUP BY day
|
||||||
|
UNION ALL
|
||||||
|
SELECT date,0,0,COALESCE(success_count,0),COALESCE(fail_count,0),untimed_count
|
||||||
|
FROM CrontabStats WHERE ref_id=$id AND date >= $first AND date < $next
|
||||||
|
) SELECT day,SUM(exitSuccess) AS exitSuccess,SUM(exitFail) AS exitFail,
|
||||||
|
SUM(bookedSuccess) AS bookedSuccess,SUM(bookedFail) AS bookedFail,SUM(supplement) AS supplement
|
||||||
|
FROM results GROUP BY day ORDER BY day DESC`, { type: QueryTypes.SELECT, bind: {
|
||||||
|
id: cronId, start: first.valueOf(), end: next.valueOf(),
|
||||||
|
first: first.format('YYYY-MM-DD'), next: next.format('YYYY-MM-DD'),
|
||||||
|
} });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run under the schema migration's write lock, before accepting new receipts.
|
||||||
|
export async function repairLegacyDailyStatistics(database: Sequelize, transaction: Transaction, priorVersion: 0 | 1 | 2) {
|
||||||
|
// Old panels updated every outstanding start with the reported PID. Identical
|
||||||
|
// copies of a start prove only one exit; conflicting rows cannot identify it.
|
||||||
|
const imported = await database.query(`WITH exits AS (
|
||||||
|
SELECT MIN(i.id) AS instance FROM RunningInstances i
|
||||||
|
JOIN Crontabs c ON c.id=i.cron_id
|
||||||
|
LEFT JOIN CronTriggers linked ON linked.instance_id=i.id
|
||||||
|
WHERE (linked.id IS NULL OR (linked.source='legacy' AND linked.owner IS NULL))
|
||||||
|
AND i.status IN (1,3) AND i.pid>0 AND i.started_at>0 AND i.finished_at>=i.started_at
|
||||||
|
AND i.finished_at*1000>=COALESCE((SELECT cutoff FROM CronHistoryRetention WHERE id=1),0)
|
||||||
|
GROUP BY i.cron_id,i.pid,i.finished_at
|
||||||
|
HAVING MIN(i.started_at)=MAX(i.started_at)
|
||||||
|
AND COUNT(DISTINCT COALESCE(i.log_path,''))=1 AND MIN(i.status)=MAX(i.status)
|
||||||
|
AND COUNT(i.exit_code)=COUNT(*) AND MIN(i.exit_code)=MAX(i.exit_code)
|
||||||
|
AND COUNT(linked.id)=0
|
||||||
|
)
|
||||||
|
INSERT INTO CronTriggers(id,cron_id,source,state,observed_at,started_at,finished_at,instance_id,log_path,exit_code,counted)
|
||||||
|
SELECT 'legacy-db-' || i.id,i.cron_id,'legacy',CASE i.status WHEN 1 THEN 'finished' ELSE 'failed' END,
|
||||||
|
i.started_at*1000,i.started_at*1000,i.finished_at*1000,i.id,i.log_path,i.exit_code,1
|
||||||
|
FROM exits JOIN RunningInstances i ON i.id=exits.instance
|
||||||
|
WHERE (i.status=1 AND i.exit_code=0) OR (i.status=3 AND i.exit_code<>0)`,
|
||||||
|
{ transaction, type: QueryTypes.BULKUPDATE });
|
||||||
|
|
||||||
|
// A newly imported last run must have the same owner it would acquire through
|
||||||
|
// the normal handoff. An older exit must never replace a more recent start.
|
||||||
|
if (imported) await database.query(`WITH ranked AS (
|
||||||
|
SELECT cron_id,instance_id,started_at/1000 AS start,(finished_at-started_at)/1000 AS elapsed,
|
||||||
|
ROW_NUMBER() OVER (PARTITION BY cron_id ORDER BY started_at DESC,instance_id DESC) AS position
|
||||||
|
FROM CronTriggers WHERE source='legacy' AND id LIKE 'legacy-db-%'
|
||||||
|
)
|
||||||
|
UPDATE Crontabs AS c SET (last_instance_id,last_execution_time,last_running_time)=
|
||||||
|
(SELECT instance_id,start,elapsed FROM ranked WHERE cron_id=c.id AND position=1)
|
||||||
|
WHERE EXISTS (SELECT 1 FROM ranked r WHERE r.cron_id=c.id AND r.position=1 AND
|
||||||
|
(c.last_execution_time IS NULL OR c.last_execution_time<r.start OR
|
||||||
|
(c.last_execution_time=r.start AND (c.last_instance_id IS NULL OR c.last_instance_id<r.instance_id)))
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM RunningInstances i WHERE i.cron_id=c.id AND
|
||||||
|
(i.started_at>r.start OR (i.started_at=r.start AND i.id>r.instance_id))))`,
|
||||||
|
{ transaction });
|
||||||
|
|
||||||
|
// Statistics retention removes whole dates across tasks. When the old panel
|
||||||
|
// retained no earlier totals, yesterday is the oldest date even a 1-day policy keeps.
|
||||||
|
const yesterday = dayjs().subtract(1,'day').format('YYYY-MM-DD');
|
||||||
|
const days = await database.query<DayCounts>(`WITH exits AS (
|
||||||
|
SELECT t.cron_id,t.state,date(t.finished_at/1000,'unixepoch','localtime') AS day,
|
||||||
|
t.started_at,i.createdAt,t.id
|
||||||
|
FROM CronTriggers t LEFT JOIN RunningInstances i ON i.id=t.instance_id
|
||||||
|
WHERE t.counted=1 AND ${confirmedResult}
|
||||||
|
), ordered AS (
|
||||||
|
SELECT e.*, CASE WHEN e.id LIKE 'legacy-db-%' AND s.untimed_count=0
|
||||||
|
AND date(s.updatedAt,'localtime')=e.day AND julianday(s.createdAt)<=julianday(s.updatedAt)
|
||||||
|
AND e.started_at/1000>CAST(strftime('%s',s.updatedAt) AS INTEGER)
|
||||||
|
AND julianday(e.createdAt)>julianday(s.updatedAt) THEN 1 ELSE 0 END AS later
|
||||||
|
FROM exits e LEFT JOIN CrontabStats s ON s.ref_id=e.cron_id AND s.date=e.day
|
||||||
|
), evidence AS (
|
||||||
|
SELECT cron_id,day,SUM(state='finished') AS successes,SUM(state='failed') AS failures,
|
||||||
|
SUM(later*(state='finished')) AS later_successes,SUM(later*(state='failed')) AS later_failures,
|
||||||
|
0 AS has_bookings,0 AS saved_successes,0 AS saved_failures FROM ordered GROUP BY cron_id,day
|
||||||
|
UNION ALL
|
||||||
|
SELECT ref_id,date,0,0,0,0,1,COALESCE(success_count,0),COALESCE(fail_count,0) FROM CrontabStats
|
||||||
|
)
|
||||||
|
SELECT cron_id,day,MAX(has_bookings) AS has_bookings,SUM(successes) AS successes,SUM(failures) AS failures,
|
||||||
|
SUM(later_successes) AS later_successes,SUM(later_failures) AS later_failures,
|
||||||
|
SUM(saved_successes) AS saved_successes,SUM(saved_failures) AS saved_failures
|
||||||
|
FROM evidence WHERE day>=MIN($yesterday,COALESCE((SELECT MIN(date) FROM CrontabStats),$yesterday))
|
||||||
|
AND day>=COALESCE((SELECT first_day FROM CronStatRetention WHERE id=1),'')
|
||||||
|
GROUP BY cron_id,day ORDER BY cron_id,day DESC`, { type: QueryTypes.SELECT, transaction, bind:{ yesterday } });
|
||||||
|
|
||||||
|
// Anonymous receipts cannot identify a particular exit. Match nearest
|
||||||
|
// eligible days first; never move already booked counts or elapsed samples.
|
||||||
|
let task: number | undefined;
|
||||||
|
let successCredit = 0, failureCredit = 0;
|
||||||
|
let priorSuccessCredit = 0, priorFailureCredit = 0;
|
||||||
|
for (const day of days) {
|
||||||
|
if (day.cron_id !== task) {
|
||||||
|
task = day.cron_id;
|
||||||
|
successCredit = failureCredit = priorSuccessCredit = priorFailureCredit = 0;
|
||||||
|
}
|
||||||
|
const successes = matchDailyCounts(day.successes, day.saved_successes, successCredit, day.later_successes);
|
||||||
|
const failures = matchDailyCounts(day.failures, day.saved_failures, failureCredit, day.later_failures);
|
||||||
|
successCredit = successes.remaining;
|
||||||
|
failureCredit = failures.remaining;
|
||||||
|
let success = successes.missing, failure = failures.missing;
|
||||||
|
if (priorVersion === 2) {
|
||||||
|
// Replaying v2 on the current snapshot could resurrect a cleaned-up day.
|
||||||
|
// Only the extra lower bound established by v3 is new evidence.
|
||||||
|
const priorSuccess = matchDailyCounts(day.successes, day.saved_successes, priorSuccessCredit);
|
||||||
|
const priorFailure = matchDailyCounts(day.failures, day.saved_failures, priorFailureCredit);
|
||||||
|
priorSuccessCredit = priorSuccess.remaining;
|
||||||
|
priorFailureCredit = priorFailure.remaining;
|
||||||
|
success = Math.max(0, success - priorSuccess.missing);
|
||||||
|
failure = Math.max(0, failure - priorFailure.missing);
|
||||||
|
}
|
||||||
|
// Old repairs did not journal deletions. An absent task/day could have
|
||||||
|
// been cleared after a supplement. Preserve the gap on repeat upgrades.
|
||||||
|
if (priorVersion && !day.has_bookings) continue;
|
||||||
|
if (!success && !failure) continue;
|
||||||
|
// Anonymous elapsed samples have no execution identity. Preserve measured
|
||||||
|
// times; adding instance wall times here could count the same sample twice.
|
||||||
|
await database.query(`INSERT INTO CrontabStats
|
||||||
|
(ref_id,date,run_count,success_count,fail_count,untimed_count,total_time,max_time,createdAt,updatedAt)
|
||||||
|
VALUES ($task,$day,$runs,$success,$failure,$runs,0,0,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP)
|
||||||
|
ON CONFLICT(ref_id,date) DO UPDATE SET run_count=COALESCE(run_count,0)+excluded.run_count,
|
||||||
|
success_count=COALESCE(success_count,0)+excluded.success_count,
|
||||||
|
fail_count=COALESCE(fail_count,0)+excluded.fail_count,
|
||||||
|
untimed_count=untimed_count+excluded.untimed_count,updatedAt=CURRENT_TIMESTAMP`,
|
||||||
|
{ transaction, bind: { task,day:day.day,runs:success+failure,success,failure } });
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
import { shellQuote } from './shellQuote';
|
||||||
|
|
||||||
|
const repositoryNames =
|
||||||
|
/^(debian(?:-security)?|ubuntu(?:-ports)?|alpine|armbian)$/;
|
||||||
|
|
||||||
|
export function normalizeMirror(mirror: string): URL {
|
||||||
|
if (/\s/.test(mirror)) throw new Error('Invalid Linux mirror URL');
|
||||||
|
const url = new URL(mirror);
|
||||||
|
if (
|
||||||
|
!['http:', 'https:'].includes(url.protocol) ||
|
||||||
|
url.username ||
|
||||||
|
url.password ||
|
||||||
|
url.search ||
|
||||||
|
url.hash
|
||||||
|
) {
|
||||||
|
throw new Error(
|
||||||
|
'Linux mirror must be an HTTP(S) URL without credentials, query or fragment',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return url;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function replaceMirrorUrl(current: string, mirror: string): string {
|
||||||
|
const target = normalizeMirror(mirror);
|
||||||
|
const previous = new URL(current);
|
||||||
|
const oldSegments = previous.pathname.split('/').filter(Boolean);
|
||||||
|
const newSegments = target.pathname.split('/').filter(Boolean);
|
||||||
|
const repositoryIndex = oldSegments.findIndex((part) =>
|
||||||
|
repositoryNames.test(part),
|
||||||
|
);
|
||||||
|
const targetIndex = newSegments.findIndex((part) =>
|
||||||
|
repositoryNames.test(part),
|
||||||
|
);
|
||||||
|
const prefix =
|
||||||
|
targetIndex >= 0 ? newSegments.slice(0, targetIndex) : newSegments;
|
||||||
|
if (repositoryIndex >= 0) {
|
||||||
|
const repository = oldSegments[repositoryIndex];
|
||||||
|
let suffix = oldSegments.slice(repositoryIndex + 1);
|
||||||
|
// Repair paths already corrupted by repeated authority-only replacement.
|
||||||
|
while (suffix[0] === repository) suffix.shift();
|
||||||
|
return `${target.origin}/${[...prefix, repository, ...suffix].join('/')}`;
|
||||||
|
}
|
||||||
|
// Unknown repository paths: a full URL replaces the old root, a hostname
|
||||||
|
// preserves it. Never append the newly entered root to the existing root.
|
||||||
|
return `${target.origin}/${(newSegments.length
|
||||||
|
? newSegments
|
||||||
|
: oldSegments
|
||||||
|
).join('/')}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function rewriteMirrorSources(content: string, mirror: string): string {
|
||||||
|
normalizeMirror(mirror);
|
||||||
|
return content
|
||||||
|
.split(/(\r?\n[ \t]*\r?\n)/)
|
||||||
|
.map((stanza) => {
|
||||||
|
const signedBy = stanza.match(/^Signed-By:\s*([^\r\n]*)/im)?.[1];
|
||||||
|
return stanza
|
||||||
|
.split('\n')
|
||||||
|
.map((line) => {
|
||||||
|
if (!/^\s*(?:URIs:|deb(?:-src)?\s|https?:\/\/)/.test(line))
|
||||||
|
return line;
|
||||||
|
const legacy = /^\s*deb(?:-src)?\s/.test(line);
|
||||||
|
const key = legacy
|
||||||
|
? line.match(/\bsigned-by=([^\]\s]+)/i)?.[1]
|
||||||
|
: signedBy;
|
||||||
|
// In .list files a # starts an inline comment; its URLs are metadata.
|
||||||
|
const comment = legacy ? line.indexOf('#') : -1;
|
||||||
|
const source = comment < 0 ? line : line.slice(0, comment);
|
||||||
|
const rewritten = source.replace(/https?:\/\/[^\s#]+/g, (url) =>
|
||||||
|
isSystemRepository(url, key) ? replaceMirrorUrl(url, mirror) : url,
|
||||||
|
);
|
||||||
|
return rewritten + (comment < 0 ? '' : line.slice(comment));
|
||||||
|
})
|
||||||
|
.join('\n');
|
||||||
|
})
|
||||||
|
.join('');
|
||||||
|
}
|
||||||
|
|
||||||
|
function isSystemRepository(url: string, signedBy?: string): boolean {
|
||||||
|
const keys =
|
||||||
|
signedBy
|
||||||
|
?.trim()
|
||||||
|
.split(/[\s,]+/)
|
||||||
|
.filter(Boolean) || [];
|
||||||
|
const archiveKeyring =
|
||||||
|
keys.length > 0 &&
|
||||||
|
keys.every((key) =>
|
||||||
|
/\/(?:debian|ubuntu|armbian)-archive-keyring\.(?:gpg|asc)$/.test(key),
|
||||||
|
);
|
||||||
|
// A separate signing key identifies a third-party archive, even if its
|
||||||
|
// repository happens to be called /debian or /ubuntu.
|
||||||
|
if (keys.length > 0 && !archiveKeyring) return false;
|
||||||
|
const segments = new URL(url).pathname.split('/').filter(Boolean);
|
||||||
|
const repositoryIndex = segments.findIndex((part) =>
|
||||||
|
repositoryNames.test(part),
|
||||||
|
);
|
||||||
|
if (repositoryIndex < 0) return false;
|
||||||
|
// Without archive metadata, accept only conventional archive roots. A
|
||||||
|
// vendor path such as Docker's /linux/debian is not the Debian archive.
|
||||||
|
return (
|
||||||
|
archiveKeyring ||
|
||||||
|
repositoryIndex === 0 ||
|
||||||
|
segments
|
||||||
|
.slice(0, repositoryIndex)
|
||||||
|
.every((part) => /^mirrors?$/.test(part)) ||
|
||||||
|
segments[repositoryIndex] === 'alpine'
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function mirrorWriteCommand(
|
||||||
|
file: string,
|
||||||
|
content: string,
|
||||||
|
prefix: string,
|
||||||
|
update: string,
|
||||||
|
): string {
|
||||||
|
return `${prefix}mkdir -p ${shellQuote(
|
||||||
|
file.slice(0, file.lastIndexOf('/')),
|
||||||
|
)} && printf '%s' ${shellQuote(content)} | ${prefix}tee ${shellQuote(
|
||||||
|
file,
|
||||||
|
)} >/dev/null && ${update}`;
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
import { resolveFileAccess } from './fileAccess';
|
||||||
|
|
||||||
|
/** Check stored paths too: old database rows are not trusted input. */
|
||||||
|
export function resolveLogPath(root: string, value: string): string {
|
||||||
|
const target = resolveFileAccess(root, [value]);
|
||||||
|
if (!target)
|
||||||
|
throw Object.assign(new Error('Log path is outside the log directory'), {
|
||||||
|
status: 400,
|
||||||
|
});
|
||||||
|
return target;
|
||||||
|
}
|
||||||
+55
-15
@@ -1,12 +1,18 @@
|
|||||||
import fs from 'fs/promises';
|
import fs from 'fs/promises';
|
||||||
|
import { createHash } from 'crypto';
|
||||||
|
|
||||||
export const DEFAULT_LOG_CHUNK_BYTES = 256 * 1024;
|
export const DEFAULT_LOG_CHUNK_BYTES = 256 * 1024;
|
||||||
export const MAX_LOG_CHUNK_BYTES = 1024 * 1024;
|
export const MAX_LOG_CHUNK_BYTES = 1024 * 1024;
|
||||||
|
|
||||||
|
// The viewer retains at most one Mi UTF-16 code units.
|
||||||
|
const CURSOR_WINDOW_BYTES = 4 * MAX_LOG_CHUNK_BYTES;
|
||||||
|
|
||||||
export interface LogReadOptions {
|
export interface LogReadOptions {
|
||||||
offset?: number;
|
offset?: number;
|
||||||
limit?: number;
|
limit?: number;
|
||||||
tail?: boolean;
|
tail?: boolean;
|
||||||
|
follow?: boolean;
|
||||||
|
cursor?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface LogChunk {
|
export interface LogChunk {
|
||||||
@@ -15,6 +21,8 @@ export interface LogChunk {
|
|||||||
nextOffset: number;
|
nextOffset: number;
|
||||||
total: number;
|
total: number;
|
||||||
truncated: boolean;
|
truncated: boolean;
|
||||||
|
cursor?: string;
|
||||||
|
reset?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
function normalizeLimit(limit?: number) {
|
function normalizeLimit(limit?: number) {
|
||||||
@@ -49,6 +57,10 @@ function completeUtf8End(buffer: Buffer, start: number, end: number) {
|
|||||||
return end - sequenceStart < expectedLength ? sequenceStart : end;
|
return end - sequenceStart < expectedLength ? sequenceStart : end;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function logDigest(identity: string, from: number, bytes: Buffer) {
|
||||||
|
return createHash('sha256').update(`${identity}:${from}:`).update(bytes).digest('hex');
|
||||||
|
}
|
||||||
|
|
||||||
export async function readLogChunk(
|
export async function readLogChunk(
|
||||||
filePath: string,
|
filePath: string,
|
||||||
options: LogReadOptions = {},
|
options: LogReadOptions = {},
|
||||||
@@ -56,23 +68,43 @@ export async function readLogChunk(
|
|||||||
let handle: fs.FileHandle | undefined;
|
let handle: fs.FileHandle | undefined;
|
||||||
try {
|
try {
|
||||||
handle = await fs.open(filePath, 'r');
|
handle = await fs.open(filePath, 'r');
|
||||||
const { size: total } = await handle.stat();
|
const stat = await handle.stat({ bigint: true });
|
||||||
|
const total = Number(stat.size);
|
||||||
|
const identity = `${stat.dev}:${stat.ino}`;
|
||||||
const limit = normalizeLimit(options.limit);
|
const limit = normalizeLimit(options.limit);
|
||||||
const requestedOffset = Number.isFinite(options.offset)
|
const requestedOffset = Number.isFinite(options.offset)
|
||||||
? Math.trunc(options.offset!)
|
? Math.max(0, Math.trunc(options.offset!))
|
||||||
: undefined;
|
: undefined;
|
||||||
const requestedStart =
|
let start = options.tail || requestedOffset === undefined
|
||||||
options.tail || requestedOffset === undefined
|
|
||||||
? Math.max(total - limit, 0)
|
? Math.max(total - limit, 0)
|
||||||
: Math.min(Math.max(requestedOffset, 0), total);
|
: Math.min(requestedOffset, total);
|
||||||
const length = Math.min(limit + 3, total - requestedStart);
|
const resume = options.cursor && !options.tail && requestedOffset !== undefined;
|
||||||
const buffer = Buffer.allocUnsafe(length);
|
const [fromText, digest] = resume ? options.cursor!.split('.') : [];
|
||||||
const { bytesRead } = await handle.read(
|
let from = resume ? Number(fromText) : start;
|
||||||
buffer,
|
let reset = Boolean(resume && (
|
||||||
0,
|
requestedOffset !== start || !Number.isSafeInteger(from) ||
|
||||||
length,
|
from < 0 || from > start || start - from > CURSOR_WINDOW_BYTES
|
||||||
requestedStart,
|
));
|
||||||
);
|
if (reset) from = start = 0;
|
||||||
|
|
||||||
|
// Read the previously displayed bytes and their continuation together.
|
||||||
|
// A truncate followed by regrowth can leave both size and inode unchanged.
|
||||||
|
const readWindow = async (position: number, end: number) => {
|
||||||
|
const data = Buffer.allocUnsafe(Math.min(end, total) - position);
|
||||||
|
const { bytesRead } = await handle!.read(data, 0, data.length, position);
|
||||||
|
return data.subarray(0, bytesRead);
|
||||||
|
};
|
||||||
|
let window = await readWindow(from, start + limit + 3);
|
||||||
|
if (resume && !reset) {
|
||||||
|
const prefix = window.subarray(0, start - from);
|
||||||
|
if (prefix.length !== start - from || logDigest(identity, from, prefix) !== digest) {
|
||||||
|
reset = true;
|
||||||
|
start = from = 0;
|
||||||
|
window = await readWindow(0, limit + 3);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const buffer = window.subarray(start - from);
|
||||||
|
const bytesRead = buffer.length;
|
||||||
let leadingBytes = 0;
|
let leadingBytes = 0;
|
||||||
while (
|
while (
|
||||||
leadingBytes < bytesRead &&
|
leadingBytes < bytesRead &&
|
||||||
@@ -80,12 +112,20 @@ export async function readLogChunk(
|
|||||||
) {
|
) {
|
||||||
leadingBytes++;
|
leadingBytes++;
|
||||||
}
|
}
|
||||||
const offset = requestedStart + leadingBytes;
|
const offset = start + leadingBytes;
|
||||||
const candidateEnd = Math.min(leadingBytes + limit, bytesRead);
|
const candidateEnd = Math.min(leadingBytes + limit, bytesRead);
|
||||||
const contentEnd = completeUtf8End(buffer, leadingBytes, candidateEnd);
|
const contentEnd = completeUtf8End(buffer, leadingBytes, candidateEnd);
|
||||||
const nextOffset = requestedStart + contentEnd;
|
const nextOffset = start + contentEnd;
|
||||||
|
|
||||||
|
let cursor: string | undefined;
|
||||||
|
if (options.follow || options.cursor) {
|
||||||
|
const retainedFrom = Math.max(from, nextOffset - CURSOR_WINDOW_BYTES);
|
||||||
|
cursor = resume && !reset && nextOffset === start ? options.cursor :
|
||||||
|
`${retainedFrom}.${logDigest(identity, retainedFrom, window.subarray(retainedFrom - from, nextOffset - from))}`;
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
...(cursor && { cursor, reset }),
|
||||||
content: buffer.subarray(leadingBytes, contentEnd).toString('utf8'),
|
content: buffer.subarray(leadingBytes, contentEnd).toString('utf8'),
|
||||||
offset,
|
offset,
|
||||||
nextOffset,
|
nextOffset,
|
||||||
|
|||||||
@@ -0,0 +1,47 @@
|
|||||||
|
const RETRY_TTL = 15 * 60 * 1000;
|
||||||
|
const MAX_SOURCES = 10000;
|
||||||
|
const MAX_RETRY_SECONDS = 5 * 60;
|
||||||
|
|
||||||
|
/** Source-specific, bounded backoff; persisted account history is not a lock. */
|
||||||
|
export class LoginThrottle {
|
||||||
|
private sources = new Map<
|
||||||
|
string,
|
||||||
|
{ failures: number; lastFailure: number }
|
||||||
|
>();
|
||||||
|
|
||||||
|
private get(source: string, now: number) {
|
||||||
|
const state = this.sources.get(source);
|
||||||
|
if (state && now - state.lastFailure >= RETRY_TTL) {
|
||||||
|
this.sources.delete(source);
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
return state;
|
||||||
|
}
|
||||||
|
|
||||||
|
retryAfter(source: string, now: number): number {
|
||||||
|
const state = this.get(source, now);
|
||||||
|
if (!state || state.failures < 3) return 0;
|
||||||
|
const seconds = Math.min(3 ** state.failures, MAX_RETRY_SECONDS);
|
||||||
|
return Math.max(
|
||||||
|
0,
|
||||||
|
Math.ceil((seconds * 1000 - (now - state.lastFailure)) / 1000),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fail(source: string, now: number): number {
|
||||||
|
const failures = Math.min((this.get(source, now)?.failures || 0) + 1, 6);
|
||||||
|
this.sources.delete(source);
|
||||||
|
if (this.sources.size >= MAX_SOURCES) {
|
||||||
|
this.sources.delete(this.sources.keys().next().value!);
|
||||||
|
}
|
||||||
|
this.sources.set(source, { failures, lastFailure: now });
|
||||||
|
return failures;
|
||||||
|
}
|
||||||
|
|
||||||
|
reset(source: string) {
|
||||||
|
this.sources.delete(source);
|
||||||
|
}
|
||||||
|
clear() {
|
||||||
|
this.sources.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,390 @@
|
|||||||
|
// Keep the command's actual exit available until its SQLite transaction commits.
|
||||||
|
const fs = require('node:fs');
|
||||||
|
const path = require('node:path');
|
||||||
|
const { createHash, randomUUID } = require('node:crypto');
|
||||||
|
|
||||||
|
const idPattern = /^[a-z0-9-]{1,120}$/;
|
||||||
|
|
||||||
|
// A full data volume can still have room for a directory entry. In this format
|
||||||
|
// the empty file publishes the tuple atomically, including its execution-bound
|
||||||
|
// checksum. Keep the encoded name when moving it into a collector-private slot.
|
||||||
|
const metadataName = /^\.pending\.([a-z0-9-]{1,120})\.full-(\d{1,3}|SIG[A-Z0-9]+)-(\d{1,16})-([0-9a-f]{64})-([a-zA-Z0-9_]+)$/;
|
||||||
|
|
||||||
|
// The digest only guards disposal of legacy bytes. It cannot establish an exit.
|
||||||
|
const legacyCapture = /^\.pending\.([a-z0-9-]{1,120})\.l-([0-9a-f]{32})-[0-9a-f]{24}$/;
|
||||||
|
const claimName = /^\.claim(-bound)?\.([a-z0-9-]{1,120})\.[a-zA-Z0-9]{6}$/;
|
||||||
|
|
||||||
|
function metadataReceipt(file) {
|
||||||
|
const name = path.basename(file);
|
||||||
|
return metadataName.exec(name === '.pending.claimed' ? path.basename(path.dirname(file)) : name);
|
||||||
|
}
|
||||||
|
|
||||||
|
function storageFull(error) {
|
||||||
|
const codes = [error?.code, error?.original?.code, error?.parent?.code];
|
||||||
|
if (codes.some(code => ['ENOSPC', 'EDQUOT', 'SQLITE_FULL'].includes(code))) return true;
|
||||||
|
const directory = receiptDirectory();
|
||||||
|
if (directory && codes.includes('SQLITE_CANTOPEN')) {
|
||||||
|
// SQLite maps a failed journal creation to CANTOPEN even when the kernel
|
||||||
|
// reported ENOSPC. Other open failures keep their normal error semantics.
|
||||||
|
try {
|
||||||
|
const capacity = fs.statfsSync(path.dirname(directory));
|
||||||
|
return capacity.bavail === 0 || (capacity.files > 0 && capacity.ffree === 0);
|
||||||
|
} catch { return false; }
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function receiptDirectory() {
|
||||||
|
const data = process.env.QL_DATA_DIR || (process.env.QL_DIR && path.join(process.env.QL_DIR, 'data'));
|
||||||
|
if (data) return path.join(data, 'db', 'cron-exits');
|
||||||
|
}
|
||||||
|
|
||||||
|
function receiptPath(id) {
|
||||||
|
const directory = receiptDirectory();
|
||||||
|
if (directory && typeof id === 'string' && idPattern.test(id)) return path.join(directory, id);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function listReceiptFiles(directory, prefix) {
|
||||||
|
if (!directory) return [];
|
||||||
|
try {
|
||||||
|
const entries = await fs.promises.readdir(directory, { withFileTypes: true });
|
||||||
|
const files = [];
|
||||||
|
for (const entry of entries) {
|
||||||
|
if (!entry.isFile() || !entry.name.startsWith(prefix)) continue;
|
||||||
|
const id = entry.name.slice(prefix.length);
|
||||||
|
if (idPattern.test(id)) files.push({ id, file: path.join(directory, entry.name) });
|
||||||
|
}
|
||||||
|
return files;
|
||||||
|
} catch (error) {
|
||||||
|
if (error.code === 'ENOENT') return [];
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function publishedExits() {
|
||||||
|
return (await listReceiptFiles(receiptDirectory(), '')).map(entry => entry.id);
|
||||||
|
}
|
||||||
|
|
||||||
|
// flock attaches to the open file description. The child inherits fd 3 and
|
||||||
|
// exits immediately. Closing the panel's handle (including on SIGKILL)
|
||||||
|
// releases the lock without leaving a lock file on a full volume.
|
||||||
|
async function lockClaim(handle, shared) {
|
||||||
|
const run = (program, args) => new Promise((resolve, reject) => {
|
||||||
|
const child = require('node:child_process').spawn(program, args, {
|
||||||
|
stdio: ['ignore', 'ignore', 'ignore', handle.fd],
|
||||||
|
});
|
||||||
|
child.once('error', reject);
|
||||||
|
child.once('exit', (code, signal) => {
|
||||||
|
if (code === 0 || code === 1) resolve(code === 0);
|
||||||
|
else reject(new Error(`Receipt directory lock failed (${signal || code})`));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
return await run('flock', ['-n', shared ? '-s' : '-x', '3']);
|
||||||
|
} catch (error) {
|
||||||
|
if (error.code !== 'ENOENT') throw error;
|
||||||
|
// Python is also shipped with the task supervisor on minimal Linux images.
|
||||||
|
const program = 'import fcntl,sys\ntry: fcntl.flock(3, fcntl.LOCK_NB | fcntl.LOCK_' +
|
||||||
|
(shared ? 'SH' : 'EX') + ')\nexcept BlockingIOError: sys.exit(1)';
|
||||||
|
return run('python3', ['-c', program]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Move the current directory entry to a fresh, recoverable name outside the
|
||||||
|
// reusable claim directory. A writer can replace that entry without flock.
|
||||||
|
// Only the bytes actually captured by this attempt authorize its final unlink.
|
||||||
|
/** @param {string} id @param {string} file @param {{outcome: string, time: number}} [booked] */
|
||||||
|
async function retireLegacyFile(id, file, booked) {
|
||||||
|
const directory = receiptDirectory();
|
||||||
|
const sourceDir = path.dirname(file);
|
||||||
|
const slot = path.basename(file);
|
||||||
|
const bound = /^\.(pending|published)\./.test(slot) && !legacyCapture.test(slot);
|
||||||
|
const metadata = metadataReceipt(file);
|
||||||
|
const parent = claimName.exec(path.basename(sourceDir));
|
||||||
|
const privateSlot = path.dirname(sourceDir) === directory &&
|
||||||
|
(parent?.[2] === id && slot === (parent[1] ? '.pending.claimed' : 'receipt') ||
|
||||||
|
metadata?.[1] === id && slot === '.pending.claimed');
|
||||||
|
if (!directory || !idPattern.test(id)) return;
|
||||||
|
const dbDir = path.dirname(directory);
|
||||||
|
const localBin = path.join(directory, '.reclaim');
|
||||||
|
const fromBin = sourceDir === dbDir || sourceDir === localBin;
|
||||||
|
if (sourceDir !== directory && !privateSlot && !(fromBin && captureId(slot) === id)) return;
|
||||||
|
|
||||||
|
const matches = exit => exit && !exit.namedExit && (exit.version === 1
|
||||||
|
? booked && exit.time === booked.time && (exit.signal ?? String(exit.code)) === booked.outcome
|
||||||
|
: !bound);
|
||||||
|
let handle, binHandle;
|
||||||
|
try {
|
||||||
|
if ((privateSlot || sourceDir === localBin) && process.platform === 'linux') {
|
||||||
|
handle = await fs.promises.open(sourceDir, fs.constants.O_DIRECTORY | fs.constants.O_NOFOLLOW);
|
||||||
|
if (privateSlot && !await lockClaim(handle, true)) return;
|
||||||
|
}
|
||||||
|
const source = handle ? `/proc/self/fd/${handle.fd}/${slot}` : file;
|
||||||
|
// A canonical publication may only be readable after the move. For a
|
||||||
|
// recovered copy, first avoid moving an unsettled witness on every scan.
|
||||||
|
const previous = await readExit(id, source, file);
|
||||||
|
if (slot !== id && !matches(previous)) return;
|
||||||
|
const nonce = randomUUID();
|
||||||
|
let name;
|
||||||
|
if (metadata) {
|
||||||
|
name = metadata[0].slice(0, -metadata[5].length) + 'r_' + nonce.replace(/-/g, '').slice(0, 16);
|
||||||
|
} else if (bound || !previous || previous.version === 1) {
|
||||||
|
name = `.pending.${id}.r-${nonce}`;
|
||||||
|
} else {
|
||||||
|
const digest = exitChecksum(id, previous.code, previous.signal, previous.time).slice(0, 32);
|
||||||
|
name = `.pending.${id}.l-${digest}-${nonce.replace(/-/g, '').slice(0, 24)}`;
|
||||||
|
}
|
||||||
|
// The database directory is outside the writers' receipt namespace and
|
||||||
|
// already exists even at zero free inodes. A separately mounted journal
|
||||||
|
// requires a disposal directory on its own filesystem.
|
||||||
|
let bin = sourceDir === localBin ? localBin : dbDir;
|
||||||
|
const pinBin = async () => {
|
||||||
|
binHandle = await fs.promises.open(localBin, fs.constants.O_DIRECTORY | fs.constants.O_NOFOLLOW);
|
||||||
|
return process.platform === 'linux' ? `/proc/self/fd/${binHandle.fd}/${name}` : path.join(localBin, name);
|
||||||
|
};
|
||||||
|
let captured = path.join(bin, name);
|
||||||
|
if (bin === localBin) captured = handle ? path.join(path.dirname(source), name) : await pinBin();
|
||||||
|
try {
|
||||||
|
await fs.promises.rename(source, captured);
|
||||||
|
} catch (error) {
|
||||||
|
if (error.code !== 'EXDEV' || bin === localBin) throw error;
|
||||||
|
await fs.promises.mkdir(localBin, { mode: 0o700, recursive: true });
|
||||||
|
bin = localBin;
|
||||||
|
captured = await pinBin();
|
||||||
|
await fs.promises.rename(source, captured);
|
||||||
|
}
|
||||||
|
const actual = await readExit(id, captured, path.join(bin, name));
|
||||||
|
// An unreadable source offers no authority to discard unbound bytes.
|
||||||
|
const sameExit = previous && actual && actual.version === previous.version &&
|
||||||
|
actual.code === previous.code && actual.signal === previous.signal && actual.time === previous.time;
|
||||||
|
if (matches(actual) && (sameExit || (!previous && actual.version === 1))) {
|
||||||
|
await fs.promises.unlink(captured);
|
||||||
|
} else if (actual !== undefined) {
|
||||||
|
// A differing, torn or unreadable file is evidence for another pass.
|
||||||
|
// Persist its new directory entry before releasing the original directory.
|
||||||
|
if (binHandle) await binHandle.sync();
|
||||||
|
else if (bin !== sourceDir || !handle) syncDirectory(bin);
|
||||||
|
if (bin === localBin) syncDirectory(directory);
|
||||||
|
if (handle) await handle.sync();
|
||||||
|
else if (sourceDir !== bin) syncDirectory(sourceDir);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
if (error.code !== 'ENOENT') throw error;
|
||||||
|
} finally {
|
||||||
|
if (binHandle) await binHandle.close();
|
||||||
|
if (handle) await handle.close();
|
||||||
|
if (privateSlot) await fs.promises.rmdir(sourceDir).catch(() => {});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function captureId(name) {
|
||||||
|
const marker = metadataName.exec(name);
|
||||||
|
if (marker && /^r_[0-9a-f]{16}$/.test(marker[5])) return marker[1];
|
||||||
|
const match = legacyCapture.exec(name) ??
|
||||||
|
/^\.pending\.([a-z0-9-]{1,120})\.r-[0-9a-f-]{36}$/.exec(name);
|
||||||
|
return match?.[1];
|
||||||
|
}
|
||||||
|
|
||||||
|
async function receiptEntries(directory) {
|
||||||
|
try { return await fs.promises.readdir(directory, { withFileTypes: true }); }
|
||||||
|
catch (error) { if (error.code === 'ENOENT') return []; throw error; }
|
||||||
|
}
|
||||||
|
|
||||||
|
async function retiringExits() {
|
||||||
|
const dir = receiptDirectory();
|
||||||
|
if (!dir) return [];
|
||||||
|
const files = [];
|
||||||
|
const bins = [path.dirname(dir)];
|
||||||
|
for (const entry of await receiptEntries(dir)) {
|
||||||
|
if (entry.isDirectory() && entry.name === '.reclaim') {
|
||||||
|
bins.push(path.join(dir, entry.name));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const claim = claimName.exec(entry.name);
|
||||||
|
const metadata = metadataName.exec(entry.name);
|
||||||
|
if ((claim || metadata) && entry.isDirectory()) {
|
||||||
|
const directory = path.join(dir, entry.name);
|
||||||
|
try { await fs.promises.rmdir(directory); continue; } catch (error) {
|
||||||
|
if (error.code === 'ENOENT') continue;
|
||||||
|
}
|
||||||
|
files.push({ id: metadata ? metadata[1] : claim[2], file: path.join(directory, metadata || claim[1] ? '.pending.claimed' : 'receipt') });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const match = metadata ?? /^\.(?:retiring|collecting)\.([a-z0-9-]{1,120})\.[0-9a-f-]{36}$/.exec(entry.name)
|
||||||
|
?? /^\.pending\.([a-z0-9-]{1,120})\.[a-z0-9_-]{1,64}$/.exec(entry.name)
|
||||||
|
?? /^\.published\.([a-z0-9-]{1,120})\.[0-9a-f]{64}$/.exec(entry.name);
|
||||||
|
if (entry.isFile() && match) files.push({ id:match[1], file:path.join(dir,entry.name) });
|
||||||
|
}
|
||||||
|
for (const bin of bins) {
|
||||||
|
for (const entry of await receiptEntries(bin)) {
|
||||||
|
const id = entry.isFile() && captureId(entry.name);
|
||||||
|
if (id) files.push({ id, file:path.join(bin,entry.name) });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return files;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function publishedScriptStarts() {
|
||||||
|
const directory = process.env.QL_DIR && path.join(process.env.QL_DIR, '.tmp');
|
||||||
|
try {
|
||||||
|
return await listReceiptFiles(directory, 'cron-start-');
|
||||||
|
} catch {
|
||||||
|
// Optional entry instrumentation must not prevent exit recovery.
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function validExit(code, signal, time) {
|
||||||
|
return Number.isSafeInteger(time) && time > 0 && Number.isFinite(new Date(time).getTime()) &&
|
||||||
|
((Number.isInteger(code) && code >= 0 && code <= 255 && signal === null) ||
|
||||||
|
(code === null && typeof signal === 'string' && /^SIG[A-Z0-9]+$/.test(signal)));
|
||||||
|
}
|
||||||
|
|
||||||
|
// The Python supervisor uses the same compact JSON array of ASCII values.
|
||||||
|
function exitChecksum(id, code, signal, time) {
|
||||||
|
return createHash('sha256').update(JSON.stringify([id, code, signal, time])).digest('hex');
|
||||||
|
}
|
||||||
|
|
||||||
|
function syncDirectory(directory) {
|
||||||
|
const fd = fs.openSync(directory, fs.constants.O_RDONLY | fs.constants.O_DIRECTORY);
|
||||||
|
try { fs.fsyncSync(fd); } finally { fs.closeSync(fd); }
|
||||||
|
}
|
||||||
|
|
||||||
|
function writeExit(id, code, signal, time) {
|
||||||
|
if (!validExit(code, signal, time)) throw new TypeError('Invalid command exit receipt');
|
||||||
|
const file = receiptPath(id);
|
||||||
|
if (!file) return;
|
||||||
|
fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
|
||||||
|
const checksum = exitChecksum(id, code, signal, time);
|
||||||
|
const receipt = JSON.stringify({ code, signal, time, version: 1, checksum });
|
||||||
|
// Creation is the first durable observation. No data block is needed for
|
||||||
|
// this witness. Keep the regular JSON publication for earlier readers.
|
||||||
|
const directory = path.dirname(file);
|
||||||
|
const prefix = '.pending.' + id + '.';
|
||||||
|
const marker = prefix + 'full-' + (signal ?? code) + '-' + time + '-' + checksum + '-';
|
||||||
|
const publish = (name, content) => {
|
||||||
|
for (;;) {
|
||||||
|
const target = path.join(directory, name + randomUUID().replace(/-/g, '').slice(0, 16));
|
||||||
|
try {
|
||||||
|
fs.writeFileSync(target, content, { flag: 'wx', mode: 0o600, flush: Boolean(content) });
|
||||||
|
return target;
|
||||||
|
} catch (error) {
|
||||||
|
if (error.code !== 'EEXIST') throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
publish(marker, '');
|
||||||
|
// ext4 journals the empty inode and directory entry together. Also fence the
|
||||||
|
// directory's own entry: another publisher may just have created it.
|
||||||
|
syncDirectory(directory);
|
||||||
|
syncDirectory(path.dirname(directory));
|
||||||
|
try {
|
||||||
|
if (fs.statfsSync(directory).bavail === 0) return;
|
||||||
|
const source = publish(marker, receipt);
|
||||||
|
try { fs.linkSync(source, file); }
|
||||||
|
catch (error) {
|
||||||
|
if (error.code === 'ENOENT') return;
|
||||||
|
if (error.code !== 'EEXIST') throw error;
|
||||||
|
// A panel from before metadata receipts can still discover this retry.
|
||||||
|
for (;;) {
|
||||||
|
try { fs.linkSync(source, path.join(directory, prefix + randomUUID())); break; }
|
||||||
|
catch (linkError) {
|
||||||
|
if (linkError.code === 'ENOENT') break;
|
||||||
|
if (linkError.code !== 'EEXIST') throw linkError;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Older panels find the JSON only through this newly created link.
|
||||||
|
syncDirectory(directory);
|
||||||
|
} catch (error) {
|
||||||
|
if (!storageFull(error)) throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseExit(id, value) {
|
||||||
|
if (!value || !validExit(value.code, value.signal, value.time)) return null;
|
||||||
|
const { code, signal, time, checksum, version } = value;
|
||||||
|
if (version !== undefined || checksum !== undefined) {
|
||||||
|
if (version !== 1 || checksum !== exitChecksum(id, code, signal, time)) return null;
|
||||||
|
} else if (Object.keys(value).length !== 3) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return { code, signal, time, version };
|
||||||
|
}
|
||||||
|
|
||||||
|
// undefined means no publication. null means a result cannot yet be trusted.
|
||||||
|
async function readExit(id, file = receiptPath(id), publication = file) {
|
||||||
|
if (!file) return;
|
||||||
|
try {
|
||||||
|
const handle = await fs.promises.open(file, fs.constants.O_RDONLY | fs.constants.O_NONBLOCK | fs.constants.O_NOFOLLOW);
|
||||||
|
try {
|
||||||
|
// Bound CPU and memory even if the damaged file is growing.
|
||||||
|
const buffer = Buffer.alloc(1025);
|
||||||
|
let size = 0;
|
||||||
|
while (size < buffer.length) {
|
||||||
|
const { bytesRead } = await handle.read(buffer, size, buffer.length - size, null);
|
||||||
|
if (!bytesRead) break;
|
||||||
|
size += bytesRead;
|
||||||
|
}
|
||||||
|
if (size === buffer.length) return null;
|
||||||
|
const metadata = metadataReceipt(publication);
|
||||||
|
const bytes = buffer.toString('utf8', 0, size);
|
||||||
|
let namedExit;
|
||||||
|
if (metadata && metadata[1] === id && (await handle.stat()).isFile()) {
|
||||||
|
const [, , outcome, end, checksum] = metadata;
|
||||||
|
const signal = outcome.startsWith('SIG') ? outcome : null;
|
||||||
|
const value = { code: signal ? null : Number(outcome), signal, time: Number(end), version: 1, checksum };
|
||||||
|
// Both current publishers serialize in this order. Accept a torn body
|
||||||
|
// only when every byte matches the tuple independently published by
|
||||||
|
// the filename. Validate replacement files on their own bytes below.
|
||||||
|
namedExit = parseExit(id, value);
|
||||||
|
const json = JSON.stringify(value);
|
||||||
|
if (json.startsWith(bytes)) {
|
||||||
|
return namedExit && { ...namedExit, file, namedExit: undefined };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const exit = parseExit(id, JSON.parse(bytes));
|
||||||
|
const guard = legacyCapture.exec(path.basename(publication));
|
||||||
|
if (guard && exit && exit.version !== 1 && (guard[1] !== id ||
|
||||||
|
guard[2] !== exitChecksum(id, exit.code, exit.signal, exit.time).slice(0, 32))) return null;
|
||||||
|
const conflictingName = exit?.version === 1 && namedExit && (exit.time !== namedExit.time ||
|
||||||
|
exit.code !== namedExit.code || exit.signal !== namedExit.signal);
|
||||||
|
return exit && { ...exit, file, namedExit: conflictingName ? namedExit : undefined };
|
||||||
|
} finally {
|
||||||
|
await handle.close();
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
if (error.code === 'ENOENT') return;
|
||||||
|
if (error.code || error instanceof SyntaxError) return null;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { isLegacyCapture: file => legacyCapture.test(path.basename(file)), isMetadataReceipt: file => Boolean(metadataReceipt(file)), receiptPath, publishedExits, publishedScriptStarts, readExit, writeExit, validExit, storageFull, retireLegacyFile, retiringExits };
|
||||||
|
|
||||||
|
if (require.main === module) {
|
||||||
|
const id = process.env.QL_TRIGGER_ID;
|
||||||
|
const exit = parseExit(id, JSON.parse(process.argv[2]));
|
||||||
|
if (!idPattern.test(id || '') || !exit) throw new Error('Invalid supervisor exit receipt');
|
||||||
|
const { code, signal, time } = exit;
|
||||||
|
// An older supervisor may only have delivered this tuple in its message.
|
||||||
|
// Journal it before waiting for the SQLite writer or loading the panel.
|
||||||
|
let journaled = false;
|
||||||
|
if (exit.version === 1) {
|
||||||
|
try { writeExit(id, code, signal, time); journaled = Boolean(receiptPath(id)); }
|
||||||
|
catch (error) { console.error('[scheduler] Could not journal supervisor message:', error); }
|
||||||
|
}
|
||||||
|
// Development workers run TypeScript directly. Released panels load the
|
||||||
|
// compiled module without starting another runtime on ordinary exits.
|
||||||
|
if (!fs.existsSync(path.join(__dirname, 'triggerHistory.js'))) {
|
||||||
|
process.env.TS_NODE_PROJECT = path.join(__dirname, '../tsconfig.json');
|
||||||
|
require('ts-node/register/transpile-only');
|
||||||
|
}
|
||||||
|
const { triggerHistory } = require('./triggerHistory');
|
||||||
|
triggerHistory().complete(id, signal || code !== 0 ? 'failed' : 'finished',
|
||||||
|
signal, code, time, exit.version === 1 ? exit : 'legacy').catch(error => {
|
||||||
|
if (journaled && storageFull(error)) return;
|
||||||
|
console.error('[scheduler] Failed to settle orphaned command:', error);
|
||||||
|
process.exitCode = storageFull(error) ? 75 : 1;
|
||||||
|
}).finally(() => require('../data').sequelize.close());
|
||||||
|
}
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
import { Op } from 'sequelize';
|
||||||
|
import {
|
||||||
|
SystemModel,
|
||||||
|
AuthDataType,
|
||||||
|
NotifyStatus,
|
||||||
|
NotifyLogInfo,
|
||||||
|
} from '../data/system';
|
||||||
|
|
||||||
|
// Both backend sends and script reports use the same bounded, best-effort sink.
|
||||||
|
export async function persistNotificationAudit(
|
||||||
|
info: NotifyLogInfo,
|
||||||
|
): Promise<void> {
|
||||||
|
try {
|
||||||
|
await SystemModel.create({
|
||||||
|
type: AuthDataType.notifyLog,
|
||||||
|
info: {
|
||||||
|
timestamp: Date.now(),
|
||||||
|
title: String(info.title || '').slice(0, 512),
|
||||||
|
content: String(info.content || '').slice(0, 8192),
|
||||||
|
status:
|
||||||
|
info.status === NotifyStatus.success
|
||||||
|
? NotifyStatus.success
|
||||||
|
: NotifyStatus.fail,
|
||||||
|
notifyType: String(info.notifyType || '').slice(0, 64),
|
||||||
|
source: info.source === 'script' ? 'script' : 'system',
|
||||||
|
scriptName: String(info.scriptName || '').slice(0, 512),
|
||||||
|
taskId: /^\d+$/.test(info.taskId || '') ? info.taskId : undefined,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const oldest = await SystemModel.findAll({
|
||||||
|
where: { type: AuthDataType.notifyLog },
|
||||||
|
attributes: ['id'],
|
||||||
|
order: [['id', 'DESC']],
|
||||||
|
offset: 200,
|
||||||
|
limit: 1,
|
||||||
|
});
|
||||||
|
if (oldest[0]?.id) {
|
||||||
|
await SystemModel.destroy({
|
||||||
|
where: {
|
||||||
|
type: AuthDataType.notifyLog,
|
||||||
|
id: { [Op.lte]: oldest[0].id },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to persist notification audit log');
|
||||||
|
}
|
||||||
|
}
|
||||||
+77
-24
@@ -1,5 +1,9 @@
|
|||||||
|
import { EventEmitter } from 'events';
|
||||||
import PQueue, { QueueAddOptions } from 'p-queue-cjs';
|
import PQueue, { QueueAddOptions } from 'p-queue-cjs';
|
||||||
import os from 'os';
|
import os from 'os';
|
||||||
|
import type { Sequelize } from 'sequelize';
|
||||||
|
import { QueueOccupants, recordQueueAdmission } from './queueDiagnostics';
|
||||||
|
import { CronWaitQueue, CronQueueOptions } from './cronWaitQueue';
|
||||||
import { AuthDataType, SystemModel } from '../data/system';
|
import { AuthDataType, SystemModel } from '../data/system';
|
||||||
import Logger from '../loaders/logger';
|
import Logger from '../loaders/logger';
|
||||||
import { Dependence } from '../data/dependence';
|
import { Dependence } from '../data/dependence';
|
||||||
@@ -23,7 +27,11 @@ class TaskLimit {
|
|||||||
private queuedCrons = new Map<string, ICronFn<any>[]>();
|
private queuedCrons = new Map<string, ICronFn<any>[]>();
|
||||||
private repeatCronNotifyMap = new Map<string, number>();
|
private repeatCronNotifyMap = new Map<string, number>();
|
||||||
private updateLogLimit = new PQueue({ concurrency: 1 });
|
private updateLogLimit = new PQueue({ concurrency: 1 });
|
||||||
private cronLimit = new PQueue({
|
private warnSampling = (error: Error) => Logger.warn('[queue] Could not save occupancy: %s', error.message);
|
||||||
|
private scheduledOccupants = new QueueOccupants('scheduled', () => this.cronLimit.concurrency, this.warnSampling);
|
||||||
|
private manualOccupants = new QueueOccupants('manual', () => this.manualCronoLimit.concurrency, this.warnSampling);
|
||||||
|
private cronLimit = new PQueue<CronWaitQueue, CronQueueOptions>({
|
||||||
|
queueClass: CronWaitQueue,
|
||||||
concurrency: Math.max(os.cpus().length, 4),
|
concurrency: Math.max(os.cpus().length, 4),
|
||||||
});
|
});
|
||||||
private manualCronoLimit = new PQueue({
|
private manualCronoLimit = new PQueue({
|
||||||
@@ -50,11 +58,9 @@ class TaskLimit {
|
|||||||
Buffer.from(tlsConfig.clientCert),
|
Buffer.from(tlsConfig.clientCert),
|
||||||
)
|
)
|
||||||
: credentials.createInsecure();
|
: credentials.createInsecure();
|
||||||
this._client = new ApiClient(
|
this._client = new ApiClient(`localhost:${config.grpcPort}`, creds, {
|
||||||
`localhost:${config.grpcPort}`,
|
'grpc.enable_http_proxy': 0,
|
||||||
creds,
|
});
|
||||||
{ 'grpc.enable_http_proxy': 0 },
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
return this._client;
|
return this._client;
|
||||||
}
|
}
|
||||||
@@ -81,29 +87,36 @@ class TaskLimit {
|
|||||||
private handleEvents() {
|
private handleEvents() {
|
||||||
this.cronLimit.on('add', () => {
|
this.cronLimit.on('add', () => {
|
||||||
Logger.info(
|
Logger.info(
|
||||||
`[schedule][任务加入队列] 运行中任务数: ${this.cronLimitActiveCount}, 等待中任务数: ${this.cronLimitPendingCount}`,
|
t('[schedule][任务加入队列] 运行中任务数: %s, 等待中任务数: %s'),
|
||||||
|
this.cronLimitActiveCount,
|
||||||
|
this.cronLimitPendingCount,
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
this.cronLimit.on('active', () => {
|
this.cronLimit.on('active', () => {
|
||||||
Logger.info(
|
Logger.info(
|
||||||
`[schedule][开始处理任务] 运行中任务数: ${
|
t('[schedule][开始处理任务] 运行中任务数: %s, 等待中任务数: %s'),
|
||||||
this.cronLimitActiveCount + 1
|
this.cronLimitActiveCount + 1,
|
||||||
}, 等待中任务数: ${this.cronLimitPendingCount}`,
|
this.cronLimitPendingCount,
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
this.cronLimit.on('completed', (param) => {
|
this.cronLimit.on('completed', (param) => {
|
||||||
Logger.info(`[schedule][任务处理成功] 参数 ${JSON.stringify(param)}`);
|
Logger.info(t('[schedule][任务处理成功] 参数 %s'), JSON.stringify(param));
|
||||||
});
|
});
|
||||||
this.cronLimit.on('error', (error) => {
|
this.cronLimit.on('error', (error) => {
|
||||||
Logger.error(`[schedule][任务处理错误] 参数 ${JSON.stringify(error)}`);
|
Logger.error(
|
||||||
|
t('[schedule][任务处理错误] 参数 %s'),
|
||||||
|
JSON.stringify(error),
|
||||||
|
);
|
||||||
});
|
});
|
||||||
this.cronLimit.on('next', () => {
|
this.cronLimit.on('next', () => {
|
||||||
Logger.info(
|
Logger.info(
|
||||||
`[schedule][任务处理结束] 运行中任务数: ${this.cronLimitActiveCount}, 等待中任务数: ${this.cronLimitPendingCount}`,
|
t('[schedule][任务处理结束] 运行中任务数: %s, 等待中任务数: %s'),
|
||||||
|
this.cronLimitActiveCount,
|
||||||
|
this.cronLimitPendingCount,
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
this.cronLimit.on('idle', () => {
|
this.cronLimit.on('idle', () => {
|
||||||
Logger.info(`[schedule][任务队列] 空闲中...`);
|
Logger.info(t(`[schedule][任务队列] 空闲中...`));
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -113,20 +126,29 @@ class TaskLimit {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public removeQueuedCron(id: string) {
|
public removeQueuedCron(id: string, completed?: ICronFn<any>) {
|
||||||
if (this.queuedCrons.has(id)) {
|
if (this.queuedCrons.has(id)) {
|
||||||
const runs = this.queuedCrons.get(id);
|
const runs = this.queuedCrons.get(id);
|
||||||
if (runs && runs.length > 0) {
|
if (runs && runs.length > 0) {
|
||||||
runs.pop();
|
const remaining = completed
|
||||||
this.queuedCrons.set(id, runs);
|
? runs.filter((run) => run !== completed)
|
||||||
|
: runs.slice(0, -1);
|
||||||
|
if (remaining.length) this.queuedCrons.set(id, remaining);
|
||||||
|
else this.queuedCrons.delete(id);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public get cronConcurrency() {
|
||||||
|
return this.manualCronoLimit.concurrency;
|
||||||
|
}
|
||||||
|
|
||||||
public async setCustomLimit(limit?: number) {
|
public async setCustomLimit(limit?: number) {
|
||||||
if (limit) {
|
if (limit) {
|
||||||
this.cronLimit.concurrency = limit;
|
this.cronLimit.concurrency = limit;
|
||||||
this.manualCronoLimit.concurrency = limit;
|
this.manualCronoLimit.concurrency = limit;
|
||||||
|
this.scheduledOccupants.refresh();
|
||||||
|
this.manualOccupants.refresh();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
await SystemModel.sync();
|
await SystemModel.sync();
|
||||||
@@ -134,8 +156,7 @@ class TaskLimit {
|
|||||||
where: { type: AuthDataType.systemConfig },
|
where: { type: AuthDataType.systemConfig },
|
||||||
});
|
});
|
||||||
if (doc?.info?.cronConcurrency) {
|
if (doc?.info?.cronConcurrency) {
|
||||||
this.cronLimit.concurrency = doc.info.cronConcurrency;
|
await this.setCustomLimit(doc.info.cronConcurrency);
|
||||||
this.manualCronoLimit.concurrency = doc.info.cronConcurrency;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -143,9 +164,23 @@ class TaskLimit {
|
|||||||
cron: TCron,
|
cron: TCron,
|
||||||
fn: ICronFn<T>,
|
fn: ICronFn<T>,
|
||||||
options?: Partial<QueueAddOptions>,
|
options?: Partial<QueueAddOptions>,
|
||||||
|
isCurrent: () => boolean = () => true,
|
||||||
|
onPending?: () => void,
|
||||||
|
waitingSignal?: EventEmitter,
|
||||||
|
triggerId?: string,
|
||||||
|
db?: Sequelize,
|
||||||
): Promise<T | void> {
|
): Promise<T | void> {
|
||||||
|
if (!isCurrent()) return;
|
||||||
fn.cron = cron;
|
fn.cron = cron;
|
||||||
let runs = this.queuedCrons.get(cron.id);
|
fn.isCurrent = isCurrent;
|
||||||
|
// Invalidated snapshots must not consume the new revision's repeat budget.
|
||||||
|
const runs = this.queuedCrons
|
||||||
|
.get(cron.id)
|
||||||
|
?.filter((run) => run.isCurrent?.() !== false);
|
||||||
|
if (cron.skip_if_pending && runs?.length) {
|
||||||
|
onPending?.();
|
||||||
|
return;
|
||||||
|
}
|
||||||
const result = runs?.length ? [...runs, fn] : [fn];
|
const result = runs?.length ? [...runs, fn] : [fn];
|
||||||
const repeatTimes = this.repeatCronNotifyMap.get(cron.id) || 0;
|
const repeatTimes = this.repeatCronNotifyMap.get(cron.id) || 0;
|
||||||
if (result?.length > 5) {
|
if (result?.length > 5) {
|
||||||
@@ -165,24 +200,42 @@ class TaskLimit {
|
|||||||
(err, res) => {
|
(err, res) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
Logger.error(
|
Logger.error(
|
||||||
`[schedule][任务重复运行] 通知失败 ${JSON.stringify(err)}`,
|
t('[schedule][任务重复运行] 通知失败 %s'),
|
||||||
|
JSON.stringify(err),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
Logger.warn(`[schedule][任务重复运行] 参数 ${JSON.stringify(cron)}`);
|
Logger.warn(t('[schedule][任务重复运行] 参数 %s'), JSON.stringify(cron));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
this.queuedCrons.set(cron.id, result);
|
this.queuedCrons.set(cron.id, result);
|
||||||
return this.cronLimit.add(fn, options);
|
const run = this.trackAdmission(this.scheduledOccupants, this.cronLimit.concurrency, fn,
|
||||||
|
{ id: triggerId, name: cron.name || cron.id }, db);
|
||||||
|
if (!waitingSignal) return this.cronLimit.add(run, options);
|
||||||
|
return new Promise<T | void>((resolve, reject) => {
|
||||||
|
this.cronLimit.add(run, { ...options, waitingSignal, onRemoved: () => { this.scheduledOccupants.forget(triggerId); resolve(); } }).then(resolve, reject);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private trackAdmission<T>(pool: QueueOccupants, limit: number, fn: () => Promise<T>, occupant: { id?: string; name: string }, db?: Sequelize) {
|
||||||
|
const admission = pool.admission(limit);
|
||||||
|
if (occupant.id && admission && db) {
|
||||||
|
pool.watch(occupant.id, db);
|
||||||
|
void recordQueueAdmission(db, occupant.id, admission)
|
||||||
|
.catch(error => Logger.warn('[queue] Failed to record wait reason: %s', error.message));
|
||||||
|
}
|
||||||
|
return pool.track(fn, occupant, db);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async manualRunWithCronLimit<T>(
|
public async manualRunWithCronLimit<T>(
|
||||||
fn: () => Promise<T>,
|
fn: () => Promise<T>,
|
||||||
options?: Partial<QueueAddOptions>,
|
options?: Partial<QueueAddOptions>,
|
||||||
|
occupant: { id?: string; name: string } = { name: '—' },
|
||||||
|
db?: Sequelize,
|
||||||
): Promise<T | void> {
|
): Promise<T | void> {
|
||||||
return this.manualCronoLimit.add(fn, options);
|
return this.manualCronoLimit.add(this.trackAdmission(this.manualOccupants, this.manualCronoLimit.concurrency, fn, occupant, db), options);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async runWithSubscriptionLimit<T>(
|
public async runWithSubscriptionLimit<T>(
|
||||||
|
|||||||
@@ -0,0 +1,127 @@
|
|||||||
|
import { QueryTypes, Sequelize } from 'sequelize';
|
||||||
|
|
||||||
|
type Occupant = { id?: string; name: string };
|
||||||
|
export type Admission = {
|
||||||
|
queue: 'manual' | 'scheduled';
|
||||||
|
limit: number;
|
||||||
|
occupied: number;
|
||||||
|
holders: Occupant[];
|
||||||
|
};
|
||||||
|
|
||||||
|
// Each object belongs to exactly one execution pool in its owning process.
|
||||||
|
export class QueueOccupants {
|
||||||
|
private active = new Set<Occupant>();
|
||||||
|
private waiting = new Map<string, Sequelize>();
|
||||||
|
private writing = false;
|
||||||
|
private changed = false;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private queue: Admission['queue'],
|
||||||
|
private limit: () => number,
|
||||||
|
private warn: (error: Error) => void,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
watch(id: string, db: Sequelize) {
|
||||||
|
this.waiting.set(id, db);
|
||||||
|
this.refresh();
|
||||||
|
}
|
||||||
|
|
||||||
|
forget(id?: string) {
|
||||||
|
if (id) this.waiting.delete(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
refresh() {
|
||||||
|
if (!this.waiting.size) return;
|
||||||
|
this.changed = true;
|
||||||
|
if (this.writing) return;
|
||||||
|
this.writing = true;
|
||||||
|
// Coalesce synchronous PQueue handoffs and keep SQLite writes in observation order.
|
||||||
|
void Promise.resolve().then(async () => {
|
||||||
|
try {
|
||||||
|
while (this.changed) {
|
||||||
|
this.changed = false;
|
||||||
|
const context = JSON.stringify(this.snapshot(this.limit()));
|
||||||
|
const groups = new Map<Sequelize, string[]>();
|
||||||
|
for (const [id, db] of this.waiting) {
|
||||||
|
const ids = groups.get(db);
|
||||||
|
if (ids) ids.push(id);
|
||||||
|
else groups.set(db, [id]);
|
||||||
|
}
|
||||||
|
for (const [db, ids] of groups) {
|
||||||
|
await db
|
||||||
|
.query(
|
||||||
|
`UPDATE CronTriggers SET queue_live_context=$context
|
||||||
|
WHERE state='queued' AND dequeued_at IS NULL AND id IN (SELECT value FROM json_each($ids))`,
|
||||||
|
{ bind: { context, ids: JSON.stringify(ids) } },
|
||||||
|
)
|
||||||
|
.catch(this.warn);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
this.writing = false;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
admission(limit: number): Admission | undefined {
|
||||||
|
return this.active.size >= limit ? this.snapshot(limit) : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
private snapshot(limit: number): Admission {
|
||||||
|
return {
|
||||||
|
queue: this.queue,
|
||||||
|
limit,
|
||||||
|
occupied: this.active.size,
|
||||||
|
holders: [...this.active].slice(0, 8),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
track<T>(job: () => Promise<T>, occupant: Occupant, db?: Sequelize) {
|
||||||
|
return async () => {
|
||||||
|
this.forget(occupant.id);
|
||||||
|
this.active.add(occupant);
|
||||||
|
this.refresh();
|
||||||
|
if (occupant.id && db) {
|
||||||
|
void db
|
||||||
|
.query(
|
||||||
|
'UPDATE CronTriggers SET dequeued_at=$at,queue_live_context=NULL WHERE id=$id AND queued_at <= $at AND dequeued_at IS NULL',
|
||||||
|
{ bind: { id: occupant.id, at: Date.now() } },
|
||||||
|
)
|
||||||
|
.catch(this.warn);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return await job();
|
||||||
|
} finally {
|
||||||
|
this.active.delete(occupant);
|
||||||
|
this.refresh();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function recordQueueAdmission(
|
||||||
|
db: Sequelize,
|
||||||
|
id: string,
|
||||||
|
admission: Admission,
|
||||||
|
) {
|
||||||
|
return db.query(
|
||||||
|
'UPDATE CronTriggers SET queue_context=$context WHERE id=$id AND queued_at <= $at AND queue_context IS NULL',
|
||||||
|
{ bind: { id, at: Date.now(), context: JSON.stringify(admission) } },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function oldestQueued(db: Sequelize) {
|
||||||
|
return db.query<{
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
source: string;
|
||||||
|
queued_at: number;
|
||||||
|
queue_context: string | null;
|
||||||
|
queue_live_context: string | null;
|
||||||
|
}>(
|
||||||
|
`SELECT t.id, COALESCE(NULLIF(c.name,''), c.command, '#' || t.cron_id) AS name, t.source,t.queued_at,t.queue_context,t.queue_live_context
|
||||||
|
FROM CronTriggers t LEFT JOIN Crontabs c ON c.id=t.cron_id
|
||||||
|
WHERE t.state='queued' AND t.dequeued_at IS NULL AND t.queued_at IS NOT NULL ORDER BY t.queued_at,t.id LIMIT 25`,
|
||||||
|
{ type: QueryTypes.SELECT },
|
||||||
|
);
|
||||||
|
}
|
||||||
+195
-33
@@ -1,100 +1,262 @@
|
|||||||
import { spawn } from 'cross-spawn';
|
import { hasLiveCronProcess } from './cronProcess';
|
||||||
|
import { EventEmitter } from 'events';
|
||||||
|
import PQueue from 'p-queue-cjs';
|
||||||
|
import { triggerHistory, TriggerState } from './triggerHistory';
|
||||||
|
import { t } from './i18n';
|
||||||
|
import { spawnCronCommand, releaseCronCommand } from './cronCommandProcess';
|
||||||
import taskLimit from './pLimit';
|
import taskLimit from './pLimit';
|
||||||
import Logger from '../loaders/logger';
|
import Logger from '../loaders/logger';
|
||||||
import { ICron } from '../protos/cron';
|
import { ICron } from '../protos/cron';
|
||||||
import { CrontabModel, CrontabStatus } from '../data/cron';
|
import { CrontabModel, CrontabStatus } from '../data/cron';
|
||||||
import { killTask } from '../config/util';
|
|
||||||
import { RunningInstanceModel, InstanceStatus } from '../data/runningInstance';
|
import { RunningInstanceModel, InstanceStatus } from '../data/runningInstance';
|
||||||
import dayjs from 'dayjs';
|
import dayjs from 'dayjs';
|
||||||
import { observeChildProcess, asError } from './childProcess';
|
import { observeChildProcess, asError } from './childProcess';
|
||||||
|
|
||||||
export function runCron(cmd: string, cron: ICron): Promise<number | void> {
|
const waiting = new Map<string, () => void>();
|
||||||
return taskLimit.runWithCronLimit(cron, async () => {
|
// A backlog of missed timers can otherwise open hundreds of competing SQLite
|
||||||
|
// writer transactions on wake. Only serialize history writes, never commands.
|
||||||
|
const historyWrites = new PQueue({ concurrency: 1 });
|
||||||
|
export function cancelWaitingTriggers(ids: string[]) {
|
||||||
|
for (const id of ids) waiting.get(id)?.();
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function runCron(
|
||||||
|
cmd: string,
|
||||||
|
cron: ICron,
|
||||||
|
isCurrent: () => boolean = () => true,
|
||||||
|
trigger?: { id: string; time: number; source: 'node' | 'recovery' },
|
||||||
|
): Promise<number | void> {
|
||||||
|
const history = trigger && triggerHistory();
|
||||||
|
const queuedAt = Date.now();
|
||||||
|
if (trigger && history) {
|
||||||
|
const cronId = Number(cron.id);
|
||||||
|
const admitted = await historyWrites.add(() => cron.skip_if_pending
|
||||||
|
? history.enqueueUnlessPending(trigger.id, cronId, trigger.source, trigger.time, queuedAt, cron.recovery_from)
|
||||||
|
: history.enqueue(trigger.id, cronId, trigger.source, trigger.time, queuedAt));
|
||||||
|
if (!admitted) return;
|
||||||
|
}
|
||||||
|
let reason = 'capacity';
|
||||||
|
let state: TriggerState = 'skipped';
|
||||||
|
let exitCode: number | null = null;
|
||||||
|
let claimed = false;
|
||||||
|
const deadline = cron.max_queue_wait ? queuedAt + cron.max_queue_wait * 1000 : Infinity;
|
||||||
|
let timer: NodeJS.Timeout | undefined;
|
||||||
|
const expiry = trigger ? new EventEmitter() : undefined;
|
||||||
|
const alive = () => isCurrent() && Date.now() < deadline;
|
||||||
|
if (trigger && expiry && Number.isFinite(deadline)) {
|
||||||
|
timer = setTimeout(() => {
|
||||||
|
reason = 'queue_timeout';
|
||||||
|
taskLimit.removeQueuedCron(cron.id, execute);
|
||||||
|
expiry.emit('expired');
|
||||||
|
}, Math.max(0, deadline - Date.now()));
|
||||||
|
timer.unref();
|
||||||
|
}
|
||||||
|
const execute = async () => {
|
||||||
|
if (trigger) waiting.delete(trigger.id);
|
||||||
|
reason = 'changed';
|
||||||
try {
|
try {
|
||||||
// Check if the cron is already running and stop it (only if multiple instances are not allowed)
|
if (!alive()) { reason = isCurrent() ? 'queue_timeout' : 'changed'; return; }
|
||||||
|
let multiple = false;
|
||||||
|
// A single-instance trigger must not interrupt work already in progress.
|
||||||
try {
|
try {
|
||||||
const existingCron = await CrontabModel.findOne({
|
const existingCron = await CrontabModel.findOne({
|
||||||
where: { id: Number(cron.id) },
|
where: { id: Number(cron.id) },
|
||||||
});
|
});
|
||||||
|
|
||||||
// Default to single instance mode (0) for backward compatibility
|
// A mutation may commit while this queued run is reading the database.
|
||||||
const allowSingleInstances =
|
if (!isCurrent() || !existingCron || existingCron.isDisabled) return;
|
||||||
existingCron?.allow_multiple_instances === 0;
|
|
||||||
|
|
||||||
|
// Default to single instance mode (0) for backward compatibility
|
||||||
|
multiple = existingCron.allow_multiple_instances === 1;
|
||||||
|
|
||||||
|
if (!multiple) {
|
||||||
|
// A queued manual run is an active reservation, even if its stored PID
|
||||||
|
// belongs to a previous execution.
|
||||||
if (
|
if (
|
||||||
allowSingleInstances &&
|
existingCron.status === CrontabStatus.queued &&
|
||||||
existingCron &&
|
existingCron.pid
|
||||||
existingCron.pid &&
|
|
||||||
(existingCron.status === CrontabStatus.running ||
|
|
||||||
existingCron.status === CrontabStatus.queued)
|
|
||||||
) {
|
) {
|
||||||
Logger.info(
|
Logger.info(
|
||||||
`[schedule][停止已运行任务] 任务ID: ${cron.id}, PID: ${existingCron.pid}`,
|
'[schedule][Skip queued single-instance task] Task ID: %s, PID: %s',
|
||||||
|
cron.id,
|
||||||
|
existingCron.pid,
|
||||||
);
|
);
|
||||||
await killTask(existingCron.pid, true);
|
reason = 'overlap';
|
||||||
// Mark old running instances as stopped
|
return;
|
||||||
const stoppedAt = dayjs().unix();
|
}
|
||||||
await RunningInstanceModel.update(
|
if (
|
||||||
{ status: InstanceStatus.stopped, finished_at: stoppedAt },
|
existingCron.status === CrontabStatus.running &&
|
||||||
{
|
existingCron.pid
|
||||||
|
) {
|
||||||
|
if (hasLiveCronProcess(existingCron.status, existingCron.pid)) {
|
||||||
|
Logger.info(
|
||||||
|
'[schedule][Skip active single-instance task] Task ID: %s, PID: %s',
|
||||||
|
cron.id,
|
||||||
|
existingCron.pid,
|
||||||
|
);
|
||||||
|
reason = 'overlap';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Capture old instance IDs before releasing the stale cron state so
|
||||||
|
// a concurrent replacement cannot have its instance marked stopped.
|
||||||
|
const staleInstances = await RunningInstanceModel.findAll({
|
||||||
|
attributes: ['id'],
|
||||||
where: {
|
where: {
|
||||||
cron_id: Number(cron.id),
|
cron_id: Number(cron.id),
|
||||||
|
pid: existingCron.pid,
|
||||||
status: InstanceStatus.running,
|
status: InstanceStatus.running,
|
||||||
},
|
},
|
||||||
|
});
|
||||||
|
if (!isCurrent()) return;
|
||||||
|
const [recovered] = await CrontabModel.update(
|
||||||
|
{
|
||||||
|
status: CrontabStatus.idle,
|
||||||
|
pid: null,
|
||||||
|
queued_token: null,
|
||||||
|
} as any,
|
||||||
|
{
|
||||||
|
where: {
|
||||||
|
id: Number(cron.id),
|
||||||
|
pid: existingCron.pid,
|
||||||
|
status: CrontabStatus.running,
|
||||||
|
...(existingCron.log_path !== undefined
|
||||||
|
? { log_path: existingCron.log_path }
|
||||||
|
: {}),
|
||||||
|
...(existingCron.queued_token !== undefined
|
||||||
|
? { queued_token: existingCron.queued_token }
|
||||||
|
: {}),
|
||||||
|
...(existingCron.last_execution_time !== undefined
|
||||||
|
? { last_execution_time: existingCron.last_execution_time }
|
||||||
|
: {}),
|
||||||
|
...(existingCron.isDisabled !== undefined
|
||||||
|
? { isDisabled: existingCron.isDisabled }
|
||||||
|
: {}),
|
||||||
|
allow_multiple_instances: 0,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
// Update the status to idle after killing
|
if (recovered !== 1) return;
|
||||||
await CrontabModel.update(
|
const staleIds = staleInstances.map((instance) => instance.id!);
|
||||||
{ status: CrontabStatus.idle, pid: undefined },
|
if (staleIds.length) {
|
||||||
{ where: { id: Number(cron.id) } },
|
await RunningInstanceModel.update(
|
||||||
|
{ status: InstanceStatus.stopped, finished_at: dayjs().unix() },
|
||||||
|
{ where: { id: staleIds, status: InstanceStatus.running } },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
Logger.info(
|
||||||
|
'[schedule][Recovered stale single-instance task] Task ID: %s, PID: %s',
|
||||||
|
cron.id,
|
||||||
|
existingCron.pid,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
Logger.error(
|
Logger.error(
|
||||||
`[schedule][检查已运行任务失败] 任务ID: ${cron.id}, 错误: ${error}`,
|
t('[schedule][检查已运行任务失败] 任务ID: %s, 错误: %s'),
|
||||||
|
cron.id,
|
||||||
|
error,
|
||||||
);
|
);
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Invalidation can happen during the asynchronous database check.
|
||||||
|
if (!isCurrent()) return;
|
||||||
Logger.info(
|
Logger.info(
|
||||||
`[schedule][开始执行任务] 参数 ${JSON.stringify({
|
t('[schedule][开始执行任务] 参数 %s'),
|
||||||
|
JSON.stringify({
|
||||||
...cron,
|
...cron,
|
||||||
command: cmd,
|
command: cmd,
|
||||||
})}`,
|
}),
|
||||||
);
|
);
|
||||||
const cp = spawn(cmd, { shell: '/bin/bash' });
|
clearTimeout(timer);
|
||||||
|
if (Date.now() >= deadline) {
|
||||||
|
reason = 'queue_timeout';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (trigger) {
|
||||||
|
claimed = await history!.start(trigger.id, Date.now(), multiple);
|
||||||
|
if (!claimed) { reason = 'overlap'; return; }
|
||||||
|
}
|
||||||
|
if (!isCurrent() || Date.now() >= deadline) {
|
||||||
|
reason = isCurrent() ? 'queue_timeout' : 'changed';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let saveStart: Promise<void> | undefined;
|
||||||
|
const cp = spawnCronCommand(cmd, trigger?.id, trigger && history ? at => {
|
||||||
|
saveStart = history.recordCommandStart(trigger.id, at).catch(error => {
|
||||||
|
Logger.warn('[schedule] Could not record command start: %s', asError(error).message);
|
||||||
|
});
|
||||||
|
} : undefined);
|
||||||
|
|
||||||
|
let denied: string | undefined;
|
||||||
const { completed } = observeChildProcess(cp, {
|
const { completed } = observeChildProcess(cp, {
|
||||||
|
onStart: async () => {
|
||||||
|
if (!trigger) return;
|
||||||
|
try {
|
||||||
|
if (await history!.bindCommand(trigger.id, cp.pid!) && !await history!.shouldStop(trigger.id)) {
|
||||||
|
if (!alive()) denied = isCurrent() ? 'queue_timeout' : 'changed';
|
||||||
|
else await releaseCronCommand(cp);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
cp.stdin.end();
|
||||||
|
}
|
||||||
|
},
|
||||||
onStderr: async (message) => {
|
onStderr: async (message) => {
|
||||||
Logger.info(
|
Logger.info(
|
||||||
'[schedule][任务标准错误] 命令: %s, 信息: %s',
|
t('[schedule][任务标准错误] 命令: %s, 信息: %s'),
|
||||||
cmd,
|
cmd,
|
||||||
message,
|
message,
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const result = await completed;
|
const result = await completed;
|
||||||
|
await saveStart;
|
||||||
|
if (denied) { reason = denied; return; }
|
||||||
|
state = result.signal ? 'stopped' : result.error || result.code !== 0 ? 'failed' : 'finished';
|
||||||
|
reason = result.error ? 'spawn' : result.signal || '';
|
||||||
|
exitCode = result.code;
|
||||||
if (result.error) {
|
if (result.error) {
|
||||||
Logger.error(
|
Logger.error(
|
||||||
'[schedule][执行任务失败] 命令: %s, 错误: %s',
|
t('[schedule][执行任务失败] 命令: %s, 错误: %s'),
|
||||||
cmd,
|
cmd,
|
||||||
result.error.message,
|
result.error.message,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
Logger.info(
|
Logger.info(
|
||||||
'[schedule][执行任务结束] 任务ID: %s, 退出码: %j',
|
t('[schedule][执行任务结束] 任务ID: %s, 退出码: %j'),
|
||||||
cron.id,
|
cron.id,
|
||||||
result.code,
|
result.code,
|
||||||
);
|
);
|
||||||
return { ...cron, command: cmd, pid: cp.pid, ...result } as any;
|
return { ...cron, command: cmd, pid: cp.pid, ...result } as any;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
state = 'failed';
|
||||||
|
reason = 'spawn';
|
||||||
Logger.error(
|
Logger.error(
|
||||||
'[schedule][创建任务失败] 命令: %s, 错误: %s',
|
t('[schedule][创建任务失败] 命令: %s, 错误: %s'),
|
||||||
cmd,
|
cmd,
|
||||||
asError(error).message,
|
asError(error).message,
|
||||||
);
|
);
|
||||||
} finally {
|
} finally {
|
||||||
taskLimit.removeQueuedCron(cron.id);
|
taskLimit.removeQueuedCron(cron.id, execute);
|
||||||
}
|
}
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
if (trigger && expiry) waiting.set(trigger.id, () => {
|
||||||
|
taskLimit.removeQueuedCron(cron.id, execute);
|
||||||
|
expiry.emit('expired');
|
||||||
});
|
});
|
||||||
|
return await taskLimit.runWithCronLimit(cron, execute, undefined, isCurrent, () => { reason = 'pending'; }, expiry, trigger?.id, CrontabModel.sequelize);
|
||||||
|
} finally {
|
||||||
|
clearTimeout(timer);
|
||||||
|
if (trigger) waiting.delete(trigger.id);
|
||||||
|
if (trigger && history) {
|
||||||
|
const outcomeReason = state === 'skipped' && !isCurrent() ? 'changed' : reason;
|
||||||
|
const finishedAt = Date.now();
|
||||||
|
await historyWrites.add(() => state === 'skipped' && !claimed
|
||||||
|
? history.skipQueued(trigger.id, outcomeReason, finishedAt)
|
||||||
|
: history.complete(trigger.id, state, outcomeReason || null, exitCode, finishedAt),
|
||||||
|
{ priority: 1 });
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,41 @@
|
|||||||
|
import { readFileSync } from 'fs';
|
||||||
|
import path from 'path';
|
||||||
|
import { createHash, randomUUID } from 'crypto';
|
||||||
|
|
||||||
|
function read(filename: string): Buffer | undefined {
|
||||||
|
try {
|
||||||
|
return readFileSync(filename);
|
||||||
|
} catch {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function manifest(): { sourceCommit?: string } {
|
||||||
|
try {
|
||||||
|
const value = JSON.parse(
|
||||||
|
read(path.resolve(__dirname, '../../build-info.json'))?.toString() ||
|
||||||
|
'{}',
|
||||||
|
);
|
||||||
|
return value && typeof value === 'object' ? value : {};
|
||||||
|
} catch {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Capture before serving requests. Reading the installed files on each request
|
||||||
|
// would let an old process claim the identity of a newly copied build.
|
||||||
|
const backend = read(path.resolve(__dirname, '../app.js'));
|
||||||
|
const root = process.env.QL_DIR || path.resolve(__dirname, '../../..');
|
||||||
|
const version = read(path.join(root, 'version.yaml'))
|
||||||
|
?.toString()
|
||||||
|
.match(/^\uFEFF?version:\s*["']?(\d+\.\d+\.\d+(?:-[\w.-]+)?(?:\+[\w.-]+)?)/m)?.[1];
|
||||||
|
|
||||||
|
export const runtimeBuild = Object.freeze({
|
||||||
|
instanceId: randomUUID(),
|
||||||
|
parentPid: process.ppid,
|
||||||
|
version,
|
||||||
|
sourceCommit: manifest().sourceCommit,
|
||||||
|
backendSha256: backend
|
||||||
|
? createHash('sha256').update(backend).digest('hex')
|
||||||
|
: undefined,
|
||||||
|
});
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user