Compare commits

..
143 changed files with 1112 additions and 12405 deletions
+68 -107
View File
@@ -2,44 +2,23 @@ name: Build And Push Docker Image
on:
push:
paths:
- "**"
- "!*.md"
# CLI is verified separately; its shared Bash upgrade engine ships in images.
- "!cli/**"
- "cli/src/internal/maintenance/upgradeArtifacts.cjs"
- "!.github/workflows/cli-package.yml"
paths-ignore:
- "*.md"
branches:
- "master"
- "develop"
tags:
- "v*"
workflow_dispatch:
inputs:
static_sync_only:
description: Synchronize an existing static snapshot to GitLab and verify Gitee
type: boolean
default: false
static_ref:
description: Static branch or version name, for example v2.22.0
type: string
default: ''
static_ref_type:
description: Synchronize a branch, or a version branch and Git tag
type: choice
options: [branch, tag]
default: branch
permissions:
contents: read
jobs:
validate:
if: ${{ !inputs.static_sync_only }}
uses: ./.github/workflows/validate.yml
code_gitlab:
if: ${{ !inputs.static_sync_only }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
@@ -56,8 +35,37 @@ jobs:
git push --force --all gitlab 2>&1 || echo "::warning::GitLab push failed"
git push --force --tags gitlab 2>&1 || echo "::warning::GitLab tags push failed"
code_gitee:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Push to Gitee
env:
GITEE_TOKEN: ${{ secrets.GITEE_TOKEN }}
run: |
set +e
mkdir -p ~/.ssh
printf '%s\n' "${{ secrets.GITLAB_SSH_PK }}" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -T 10 gitee.com >> ~/.ssh/known_hosts 2>/dev/null
git remote add gitee git@gitee.com:whyour/qinglong.git 2>/dev/null
if git push --force --all gitee 2>&1; then
echo "::notice::Gitee push --all succeeded"
else
echo "::warning::Push failed, trying to create repo via API..."
curl -sS --connect-timeout 30 --max-time 60 \
-X POST "https://gitee.com/api/v5/user/repos" \
-H "Content-Type: application/json" \
-d '{"name":"qinglong","private":"false"}' \
"?access_token=$GITEE_TOKEN" 2>/dev/null
git push --force --all gitee 2>&1 && echo "::notice::Gitee push succeeded after repo creation" || echo "::warning::Gitee push failed after retry"
fi
git push --force --tags gitee 2>&1 || echo "::warning::Gitee tags push failed"
build-static:
if: ${{ !inputs.static_sync_only }}
needs: validate
runs-on: ubuntu-latest
steps:
@@ -90,109 +98,62 @@ jobs:
env:
GITHUB_REPO: github.com/${{ github.repository_owner }}/qinglong-static
GITHUB_BRANCH: ${{ github.ref_name }}
GITHUB_REF_TYPE: ${{ github.ref_type }}
run: |
bash scripts/publish-static.sh "https://${{ secrets.API_TOKEN }}@${GITHUB_REPO}.git" "$GITHUB_BRANCH" "$GITHUB_REF_TYPE"
mkdir -p tmp
cd ./tmp
cp -rf ../static/* ./
git init -b ${GITHUB_BRANCH} && git add .
git config --local user.name 'github-actions[bot]'
git config --local user.email 'github-actions[bot]@users.noreply.github.com'
git commit --allow-empty -m "copy static at $(date +'%Y-%m-%d %H:%M:%S')"
git push --force --quiet "https://${{ secrets.API_TOKEN }}@${GITHUB_REPO}.git" ${GITHUB_BRANCH}:${GITHUB_BRANCH}
static_gitlab:
needs: build-static
runs-on: ubuntu-latest
concurrency:
group: static-gitlab-${{ github.ref_name }}
cancel-in-progress: false
steps:
- uses: actions/checkout@v6
- name: Push qinglong-static to GitLab
env:
STATIC_BRANCH: ${{ github.ref_name }}
STATIC_REF_TYPE: ${{ github.ref_type }}
MIRROR_SSH_KEY: ${{ secrets.GITLAB_SSH_PK }}
run: |
set -euo pipefail
set +e
mkdir -p ~/.ssh
printf '%s\n' "$MIRROR_SSH_KEY" > ~/.ssh/id_rsa
printf '%s\n' "${{ secrets.GITLAB_SSH_PK }}" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -T 10 gitlab.com >> ~/.ssh/known_hosts 2>/dev/null
bash scripts/sync-static.sh git@gitlab.com:whyour/qinglong-static.git "$STATIC_BRANCH" "$STATIC_REF_TYPE"
git clone --depth=1 --single-branch https://github.com/whyour/qinglong-static.git static-mirror
cd static-mirror
git remote add gitlab git@gitlab.com:whyour/qinglong-static.git 2>/dev/null
git push --force --all gitlab 2>&1 || echo "::warning::GitLab static push failed"
git push --force --tags gitlab 2>&1 || echo "::warning::GitLab static tags push failed"
verify-gitee-mirrors:
static_gitee:
needs: build-static
runs-on: ubuntu-latest
timeout-minutes: 14
permissions:
contents: write
strategy:
fail-fast: false
matrix:
repository: [qinglong, qinglong-static]
steps:
- uses: actions/checkout@v6
# Gitee Pull mirrors and GitHub push webhooks own synchronization.
# Verification runs alongside Docker builds and needs no Gitee credentials.
- name: Wait for Gitee Pull mirror
if: ${{ matrix.repository == 'qinglong' }}
- name: Push qinglong-static to Gitee
env:
GH_TOKEN: ${{ github.token }}
QL_MIRROR_NOTIFY_REPO: whyour/qinglong
MIRROR_REPOSITORY: ${{ matrix.repository }}
MIRROR_REF: ${{ github.ref_name }}
MIRROR_REF_TYPE: ${{ github.ref_type }}
QL_MIRROR_INCLUDE_VERSION_BRANCH: ${{ matrix.repository == 'qinglong-static' }}
QL_MIRROR_NOTIFY_ON_START: 'false'
GITEE_TOKEN: ${{ secrets.GITEE_TOKEN }}
run: |
bash scripts/wait-for-mirror.sh "https://github.com/whyour/$MIRROR_REPOSITORY.git" "https://gitee.com/whyour/$MIRROR_REPOSITORY.git" "$MIRROR_REF" "$MIRROR_REF_TYPE"
- name: Notify and wait for Gitee static mirror
if: ${{ matrix.repository == 'qinglong-static' }}
env:
GH_TOKEN: ${{ secrets.API_TOKEN }}
QL_MIRROR_NOTIFY_REPO: whyour/qinglong-static
MIRROR_REF: ${{ github.ref_name }}
MIRROR_REF_TYPE: ${{ github.ref_type }}
QL_MIRROR_INCLUDE_VERSION_BRANCH: 'true'
run: |
bash scripts/wait-for-mirror.sh https://github.com/whyour/qinglong-static.git https://gitee.com/whyour/qinglong-static.git "$MIRROR_REF" "$MIRROR_REF_TYPE"
sync-existing-static:
if: ${{ github.event_name == 'workflow_dispatch' && inputs.static_sync_only }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
mirror: [gitlab]
concurrency:
group: static-${{ matrix.mirror }}-${{ inputs.static_ref }}
cancel-in-progress: false
steps:
- uses: actions/checkout@v6
- name: Synchronize existing static snapshot
env:
STATIC_REF: ${{ inputs.static_ref }}
STATIC_REF_TYPE: ${{ inputs.static_ref_type }}
MIRROR_HOST: ${{ matrix.mirror }}.com
MIRROR_SSH_KEY: ${{ secrets.GITLAB_SSH_PK }}
run: |
set -euo pipefail
set +e
mkdir -p ~/.ssh
printf '%s\n' "$MIRROR_SSH_KEY" > ~/.ssh/id_rsa
printf '%s\n' "${{ secrets.GITLAB_SSH_PK }}" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -T 10 "$MIRROR_HOST" >> ~/.ssh/known_hosts 2>/dev/null
bash scripts/sync-static.sh "git@$MIRROR_HOST:whyour/qinglong-static.git" "$STATIC_REF" "$STATIC_REF_TYPE"
ssh-keyscan -T 10 gitee.com >> ~/.ssh/known_hosts 2>/dev/null
git clone --depth=1 --single-branch https://github.com/whyour/qinglong-static.git static-mirror
cd static-mirror
git remote add gitee git@gitee.com:whyour/qinglong-static.git 2>/dev/null
verify-existing-static-gitee:
if: ${{ github.event_name == 'workflow_dispatch' && inputs.static_sync_only }}
runs-on: ubuntu-latest
timeout-minutes: 14
steps:
- uses: actions/checkout@v6
- name: Wait for existing Gitee static snapshot
env:
GH_TOKEN: ${{ secrets.API_TOKEN }}
QL_MIRROR_NOTIFY_REPO: whyour/qinglong-static
MIRROR_REF: ${{ inputs.static_ref }}
MIRROR_REF_TYPE: ${{ inputs.static_ref_type }}
QL_MIRROR_INCLUDE_VERSION_BRANCH: 'true'
run: |
bash scripts/wait-for-mirror.sh https://github.com/whyour/qinglong-static.git https://gitee.com/whyour/qinglong-static.git "$MIRROR_REF" "$MIRROR_REF_TYPE"
if git push --force --all gitee 2>&1; then
echo "::notice::Gitee static push succeeded"
else
echo "::warning::Push failed, trying to create repo via API..."
curl -sS --connect-timeout 30 --max-time 60 \
-X POST "https://gitee.com/api/v5/user/repos" \
-H "Content-Type: application/json" \
-d '{"name":"qinglong-static","private":"false"}' \
"?access_token=$GITEE_TOKEN" 2>/dev/null
git push --force --all gitee 2>&1 && echo "::notice::Gitee static push succeeded after repo creation" || echo "::warning::Gitee static push failed after retry"
fi
git push --force --tags gitee 2>&1 || echo "::warning::Gitee static tags push failed"
build-alpine:
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
+8 -54
View File
@@ -21,27 +21,16 @@ on:
workflow_dispatch:
inputs:
publish:
description: Publish the verified CLI package (master stable / develop prerelease)
description: Publish the verified CLI package (master only)
type: boolean
default: false
prerelease_version:
description: Develop only, e.g. 0.1.2-beta.0 (alpha/beta/rc)
type: string
default: ''
permissions:
contents: read
concurrency:
# Keep validation runs separate from releases; never interrupt a running release.
# Master push releases share the same group as manually requested releases.
group: >-
cli-package-${{ github.workflow }}-${{ github.ref }}-${{
((github.event_name == 'workflow_dispatch' && inputs.publish) ||
(github.event_name == 'push' && github.ref == 'refs/heads/master')) && 'publish' || 'check' }}
cancel-in-progress: >-
${{ github.ref != 'refs/heads/master' &&
!(github.event_name == 'workflow_dispatch' && inputs.publish) }}
group: cli-package-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
jobs:
package:
@@ -63,20 +52,6 @@ jobs:
sudo apt-get update
sudo apt-get install -y --no-install-recommends bash ca-certificates curl git jq perl procps python3 unzip zip
npm install --global --ignore-scripts --no-audit --no-fund ts-node@10.9.2 typescript@5.2.2
- name: Set requested prerelease version before verification
if: github.event_name == 'workflow_dispatch' && inputs.prerelease_version != ''
working-directory: cli
env:
PRERELEASE_VERSION: ${{ inputs.prerelease_version }}
shell: bash
run: |
set -euo pipefail
if [[ "$GITHUB_REF" != refs/heads/develop ||
! "$PRERELEASE_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+-(alpha|beta|rc)\.[0-9]+$ ]]; then
echo '::error::Prereleases require develop and an explicit alpha/beta/rc version.'
exit 1
fi
npm version "$PRERELEASE_VERSION" --no-git-tag-version --ignore-scripts
- run: npm ci --prefix cli --no-audit --no-fund
- run: npm run check:cli
- run: npm run test:cli
@@ -97,11 +72,9 @@ jobs:
needs: package
if: >-
github.repository == 'whyour/qinglong' &&
((github.ref == 'refs/heads/master' &&
(github.event_name == 'push' ||
(github.event_name == 'workflow_dispatch' && inputs.publish))) ||
(github.ref == 'refs/heads/develop' && github.event_name == 'workflow_dispatch' &&
inputs.publish && inputs.prerelease_version != ''))
github.ref == 'refs/heads/master' &&
(github.event_name == 'push' ||
(github.event_name == 'workflow_dispatch' && inputs.publish))
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
@@ -118,8 +91,6 @@ jobs:
name: qinglong-cli-${{ github.sha }}
path: cli-package
- name: Publish verified npm archive
env:
PRERELEASE_VERSION: ${{ inputs.prerelease_version }}
shell: bash
run: |
set -euo pipefail
@@ -132,30 +103,13 @@ jobs:
archive="${archives[0]}"
metadata=$(tar -xOf "$archive" package/package.json)
name=$(jq -er '.name | select(. == "@whyour/qinglong-cli")' <<< "$metadata")
version=$(jq -er '.version | select(type == "string")' <<< "$metadata")
if [[ "$GITHUB_REF" == refs/heads/develop ]]; then
if [[ "$version" != "$PRERELEASE_VERSION" ||
! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+-(alpha|beta|rc)\.[0-9]+$ ]]; then
echo '::error::Archive version does not match the requested prerelease.'
exit 1
fi
tag="${BASH_REMATCH[1]}"
elif [[ "$GITHUB_REF" == refs/heads/master && "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
tag=latest
else
echo '::error::Refusing to publish this branch/version combination.'
exit 1
fi
version=$(jq -er '.version | select(type == "string" and test("^[0-9]+\\.[0-9]+\\.[0-9]+$"))' <<< "$metadata")
if npm view "$name@$version" version --json --registry=https://registry.npmjs.org > version.json 2> version-error.log; then
jq -e --arg version "$version" '. == $version' version.json > /dev/null
if [[ "$tag" != latest ]]; then
echo '::error::Prerelease version already exists; choose a new version to test this build.'
exit 1
fi
echo "::notice::$name@$version is already published; bump cli/package.json and its lockfile to release changes."
exit 0
elif ! jq -e '.error.code == "E404"' version.json > /dev/null; then
echo '::error::Could not check the published CLI version; refusing to publish.'
exit 1
fi
npm publish "./$archive" --access public --tag "$tag" --ignore-scripts --registry=https://registry.npmjs.org
npm publish "./$archive" --access public --tag latest --ignore-scripts --registry=https://registry.npmjs.org
-136
View File
@@ -1,136 +0,0 @@
name: Release readiness
on:
pull_request:
paths:
- '.github/workflows/release-readiness.yml'
- 'package.json'
- 'pnpm-lock.yaml'
workflow_dispatch:
permissions:
contents: read
concurrency:
group: release-readiness-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
regression:
uses: ./.github/workflows/validate.yml
static:
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v6
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
- uses: pnpm/action-setup@v6
with:
version: '8.3.1'
- uses: actions/setup-node@v6
with:
node-version: '20'
cache: pnpm
- run: pnpm install --frozen-lockfile
- run: pnpm build:front && pnpm build:back && pnpm build:info
- uses: actions/upload-artifact@v6
with:
name: release-readiness-static
path: static/
include-hidden-files: true
if-no-files-found: error
images:
needs: [regression, static]
runs-on: ubuntu-22.04
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
include:
- image: alpine
file: docker/Dockerfile
cache: alpine
platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x,linux/386
- image: debian
file: docker/Dockerfile.debian
cache: debian
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
- image: alpine-python3.10
file: docker/Dockerfile.310
cache: alpine-python3.10
platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x,linux/386
- image: debian-python3.10
file: docker/Dockerfile.debian310
cache: debian-python3.10
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
steps:
- uses: actions/checkout@v6
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
- name: Record source
id: source
env:
SOURCE_URL: ${{ github.event.pull_request.head.repo.clone_url || format('https://github.com/{0}.git', github.repository) }}
SOURCE_BRANCH: ${{ github.head_ref || github.ref_name }}
run: |
echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
echo "url=$SOURCE_URL" >> "$GITHUB_OUTPUT"
echo "branch=$SOURCE_BRANCH" >> "$GITHUB_OUTPUT"
- uses: actions/download-artifact@v7
with:
name: release-readiness-static
path: static/
- uses: docker/setup-qemu-action@v4
with:
cache-image: false
- uses: docker/setup-buildx-action@v4
- name: Build every release architecture
uses: docker/build-push-action@v7
with:
context: .
file: ${{ matrix.file }}
platforms: ${{ matrix.platforms }}
build-args: |
QL_URL=${{ steps.source.outputs.url }}
QL_BRANCH=${{ steps.source.outputs.branch }}
SOURCE_COMMIT=${{ steps.source.outputs.sha }}
push: false
provenance: false
sbom: false
cache-from: type=registry,ref=whyour/qinglong:cache-${{ matrix.cache }}
- name: Load native image for runtime verification
uses: docker/build-push-action@v7
with:
context: .
file: ${{ matrix.file }}
platforms: linux/amd64
build-args: |
QL_URL=${{ steps.source.outputs.url }}
QL_BRANCH=${{ steps.source.outputs.branch }}
SOURCE_COMMIT=${{ steps.source.outputs.sha }}
tags: qinglong-readiness:${{ matrix.image }}
load: true
push: false
provenance: false
sbom: false
- name: Verify native production runtime without external networking
run: |
docker run --rm --network none --entrypoint node \
qinglong-readiness:${{ matrix.image }} \
-e 'const assert=require("node:assert/strict"); const {createRequire}=require("node:module"); for(const [name,want] of Object.entries({"@grpc/grpc-js":"1.14.5",compression:"1.8.2",multer:"2.3.0",sequelize:"6.37.8"}))assert.equal(require(name+"/package.json").version,want); const expressRequire=createRequire(require.resolve("express")); assert.equal(expressRequire("proxy-addr/package.json").version,"2.0.8"); const sockRequire=createRequire(require.resolve("sockjs")); assert.equal(sockRequire("websocket-driver/package.json").version,"0.7.5"); require("sqlite3"); require("nodemailer"); require("undici"); console.log("Minimal production patches and runtime imports verified")'
- name: Export native Debian image for isolated dev validation
if: matrix.image == 'debian'
run: |
docker save qinglong-readiness:debian | gzip -1 > release-readiness-debian-image.tar.gz
sha256sum release-readiness-debian-image.tar.gz > release-readiness-debian-image.sha256
docker image inspect qinglong-readiness:debian > release-readiness-debian-image.json
- uses: actions/upload-artifact@v6
if: matrix.image == 'debian'
with:
name: release-readiness-debian-image
path: release-readiness-debian-image.*
compression-level: 0
if-no-files-found: error
+7 -3
View File
@@ -4,6 +4,7 @@ import { Logger } from 'winston';
import config from '../config';
import * as fs from 'fs/promises';
import { celebrate, Joi } from 'celebrate';
import { join, basename } from 'path';
import { SAMPLE_FILES } from '../config/const';
import { t } from '../shared/i18n';
import ConfigService from '../services/config';
@@ -72,12 +73,15 @@ export default (app: Router) => {
const logger: Logger = Container.get('logger');
try {
const { name, content } = req.body;
// Resolve path first to prevent traversal attacks
let basePath = config.configPath;
if (name.startsWith('data/scripts/')) {
return res.send({ code: 403, message: t('文件无法访问') });
basePath = join(config.rootPath, 'data/scripts');
}
const cleanName = name.replace(/^data\/scripts\//, '');
const normalized = resolveFileAccess(
config.configPath,
[name],
basePath,
[cleanName],
config.blackFileList,
);
if (!normalized) {
+14 -17
View File
@@ -1,16 +1,24 @@
import { Joi, celebrate } from 'celebrate';
import { NextFunction, Request, Response, Router } from 'express';
import fs from 'fs';
import multer from 'multer';
import { Container } from 'typedi';
import { Logger } from 'winston';
import config from '../config';
import { safeJSONParse } from '../config/util';
import { t } from '../shared/i18n';
import EnvService from '../services/env';
const route = Router();
const upload = multer({
storage: multer.memoryStorage(),
limits: { fileSize: 5 * 1024 * 1024, files: 1 },
const storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, config.scriptPath);
},
filename: function (req, file, cb) {
cb(null, file.originalname);
},
});
const upload = multer({ storage: storage });
const labelSchema = Joi.array()
.items(Joi.string().trim().required())
.min(1)
@@ -274,23 +282,12 @@ export default (app: Router) => {
const logger: Logger = Container.get('logger');
try {
const envService = Container.get(EnvService);
let parseContent: unknown;
try {
parseContent = JSON.parse(req.file?.buffer.toString('utf8') || '');
} catch {
return res.status(400).send({ code: 400, message: t('参数错误') });
}
const fileContent = await fs.promises.readFile(req!.file!.path, 'utf8');
const parseContent = safeJSONParse(fileContent);
const data = Array.isArray(parseContent)
? parseContent
: [parseContent];
if (
data.length > 0 &&
data.every((x) =>
x && typeof x.name === 'string' &&
/^[a-zA-Z_][0-9a-zA-Z_]*$/.test(x.name) &&
typeof x.value === 'string' && x.value.length > 0,
)
) {
if (data.every((x) => x.name && x.value)) {
const result = await envService.create(
data.map((x) => ({
name: x.name,
-2
View File
@@ -3,7 +3,6 @@ import { Container } from 'typedi';
import OpenService from '../services/open';
import { Logger } from 'winston';
import { celebrate, Joi } from 'celebrate';
import { rateLimit } from 'express-rate-limit';
const route = Router();
export default (app: Router) => {
@@ -101,7 +100,6 @@ export default (app: Router) => {
route.get(
'/auth/token',
rateLimit({ windowMs: 15 * 60 * 1000, limit: 100 }),
celebrate({
query: {
client_id: Joi.string().required(),
+19 -99
View File
@@ -7,7 +7,7 @@ import { Logger } from 'winston';
import config from '../config';
import * as fs from 'fs/promises';
import { celebrate, Joi } from 'celebrate';
import path, { dirname, join, parse, resolve, sep } from 'path';
import path, { join, parse } from 'path';
import ScriptService from '../services/script';
import ScriptHistoryService from '../services/scriptHistory';
import { t } from '../shared/i18n';
@@ -22,64 +22,10 @@ const route = Router();
function isPathAllowed(targetPath: string): boolean {
const resolved = path.resolve(targetPath);
return config.writePathList.some((x) =>
Boolean(
resolveFileAccess(
x,
[resolved],
// Panel configuration secrets must not restrict user script filenames.
path.resolve(x) === path.resolve(config.configPath)
? config.blackFileList
: [],
),
),
Boolean(resolveFileAccess(x, [resolved], config.blackFileList)),
);
}
async function writeScriptFile(
targetPath: string,
content: string,
): Promise<boolean> {
const target = resolve(targetPath);
for (const writableRoot of config.writePathList) {
const root = resolve(writableRoot);
const prefix = root.endsWith(sep) ? root : root + sep;
// Check the actual write target, including filename, at the write boundary.
if (!target.startsWith(prefix)) continue;
if (!isPathAllowed(target)) return false;
await writeFileWithLock(target, content);
return true;
}
return false;
}
async function existingWritableRealPath(targetPath: string): Promise<string> {
if (!isPathAllowed(targetPath)) {
throw new ScriptHistoryError(403, '暂无权限');
}
const resolved = resolve(targetPath);
for (const writableRoot of config.writePathList) {
const root = resolve(writableRoot);
const rootPrefix = root.endsWith(sep) ? root : root + sep;
// Keep an explicit normalized boundary check at the filesystem operation.
if (!resolved.startsWith(rootPrefix)) continue;
const realPath = await fs.realpath(resolved);
const realRoot = await fs.realpath(root);
const realRootPrefix = realRoot.endsWith(sep) ? realRoot : realRoot + sep;
if (
!realPath.startsWith(realRootPrefix) ||
!resolveFileAccess(
realRoot,
[realPath],
root === resolve(config.configPath) ? config.blackFileList : [],
)
) {
throw new ScriptHistoryError(403, '暂无权限');
}
return realPath;
}
throw new ScriptHistoryError(403, '暂无权限');
}
const storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, config.tmpPath);
@@ -114,7 +60,11 @@ export default (app: Router) => {
];
if (req.query.path) {
if (
!resolveFileAccess(config.scriptPath, [req.query.path as string])
!resolveFileAccess(
config.scriptPath,
[req.query.path as string],
config.blackFileList,
)
) {
return res.send({ code: 403, message: t('暂无权限') });
}
@@ -331,45 +281,28 @@ export default (app: Router) => {
if (!isPathAllowed(filePath) || !isPathAllowed(originFilePath)) {
return res.send({ code: 403, message: t('暂无权限') });
}
// Check the canonical parent directly at the directory creation boundary.
const parentPath = resolve(dirname(filePath));
let parentCreated = false;
for (const writableRoot of config.writePathList) {
const root = resolve(writableRoot);
const rootPrefix = root.endsWith(sep) ? root : root + sep;
if (parentPath === root) {
await fs.mkdir(root, { recursive: true });
} else if (parentPath.startsWith(rootPrefix)) {
await fs.mkdir(parentPath, { recursive: true });
} else {
continue;
}
parentCreated = true;
break;
}
if (!parentCreated) {
return res.send({ code: 403, message: t('暂无权限') });
}
await fs.mkdir(path, { recursive: true });
const fileExists = await fileExist(filePath);
if (fileExists && resolveFileAccess(config.scriptPath, [filePath])) {
if (
fileExists &&
resolveFileAccess(config.scriptPath, [filePath], config.blackFileList)
) {
let removeSource = filename !== originFilename;
let backupSource = '';
if (removeSource) {
const [originRealPath, targetRealPath] = await Promise.all([
existingWritableRealPath(originFilePath),
existingWritableRealPath(filePath),
fs.realpath(originFilePath),
fs.realpath(filePath),
]);
// Aliases of one script are an in-place save. Removing the source
// would also remove the destination behind a target symlink.
removeSource = originRealPath !== targetRealPath;
backupSource = originRealPath;
}
// Save-as removes the source after committing the destination. Keep
// its original content too: destination history only protects the
// file being overwritten, not the source being deleted.
if (removeSource) {
await fs.copyFile(
backupSource,
originFilePath,
join(config.bakPath, originFilename.replace(/\//g, '')),
);
}
@@ -390,9 +323,7 @@ export default (app: Router) => {
await rmPath(originFilePath);
}
}
if (!(await writeScriptFile(filePath, content))) {
return res.send({ code: 403, message: t('暂无权限') });
}
await writeFileWithLock(filePath, content);
return res.send({ code: 200 });
} catch (e) {
return historyError(e, res, next);
@@ -532,9 +463,7 @@ export default (app: Router) => {
if (!isPathAllowed(filePath)) {
return res.send({ code: 403, message: t('暂无权限') });
}
if (!(await writeScriptFile(filePath, content || ''))) {
return res.send({ code: 403, message: t('暂无权限') });
}
await writeFileWithLock(filePath, content || '');
const scriptService = Container.get(ScriptService);
const result = await scriptService.runScript(filePath);
@@ -565,21 +494,12 @@ export default (app: Router) => {
if (!isPathAllowed(filePath)) {
return res.send({ code: 403, message: t('暂无权限') });
}
const logPath = resolveFileAccess(config.logPath, [
path,
`${name}.swap`,
]);
if (!logPath) {
return res.send({ code: 403, message: t('暂无权限') });
}
const logPath = join(config.logPath, path, `${name}.swap`);
const scriptService = Container.get(ScriptService);
const result = await scriptService.stopScript(filePath, pid);
setTimeout(() => {
const cleanupPath = resolveFileAccess(config.logPath, [logPath]);
if (cleanupPath) {
void rmPath(cleanupPath);
}
rmPath(logPath);
}, 3000);
res.send(result);
} catch (e) {
+10 -3
View File
@@ -3,7 +3,7 @@ import { Container } from 'typedi';
import { Logger } from 'winston';
import SubscriptionService from '../services/subscription';
import { celebrate, Joi } from 'celebrate';
import { isValidCronSchedule } from '../shared/cronSchedule';
import CronExpressionParser from 'cron-parser';
const route = Router();
export default (app: Router) => {
@@ -58,7 +58,10 @@ export default (app: Router) => {
async (req: Request, res: Response, next: NextFunction) => {
const logger: Logger = Container.get('logger');
try {
if (!req.body.schedule || isValidCronSchedule(req.body.schedule)) {
if (
!req.body.schedule ||
CronExpressionParser.parse(req.body.schedule).hasNext()
) {
const subscriptionService = Container.get(SubscriptionService);
const data = await subscriptionService.create(req.body);
return res.send({ code: 200, data });
@@ -201,7 +204,11 @@ export default (app: Router) => {
async (req: Request, res: Response, next: NextFunction) => {
const logger: Logger = Container.get('logger');
try {
if (!req.body.schedule || isValidCronSchedule(req.body.schedule)) {
if (
!req.body.schedule ||
typeof req.body.schedule === 'object' ||
CronExpressionParser.parse(req.body.schedule).hasNext()
) {
const subscriptionService = Container.get(SubscriptionService);
const data = await subscriptionService.update(req.body);
return res.send({ code: 200, data });
+9 -14
View File
@@ -8,12 +8,15 @@ import { celebrate, Joi } from 'celebrate';
import UserService from '../services/user';
import { t } from '../shared/i18n';
import { isDefaultAuthInfo } from '../shared/auth';
import { getUniqPath, handleLogPath, parseVersion } from '../config/util';
import { findCronId } from '../shared/cronCommand';
import {
getUniqPath,
handleLogPath,
parseVersion,
promiseExec,
} from '../config/util';
import dayjs from 'dayjs';
import multer from 'multer';
import { logStreamManager } from '../shared/logStreamManager';
import { rateLimit } from 'express-rate-limit';
const route = Router();
const storage = multer.diskStorage({
@@ -246,7 +249,6 @@ export default (app: Router) => {
route.put(
'/command-run',
rateLimit({ windowMs: 60 * 1000, limit: 60 }),
celebrate({
body: Joi.object({
command: Joi.string().required(),
@@ -256,13 +258,8 @@ export default (app: Router) => {
try {
const systemService = Container.get(SystemService);
const command = req.body.command;
const crontab = await fs
.readFile(config.crontabFile, 'utf8')
.catch((e) => {
if (e.code === 'ENOENT') return '';
throw e;
});
const id = findCronId(crontab, command);
const idStr = `cat ${config.crontabFile} | grep -E "${command}" | perl -pe "s|.*ID=(.*) ${command}.*|\\1|" | head -1 | awk -F " " '{print $1}' | xargs echo -n`;
let id = await promiseExec(idStr);
const uniqPath = await getUniqPath(command, id);
const logTime = dayjs().format('YYYY-MM-DD-HH-mm-ss-SSS');
const logPath = `${uniqPath}/${logTime}.log`;
@@ -277,9 +274,7 @@ export default (app: Router) => {
onEnd: async (cp, endTime, diff) => {
// Close the stream after task completion
try {
await logStreamManager.closeStream(
await handleLogPath(logPath),
);
await logStreamManager.closeStream(await handleLogPath(logPath));
} finally {
res.end();
}
+47 -95
View File
@@ -1,5 +1,6 @@
import { execFileSync } from 'child_process';
import { execSync } from 'child_process';
import * as fs from 'fs/promises';
import * as os from 'os';
import path from 'path';
import config from './index';
import { fileExist } from './util';
@@ -23,120 +24,71 @@ const clientCertPath = path.join(certDir, 'client.crt');
let cachedConfig: GrpcTlsConfig | null = null;
function run(args: string[]): string {
return execFileSync('openssl', args, {
stdio: 'pipe',
timeout: 30000,
encoding: 'utf-8',
}).trim();
function run(cmd: string, execOpts?: Record<string, unknown>): string {
const opts = { stdio: 'pipe', timeout: 30000, encoding: 'utf-8', ...execOpts } as any;
return (execSync(cmd, opts) as string).trim();
}
async function tmpFile(prefix: string): Promise<string> {
const dir = (await fileExist(certDir)) ? certDir : os.tmpdir();
await fs.mkdir(dir, { recursive: true });
return path.join(dir, `.${prefix}_${Date.now()}_${Math.random().toString(36).slice(2)}.pem`);
}
async function generateAllCerts(): Promise<GrpcTlsConfig> {
Logger.info('[boot] Generating gRPC mTLS certificates...');
await fs.mkdir(certDir, { recursive: true, mode: 0o700 });
// Atomically reserve a private workspace for keys, CSRs and OpenSSL side files.
const workspace = await fs.mkdtemp(path.join(certDir, '.generate-'));
const caKeyTmp = path.join(workspace, 'ca.key');
const caCertTmp = path.join(workspace, 'ca.crt');
const serverKeyTmp = path.join(workspace, 'server.key');
const serverCsrTmp = path.join(workspace, 'server.csr');
const serverExtTmp = path.join(workspace, 'server.ext');
const clientKeyTmp = path.join(workspace, 'client.key');
const clientCsrTmp = path.join(workspace, 'client.csr');
const clientExtTmp = path.join(workspace, 'client.ext');
const caKeyTmp = await tmpFile('ca_key');
const caCertTmp = await tmpFile('ca_cert');
const serverKeyTmp = await tmpFile('server_key');
const serverCsrTmp = await tmpFile('server_csr');
const serverExtTmp = await tmpFile('server_ext');
const clientKeyTmp = await tmpFile('client_key');
const clientCsrTmp = await tmpFile('client_csr');
const clientExtTmp = await tmpFile('client_ext');
const srlTmp = path.join(path.dirname(caKeyTmp), '.grpc_ca.srl');
const cleanup = async () => {
await fs.rm(workspace, { recursive: true, force: true });
for (const f of [caKeyTmp, caCertTmp, serverKeyTmp, serverCsrTmp, serverExtTmp,
clientKeyTmp, clientCsrTmp, clientExtTmp, srlTmp]) {
try { await fs.unlink(f); } catch {}
}
};
try {
run(['genrsa', '-out', caKeyTmp, '2048']);
run([
'req',
'-new',
'-x509',
'-days',
'3650',
'-key',
caKeyTmp,
'-out',
caCertTmp,
'-subj',
'/CN=qinglong-ca/O=qinglong/C=CN',
]);
// 1. CA(私钥直接存盘,证书写入临时文件供签发使用)
run(`openssl genrsa -out '${caKeyTmp}' 2048 2>/dev/null`);
run(`openssl req -new -x509 -days 3650 -key '${caKeyTmp}' -out '${caCertTmp}' -subj '/CN=qinglong-ca/O=qinglong/C=CN' 2>/dev/null`);
const caKey = await fs.readFile(caKeyTmp, 'utf-8');
const caCert = await fs.readFile(caCertTmp, 'utf-8');
await fs.mkdir(certDir, { recursive: true });
await fs.writeFile(caKeyPath, caKey, { mode: 0o600 });
run(['genrsa', '-out', serverKeyTmp, '2048']);
run([
'req',
'-new',
'-key',
serverKeyTmp,
'-out',
serverCsrTmp,
'-subj',
'/CN=grpc-server',
]);
await fs.writeFile(
serverExtTmp,
'subjectAltName=DNS:localhost,IP:127.0.0.1,IP:::1\n',
// 2. 服务端
run(`openssl genrsa -out '${serverKeyTmp}' 2048 2>/dev/null`);
run(`openssl req -new -key '${serverKeyTmp}' -out '${serverCsrTmp}' -subj '/CN=grpc-server' 2>/dev/null`);
await fs.writeFile(serverExtTmp, 'subjectAltName=DNS:localhost,IP:127.0.0.1,IP:::1\n');
const serverCert = run(
`openssl x509 -req -days 3650 -in '${serverCsrTmp}' -CA '${caCertTmp}' -CAkey '${caKeyTmp}' -CAcreateserial -extfile '${serverExtTmp}' 2>/dev/null`,
);
const serverCert = run([
'x509',
'-req',
'-days',
'3650',
'-in',
serverCsrTmp,
'-CA',
caCertTmp,
'-CAkey',
caKeyTmp,
'-CAserial',
path.join(workspace, 'ca.srl'),
'-CAcreateserial',
'-extfile',
serverExtTmp,
]);
const serverKey = await fs.readFile(serverKeyTmp, 'utf-8');
run(['genrsa', '-out', clientKeyTmp, '2048']);
run([
'req',
'-new',
'-key',
clientKeyTmp,
'-out',
clientCsrTmp,
'-subj',
'/CN=grpc-client',
]);
// 3. 客户端
run(`openssl genrsa -out '${clientKeyTmp}' 2048 2>/dev/null`);
run(`openssl req -new -key '${clientKeyTmp}' -out '${clientCsrTmp}' -subj '/CN=grpc-client' 2>/dev/null`);
await fs.writeFile(clientExtTmp, 'extendedKeyUsage=clientAuth\n');
const clientCert = run([
'x509',
'-req',
'-days',
'3650',
'-in',
clientCsrTmp,
'-CA',
caCertTmp,
'-CAkey',
caKeyTmp,
'-CAserial',
path.join(workspace, 'ca.srl'),
'-CAcreateserial',
'-extfile',
clientExtTmp,
]);
const clientCert = run(
`openssl x509 -req -days 3650 -in '${clientCsrTmp}' -CA '${caCertTmp}' -CAkey '${caKeyTmp}' -CAcreateserial -extfile '${clientExtTmp}' 2>/dev/null`,
);
const clientKey = await fs.readFile(clientKeyTmp, 'utf-8');
Logger.info('[boot] gRPC mTLS certificates generated successfully');
return { caCert, serverCert, serverKey, clientCert, clientKey };
} finally {
await cleanup();
Logger.info('[boot] gRPC mTLS certificates generated successfully');
return { caCert, serverCert, serverKey, clientCert, clientKey };
} catch (e) {
await cleanup();
throw e;
}
}
+26 -23
View File
@@ -1,7 +1,5 @@
import { Subscription } from '../data/subscription';
import isNil from 'lodash/isNil';
import { shellQuote } from '../shared/shellQuote';
import { getSubscriptionSshAlias } from '../shared/subscriptionPath';
export function formatUrl(doc: Subscription) {
let url = doc.url;
@@ -9,7 +7,7 @@ export function formatUrl(doc: Subscription) {
if (doc.type === 'private-repo') {
if (doc.pull_type === 'ssh-key') {
host = doc.url!.replace(/.*\@([^\:]+)\:.*/, '$1');
url = doc.url!.replace(host, getSubscriptionSshAlias(doc.alias));
url = doc.url!.replace(host, doc.alias);
} else {
host = doc.url!.replace(/.*\:\/\/([^\/]+)\/.*/, '$1');
const { username, password } = doc.pull_option as any;
@@ -20,24 +18,29 @@ export function formatUrl(doc: Subscription) {
}
export function formatCommand(doc: Subscription, url?: string) {
const args =
doc.type === 'file'
? ['raw', url || formatUrl(doc).url, doc.proxy || '']
: [
'repo',
url || formatUrl(doc).url,
doc.whitelist || '',
doc.blacklist || '',
doc.dependences || '',
doc.branch || '',
doc.extensions || '',
doc.proxy || '',
];
args.push(
String(isNil(doc.autoAddCron) ? true : Boolean(doc.autoAddCron)),
String(isNil(doc.autoDelCron) ? true : Boolean(doc.autoDelCron)),
);
return `SUB_ID=${shellQuote(String(doc.id))} ql ${args
.map((arg) => shellQuote(arg!))
.join(' ')}`;
let command = `SUB_ID=${doc.id} ql `;
let _url = url || formatUrl(doc).url;
const {
type,
whitelist,
blacklist,
dependences,
branch,
extensions,
proxy,
autoAddCron,
autoDelCron,
} = doc;
if (type === 'file') {
command += `raw "${_url}" "${proxy || ''}" "${
isNil(autoAddCron) ? true : Boolean(autoAddCron)
}" "${isNil(autoDelCron) ? true : Boolean(autoDelCron)}"`;
} else {
command += `repo "${_url}" "${whitelist || ''}" "${blacklist || ''}" "${
dependences || ''
}" "${branch || ''}" "${extensions || ''}" "${proxy || ''}" "${
isNil(autoAddCron) ? true : Boolean(autoAddCron)
}" "${isNil(autoDelCron) ? true : Boolean(autoDelCron)}"`;
}
return command;
}
+4 -6
View File
@@ -846,25 +846,23 @@ async function _updateLinuxMirror(
mirrorDomainWithScheme: string,
): Promise<string> {
const S = isInContainer() ? 'sudo ' : '';
// APT otherwise returns zero when index downloads fail with transient errors.
const aptUpdate = `${S}apt-get update --error-on=any`;
let filePath: string, currentDomainWithScheme: string | null;
switch (osType) {
case 'Debian':
filePath = '/etc/apt/sources.list.d/debian.sources';
currentDomainWithScheme = await getCurrentMirrorDomain(filePath);
if (currentDomainWithScheme) {
return `${S}sed -i 's|${currentDomainWithScheme}|${mirrorDomainWithScheme || 'http://deb.debian.org'}|g' ${filePath} || (${S}mkdir -p /etc/apt/sources.list.d && echo -e "Types: deb\\nURIs: ${mirrorDomainWithScheme || 'http://deb.debian.org'}\\nSuites: \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-updates\\nComponents: main\\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg" | ${S}tee ${filePath}) && ${aptUpdate}`;
return `${S}sed -i 's|${currentDomainWithScheme}|${mirrorDomainWithScheme || 'http://deb.debian.org'}|g' ${filePath} || (${S}mkdir -p /etc/apt/sources.list.d && echo -e "Types: deb\\nURIs: ${mirrorDomainWithScheme || 'http://deb.debian.org'}\\nSuites: \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-updates\\nComponents: main\\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg" | ${S}tee ${filePath}) && ${S}apt-get update`;
} else {
return `${S}mkdir -p /etc/apt/sources.list.d && echo -e "Types: deb\\nURIs: ${mirrorDomainWithScheme || 'http://deb.debian.org'}\\nSuites: \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-updates\\nComponents: main\\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg" | ${S}tee ${filePath} && ${aptUpdate}`;
return `${S}mkdir -p /etc/apt/sources.list.d && echo -e "Types: deb\\nURIs: ${mirrorDomainWithScheme || 'http://deb.debian.org'}\\nSuites: \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-updates\\nComponents: main\\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg" | ${S}tee ${filePath} && ${S}apt-get update`;
}
case 'Ubuntu':
filePath = '/etc/apt/sources.list.d/ubuntu.sources';
currentDomainWithScheme = await getCurrentMirrorDomain(filePath);
if (currentDomainWithScheme) {
return `${S}sed -i 's|${currentDomainWithScheme}|${mirrorDomainWithScheme || 'http://archive.ubuntu.com'}|g' ${filePath} || (${S}mkdir -p /etc/apt/sources.list.d && echo -e "Types: deb\\nURIs: ${mirrorDomainWithScheme || 'http://archive.ubuntu.com'}\\nSuites: \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-updates \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-backports\\nComponents: main restricted universe multiverse\\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg" | ${S}tee ${filePath}) && ${aptUpdate}`;
return `${S}sed -i 's|${currentDomainWithScheme}|${mirrorDomainWithScheme || 'http://archive.ubuntu.com'}|g' ${filePath} || (${S}mkdir -p /etc/apt/sources.list.d && echo -e "Types: deb\\nURIs: ${mirrorDomainWithScheme || 'http://archive.ubuntu.com'}\\nSuites: \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-updates \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-backports\\nComponents: main restricted universe multiverse\\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg" | ${S}tee ${filePath}) && ${S}apt-get update`;
} else {
return `${S}mkdir -p /etc/apt/sources.list.d && echo -e "Types: deb\\nURIs: ${mirrorDomainWithScheme || 'http://archive.ubuntu.com'}\\nSuites: \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-updates \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-backports\\nComponents: main restricted universe multiverse\\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg" | ${S}tee ${filePath} && ${aptUpdate}`;
return `${S}mkdir -p /etc/apt/sources.list.d && echo -e "Types: deb\\nURIs: ${mirrorDomainWithScheme || 'http://archive.ubuntu.com'}\\nSuites: \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-updates \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-backports\\nComponents: main restricted universe multiverse\\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg" | ${S}tee ${filePath} && ${S}apt-get update`;
}
case 'Alpine':
filePath = '/etc/apk/repositories';
-2
View File
@@ -14,7 +14,6 @@ import { AuthInfo } from '../data/system';
import path from 'path';
import { t } from '../shared/i18n';
import { AppScope } from '../data/open';
import uniqueConstraintError from '../middlewares/uniqueConstraintError';
import protectedPathCase from '../middlewares/protectedPathCase';
function resolveTrustProxy(value = process.env.QL_TRUST_PROXY) {
@@ -191,7 +190,6 @@ export default ({ app }: { app: Application }) => {
});
app.use(errors());
app.use(uniqueConstraintError);
app.use(
(
+26 -12
View File
@@ -13,11 +13,12 @@ import { AuthDataType, SystemModel } from '../data/system';
import SystemService from '../services/system';
import UserService from '../services/user';
import { writeFile, readFile } from 'fs/promises';
import { fileExist, isDemoEnv, safeJSONParse } from '../config/util';
import { createRandomString, fileExist, isDemoEnv, safeJSONParse } from '../config/util';
import OpenService from '../services/open';
import { shareStore } from '../shared/store';
import Logger from './logger';
import cronClient from '../schedule/client';
import { AppModel } from '../data/open';
import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
import { setLang, systemLang } from '../shared/i18n';
@@ -30,7 +31,24 @@ export default async () => {
const openService = Container.get(OpenService);
// 初始化增加系统配置
await openService.initializeSystemApp();
let systemApp = (
await AppModel.findOne({
where: { name: 'system' },
})
)?.get({ plain: true });
if (!systemApp) {
systemApp = await AppModel.create({
name: 'system',
scopes: ['crons', 'system', 'dashboard'],
client_id: createRandomString(12, 12),
client_secret: createRandomString(24, 24),
});
} else if (!systemApp.scopes.includes('dashboard')) {
await AppModel.update(
{ scopes: [...systemApp.scopes, 'dashboard'] },
{ where: { name: 'system' } },
);
}
const [systemConfig] = await SystemModel.findOrCreate({
where: { type: AuthDataType.systemConfig },
});
@@ -126,11 +144,7 @@ export default async () => {
// 清空所有运行中的实例记录(服务重启后进程已不存在)
await RunningInstanceModel.update(
{
status: InstanceStatus.stopped,
finished_at: Math.floor(Date.now() / 1000),
// The previous process's exit code cannot be recovered after a restart.
},
{ status: InstanceStatus.stopped },
{ where: { status: InstanceStatus.running } },
);
@@ -223,15 +237,15 @@ export default async () => {
} catch { }
// 初始化保存一次ck和定时任务数据
cronClient.readiness.configure(async () => {
await cronService.autosave_crontab(true);
await systemService.restoreCronConcurrency();
});
cronClient.readiness.configure(() => cronService.autosave_crontab(true));
await cronClient.readiness.recover();
await envService.set_envs();
const authInfo = await userService.getAuthInfo();
const apps = await openService.findApps();
await shareStore.updateAuthInfo(authInfo);
await openService.refreshApps();
if (apps?.length) {
await shareStore.updateApps(apps);
}
};
+1 -10
View File
@@ -7,8 +7,6 @@ import config from '../config';
import { fileExist } from '../config/util';
import { join } from 'path';
import { t } from '../shared/i18n';
import Logger from './logger';
import { assertSubscriptionAlias } from '../shared/subscriptionPath';
export default async () => {
const systemService = Container.get(SystemService);
@@ -26,7 +24,6 @@ export default async () => {
if (await fileExist(tokenFile)) {
tokenCommand = `node ${tokenFile}`;
}
tokenCommand += ' --renew';
const cron = {
id: NaN,
name: t('生成token'),
@@ -73,12 +70,6 @@ export default async () => {
await subscriptionService.setSshConfig();
const subs = await subscriptionService.list();
for (const sub of subs) {
try {
const doc = sub.get({ plain: true });
assertSubscriptionAlias(doc.alias);
await subscriptionService.handleTask(doc, !sub.is_disabled);
} catch (error) {
Logger.warn('跳过订阅 %s 的无效定时配置: %s', sub.id, error);
}
subscriptionService.handleTask(sub.get({ plain: true }), !sub.is_disabled);
}
};
-14
View File
@@ -1,14 +0,0 @@
import { ErrorRequestHandler } from 'express';
import { UniqueConstraintError } from 'sequelize';
import { t } from '../shared/i18n';
// Handle the database constraint itself so concurrent writes are covered too.
const uniqueConstraintError: ErrorRequestHandler = (err, req, res, next) => {
if (!(err instanceof UniqueConstraintError)) return next(err);
// Constraint details can contain environment values or application credentials.
res
.status(409)
.json({ code: 409, message: t('资源已存在,请检查重复的名称或值') });
};
export default uniqueConstraintError;
-4
View File
@@ -5,7 +5,6 @@ package com.ql.cron;
service Cron {
rpc addCron(AddCronRequest) returns (AddCronResponse);
rpc delCron(DeleteCronRequest) returns (DeleteCronResponse);
rpc setConcurrency(SetConcurrencyRequest) returns (AddCronResponse);
}
message ISchedule { string schedule = 1; }
@@ -28,6 +27,3 @@ message AddCronResponse {}
message DeleteCronRequest { repeated string ids = 1; }
message DeleteCronResponse {}
// Zero restores the default limit.
message SetConcurrencyRequest { int32 concurrency = 1; }
+1 -89
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-ts_proto. DO NOT EDIT.
// versions:
// protoc-gen-ts_proto v2.6.1
// protoc v5.28.3
// protoc v3.17.3
// source: back/protos/cron.proto
/* eslint-disable */
@@ -48,11 +48,6 @@ export interface DeleteCronRequest {
export interface DeleteCronResponse {
}
/** Zero restores the default limit. */
export interface SetConcurrencyRequest {
concurrency: number;
}
function createBaseISchedule(): ISchedule {
return { schedule: "" };
}
@@ -457,64 +452,6 @@ export const DeleteCronResponse: MessageFns<DeleteCronResponse> = {
},
};
function createBaseSetConcurrencyRequest(): SetConcurrencyRequest {
return { concurrency: 0 };
}
export const SetConcurrencyRequest: MessageFns<SetConcurrencyRequest> = {
encode(message: SetConcurrencyRequest, writer: BinaryWriter = new BinaryWriter()): BinaryWriter {
if (message.concurrency !== 0) {
writer.uint32(8).int32(message.concurrency);
}
return writer;
},
decode(input: BinaryReader | Uint8Array, length?: number): SetConcurrencyRequest {
const reader = input instanceof BinaryReader ? input : new BinaryReader(input);
let end = length === undefined ? reader.len : reader.pos + length;
const message = createBaseSetConcurrencyRequest();
while (reader.pos < end) {
const tag = reader.uint32();
switch (tag >>> 3) {
case 1: {
if (tag !== 8) {
break;
}
message.concurrency = reader.int32();
continue;
}
}
if ((tag & 7) === 4 || tag === 0) {
break;
}
reader.skip(tag & 7);
}
return message;
},
fromJSON(object: any): SetConcurrencyRequest {
return { concurrency: isSet(object.concurrency) ? globalThis.Number(object.concurrency) : 0 };
},
toJSON(message: SetConcurrencyRequest): unknown {
const obj: any = {};
if (message.concurrency !== 0) {
obj.concurrency = Math.round(message.concurrency);
}
return obj;
},
create<I extends Exact<DeepPartial<SetConcurrencyRequest>, I>>(base?: I): SetConcurrencyRequest {
return SetConcurrencyRequest.fromPartial(base ?? ({} as any));
},
fromPartial<I extends Exact<DeepPartial<SetConcurrencyRequest>, I>>(object: I): SetConcurrencyRequest {
const message = createBaseSetConcurrencyRequest();
message.concurrency = object.concurrency ?? 0;
return message;
},
};
export type CronService = typeof CronService;
export const CronService = {
addCron: {
@@ -535,21 +472,11 @@ export const CronService = {
responseSerialize: (value: DeleteCronResponse) => Buffer.from(DeleteCronResponse.encode(value).finish()),
responseDeserialize: (value: Buffer) => DeleteCronResponse.decode(value),
},
setConcurrency: {
path: "/com.ql.cron.Cron/setConcurrency",
requestStream: false,
responseStream: false,
requestSerialize: (value: SetConcurrencyRequest) => Buffer.from(SetConcurrencyRequest.encode(value).finish()),
requestDeserialize: (value: Buffer) => SetConcurrencyRequest.decode(value),
responseSerialize: (value: AddCronResponse) => Buffer.from(AddCronResponse.encode(value).finish()),
responseDeserialize: (value: Buffer) => AddCronResponse.decode(value),
},
} as const;
export interface CronServer extends UntypedServiceImplementation {
addCron: handleUnaryCall<AddCronRequest, AddCronResponse>;
delCron: handleUnaryCall<DeleteCronRequest, DeleteCronResponse>;
setConcurrency: handleUnaryCall<SetConcurrencyRequest, AddCronResponse>;
}
export interface CronClient extends Client {
@@ -583,21 +510,6 @@ export interface CronClient extends Client {
options: Partial<CallOptions>,
callback: (error: ServiceError | null, response: DeleteCronResponse) => void,
): ClientUnaryCall;
setConcurrency(
request: SetConcurrencyRequest,
callback: (error: ServiceError | null, response: AddCronResponse) => void,
): ClientUnaryCall;
setConcurrency(
request: SetConcurrencyRequest,
metadata: Metadata,
callback: (error: ServiceError | null, response: AddCronResponse) => void,
): ClientUnaryCall;
setConcurrency(
request: SetConcurrencyRequest,
metadata: Metadata,
options: Partial<CallOptions>,
callback: (error: ServiceError | null, response: AddCronResponse) => void,
): ClientUnaryCall;
}
export const CronClient = makeGenericClientConstructor(CronService, "com.ql.cron.Cron") as unknown as {
+73 -52
View File
@@ -1,6 +1,6 @@
import { ServerUnaryCall, sendUnaryData, status } from '@grpc/grpc-js';
import { AddCronRequest, AddCronResponse } from '../protos/cron';
import { createCronJob, CronJob } from '../shared/cronScheduler';
import nodeSchedule from 'node-schedule';
import { isValidCronSchedule } from '../shared/cronSchedule';
import { scheduleStacks } from './data';
import { runCron } from '../shared/runCron';
@@ -24,7 +24,11 @@ const addCron = (
if (!isValidCronField(schedule)) {
validationErrors.push(
tf('任务ID %s: 无效的 cron 表达式 "%s"', String(id), schedule),
tf(
'任务ID %s: 无效的 cron 表达式 "%s"',
String(id),
schedule,
),
);
}
@@ -52,62 +56,79 @@ const addCron = (
return;
}
// Prepare stopped jobs first: construction errors must preserve the old snapshot.
const prepared = new Map<string, CronJob[]>();
try {
for (const item of call.request.crons) {
const jobs: CronJob[] = [];
prepared.get(item.id)?.forEach((job) => job.cancel());
prepared.set(item.id, jobs);
for (const schedule of [
item.schedule,
...(item.extra_schedules || []).map((x) => x.schedule),
]) {
jobs.push(
createCronJob(
schedule,
async () => {
Logger.info('[schedule][准备运行任务] 命令: %s', item.command);
await runCron(
item.command,
item,
() => scheduleStacks.get(item.id) === jobs,
);
},
{
name: `${item.id}: ${item.name || ''}`,
logger: Logger,
start: false,
},
),
);
}
}
} catch (error) {
for (const jobs of prepared.values()) jobs.forEach((job) => job.cancel());
const err: any = new Error(
error instanceof Error ? error.message : String(error),
);
err.code = status.INVALID_ARGUMENT;
err.details = err.message;
callback(err, null);
return;
}
// Recovery replaces the whole snapshot, including deletions and disabled jobs.
// Validation above must finish before touching the previous schedule.
if (call.request.replace) {
for (const jobs of scheduleStacks.values())
jobs.forEach((job) => job?.cancel());
for (const jobs of scheduleStacks.values()) {
for (const job of jobs) job?.cancel();
}
scheduleStacks.clear();
}
for (const [id, jobs] of prepared) {
scheduleStacks.get(id)?.forEach((job) => job.cancel());
scheduleStacks.set(id, jobs);
jobs.forEach((job) => job.start());
// ===== 第二遍:注册所有任务 =====
for (const item of call.request.crons) {
const { id, schedule, command, extra_schedules, name } = item;
// 取消该 id 已有的旧任务
if (scheduleStacks.has(id)) {
scheduleStacks.get(id)?.forEach((x) => x.cancel());
}
Logger.info(
'[schedule][创建定时任务] 任务ID: %s, 规则数: %s',
'[schedule][创建定时任务] 任务ID: %s, 名称: %s, cron: %s, 执行命令: %s',
id,
jobs.length,
name,
schedule,
command,
);
if (extra_schedules?.length) {
extra_schedules.forEach((x) => {
Logger.info(
'[schedule][创建定时任务] 任务ID: %s, 名称: %s, cron: %s, 执行命令: %s',
id,
name,
x.schedule,
command,
);
});
}
const mainJob = nodeSchedule.scheduleJob(id, schedule, async () => {
Logger.info(`[schedule][准备运行任务] 命令: ${command}`);
runCron(command, item);
});
if (!mainJob) {
Logger.warn(
'[schedule][创建定时任务] scheduleJob 返回 null(不符合预期,已通过预校验): 任务ID: %s, cron: %s',
id,
schedule,
);
}
const extraJobs = extra_schedules?.length
? extra_schedules.map((x) => {
const job = nodeSchedule.scheduleJob(id, x.schedule, async () => {
Logger.info(`[schedule][准备运行任务] 命令: ${command}`);
runCron(command, item);
});
if (!job) {
Logger.warn(
'[schedule][创建定时任务] scheduleJob 返回 null(不符合预期,已通过预校验): 任务ID: %s, cron: %s',
id,
x.schedule,
);
}
return job;
})
: [];
// 过滤 null(兜底保护,正常情况下预校验已拦截)
const jobs = [mainJob, ...extraJobs].filter((x) => x != null);
if (jobs.length > 0) {
scheduleStacks.set(id, jobs);
}
}
callback(null, null);
+1 -26
View File
@@ -62,41 +62,16 @@ class Client {
return this._client;
}
async setConcurrency(concurrency: number): Promise<void> {
await this.waitForReady(2000);
return new Promise((resolve, reject) => {
this.client.setConcurrency(
{ concurrency }, new Metadata(), { deadline: Date.now() + 5000 },
(err) => {
if (err) {
if (err.code === status.UNAVAILABLE || err.code === status.DEADLINE_EXCEEDED) {
this.readiness.invalidate();
Object.assign(err, { status: 503 });
}
return reject(err);
}
resolve();
},
);
});
}
async addCron(
request: AddCronRequest['crons'],
replace = false
): Promise<AddCronResponse> {
await this.waitForReady(2000);
// Include every rule in a recovery snapshot, but bound a stuck worker.
const ruleCount = request.reduce(
(count, item) => count + 1 + (item.extra_schedules?.length || 0),
0,
);
const registrationTimeoutMs = Math.min(120000, 5000 + ruleCount * 5);
return new Promise((resolve, reject) => {
this.client.addCron(
{ crons: request, replace },
new Metadata(),
{ deadline: Date.now() + registrationTimeoutMs },
{ deadline: Date.now() + 5000 },
(err, res) => {
if (err) {
if (err.code === status.UNAVAILABLE || err.code === status.DEADLINE_EXCEEDED) {
+2 -2
View File
@@ -1,6 +1,6 @@
import { CronJob } from '../shared/cronScheduler';
import nodeSchedule from 'node-schedule';
import { ToadScheduler } from 'toad-scheduler';
export const scheduleStacks = new Map<string, CronJob[]>();
export const scheduleStacks = new Map<string, nodeSchedule.Job[]>();
export const intervalSchedule = new ToadScheduler();
+1 -1
View File
@@ -13,7 +13,7 @@ const delCron = (
'[schedule][取消定时任务] 任务ID: %s',
id,
);
// 防御性过滤历史调度栈中的空任务,
// 过滤掉 nodeSchedule.scheduleJob() 对无效表达式返回的 null,
// 否则对 null 调 cancel() 会让整个取消流程抛出 UNKNOWN 错误,
// 进而导致 HTTP 端的 remove() 跳过 setCrontab(),造成 crontab.list 残留。
scheduleStacks.get(id)?.filter((x) => x != null).forEach((x) => {
-26
View File
@@ -1,26 +0,0 @@
import { ServerUnaryCall, sendUnaryData, status } from '@grpc/grpc-js';
import os from 'os';
import { AddCronResponse, SetConcurrencyRequest } from '../protos/cron';
import taskLimit from '../shared/pLimit';
export async function setConcurrency(
call: ServerUnaryCall<SetConcurrencyRequest, AddCronResponse>,
callback: sendUnaryData<AddCronResponse>,
) {
const limit = call.request.concurrency;
if (!Number.isInteger(limit) || limit < 0 || limit > 2147483647) {
callback(
Object.assign(new Error('Invalid concurrency'), {
code: status.INVALID_ARGUMENT,
}),
null,
);
return;
}
try {
await taskLimit.setCustomLimit(limit || Math.max(os.cpus().length, 4));
callback(null, {});
} catch (error) {
callback(error as Error, null);
}
}
+4 -3
View File
@@ -12,12 +12,13 @@ export default class ConfigService {
public async getFile(filePath: string, res: Response) {
let content = '';
if (!filePath || filePath.startsWith('data/scripts/')) {
if (!filePath) {
return res.send({ code: 403, message: t('文件无法访问') });
}
const scriptFile = filePath.startsWith('data/scripts/');
const resolved = resolveFileAccess(
config.configPath,
[filePath],
scriptFile ? config.scriptPath : config.configPath,
[scriptFile ? filePath.slice('data/scripts/'.length) : filePath],
config.blackFileList,
);
if (!resolved) {
+9 -22
View File
@@ -1,5 +1,5 @@
import { randomUUID } from 'crypto';
import { getInvalidCronSchedules, isValidCronSchedule } from '../shared/cronSchedule';
import { getInvalidCronSchedules } from '../shared/cronSchedule';
import {
withSchedulerMutation,
schedulerRegistrationError,
@@ -7,7 +7,6 @@ import {
import { Service, Inject } from 'typedi';
import winston from 'winston';
import config from '../config';
import { resolveLogPath } from '../shared/logPath';
import { Crontab, CrontabModel, CrontabStatus } from '../data/cron';
import {
RunningInstanceModel,
@@ -15,6 +14,7 @@ import {
} from '../data/runningInstance';
import { exec, execSync } from 'child_process';
import fs from 'fs/promises';
import CronExpressionParser from 'cron-parser';
import {
getFileContentByName,
fileExist,
@@ -47,20 +47,12 @@ export default class CronService {
private isNodeCron(cron: Crontab) {
const { schedule, extra_schedules } = cron;
const fields = schedule?.trim().split(/\s+/) || [];
// System crontab only receives portable numeric five-field expressions.
// Extended syntax, macros and legacy shorthand use the Node scheduler.
// Extended syntax, macros and legacy shorthand belong to node-schedule.
return (
fields.length !== 5 ||
schedule?.trim().split(/\s+/).length !== 5 ||
/[^\d\s*,/\-]/.test(schedule || '') ||
Boolean(extra_schedules?.length) ||
// BusyBox treats N/step as a single value and doesn't support Sunday=7.
fields.some((field) => field.split(',').some((part) => /^\d+\//.test(part))) ||
fields[4].includes('7') ||
// BusyBox steps DOM from zero; its full-range day/week wildcard
// handling also differs from the legacy Node parser's OR semantics.
fields[2].includes('/') ||
(fields[2] !== '*' && fields[4] !== '*')
Boolean(extra_schedules?.length)
);
}
@@ -247,9 +239,6 @@ export default class CronService {
last_execution_time: number;
exit_code?: number;
}) {
if (log_path && log_path !== '/dev/null') {
resolveLogPath(config.logPath, log_path);
}
let options: any = {
status,
pid,
@@ -929,9 +918,7 @@ export default class CronService {
truncated: false,
};
}
const absolutePath = doc.log_path
? resolveLogPath(config.logPath, doc.log_path)
: '';
const absolutePath = path.resolve(config.logPath, `${doc.log_path}`);
const logFileExist = doc.log_path && (await fileExist(absolutePath));
if (logFileExist) {
const chunk = await readLogChunk(`${absolutePath}`, options);
@@ -961,12 +948,12 @@ export default class CronService {
public async logs(id: number) {
const doc = await this.getDb({ id });
if (!doc || !doc.log_path || doc.log_path === '/dev/null') {
if (!doc || !doc.log_path) {
return [];
}
const relativeDir = path.dirname(`${doc.log_path}`);
const dir = path.dirname(resolveLogPath(config.logPath, doc.log_path));
const dir = path.resolve(config.logPath, relativeDir);
const dirExist = await fileExist(dir);
if (dirExist) {
let files = await fs.readdir(dir);
@@ -1070,7 +1057,7 @@ export default class CronService {
if (
command &&
schedule &&
isValidCronSchedule(schedule)
CronExpressionParser.parse(schedule).hasNext()
) {
const name = namePrefix + '_' + index;
+4 -3
View File
@@ -262,9 +262,10 @@ export default class EnvService {
.map((x) => x.value)
.join('&')
.replace(/\\/g, '\\\\')}`;
js_env_string += `process.env[${JSON.stringify(key)}]=${JSON.stringify(
group.map((x) => x.value).join('&'),
)};\n`;
js_env_string += `process.env.${key}=\`${_env_value.replace(
/\`/g,
'\\`',
)}\`;\n`;
py_env_string += `os.environ['${key}']='''${_env_value.replace(
/\'/g,
"\\'",
+1 -2
View File
@@ -4,7 +4,6 @@ import { HealthService } from '../protos/health';
import { ApiService } from '../protos/api';
import { addCron } from '../schedule/addCron';
import { delCron } from '../schedule/delCron';
import { setConcurrency } from '../schedule/setConcurrency';
import { check } from '../schedule/health';
import * as Api from '../schedule/api';
import Logger from '../loaders/logger';
@@ -28,7 +27,7 @@ export class GrpcServerService {
async initialize() {
try {
this.server.addService(HealthService, { check });
this.server.addService(CronService, { addCron, delCron, setConcurrency });
this.server.addService(CronService, { addCron, delCron });
this.server.addService(ApiService, Api);
const tlsConfig = await initGrpcCerts();
+12 -28
View File
@@ -12,7 +12,7 @@ export class HttpServerService {
async initialize(expressApp: express.Application, port: number) {
const hostsToTry = [
config.bindHost,
...(config.bindHost === '::' ? ['0.0.0.0'] : []),
...(config.bindHost === '::' ? ['0.0.0.0'] : [])
];
let lastError: Error | null = null;
@@ -20,21 +20,16 @@ export class HttpServerService {
for (const host of hostsToTry) {
try {
const server = await this.tryListen(expressApp, port, host);
Logger.debug(
`[boot] HTTP service started successfully on ${host}:${port}`,
);
Logger.debug(`[boot] HTTP service started successfully on ${host}:${port}`);
metricsService.record('http_service_start', 1, {
port: port.toString(),
host,
host
});
this.server = server;
return server;
} catch (err) {
lastError = err as Error;
Logger.warn(
`Failed to bind HTTP on ${host}:${port}, trying next...`,
err,
);
Logger.warn(`Failed to bind HTTP on ${host}:${port}, trying next...`, err);
}
}
@@ -42,28 +37,17 @@ export class HttpServerService {
throw lastError || new Error('Failed to start HTTP service');
}
private async tryListen(
expressApp: express.Application,
port: number,
host: string,
): Promise<Server> {
private async tryListen(expressApp: express.Application, port: number, host: string): Promise<Server> {
return new Promise((resolve, reject) => {
// There is one HTTP worker; accepting here avoids primary IPC handoff
// for every connection. Restore shared listening for custom clusters.
const server = expressApp.listen(
{
port,
host,
exclusive: process.env.QL_HTTP_SHARED_LISTEN !== 'true',
},
() => {
resolve(server);
},
);
server.requestTimeout = 300000;
server.headersTimeout = 120000;
server.keepAliveTimeout = 65000;
const server = expressApp.listen({
port,
host,
exclusive: process.env.QL_HTTP_SHARED_LISTEN !== 'true',
}, () => {
resolve(server);
});
server.on('error', (err: Error) => {
server.close();
+6 -1
View File
@@ -1,4 +1,5 @@
import { resolveFileAccess } from '../shared/fileAccess';
import path from 'path';
import { Inject, Service } from 'typedi';
import winston from 'winston';
import config from '../config';
@@ -8,6 +9,10 @@ export default class LogService {
constructor(@Inject('logger') private logger: winston.Logger) {}
public checkFilePath(filePath: string, fileName: string) {
return resolveFileAccess(config.logPath, [filePath || '', fileName]);
return resolveFileAccess(
config.logPath,
[filePath || '', fileName],
config.blackFileList,
);
}
}
+34 -116
View File
@@ -6,77 +6,27 @@ import { v4 as uuidV4 } from 'uuid';
import sequelize, { Op } from 'sequelize';
import { shareStore } from '../shared/store';
import { t } from '../shared/i18n';
import { lock } from 'proper-lockfile';
import path from 'path';
import os from 'os';
import { writeFile, rename, rm, stat, chmod } from 'fs/promises';
import config from '../config';
@Service()
export default class OpenService {
constructor(@Inject('logger') private logger: winston.Logger) {}
// Every database mutation and full authentication-cache publication shares
// this cross-process lock. Lock a separate marker, not token.json itself.
private async withAppsLock<T>(operation: () => Promise<T>): Promise<T> {
const target = path.join(config.configPath, '.apps-state');
const release = await lock(target, {
realpath: false,
stale: 10_000,
retries: { retries: 10, factor: 2, minTimeout: 100, maxTimeout: 3000 },
});
try {
return await operation();
} finally {
await release();
}
}
public async initializeSystemApp(): Promise<void> {
await this.withAppsLock(async () => {
const doc = await AppModel.findOne({ where: { name: 'system' } });
const app = doc?.get({ plain: true });
if (!app) {
await this.insert({
name: 'system',
scopes: ['crons', 'system', 'dashboard'],
client_id: createRandomString(12, 12),
client_secret: createRandomString(24, 24),
});
} else if (!app.scopes.includes('dashboard')) {
await AppModel.update(
{ scopes: [...app.scopes, 'dashboard'] },
{ where: { id: app.id } },
);
}
await shareStore.updateApps(await this.find({}));
});
}
public async refreshApps(): Promise<void> {
await this.withAppsLock(async () => {
await shareStore.updateApps(await this.find({}));
});
}
public async findApps(): Promise<App[] | null> {
const docs = await this.find({});
return docs;
}
public async create(payload: App): Promise<App> {
return this.withAppsLock(async () => {
const tab = { ...payload };
tab.client_id = createRandomString(12, 12);
tab.client_secret = createRandomString(24, 24);
const doc = await this.insert(tab);
const apps = await this.find({});
await shareStore.updateApps(apps);
return { ...doc, tokens: [] };
});
const tab = { ...payload };
tab.client_id = createRandomString(12, 12);
tab.client_secret = createRandomString(24, 24);
const doc = await this.insert(tab);
const apps = await this.find({});
await shareStore.updateApps(apps);
return { ...doc, tokens: [] };
}
private async insert(payload: App): Promise<App> {
public async insert(payload: App): Promise<App> {
const doc = await AppModel.create(payload, { returning: true });
return doc.get({ plain: true });
}
@@ -91,12 +41,10 @@ export default class OpenService {
}
private async updateDb(payload: Partial<App>): Promise<App> {
return this.withAppsLock(async () => {
await AppModel.update(payload, { where: { id: payload.id } });
const apps = await this.find({});
await shareStore.updateApps(apps);
return apps?.find((x) => x.id === payload.id) as App;
});
await AppModel.update(payload, { where: { id: payload.id } });
const apps = await this.find({});
await shareStore.updateApps(apps);
return apps?.find((x) => x.id === payload.id) as App;
}
public async getDb(query: Record<string, any>): Promise<App> {
@@ -108,11 +56,9 @@ export default class OpenService {
}
public async remove(ids: number[]) {
return this.withAppsLock(async () => {
await AppModel.destroy({ where: { id: ids } });
const apps = await this.find({});
await shareStore.updateApps(apps);
});
await AppModel.destroy({ where: { id: ids } });
const apps = await this.find({});
await shareStore.updateApps(apps);
}
public async resetSecret(id: number): Promise<App> {
@@ -166,15 +112,7 @@ export default class OpenService {
return docs.map((x) => x.get({ plain: true }));
}
public async authToken(credentials: {
client_id: string;
client_secret: string;
}): Promise<any> {
return this.withAppsLock(() => this.issueToken(credentials));
}
// Caller holds withAppsLock through database update and cache publication.
private async issueToken({
public async authToken({
client_id,
client_secret,
}: {
@@ -218,47 +156,27 @@ export default class OpenService {
}
}
public async generateSystemToken(renew = false): Promise<{
public async generateSystemToken(): Promise<{
value: string;
expiration: number;
}> {
return this.withAppsLock(async () => {
// Read after acquiring the lock; a concurrent issuer may have renewed.
const systemApp = await this.getDb({ name: 'system' });
const now = Math.round(Date.now() / 1000);
let appToken = systemApp.tokens
?.filter((token) => token.expiration > now)
.sort((left, right) => right.expiration - left.expiration)[0];
if (renew || !appToken) {
const { data } = await this.issueToken({
client_id: systemApp.client_id,
client_secret: systemApp.client_secret,
});
appToken = { value: data.token, expiration: data.expiration };
} else {
// Recover a prior database commit whose cache publication failed.
await shareStore.updateApps(await this.find({}));
}
const tokenFile = path.join(config.configPath, 'token.json');
const temporaryFile = `${tokenFile}.${process.pid}.tmp`;
const mode = await stat(tokenFile).then(
(file) => file.mode & 0o777,
(error: NodeJS.ErrnoException) => {
if (error.code === 'ENOENT') return 0o600;
throw error;
},
);
try {
await writeFile(temporaryFile, `${JSON.stringify(appToken)}${os.EOL}`, {
mode,
});
await chmod(temporaryFile, mode);
// Shell readers see a complete old or new document, never a partial one.
await rename(temporaryFile, tokenFile);
} finally {
await rm(temporaryFile, { force: true });
}
return appToken;
const apps = await shareStore.getApps();
const systemApp = apps?.find((x) => x.name === 'system');
if (!systemApp) {
throw new Error('system app not found');
}
const now = Math.round(Date.now() / 1000);
const currentToken = systemApp.tokens?.find((x) => x.expiration > now);
if (currentToken) {
return currentToken;
}
const { data } = await this.authToken({
client_id: systemApp.client_id,
client_secret: systemApp.client_secret,
});
return {
...data,
value: data.token,
};
}
}
+14 -33
View File
@@ -1,6 +1,6 @@
import { Service, Inject } from 'typedi';
import winston from 'winston';
import { createCronJob, CronJob } from '../shared/cronScheduler';
import nodeSchedule from 'node-schedule';
import { ChildProcessWithoutNullStreams } from 'child_process';
import {
ToadScheduler,
@@ -11,11 +11,7 @@ import {
import dayjs from 'dayjs';
import taskLimit from '../shared/pLimit';
import { spawn } from 'cross-spawn';
import {
observeChildProcess,
asError,
ProcessResult,
} from '../shared/childProcess';
import { observeChildProcess, asError, ProcessResult } from '../shared/childProcess';
export interface ScheduleTaskType {
id?: number;
@@ -42,7 +38,7 @@ export interface TaskCallbacks {
@Service()
export default class ScheduleService {
private scheduleStacks = new Map<string, CronJob>();
private scheduleStacks = new Map<string, nodeSchedule.Job>();
private intervalSchedule = new ToadScheduler();
@@ -163,33 +159,18 @@ export default class ScheduleService {
command,
);
if (schedule) {
try {
const job = createCronJob(
this.scheduleStacks.set(
_id,
nodeSchedule.scheduleJob(_id, schedule, async () => {
this.runTask(command, callbacks, {
name,
schedule,
() =>
this.runTask(command, callbacks, {
name,
schedule,
command,
id: _id,
runOrigin,
}),
{ name: `${_id}: ${name || ''}`, logger: this.logger, start: false },
);
this.scheduleStacks.get(_id)?.cancel();
this.scheduleStacks.set(_id, job);
job.start();
} catch (error) {
// A persisted invalid subscription must not become an unhandled startup rejection.
this.logger.warn(
'[panel][跳过无效定时任务] 任务ID: %s, cron: %s, 错误: %s',
_id,
schedule,
asError(error).message,
);
}
}
command,
id: _id,
runOrigin,
});
}),
);
if (runImmediately) {
this.runTask(command, callbacks, {
+5 -1
View File
@@ -66,7 +66,11 @@ export default class ScriptService {
}
public checkFilePath(filePath: string, fileName: string) {
return resolveFileAccess(config.scriptPath, [filePath || '', fileName]);
return resolveFileAccess(
config.scriptPath,
[filePath || '', fileName],
config.blackFileList,
);
}
public async getFile(filePath: string, fileName: string) {
+5 -1
View File
@@ -7,6 +7,10 @@ import { ScriptHistory } from '../shared/scriptHistory';
@Service()
export default class ScriptHistoryService extends ScriptHistory {
constructor() {
super(config.scriptPath, path.join(config.dataPath, 'script-history'));
super(
config.scriptPath,
path.join(config.dataPath, 'script-history'),
config.blackFileList,
);
}
}
+23 -43
View File
@@ -8,22 +8,10 @@ import { formatUrl } from '../config/subscription';
import config from '../config';
import { fileExist, rmPath } from '../config/util';
import { writeFileWithLock } from '../shared/utils';
import {
getSubscriptionSshAlias,
resolveSubscriptionPath,
} from '../shared/subscriptionPath';
@Service()
export default class SshKeyService {
// OpenSSH uses the effective user's passwd entry, not the HOME override.
private get homedir(): string {
try {
return os.userInfo().homedir;
} catch {
// Custom container UIDs may have no passwd entry. Keep startup working.
return os.homedir();
}
}
private homedir = os.homedir();
private sshPath = config.sshdPath;
private sshConfigFilePath = path.resolve(this.homedir, '.ssh', 'config');
private sshConfigHeader = `Include ${path.join(this.sshPath, '*.config')}`;
@@ -33,10 +21,6 @@ export default class SshKeyService {
}
private async initSshConfigFile() {
await fs.mkdir(path.dirname(this.sshConfigFilePath), {
recursive: true,
mode: 0o700,
});
let config = '';
const _exist = await fileExist(this.sshConfigFilePath);
if (_exist) {
@@ -58,7 +42,7 @@ export default class SshKeyService {
key: string,
): Promise<void> {
try {
const filePath = resolveSubscriptionPath(this.sshPath, alias);
const filePath = path.join(this.sshPath, alias);
try {
await rmPath(filePath);
} catch { }
@@ -70,7 +54,7 @@ export default class SshKeyService {
private async removePrivateKeyFile(alias: string): Promise<void> {
try {
const filePath = resolveSubscriptionPath(this.sshPath, alias);
const filePath = path.join(this.sshPath, alias);
await rmPath(filePath);
} catch (error) {
this.logger.error('删除私钥文件失败', error);
@@ -82,19 +66,18 @@ export default class SshKeyService {
host: string,
proxy?: string,
) {
if (!/^[a-zA-Z0-9.-]+$/.test(host) ||
(proxy && !/^[a-zA-Z0-9.:[\]-]+$/.test(proxy))) {
throw Object.assign(new Error('Invalid SSH host or proxy'), { status: 400 });
}
if (host === 'github.com') {
host = `ssh.github.com\n Port 443\n HostkeyAlgorithms +ssh-rsa`;
}
const proxyStr = proxy
? ` ProxyCommand nc -v -x ${proxy} %h %p 2>/dev/null\n`
: '';
const config = `Host ${alias}\n Hostname ${host}\n IdentityFile ${resolveSubscriptionPath(this.sshPath, alias)}\n StrictHostKeyChecking no\n${proxyStr}`;
const config = `Host ${alias}\n Hostname ${host}\n IdentityFile ${path.join(
this.sshPath,
alias,
)}\n StrictHostKeyChecking no\n${proxyStr}`;
await writeFileWithLock(
`${resolveSubscriptionPath(this.sshPath, alias, '.config')}`,
`${path.join(this.sshPath, `${alias}.config`)}`,
config,
{
encoding: 'utf8',
@@ -105,7 +88,7 @@ export default class SshKeyService {
private async removeSshConfig(alias: string) {
try {
const filePath = resolveSubscriptionPath(this.sshPath, alias, '.config');
const filePath = path.join(this.sshPath, `${alias}.config`);
await rmPath(filePath);
} catch (error) {
this.logger.error(`删除ssh配置文件${alias}失败`, error);
@@ -118,7 +101,6 @@ export default class SshKeyService {
host: string,
proxy?: string,
): Promise<void> {
alias = getSubscriptionSshAlias(alias);
await this.generatePrivateKeyFile(alias, key);
await this.generateSingleSshConfig(alias, host, proxy);
}
@@ -128,7 +110,6 @@ export default class SshKeyService {
host: string,
proxy?: string,
): Promise<void> {
alias = getSubscriptionSshAlias(alias);
await this.removePrivateKeyFile(alias);
await this.removeSshConfig(alias);
}
@@ -136,19 +117,15 @@ export default class SshKeyService {
public async setSshConfig(docs: Subscription[]) {
for (const doc of docs) {
if (doc.type === 'private-repo' && doc.pull_type === 'ssh-key') {
try {
const alias = getSubscriptionSshAlias(doc.alias);
const { host } = formatUrl(doc);
await this.removePrivateKeyFile(alias);
await this.removeSshConfig(alias);
await this.generatePrivateKeyFile(
alias,
(doc.pull_option as any).private_key,
);
await this.generateSingleSshConfig(alias, host, doc.proxy);
} catch (error) {
this.logger.warn('跳过订阅 %s 的无效 SSH 配置: %s', doc.id, error);
}
const { alias, proxy } = doc;
const { host } = formatUrl(doc);
await this.removePrivateKeyFile(alias);
await this.removeSshConfig(alias);
await this.generatePrivateKeyFile(
alias,
(doc.pull_option as any).private_key,
);
await this.generateSingleSshConfig(alias, host, proxy);
}
}
}
@@ -167,9 +144,12 @@ export default class SshKeyService {
private async generateGlobalSshConfig(alias: string) {
// Create a config that matches all hosts, making this key globally available
const config = `Host *\n IdentityFile ${resolveSubscriptionPath(this.sshPath, alias)}\n StrictHostKeyChecking no\n`;
const config = `Host *\n IdentityFile ${path.join(
this.sshPath,
alias,
)}\n StrictHostKeyChecking no\n`;
await writeFileWithLock(
`${resolveSubscriptionPath(this.sshPath, alias, '.config')}`,
`${path.join(this.sshPath, `${alias}.config`)}`,
config,
{
encoding: 'utf8',
+33 -108
View File
@@ -20,7 +20,7 @@ import {
} from '../config/util';
import fs from 'fs/promises';
import { FindOptions, Op } from 'sequelize';
import path from 'path';
import path, { join } from 'path';
import ScheduleService, { TaskCallbacks } from './schedule';
import { SimpleIntervalSchedule } from 'toad-scheduler';
import SockService from './sock';
@@ -30,14 +30,6 @@ import dayjs from 'dayjs';
import { LOG_END_SYMBOL } from '../config/const';
import { formatCommand, formatUrl } from '../config/subscription';
import { CrontabModel } from '../data/cron';
import { assertSubscriptionAlias } from '../shared/subscriptionPath';
import { resolveLogPath } from '../shared/logPath';
import {
getSubscriptionStorageCandidates,
isSubscriptionStorageName,
resolveSubscriptionStoragePath,
} from '../shared/subscriptionStorage';
import { withSubscriptionMutation } from '../shared/subscriptionMutationLock';
import CrontabService from './cron';
import taskLimit from '../shared/pLimit';
import { logStreamManager } from '../shared/logStreamManager';
@@ -96,11 +88,9 @@ export default class SubscriptionService {
needCreate = true,
runImmediately = false,
) {
// Cancelling by ID must also work for invalid historical subscriptions.
if (needCreate) {
const { url } = formatUrl(doc);
doc.command = formatCommand(doc, url as string);
}
const { url } = formatUrl(doc);
doc.command = formatCommand(doc, url as string);
if (doc.schedule_type === 'crontab') {
this.scheduleService.cancelCronTask(doc as any);
@@ -232,14 +222,11 @@ export default class SubscriptionService {
}
public async create(payload: Subscription): Promise<Subscription> {
assertSubscriptionAlias(payload.alias);
return withSubscriptionMutation(async () => {
const tab = new Subscription(payload);
const doc = await this.insert(tab);
await this.handleTask(doc.get({ plain: true }));
await this.setSshConfig();
return doc;
});
const tab = new Subscription(payload);
const doc = await this.insert(tab);
await this.handleTask(doc.get({ plain: true }));
await this.setSshConfig();
return doc;
}
public async insert(payload: Subscription): Promise<SubscriptionInstance> {
@@ -247,27 +234,12 @@ export default class SubscriptionService {
}
public async update(payload: Subscription): Promise<Subscription> {
assertSubscriptionAlias(payload.alias);
return withSubscriptionMutation(async () => {
const doc = await this.getDb({ id: payload.id });
const tab = new Subscription({ ...doc, ...payload });
const newDoc = await this.updateDb(tab);
await this.handleTask(newDoc, !newDoc.is_disabled);
await this.removeSshConfigForSubscription(doc);
await this.setSshConfig();
return newDoc;
});
}
private async removeSshConfigForSubscription(doc: Subscription) {
if (doc.type !== 'private-repo' || doc.pull_type !== 'ssh-key') return;
// Invalid historical rows must remain deletable without deriving unsafe paths.
try {
assertSubscriptionAlias(doc.alias);
} catch {
return;
}
await this.sshKeyService.removeSSHKey(doc.alias, '');
const doc = await this.getDb({ id: payload.id });
const tab = new Subscription({ ...doc, ...payload });
const newDoc = await this.updateDb(tab);
await this.handleTask(newDoc, !newDoc.is_disabled);
await this.setSshConfig();
return newDoc;
}
public async updateDb(payload: Subscription): Promise<Subscription> {
@@ -290,7 +262,6 @@ export default class SubscriptionService {
last_running_time: number;
last_execution_time: number;
}) {
if (log_path) resolveLogPath(config.logPath, log_path);
const options: any = {
status,
pid,
@@ -308,71 +279,25 @@ export default class SubscriptionService {
}
public async remove(ids: number[], query: { force?: boolean }) {
return withSubscriptionMutation(async () => {
const docs = await SubscriptionModel.findAll({ where: { id: ids } });
const storagePaths: string[] = [];
const retainedPaths: string[] = [];
const collectPaths = (
doc: Subscription,
targets: string[],
deleting = false,
) => {
for (const names of getSubscriptionStorageCandidates(doc)) {
// A URL accepted by one engine can produce an invalid name in another.
// Validate each candidate independently, retaining the safe candidate.
if (!Object.values(names).every(isSubscriptionStorageName)) continue;
const resolve = deleting ? resolveSubscriptionStoragePath : path.resolve;
const scriptPath = resolve(config.scriptPath, names.script);
if (scriptPath) targets.push(scriptPath);
if (names.repo) {
const repoPath = resolve(config.repoPath, names.repo);
if (repoPath) targets.push(repoPath);
}
if (names.raw) {
const rawPath = resolve(path.join(config.dataPath, 'raw'), names.raw);
if (rawPath) targets.push(rawPath);
}
}
};
// Validate all deletion targets before changing rows or removing any files.
if (query?.force === true) {
for (const doc of docs) {
assertSubscriptionAlias(doc.alias);
resolveSubscriptionStoragePath(config.scriptPath, doc.alias);
resolveSubscriptionStoragePath(config.repoPath, doc.alias);
collectPaths(doc, storagePaths, true);
}
const remaining = await SubscriptionModel.findAll({ where: {} });
for (const doc of remaining) {
if (!ids.includes(doc.id!)) collectPaths(doc, retainedPaths);
}
}
for (const doc of docs) {
await this.handleTask(doc.get({ plain: true }), false);
}
await SubscriptionModel.destroy({ where: { id: ids } });
for (const doc of docs) {
await this.removeSshConfigForSubscription(doc);
}
await this.setSshConfig();
const docs = await SubscriptionModel.findAll({ where: { id: ids } });
for (const doc of docs) {
await this.handleTask(doc.get({ plain: true }), false);
}
await SubscriptionModel.destroy({ where: { id: ids } });
await this.setSshConfig();
if (query?.force === true) {
const crons = await CrontabModel.findAll({ where: { sub_id: ids } });
if (crons?.length) {
await this.crontabService.remove(crons.map((x) => x.id!));
}
for (const storagePath of new Set(storagePaths)) {
// rm is recursive: an ancestor must also be retained if a surviving
// subscription uses a nested branch, and vice versa.
const shared = retainedPaths.some((retained) =>
retained === storagePath ||
retained.startsWith(`${storagePath}${path.sep}`) ||
storagePath.startsWith(`${retained}${path.sep}`),
);
if (!shared) await rmPath(storagePath);
}
if (query?.force === true) {
const crons = await CrontabModel.findAll({ where: { sub_id: ids } });
if (crons?.length) {
await this.crontabService.remove(crons.map((x) => x.id!));
}
});
for (const doc of docs) {
const filePath = join(config.scriptPath, doc.alias);
const repoPath = join(config.repoPath, doc.alias);
await rmPath(filePath);
await rmPath(repoPath);
}
}
}
public async getDb(
@@ -474,7 +399,7 @@ export default class SubscriptionService {
if (doc.log_path) {
const relativeDir = path.dirname(`${doc.log_path}`);
const dir = path.dirname(resolveLogPath(config.logPath, doc.log_path));
const dir = path.resolve(config.logPath, relativeDir);
const _exist = await fileExist(dir);
if (_exist) {
let files = await fs.readdir(dir);
+22 -52
View File
@@ -1,7 +1,4 @@
import { spawn } from 'cross-spawn';
import { execFile } from 'child_process';
import { promisify } from 'util';
import { shellQuote } from '../shared/shellQuote';
import { Response } from 'express';
import fs from 'fs';
import { Agent, request } from 'undici';
@@ -34,8 +31,6 @@ import {
SystemModelInfo,
} from '../data/system';
import taskLimit from '../shared/pLimit';
import cronClient from '../schedule/client';
import { withSchedulerMutation } from '../shared/schedulerMutationLock';
import NotificationService from './notify';
import ScheduleService, { TaskCallbacks } from './schedule';
import SockService from './sock';
@@ -124,29 +119,15 @@ export default class SystemService {
}
public async updateCronConcurrency(info: SystemModelInfo) {
const limit = info.cronConcurrency ?? 0;
if (!Number.isInteger(limit) || limit < 0 || limit > 2147483647) {
throw Object.assign(new Error(t('参数错误')), { status: 400 });
const oDoc = await this.getSystemConfig();
await this.updateAuthDb({
...oDoc,
info: { ...oDoc.info, ...info },
});
if (info.cronConcurrency) {
await taskLimit.setCustomLimit(info.cronConcurrency);
}
return withSchedulerMutation(async () => {
const oDoc = await this.getSystemConfig();
await this.updateAuthDb({
...oDoc,
info: { ...oDoc.info, ...info, cronConcurrency: limit },
});
await taskLimit.setCustomLimit(limit || Math.max(os.cpus().length, 4));
await cronClient.setConcurrency(limit);
return { code: 200, data: info };
});
}
public async restoreCronConcurrency() {
await withSchedulerMutation(async () => {
const doc = await this.getSystemConfig();
const limit = doc.info?.cronConcurrency ?? 0;
await taskLimit.setCustomLimit(limit || Math.max(os.cpus().length, 4));
await cronClient.setConcurrency(limit);
});
return { code: 200, data: info };
}
public async updateDependenceProxy(info: SystemModelInfo) {
@@ -158,7 +139,7 @@ export default class SystemService {
if (info.dependenceProxy) {
await fs.promises.writeFile(
config.dependenceProxyFile,
`export http_proxy=${shellQuote(info.dependenceProxy)}\nexport https_proxy=${shellQuote(info.dependenceProxy)}`,
`export http_proxy="${info.dependenceProxy}"\nexport https_proxy="${info.dependenceProxy}"`,
);
} else {
await fs.promises.rm(config.dependenceProxyFile);
@@ -174,7 +155,7 @@ export default class SystemService {
});
let cmd = 'pnpm config delete registry';
if (info.nodeMirror) {
cmd = `pnpm config set registry ${shellQuote(info.nodeMirror)}`;
cmd = `pnpm config set registry ${info.nodeMirror}`;
}
let command = `cd && ${cmd}`;
const docs = await DependenceModel.findAll({
@@ -223,7 +204,7 @@ export default class SystemService {
});
let cmd = 'pip config unset global.index-url';
if (info.pythonMirror) {
cmd = `pip3 config set global.index-url ${shellQuote(info.pythonMirror)}`;
cmd = `pip3 config set global.index-url ${info.pythonMirror}`;
}
await promiseExec(cmd);
return { code: 200, data: info };
@@ -239,6 +220,7 @@ export default class SystemService {
return;
}
const command = await updateLinuxMirrorFile(info.linuxMirror || '');
let hasError = false;
this.scheduleService.runTask(
command,
{
@@ -246,15 +228,14 @@ export default class SystemService {
res?.setHeader('QL-Task-Pid', `${cp.pid}`);
res?.end();
},
onEnd: async (cp) => {
onEnd: async () => {
this.sockService.sendMessage({
type: 'updateLinuxMirror',
message: 'update linux mirror end',
status: 'completed',
});
onEnd?.();
// Tools such as sudo and apt may write warnings to stderr on success.
if (cp?.exitCode === 0 && cp.signalCode === null) {
if (!hasError) {
await this.updateAuthDb({
...oDoc,
info: { ...oDoc.info, ...info },
@@ -262,6 +243,7 @@ export default class SystemService {
}
},
onError: async (message: string) => {
hasError = true;
this.sockService.sendMessage({ type: 'updateLinuxMirror', message });
},
onLog: async (message: string) => {
@@ -375,10 +357,7 @@ export default class SystemService {
}
public async reloadSystem(target?: 'system' | 'data') {
if (target && !['system', 'data'].includes(target)) {
return { code: 400, message: t('参数错误') };
}
const cmd = `real_time=true ql reload ${shellQuote(target || '')}`;
const cmd = `real_time=true ql reload ${target || ''}`;
const cp = spawn(cmd, {
shell: '/bin/bash',
detached: true,
@@ -459,20 +438,11 @@ export default class SystemService {
if (type && type.length) {
dataDirs = dataDirs.concat(type.filter((x) => x !== 'base'));
}
const allowed = new Set(['db', 'upload', 'config', 'scripts', 'log', 'deps',
'syslog', 'dep_cache', 'raw', 'repo', 'ssh.d', 'script-history']);
if (dataDirs.some((dir) => !allowed.has(dir))) {
return res.status(400).send({ code: 400, message: t('参数错误') });
}
if (dataDirs.includes('scripts')) dataDirs.push('script-history');
// History is created lazily; old/fresh installations must still export.
if (!fs.existsSync(path.join(config.dataPath, 'script-history'))) {
dataDirs = dataDirs.filter((dir) => dir !== 'script-history');
}
const dataPaths = [...new Set(dataDirs)].map((dir) => `data/${dir}`);
await promisify(execFile)('tar', ['-zcf', config.dataTgzFile, '--', ...dataPaths], {
cwd: path.dirname(config.dataPath),
});
const dataPaths = dataDirs.map((dir) => `data/${dir}`);
await promiseExec(
`cd ${config.dataPath} && cd ../ && tar -zcvf ${config.dataTgzFile
} ${dataPaths.join(' ')}`,
);
res.download(config.dataTgzFile);
} catch (error: any) {
return res.send({ code: 400, message: error.message });
@@ -595,7 +565,7 @@ export default class SystemService {
try {
await fs.promises.writeFile(
config.langEnvFile,
`export QL_LANG=${shellQuote(lang)}\n`,
`export QL_LANG='${lang}'\n`,
);
} catch (error) {
this.logger.error(`Failed to write lang_env.sh: ${error}`);
+12 -15
View File
@@ -33,11 +33,9 @@ import {
verifyPassword,
} from '../shared/password';
import { serializeAuthMutation } from '../shared/authMutation';
import { LoginThrottle } from '../shared/loginThrottle';
@Service()
export default class UserService {
private loginThrottle = new LoginThrottle();
@Inject((type) => NotificationService)
private notificationService!: NotificationService;
@@ -108,8 +106,11 @@ export default class UserService {
return { code: 403, message: t('该 IP 已被列入黑名单') };
}
const waitTime = this.loginThrottle.retryAfter(ip, timestamp);
if (waitTime > 0) {
const retriesTime = Math.pow(3, retries) * 1000;
if (retries > 2 && timestamp - lastlogon < retriesTime) {
const waitTime = Math.ceil(
(retriesTime - (timestamp - lastlogon)) / 1000,
);
return {
code: 410,
message: tf('失败次数过多,请%s秒后重试', waitTime),
@@ -132,7 +133,6 @@ export default class UserService {
}
if (passwordMatches) {
this.loginThrottle.reset(ip);
const data = createRandomString(50, 100);
const expiration = twoFactorActivated ? '60d' : '20d';
let token = jwt.sign({ data }, config.jwt.secret, {
@@ -206,9 +206,8 @@ export default class UserService {
},
};
} else {
const failures = this.loginThrottle.fail(ip, timestamp);
await this.updateAuthInfo(content, {
retries: failures,
retries: retries + 1,
lastlogon: timestamp,
lastip: ip,
lastaddr: address,
@@ -240,8 +239,8 @@ export default class UserService {
},
});
this.getLoginLog();
if (failures > 3) {
const waitTime = this.loginThrottle.retryAfter(ip, timestamp);
if (retries > 2) {
const waitTime = Math.round(Math.pow(3, retries + 1));
return {
code: 410,
message: tf('失败次数过多,请%s秒后重试', waitTime),
@@ -440,8 +439,8 @@ export default class UserService {
const authInfo = await this.getAuthInfo();
const { isTwoFactorChecking, twoFactorSecret } = authInfo;
const now = Date.now();
const source = getClientIp(req);
if (this.loginThrottle.retryAfter(source, now) > 0) {
const retries = authInfo.retries || 0;
if (retries > 2 && now - authInfo.lastlogon < Math.pow(3, retries) * 1000) {
return { code: 410, message: t('失败次数过多,请稍后重试') };
}
if (
@@ -473,8 +472,9 @@ export default class UserService {
.join(' ');
}
await this.updateAuthInfo(authInfo, {
retries: this.loginThrottle.fail(source, now),
retries: retries + 1,
lastlogon: now,
isTwoFactorChecking: retries + 1 < 5,
lastip: ip,
lastaddr: address,
platform: req.platform,
@@ -690,8 +690,5 @@ export default class UserService {
});
}
await this.updateAuthInfo(authInfo, payload);
if (retries === 0 || password !== undefined || username !== undefined) {
this.loginThrottle.clear();
}
}
}
-11
View File
@@ -1,11 +0,0 @@
// Match command text literally: looking up a task must never execute its input.
export function findCronId(crontab: string, command: string): string {
if (!command) return '';
for (const line of crontab.split('\n')) {
const id = /\bID=(\d+)\s/.exec(line);
if (id && line.includes(command, id.index + id[0].length)) {
return id[1];
}
}
return '';
}
+7 -90
View File
@@ -1,93 +1,12 @@
import CronExpressionParser from 'cron-parser';
import { validate } from 'node-cron';
import cronParser from 'cron-parser-v4';
import { ScheduleType } from '../interface/schedule';
const aliases: Record<string, string> = {
'@yearly': '0 0 0 1 1 *',
'@annually': '0 0 0 1 1 *',
'@monthly': '0 0 0 1 * *',
'@weekly': '0 0 0 * * 0',
'@daily': '0 0 0 * * *',
'@midnight': '0 0 0 * * *',
'@hourly': '0 0 * * * *',
'@minutely': '0 * * * * *',
};
export interface CronSchedule {
patterns: string[];
dayCount: number;
nthDay: number;
}
// Validation and registration repeatedly see identical rules in large snapshots.
// Cache only immutable syntax, never dates or running tasks, and bound user input.
const scheduleCache = new Map<string, CronSchedule>();
// Canonicalize legacy shorthand, aliases, names and numeric-start steps before
// node-cron sees them. Extend legacy aliases explicitly; reject H and bare /N.
export function parseCronSchedule(schedule: unknown): CronSchedule {
if (typeof schedule !== 'string' || !schedule.trim()) {
throw new Error('Invalid cron schedule');
}
let source = schedule.trim();
const cacheKey = source;
const cached = scheduleCache.get(cacheKey);
if (cached) return cached;
if (source.startsWith('@')) {
if (!Object.hasOwn(aliases, source)) throw new Error('Invalid cron alias');
source = aliases[source];
}
let parts = source.split(/\s+/);
if (
parts.length > 6 ||
parts.some((part) => /(^|,)\//.test(part) || /H(?:\(|\/|$)/i.test(part))
) {
throw new Error('Unsupported cron syntax');
}
parts = [
...['0', '*', '*', '*', '*', '*'].slice(0, 6 - parts.length),
...parts,
];
if (
parts.some(
(part, index) => index !== 3 && index !== 5 && part.includes('?'),
)
) {
throw new Error('Question mark is only valid in day fields');
}
const expression = CronExpressionParser.parse(parts.join(' '));
if (!expression.hasNext())
throw new Error('Cron schedule has no next execution');
const normalized = expression.stringify(true).replace(/\?/g, '*').split(' ');
const dayCount = expression.fields.dayOfMonth.values.length;
const weekCount = expression.fields.dayOfWeek.values.length;
const patterns =
dayCount < 31 && weekCount < 8
? [
[...normalized.slice(0, 5), '*'].join(' '),
[...normalized.slice(0, 3), '*', ...normalized.slice(4)].join(' '),
]
: [normalized.join(' ')];
if (!patterns.every((pattern) => validate(pattern))) {
throw new Error('Unsupported cron schedule');
}
const parsed = {
patterns,
dayCount,
nthDay: expression.fields.dayOfWeek.nthDay,
};
Object.freeze(patterns);
Object.freeze(parsed);
if (scheduleCache.size >= 512)
scheduleCache.delete(scheduleCache.keys().next().value);
scheduleCache.set(cacheKey, parsed);
return parsed;
}
// Keep validation aligned with node-schedule's locked cron-parser version.
// cron-parser 5 accepts syntax (e.g. H) that the scheduler cannot execute.
export function isValidCronSchedule(schedule: unknown): schedule is string {
if (typeof schedule !== 'string' || !schedule.trim()) return false;
try {
parseCronSchedule(schedule);
return true;
return cronParser.parseExpression(schedule).hasNext();
} catch {
return false;
}
@@ -103,8 +22,6 @@ export function getInvalidCronSchedules(cron: {
) {
return [];
}
return [
cron.schedule,
...(cron.extra_schedules || []).map((x) => x.schedule),
].filter((schedule) => !isValidCronSchedule(schedule));
return [cron.schedule, ...(cron.extra_schedules || []).map((x) => x.schedule)]
.filter((schedule) => !isValidCronSchedule(schedule));
}
-89
View File
@@ -1,89 +0,0 @@
import { createTask, ScheduledTask } from 'node-cron';
import { parseCronSchedule } from './cronSchedule';
interface SchedulerLogger {
warn(message: string, ...args: unknown[]): unknown;
error(message: string, ...args: unknown[]): unknown;
}
export interface CronJob {
start(): void;
cancel(): void;
}
// node-cron owns timers and calendar calculation; Qinglong owns execution
// concurrency and process lifetime. Missed in-process slots join that same queue.
export function createCronJob(
schedule: string,
callback: (date: Date) => unknown | Promise<unknown>,
options: { name: string; logger: SchedulerLogger; start?: boolean },
): CronJob {
const parsed = parseCronSchedule(schedule);
const tasks: ScheduledTask[] = [];
let cancelled = false;
let started = false;
const accepts = (index: number, date: Date) => {
if (tasks.length === 1) return true;
// Preserve cron-parser 4's day/weekday OR semantics, including its
// month-dependent full-day range and the global nth-week constraint.
if (parsed.nthDay && Math.ceil(date.getDate() / 7) !== parsed.nthDay)
return false;
if (index === 1) return !tasks[0].match(date);
const monthDays = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
return parsed.dayCount < monthDays[date.getMonth()] || tasks[1].match(date);
};
const dispatch = async (index: number, date: Date, missed: boolean) => {
if (cancelled || !started || !accepts(index, date)) return;
if (missed) {
options.logger.warn(
'[schedule][补执行迟到任务] 任务: %s, 计划时间: %s',
options.name,
date.toISOString(),
);
}
try {
await callback(date);
} catch (error) {
options.logger.error(
'[schedule][定时回调失败] 任务: %s, 错误: %s',
options.name,
error instanceof Error ? error.message : String(error),
);
}
};
try {
for (const [index, pattern] of parsed.patterns.entries()) {
const task = createTask(
pattern,
(context) => dispatch(index, context.date, false),
{
noOverlap: false,
missedExecutionTolerance: 5000,
},
);
tasks.push(task);
task.on('execution:missed', (context) =>
dispatch(index, context.date, true),
);
// Fail before an existing schedule is cancelled, including impossible dates.
task.getNextRuns(1);
}
} catch (error) {
tasks.forEach((task) => task.destroy());
throw error;
}
const job: CronJob = {
start() {
if (started || cancelled) return;
started = true;
tasks.forEach((task) => task.start());
},
cancel() {
if (cancelled) return;
cancelled = true;
tasks.forEach((task) => task.destroy());
},
};
if (options.start !== false) job.start();
return job;
}
-1
View File
@@ -3,7 +3,6 @@ import { shareStore } from './store';
const messages: Record<string, Record<string, string>> = {
zh: {},
en: {
'资源已存在,请检查重复的名称或值': 'Resource already exists; check for duplicate names or values',
'暂无权限': 'Access denied',
'参数错误': 'Invalid parameter',
'参数不正确': 'Invalid parameter',
-1
View File
@@ -18,7 +18,6 @@ export interface IDependencyFn<T> {
export interface ICronFn<T> {
(): Promise<T>;
cron?: TCron;
isCurrent?: () => boolean;
}
export interface ISchedule {
-11
View File
@@ -1,11 +0,0 @@
import { resolveFileAccess } from './fileAccess';
/** Check stored paths too: old database rows are not trusted input. */
export function resolveLogPath(root: string, value: string): string {
const target = resolveFileAccess(root, [value]);
if (!target)
throw Object.assign(new Error('Log path is outside the log directory'), {
status: 400,
});
return target;
}
-47
View File
@@ -1,47 +0,0 @@
const RETRY_TTL = 15 * 60 * 1000;
const MAX_SOURCES = 10000;
const MAX_RETRY_SECONDS = 5 * 60;
/** Source-specific, bounded backoff; persisted account history is not a lock. */
export class LoginThrottle {
private sources = new Map<
string,
{ failures: number; lastFailure: number }
>();
private get(source: string, now: number) {
const state = this.sources.get(source);
if (state && now - state.lastFailure >= RETRY_TTL) {
this.sources.delete(source);
return undefined;
}
return state;
}
retryAfter(source: string, now: number): number {
const state = this.get(source, now);
if (!state || state.failures < 3) return 0;
const seconds = Math.min(3 ** state.failures, MAX_RETRY_SECONDS);
return Math.max(
0,
Math.ceil((seconds * 1000 - (now - state.lastFailure)) / 1000),
);
}
fail(source: string, now: number): number {
const failures = Math.min((this.get(source, now)?.failures || 0) + 1, 6);
this.sources.delete(source);
if (this.sources.size >= MAX_SOURCES) {
this.sources.delete(this.sources.keys().next().value!);
}
this.sources.set(source, { failures, lastFailure: now });
return failures;
}
reset(source: string) {
this.sources.delete(source);
}
clear() {
this.sources.clear();
}
}
+9 -16
View File
@@ -50,9 +50,11 @@ class TaskLimit {
Buffer.from(tlsConfig.clientCert),
)
: credentials.createInsecure();
this._client = new ApiClient(`localhost:${config.grpcPort}`, creds, {
'grpc.enable_http_proxy': 0,
});
this._client = new ApiClient(
`localhost:${config.grpcPort}`,
creds,
{ 'grpc.enable_http_proxy': 0 },
);
}
return this._client;
}
@@ -111,15 +113,12 @@ class TaskLimit {
}
}
public removeQueuedCron(id: string, completed?: ICronFn<any>) {
public removeQueuedCron(id: string) {
if (this.queuedCrons.has(id)) {
const runs = this.queuedCrons.get(id);
if (runs && runs.length > 0) {
const remaining = completed
? runs.filter((run) => run !== completed)
: runs.slice(0, -1);
if (remaining.length) this.queuedCrons.set(id, remaining);
else this.queuedCrons.delete(id);
runs.pop();
this.queuedCrons.set(id, runs);
}
}
}
@@ -144,15 +143,9 @@ class TaskLimit {
cron: TCron,
fn: ICronFn<T>,
options?: Partial<QueueAddOptions>,
isCurrent: () => boolean = () => true,
): Promise<T | void> {
if (!isCurrent()) return;
fn.cron = cron;
fn.isCurrent = isCurrent;
// Invalidated snapshots must not consume the new revision's repeat budget.
const runs = this.queuedCrons
.get(cron.id)
?.filter((run) => run.isCurrent?.() !== false);
let runs = this.queuedCrons.get(cron.id);
const result = runs?.length ? [...runs, fn] : [fn];
const repeatTimes = this.repeatCronNotifyMap.get(cron.id) || 0;
if (result?.length > 5) {
+4 -15
View File
@@ -8,23 +8,15 @@ import { RunningInstanceModel, InstanceStatus } from '../data/runningInstance';
import dayjs from 'dayjs';
import { observeChildProcess, asError } from './childProcess';
export function runCron(
cmd: string,
cron: ICron,
isCurrent: () => boolean = () => true,
): Promise<number | void> {
const execute = async () => {
export function runCron(cmd: string, cron: ICron): Promise<number | void> {
return taskLimit.runWithCronLimit(cron, async () => {
try {
if (!isCurrent()) return;
// Check if the cron is already running and stop it (only if multiple instances are not allowed)
try {
const existingCron = await CrontabModel.findOne({
where: { id: Number(cron.id) },
});
// A mutation may commit while this queued run is reading the database.
if (!isCurrent() || !existingCron || existingCron.isDisabled) return;
// Default to single instance mode (0) for backward compatibility
const allowSingleInstances =
existingCron?.allow_multiple_instances === 0;
@@ -64,8 +56,6 @@ export function runCron(
throw error;
}
// In particular, invalidation can happen while stopping a previous run.
if (!isCurrent()) return;
Logger.info(
`[schedule][开始执行任务] 参数 ${JSON.stringify({
...cron,
@@ -104,8 +94,7 @@ export function runCron(
asError(error).message,
);
} finally {
taskLimit.removeQueuedCron(cron.id, execute);
taskLimit.removeQueuedCron(cron.id);
}
};
return taskLimit.runWithCronLimit(cron, execute, undefined, isCurrent);
});
}
-5
View File
@@ -1,5 +0,0 @@
/** Quote one literal POSIX shell argument, including quotes and substitutions. */
export function shellQuote(value: string): string {
if (value.includes('\0')) throw new Error('Invalid shell argument');
return "'" + value.replace(/'/g, "'\\''") + "'";
}
-30
View File
@@ -1,30 +0,0 @@
import lockfile from 'proper-lockfile';
import path from 'path';
import config from '../config';
// Protect subscription storage ownership from the first database read through
// cleanup. Creation and URL/branch updates must use the same cross-process lock.
export async function withSubscriptionMutation<T>(
operation: () => Promise<T>,
): Promise<T> {
const target = `${path.resolve(config.scriptPath)}.subscriptions`;
let release: () => Promise<void>;
try {
release = await lockfile.lock(target, {
realpath: false,
stale: 30000,
update: 10000,
retries: { retries: 50, factor: 1, minTimeout: 100, maxTimeout: 100 },
});
} catch (cause) {
throw Object.assign(
new Error('Subscription configuration is busy', { cause }),
{ status: 503 },
);
}
try {
return await operation();
} finally {
await release();
}
}
-45
View File
@@ -1,45 +0,0 @@
import { resolveFileAccess } from './fileAccess';
import { createHash } from 'crypto';
export function assertSubscriptionAlias(alias: string): void {
if (
typeof alias !== 'string' ||
alias.length > 200 ||
!/^[^\s\\\p{C}]+$/u.test(alias) ||
alias.startsWith('~') ||
alias.split('/').some((part) => !part || part === '.' || part === '..')
) {
throw Object.assign(new Error('Invalid subscription alias'), {
status: 400,
});
}
}
/** Keep ordinary SSH names and use a reserved namespace for path-like aliases.
* The database alias and its script/log directories remain unchanged. */
export function getSubscriptionSshAlias(alias: string): string {
assertSubscriptionAlias(alias);
if (/^[\p{L}\p{N}_.+-]+$/u.test(alias) && !alias.endsWith('.config')) {
return alias;
}
return `~sub_${createHash('sha256').update(alias).digest('hex')}`;
}
export function resolveSubscriptionPath(
root: string,
alias: string,
suffix = '',
): string {
// Internal SSH filenames cannot collide with user-supplied aliases.
if (!/^~sub_[a-f0-9]{64}$/.test(alias)) {
assertSubscriptionAlias(
alias.startsWith('~global_') ? alias.slice(8) : alias,
);
}
const target = resolveFileAccess(root, [alias + suffix]);
if (!target)
throw Object.assign(new Error('Invalid subscription path'), {
status: 400,
});
return target;
}
-106
View File
@@ -1,106 +0,0 @@
import type { Subscription } from '../data/subscription';
import { formatUrl } from '../config/subscription';
import path from 'path';
import fs from 'fs';
import { resolveFileAccess } from './fileAccess';
// Storage namespaces come from URLs and branches, not user aliases. In
// particular, a legal Git directory can exceed the alias's 200-character limit.
export function isSubscriptionStorageName(name: string): boolean {
return (
typeof name === 'string' &&
!!name &&
!path.isAbsolute(name) &&
!/[\0\r\n\\]/.test(name) &&
!name.split('/').some((part) => !part || part === '.' || part === '..')
);
}
export function resolveSubscriptionStoragePath(
root: string,
name: string,
): string | undefined {
const target = isSubscriptionStorageName(name)
? resolveFileAccess(root, [name])
: '';
if (!target) {
if (isSubscriptionStorageName(name)) {
try {
fs.lstatSync(path.resolve(root, name));
} catch (error: any) {
// A long query can make the Shell candidate unrepresentable while
// the CLI's pathname-based file exists. Use the actual filesystem
// limit: macOS permits Unicode names exceeding 255 UTF-8 bytes.
if (error.code === 'ENAMETOOLONG') return undefined;
}
}
throw Object.assign(new Error('Invalid subscription path'), {
status: 400,
});
}
return target;
}
/** Match shell/update.sh get_uniq_path and update_raw, including SCP URLs. */
export function getSubscriptionStorageNames(
doc: Subscription,
engine: 'shell' | 'cli' = 'shell',
): {
script: string;
repo?: string;
raw?: string;
} {
// Old imported rows may only have an alias and no URL.
if (!doc.url) return { script: doc.alias, repo: doc.alias };
const url = formatUrl(doc).url!;
const trimmed = url.endsWith('/') ? url.slice(0, -1) : url;
const basename = trimmed.slice(trimmed.lastIndexOf('/') + 1);
const dot = basename.lastIndexOf('.');
const repo = dot < 0 ? basename : basename.slice(0, dot);
const parentUrl = engine === 'cli' ? trimmed : url;
const slash = parentUrl.lastIndexOf('/');
const parent =
engine === 'shell' && slash < 0 ? url : parentUrl.slice(0, slash);
const owner = parent.slice(parent.lastIndexOf('/') + 1);
const author = owner.slice(owner.lastIndexOf(':') + 1);
const stem = `${author.slice(author.lastIndexOf('.') + 1)}_${repo}`;
if (doc.type === 'file') {
const suffix =
engine === 'cli'
? path.extname(new URL(url).pathname)
: `.${url.slice(url.lastIndexOf('.') + 1)}`;
const raw = `${stem}${suffix}`;
return { script: `raw_${raw}`, raw };
}
const directory = doc.branch ? `${stem}_${doc.branch}` : stem;
return { script: directory, repo: directory };
}
// A subscription may have run through either engine before being deleted.
export function getSubscriptionStorageCandidates(doc: Subscription) {
const candidates: ReturnType<typeof getSubscriptionStorageNames>[] = [];
for (const engine of ['shell', 'cli'] as const) {
try {
candidates.push(getSubscriptionStorageNames(doc, engine));
} catch {
// Historical rows may have an invalid SSH alias or missing credentials.
// Retain their URL-derived storage without needing an executable SSH URL.
try {
candidates.push(
getSubscriptionStorageNames(
{
url: doc.url,
alias: doc.alias,
branch: doc.branch,
type: doc.type === 'private-repo' ? 'public-repo' : doc.type,
},
engine,
),
);
} catch {
// One engine's invalid URL must not hide the other engine's candidate.
}
}
}
return candidates;
}
+5 -8
View File
@@ -21,7 +21,7 @@ export async function writeFileWithLock(
}
if (!(await fileExist(filePath))) {
const fileHandle = await open(filePath, 'w');
await fileHandle.close();
fileHandle.close();
}
const lockfilePath = getUniqueLockPath(filePath);
@@ -34,12 +34,9 @@ export async function writeFileWithLock(
},
lockfilePath,
});
try {
await writeFile(filePath, content, { encoding: 'utf8', ...options });
if (options?.mode) {
await chmod(filePath, options.mode);
}
} finally {
await release();
await writeFile(filePath, content, { encoding: 'utf8', ...options });
if (options?.mode) {
await chmod(filePath, options.mode);
}
await release();
}
+16 -5
View File
@@ -2,19 +2,30 @@ import 'reflect-metadata';
import OpenService from './services/open';
import { Container } from 'typedi';
import LoggerInstance from './loaders/logger';
import config from './config';
import path from 'path';
import os from 'os';
import { writeFileWithLock } from './shared/utils';
const tokenFile = path.join(config.configPath, 'token.json');
async function getToken() {
try {
Container.set('logger', LoggerInstance);
const openService = Container.get(OpenService);
const appToken = await openService.generateSystemToken(
process.argv.includes('--renew'),
);
const appToken = await openService.generateSystemToken();
console.log(appToken.value);
await writeFile({
value: appToken.value,
expiration: appToken.expiration,
});
} catch (error) {
console.error(error);
process.exitCode = 1;
console.log(error);
}
}
async function writeFile(data: any) {
await writeFileWithLock(tokenFile, `${JSON.stringify(data)}${os.EOL}`);
}
getToken();
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "@whyour/qinglong-cli",
"version": "0.1.2",
"version": "0.1.1",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@whyour/qinglong-cli",
"version": "0.1.2",
"version": "0.1.1",
"bin": {
"ql": "dist/npm/ql.js"
},
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@whyour/qinglong-cli",
"version": "0.1.2",
"version": "0.1.1",
"description": "Authenticated remote management CLI for QingLong 2.x",
"type": "commonjs",
"bin": {
+1 -7
View File
@@ -13,10 +13,6 @@ execFileSync(
],
{ stdio: 'inherit' },
);
fs.copyFileSync(
path.join(root, 'src/internal/maintenance/upgradeArtifacts.cjs'),
path.join(root, 'dist/internal/maintenance/upgradeArtifacts.cjs'),
);
require('esbuild').buildSync({
entryPoints: [path.join(root, 'src/shared/cli/commander.ts')],
outfile: path.join(root, 'dist/shared/cli/commander.js'),
@@ -49,9 +45,7 @@ const remote = require('esbuild').buildSync({
metafile: true,
});
for (const input of Object.keys(remote.metafile.inputs)) {
if (
/(?:^|[\\/])src[\\/](?:internal|compatibility|developer)[\\/]/.test(input)
)
if (/(?:^|[\\/])src[\\/](?:internal|compatibility|developer)[\\/]/.test(input))
throw new Error(
'Local implementation leaked into remote npm bundle: ' + input,
);
+2 -18
View File
@@ -70,26 +70,10 @@ if [ -n "$__ql_timeout" ]; then
fi
if [ "$__ql_command" = true ]; then
"\${__ql_args[@]}" "\${__ql_script_args[@]}"
_task_exit_code=$?
else
# Match task.sh: keep script exit/exec/options inside a child shell while
# retaining before-hook variables and functions for the script.
# Keep the supervisor alive until the child's signal/EXIT traps finish, but
# do not turn user cancellation into an ordinary completion with after hooks.
__ql_saved_signals=$(trap -p INT TERM HUP QUIT ALRM TSTP)
__ql_cancel_code=''
trap '__ql_cancel_code=$((128 + $(kill -l INT)))' INT
trap '__ql_cancel_code=$((128 + $(kill -l TERM)))' TERM
trap '__ql_cancel_code=$((128 + $(kill -l HUP)))' HUP
trap '__ql_cancel_code=$((128 + $(kill -l QUIT)))' QUIT
trap '__ql_cancel_code=$((128 + $(kill -l ALRM)))' ALRM
trap '__ql_cancel_code=$((128 + $(kill -l TSTP)))' TSTP
( . "\${__ql_args[0]}" "\${__ql_script_args[@]}" )
_task_exit_code=$?
trap - INT TERM HUP QUIT ALRM TSTP
eval "$__ql_saved_signals"
if [ -n "$__ql_cancel_code" ] && ! [ -f "$__ql_timeout" ]; then exit "$__ql_cancel_code"; fi
. "\${__ql_args[0]}" "\${__ql_script_args[@]}"
fi
_task_exit_code=$?
if [ -n "$__ql_timeout" ] && [ -f "$__ql_timeout" ]; then _task_exit_code=124; fi
__ql_run_after
exit "$_task_exit_code"
+129 -51
View File
@@ -8,33 +8,12 @@ import { checkedProcess } from '../runtime/process';
import { operationSignal, withoutCancellation } from '../runtime/cancellation';
import { installPanelDependencies, startPanel, stopPanel } from './operator';
// The Bash image updater and local CLI must enforce the same build provenance.
const artifacts = require('./upgradeArtifacts.cjs') as {
releaseBranch(env: NodeJS.ProcessEnv): string;
verifyUpgrade(source: string, staticRoot: string): Promise<unknown>;
waitForHealth(
env: NodeJS.ProcessEnv,
timeout?: number,
signal?: AbortSignal,
): Promise<void>;
withUpgradeLock<T>(tmp: string, operation: () => Promise<T>): Promise<T>;
stageUpgrade(options: {
root: string;
tmp: string;
env: NodeJS.ProcessEnv;
mirror: 'github' | 'gitee';
signal?: AbortSignal;
run(program: string, args: string[], capture?: boolean): Promise<unknown>;
install(source: string): Promise<void>;
}): Promise<{ source: string; static: string }>;
selectedUpgrade(
tmp: string,
branch: string,
): Promise<{ source: string; static: string }>;
};
export function releaseBranch(context: LocalContext): string {
return artifacts.releaseBranch(context.env);
return ['develop', 'debian', 'debian-dev'].includes(
context.env.QL_BRANCH ?? '',
)
? context.env.QL_BRANCH!
: 'master';
}
async function validateTree(
@@ -69,27 +48,134 @@ export async function stageUpgrade(
context: LocalContext,
mirror: 'github' | 'gitee',
): Promise<{ source: string; static: string }> {
return artifacts.withUpgradeLock(context.paths.dir_tmp!, () =>
artifacts.stageUpgrade({
root: context.root,
tmp: context.paths.dir_tmp!,
env: context.env,
mirror,
signal: operationSignal(),
run: (program, args, capture) =>
checkedProcess(program, args, { env: context.env, capture }),
install: (source) => installPanelDependencies(context, source),
}),
await fs.mkdir(context.paths.dir_tmp!, { recursive: true });
const workspace = await fs.mkdtemp(
path.join(context.paths.dir_tmp!, 'upgrade-'),
);
const branch = releaseBranch(context);
try {
for (const repository of ['qinglong', 'qinglong-static']) {
const url =
mirror === 'github'
? `https://github.com/whyour/${repository}/archive/refs/heads/${branch}.zip`
: `https://gitee.com/whyour/${repository}/repository/archive/${branch}.zip`;
const archive = path.join(workspace, `${repository}.zip`);
await checkedProcess(
'curl',
[
'--fail',
'--location',
'--proto',
'=https',
'--proto-redir',
'=https',
'--output',
archive,
url,
],
{ env: context.env },
);
const listing = await checkedProcess('unzip', ['-Z1', archive], {
env: context.env,
capture: true,
});
const expected = `${repository}-${branch}`;
for (const name of listing.stdout.split('\n').filter(Boolean)) {
if (
name.includes('\\') ||
name.includes('\r') ||
name.startsWith('/') ||
name.split('/').includes('..') ||
name.split('/')[0] !== expected
)
throw new Error(translate(context.env, '升级归档包含无效路径。'));
}
// Reject Unix symlinks before extraction, including links that could redirect
// a later archive entry outside the staging directory.
const modes = await checkedProcess('unzip', ['-Z', '-l', archive], {
env: context.env,
capture: true,
});
if (modes.stdout.split('\n').some((line) => /^l[rwx-]{9}\s/.test(line)))
throw new Error(translate(context.env, '升级归档包含符号链接。'));
await checkedProcess('unzip', ['-oq', archive, '-d', workspace], {
env: context.env,
});
await validateTree(path.join(workspace, expected), context.env);
}
const source = path.join(workspace, `qinglong-${branch}`);
const staticRoot = path.join(workspace, `qinglong-static-${branch}`);
await fs.access(path.join(staticRoot, 'build/app.js'));
const manifest = await fs.readFile(path.join(source, 'package.json'));
const existing = await fs.readFile(path.join(context.root, 'package.json'));
if (!manifest.equals(existing))
await installPanelDependencies(context, source);
// Publish a marker only after both archives and dependency installation succeed.
await fs.writeFile(
path.join(workspace, 'ready.json'),
JSON.stringify({ source, static: staticRoot }),
{ flag: 'wx', mode: 0o600 },
);
const pointer = path.join(workspace, 'pointer.json');
await fs.writeFile(
pointer,
JSON.stringify({ directory: path.basename(workspace) }),
{
flag: 'wx',
mode: 0o600,
},
);
await fs.rename(
pointer,
path.join(context.paths.dir_tmp!, `upgrade-ready-${branch}.json`),
);
return { source, static: staticRoot };
} catch (error) {
await fs.rm(workspace, { recursive: true, force: true });
throw error;
}
}
async function selectedUpgrade(
context: LocalContext,
): Promise<{ source: string; static: string }> {
return artifacts.selectedUpgrade(
context.paths.dir_tmp!,
releaseBranch(context),
const branch = releaseBranch(context);
const fallback = {
source: path.join(context.paths.dir_tmp!, `qinglong-${branch}`),
static: path.join(context.paths.dir_tmp!, `qinglong-static-${branch}`),
};
const pointer = await fs
.readFile(
path.join(context.paths.dir_tmp!, `upgrade-ready-${branch}.json`),
'utf8',
)
.catch((error: NodeJS.ErrnoException) => {
if (error.code === 'ENOENT') return undefined;
throw error;
});
if (pointer === undefined) return fallback;
const directory: unknown = JSON.parse(pointer)?.directory;
if (
typeof directory !== 'string' ||
!/^upgrade-[A-Za-z0-9_-]+$/.test(directory)
)
throw new Error(translate(context.env, '暂存升级指针无效。'));
const workspace = path.join(context.paths.dir_tmp!, directory);
const selected = {
source: path.join(workspace, `qinglong-${branch}`),
static: path.join(workspace, `qinglong-static-${branch}`),
};
for (const location of [workspace, selected.source, selected.static]) {
const stat = await fs.lstat(location);
if (!stat.isDirectory() || stat.isSymbolicLink())
throw new Error(translate(context.env, '暂存升级目录无效。'));
}
const ready = JSON.parse(
await fs.readFile(path.join(workspace, 'ready.json'), 'utf8'),
);
if (ready?.source !== selected.source || ready?.static !== selected.static)
throw new Error(translate(context.env, '暂存升级就绪记录与目录不匹配。'));
return selected;
}
export interface ReloadLifecycle {
@@ -259,7 +345,8 @@ export async function reloadPanel(
const staticRoot = selected.static;
await validateTree(source, context.env);
await validateTree(staticRoot, context.env);
await artifacts.verifyUpgrade(source, staticRoot);
await fs.access(path.join(source, 'package.json'));
await fs.access(path.join(staticRoot, 'build/app.js'));
const entries = await fs.readdir(source);
const reserved = new Set(['data', '.tmp', 'static', '.git', '.env']);
const relativeData = path.relative(context.root, context.data);
@@ -284,16 +371,7 @@ export async function reloadPanel(
if (selection)
replacements.push({ source: selection.source, target: selection.target });
try {
return await artifacts.withUpgradeLock(context.paths.dir_tmp!, () =>
replaceAndReload(context, replacements, {
stop: stopPanel,
start: async (ctx) => {
const service = await startPanel(ctx);
await artifacts.waitForHealth(ctx.env, 60000, operationSignal());
return service;
},
}),
);
return await replaceAndReload(context, replacements);
} finally {
if (selection)
await fs.rm(selection.directory, { recursive: true, force: true });
@@ -1,657 +0,0 @@
// Shared by the image's Bash updater and the optional local CLI. Use only
// installed code to validate downloads; never execute a downloaded verifier.
const fs = require('node:fs/promises');
const path = require('node:path');
const { createHash, randomUUID } = require('node:crypto');
const { spawn } = require('node:child_process');
const http = require('node:http');
const { setTimeout: delay } = require('node:timers/promises');
const hash = (bytes) => createHash('sha256').update(bytes).digest('hex');
const releaseBranch = (env) =>
['develop', 'debian-dev'].includes(env.QL_BRANCH) ? 'develop' : 'master';
async function manifestAt(staticRoot) {
const manifest = JSON.parse(
await fs.readFile(path.join(staticRoot, 'build-info.json'), 'utf8'),
);
if (
manifest.version !== 1 ||
manifest.dirty !== false ||
!/^[a-f0-9]{40}$/.test(manifest.sourceCommit) ||
!/^[a-f0-9]{64}$/.test(manifest.lockfileSha256) ||
!manifest.files ||
typeof manifest.files !== 'object' ||
Array.isArray(manifest.files)
)
throw new Error('Invalid or dirty build manifest / 构建清单无效。');
for (const required of ['build/app.js', 'dist/index.html'])
if (!Object.hasOwn(manifest.files, required))
throw new Error(`Missing build output / 缺少构建产物: ${required}`);
return manifest;
}
async function verifyUpgrade(source, staticRoot) {
const manifest = await manifestAt(staticRoot);
if (
hash(await fs.readFile(path.join(source, 'pnpm-lock.yaml'))) !==
manifest.lockfileSha256
)
throw new Error(
'Source and build lockfiles differ / 源码与构建依赖不一致。',
);
await fs.access(path.join(source, 'package.json'));
await fs.access(path.join(source, 'sample/config.sample.sh'));
const files = {};
async function visit(directory, prefix = '') {
for (const entry of await fs.readdir(directory, { withFileTypes: true })) {
const name = prefix + entry.name;
if (name === 'build-info.json') continue;
const filename = path.join(directory, entry.name);
if (entry.isDirectory()) await visit(filename, `${name}/`);
else if (entry.isFile()) files[name] = hash(await fs.readFile(filename));
else
throw new Error(`Unsupported build entry / 构建包含非法文件: ${name}`);
}
}
await visit(staticRoot);
const actual = Object.keys(files).sort(),
expected = Object.keys(manifest.files).sort();
if (
JSON.stringify(actual) !== JSON.stringify(expected) ||
actual.some((name) => files[name] !== manifest.files[name])
)
throw new Error(
'Build checksum mismatch / 构建文件缺失、损坏或混入旧文件。',
);
return manifest;
}
async function withUpgradeLock(tmp, operation) {
await fs.mkdir(tmp, { recursive: true });
const lock = path.join(tmp, 'upgrade.lock');
const token = randomUUID();
const owner = `owner-${process.pid}-${token}`;
const candidate = path.join(tmp, `upgrade-lock-${token}`);
let acquired = false;
await fs.mkdir(candidate, { mode: 0o700 });
try {
// Publish a nonempty directory atomically: a live lock can never be
// replaced, and a crash before publication cannot leave an ownerless lock.
await fs.writeFile(path.join(candidate, owner), '', {
flag: 'wx',
mode: 0o600,
});
for (;;) {
try {
await fs.rename(candidate, lock);
acquired = true;
break;
} catch (error) {
if (!['ENOTEMPTY', 'EEXIST'].includes(error.code)) throw error;
}
let entries;
try {
entries = await fs.readdir(lock);
} catch (error) {
if (error.code === 'ENOENT') continue;
throw error;
}
if (!entries.length) continue;
const heldOwner = entries[0];
const match = /^owner-([1-9]\d*)-[a-f0-9-]{36}$/.exec(heldOwner);
let pid = match ? Number(match[1]) : 0;
if (entries.length === 1 && heldOwner === 'pid') {
try {
pid = Number(await fs.readFile(path.join(lock, heldOwner), 'utf8'));
} catch (error) {
if (error.code === 'ENOENT') continue;
throw error;
}
}
let stale = false;
if (
entries.length === 1 &&
Number.isSafeInteger(pid) &&
pid > 0 &&
pid <= 0x7fffffff
) {
try {
process.kill(pid, 0);
} catch (error) {
if (error.code === 'ESRCH') stale = true;
}
}
if (!stale)
throw new Error(
'Another upgrade is running / 已有更新或重载正在执行。',
);
// Removing the observed unique marker is a compare-and-delete. A
// contender cannot unlink the marker belonging to a replacement lock.
// Legacy pid files are supported; new owners never reuse that filename.
try {
await fs.unlink(path.join(lock, heldOwner));
} catch (error) {
if (error.code !== 'ENOENT') throw error;
}
}
return await operation();
} finally {
if (acquired) {
await fs.unlink(path.join(lock, owner)).catch((error) => {
if (error.code !== 'ENOENT') throw error;
});
await fs.rmdir(lock).catch((error) => {
if (!['ENOENT', 'ENOTEMPTY', 'EEXIST'].includes(error.code))
throw error;
});
}
await fs.rm(candidate, { recursive: true, force: true });
}
}
async function stageUpgrade({ root, tmp, env, mirror, run, install, signal }) {
const workspace = await fs.mkdtemp(path.join(tmp, 'upgrade-'));
const branch = releaseBranch(env);
try {
async function extract(repository, ref) {
const archive = path.join(workspace, `${repository}.zip`);
const url =
mirror === 'github'
? `https://github.com/whyour/${repository}/archive/${ref}.zip`
: `https://gitee.com/whyour/${repository}/repository/archive/${ref}.zip`;
await run('curl', [
'--fail',
'--location',
'--proto',
'=https',
'--proto-redir',
'=https',
'--connect-timeout',
'30',
'--max-time',
'600',
'--output',
archive,
url,
]);
const listing = await run('unzip', ['-Z1', archive], true);
const expected = `${repository}-${ref}`;
const names = listing.stdout.split('\n').filter(Boolean);
if (
!names.length ||
names.some(
(name) =>
name.includes('\\') ||
name.includes('\r') ||
name.startsWith('/') ||
name.split('/').includes('..') ||
name.split('/')[0] !== expected,
)
)
throw new Error('Invalid archive paths / 升级归档路径无效。');
const modes = await run('unzip', ['-Z', '-l', archive], true);
if (modes.stdout.split('\n').some((line) => /^l[rwx-]{9}\s/.test(line)))
throw new Error('Archive contains symlinks / 升级归档包含符号链接。');
await run('unzip', ['-oq', archive, '-d', workspace]);
return path.join(workspace, expected);
}
// Static branches are mutable snapshots. Their manifest pins the exact
// source revision, avoiding a race between two independent branch reads.
const staticRoot = await extract('qinglong-static', branch);
const manifest = await manifestAt(staticRoot);
const downloadedSource = await extract('qinglong', manifest.sourceCommit);
const source = path.join(workspace, `qinglong-${branch}`);
await fs.rename(downloadedSource, source);
await verifyUpgrade(source, staticRoot);
let dependenciesChanged = false;
for (const name of ['package.json', 'pnpm-lock.yaml']) {
const current = await fs
.readFile(path.join(root, name))
.catch((error) => {
if (error.code !== 'ENOENT') throw error;
return Buffer.alloc(0);
});
if (!(await fs.readFile(path.join(source, name))).equals(current))
dependenciesChanged = true;
}
if (
!(await fs.stat(path.join(root, 'node_modules')).catch(() => undefined))
)
dependenciesChanged = true;
if (dependenciesChanged) {
await install(source);
await fs.access(path.join(source, 'node_modules'));
await verifyUpgrade(source, staticRoot);
}
signal?.throwIfAborted();
const selected = { source, static: staticRoot };
await fs.writeFile(
path.join(workspace, 'ready.json'),
JSON.stringify({
...selected,
sourceCommit: manifest.sourceCommit,
dependenciesChanged,
}),
{ flag: 'wx', mode: 0o600 },
);
const pointer = path.join(workspace, 'pointer.json');
await fs.writeFile(
pointer,
JSON.stringify({ directory: path.basename(workspace) }),
{ flag: 'wx', mode: 0o600 },
);
const readyPointer = path.join(tmp, `upgrade-ready-${branch}.json`);
const previous = await fs
.readFile(readyPointer, 'utf8')
.catch(() => undefined);
signal?.throwIfAborted();
await fs.rename(pointer, readyPointer);
// Retire only our previous successful stage, after publishing its replacement.
try {
const directory = previous && JSON.parse(previous).directory;
if (
typeof directory === 'string' &&
/^upgrade-[A-Za-z0-9_-]+$/.test(directory) &&
directory !== path.basename(workspace)
)
await fs.rm(path.join(tmp, directory), {
recursive: true,
force: true,
});
} catch {
/* Cleanup cannot invalidate the newly published stage. */
}
return selected;
} catch (error) {
await fs.rm(workspace, { recursive: true, force: true });
throw error;
}
}
async function selectedUpgrade(tmp, branch) {
const pointer = JSON.parse(
await fs.readFile(path.join(tmp, `upgrade-ready-${branch}.json`), 'utf8'),
);
if (
typeof pointer.directory !== 'string' ||
!/^upgrade-[A-Za-z0-9_-]+$/.test(pointer.directory)
)
throw new Error('Invalid staged upgrade pointer / 暂存升级指针无效。');
const workspace = path.join(tmp, pointer.directory);
const selected = {
source: path.join(workspace, `qinglong-${branch}`),
static: path.join(workspace, `qinglong-static-${branch}`),
};
for (const directory of [workspace, selected.source, selected.static]) {
const stat = await fs.lstat(directory);
if (!stat.isDirectory() || stat.isSymbolicLink())
throw new Error('Invalid staging directory / 暂存目录无效。');
}
const ready = JSON.parse(
await fs.readFile(path.join(workspace, 'ready.json'), 'utf8'),
);
if (ready.source !== selected.source || ready.static !== selected.static)
throw new Error('Invalid upgrade readiness / 暂存升级就绪记录不匹配。');
const manifest = await verifyUpgrade(selected.source, selected.static);
if (ready.sourceCommit !== manifest.sourceCommit)
throw new Error('Invalid upgrade readiness / 暂存升级就绪记录不匹配。');
if (ready.dependenciesChanged)
await fs.access(path.join(selected.source, 'node_modules'));
return selected;
}
async function replaceAndReload(replacements, lifecycle) {
const changed = [];
let attemptedStart = false;
lifecycle.signal?.throwIfAborted();
try {
await lifecycle.stop();
for (const { source, target } of replacements) {
lifecycle.signal?.throwIfAborted();
await fs.mkdir(path.dirname(target), { recursive: true });
const backup = `${target}.ql-backup-${randomUUID()}`;
const existed = !!(await fs.lstat(target).catch((error) => {
if (error.code !== 'ENOENT') throw error;
return undefined;
}));
if (existed) {
try {
await fs.rename(target, backup);
} catch (error) {
if (error.code !== 'EXDEV') throw error;
try {
await fs.cp(target, backup, {
recursive: true,
verbatimSymlinks: true,
});
} catch (copyError) {
await fs.rm(backup, { recursive: true, force: true });
throw copyError;
}
changed.push({ target, backup, existed });
await fs.rm(target, { recursive: true });
}
}
if (changed.at(-1)?.target !== target)
changed.push({ target, backup, existed });
await fs.cp(source, target, {
recursive: true,
verbatimSymlinks: true,
force: false,
errorOnExist: true,
});
}
lifecycle.signal?.throwIfAborted();
attemptedStart = true;
await lifecycle.start();
lifecycle.signal?.throwIfAborted();
} catch (error) {
// Do not modify files under a still-running failed startup. Keep backups
// for recovery if stopping the new process itself fails.
if (attemptedStart) await lifecycle.stop();
for (const item of changed.reverse()) {
await fs.rm(item.target, { recursive: true, force: true });
if (item.existed) {
try {
await fs.rename(item.backup, item.target);
} catch (restoreError) {
if (restoreError.code !== 'EXDEV') throw restoreError;
await fs.cp(item.backup, item.target, {
recursive: true,
verbatimSymlinks: true,
});
await fs.rm(item.backup, { recursive: true });
}
}
}
await lifecycle.start(true);
throw error;
}
for (const item of changed)
if (item.existed)
await fs
.rm(item.backup, { recursive: true, force: true })
.catch(() =>
console.error(`Retained backup / 保留备份: ${item.backup}`),
);
}
async function reloadSystem({
root,
data,
tmp,
staticRoot,
config,
env,
lifecycle,
}) {
const selected = await selectedUpgrade(tmp, releaseBranch(env));
const reserved = new Set(['data', '.tmp', 'static', '.git', '.env']);
const relative = path.relative(root, data);
if (relative && !relative.startsWith('..') && !path.isAbsolute(relative))
reserved.add(relative.split(path.sep)[0]);
const replacements = (await fs.readdir(selected.source))
.filter((name) => !reserved.has(name))
.map((name) => ({
source: path.join(selected.source, name),
target: path.join(root, name),
}));
replacements.push(
{ source: selected.static, target: staticRoot },
{
source: path.join(selected.source, 'sample/config.sample.sh'),
target: path.join(config, 'config.sample.sh'),
},
);
await replaceAndReload(replacements, lifecycle);
// Cleanup after a committed installation must not report a failed update.
await fs
.rm(path.join(tmp, `upgrade-ready-${releaseBranch(env)}.json`), {
force: true,
})
.catch(() => {});
await fs
.rm(path.dirname(selected.source), { recursive: true, force: true })
.catch(() => {});
}
async function waitForHealth(env, timeout = 60000, signal) {
const port = Number(env.QlPort || 5700);
let base = env.QlBaseUrl || '';
if (base && !base.startsWith('/')) base = `/${base}`;
if (base.endsWith('/')) base = base.slice(0, -1);
const deadline = Date.now() + timeout;
while (Date.now() < deadline) {
signal?.throwIfAborted();
const healthy = await new Promise((resolve) => {
let timer;
const finish = (healthy) => {
clearTimeout(timer);
resolve(healthy);
};
const req = http.get(
{ hostname: '127.0.0.1', port, path: `${base}/api/health`, signal },
(res) => {
let body = '';
res.on('data', (chunk) => {
body += chunk;
if (body.length > 65536) req.destroy();
});
res.on('end', () => {
try {
const result = JSON.parse(body);
finish(
res.statusCode === 200 &&
result.code === 200 &&
result.data?.status === 'ok',
);
} catch {
finish(false);
}
});
res.on('error', () => finish(false));
},
);
// Socket inactivity timeouts do not bound a response that keeps streaming.
timer = setTimeout(
() => req.destroy(new Error('Health probe timed out')),
Math.min(1000, Math.max(1, deadline - Date.now())),
);
req.on('error', () => finish(false));
});
signal?.throwIfAborted();
if (healthy) return;
const remaining = deadline - Date.now();
if (remaining > 0) {
try {
await delay(Math.min(500, remaining), undefined, { signal });
} catch (error) {
signal?.throwIfAborted();
throw error;
}
}
}
throw new Error('Backend failed health check / 后端启动检查失败。');
}
function runProcess(env, program, args, capture = false, signal) {
if (signal?.aborted) return Promise.reject(signal.reason);
return new Promise((resolve, reject) => {
const child = spawn(program, args, {
env,
detached: process.platform !== 'win32',
// Pipes keep close pending until children have released their output.
stdio: ['ignore', 'pipe', 'pipe'],
});
let stdout = '';
let aborted = false;
let failure;
let escalation;
const kill = (name) => {
if (!child.pid) return;
try {
process.kill(
process.platform === 'win32' ? child.pid : -child.pid,
name,
);
} catch (error) {
if (error.code === 'EPERM') {
child.kill(name);
return;
}
if (error.code !== 'ESRCH') throw error;
}
};
const abort = () => {
if (aborted) return;
aborted = true;
kill('SIGTERM');
escalation = setTimeout(() => kill('SIGKILL'), 1000);
escalation.unref();
};
child.stdout.on('data', (chunk) => {
if (capture) stdout += chunk;
else process.stdout.write(chunk);
});
child.stderr.on('data', (chunk) => process.stderr.write(chunk));
child.once('error', (error) => {
failure = error;
});
child.once('exit', () => {
// The shell can exit before a descendant that ignores SIGTERM.
if (aborted) kill('SIGKILL');
});
child.once('close', (code) => {
if (aborted) kill('SIGKILL');
clearTimeout(escalation);
signal?.removeEventListener('abort', abort);
if (aborted) reject(signal.reason);
else if (failure) reject(failure);
else if (code === 0) resolve({ stdout });
else reject(new Error(`${program} failed (exit ${code})`));
});
signal?.addEventListener('abort', abort, { once: true });
if (signal?.aborted) abort();
});
}
async function withLifecycleHooks(root, tmp, env, operation) {
const helpers = await fs.readFile(path.join(root, 'shell/share.sh'), 'utf8');
const environment = await fs.readFile(
path.join(root, 'shell/env.sh'),
'utf8',
);
const directory = await fs.mkdtemp(path.join(tmp, 'upgrade-hooks-'));
try {
// Keep the installed functions available through replacement and rollback.
// Their pkill patterns must never become part of the bash command line.
const snapshot = path.join(directory, 'lifecycle.sh');
await fs.writeFile(
snapshot,
`${helpers}\n${environment}\nimport_config
case "$1" in
stop) delete_pm2 ;;
start) reload_pm2 ;;
install) npm_install_2 "$2"; exit "$exit_status" ;;
*) printf '%s\\n' 'Unknown lifecycle operation' >&2; exit 2 ;;
esac
`,
{
flag: 'wx',
mode: 0o600,
},
);
const hook = (operation, args = [], signal) =>
runProcess(
env,
'bash',
['--', snapshot, operation, ...args],
false,
signal,
);
return await operation(hook);
} finally {
await fs
.rm(directory, { recursive: true, force: true })
.catch(() =>
console.error(
`Retained lifecycle snapshot / 保留生命周期快照: ${directory}`,
),
);
}
}
async function main() {
const env = process.env,
root = env.QL_DIR;
if (!root || !path.isAbsolute(root))
throw new Error('QL_DIR must be absolute.');
const tmp = env.dir_tmp || path.join(root, '.tmp');
const controller = new AbortController();
for (const name of ['SIGINT', 'SIGTERM'])
process.on(name, () =>
controller.abort(new Error(`Upgrade interrupted (${name})`)),
);
const run = (program, args, capture) =>
runProcess(env, program, args, capture, controller.signal);
await withUpgradeLock(tmp, () =>
withLifecycleHooks(root, tmp, env, async (hook) => {
const lifecycle = {
signal: controller.signal,
stop: () => hook('stop'),
start: async (recover) => {
await hook('start', [], recover ? undefined : controller.signal);
await waitForHealth(
env,
60000,
recover ? undefined : controller.signal,
);
},
};
if (process.argv[2] === 'update') {
await stageUpgrade({
root,
tmp,
env,
mirror: process.argv[3],
run,
signal: controller.signal,
install: (source) => hook('install', [source], controller.signal),
});
console.log('更新包下载及校验成功...');
if (process.argv[4] !== 'true') return;
} else if (process.argv[2] !== 'reload')
throw new Error('Unknown upgrade command.');
await reloadSystem({
root,
tmp,
env,
data: env.dir_data || path.join(root, 'data'),
staticRoot: env.dir_static || path.join(root, 'static'),
config: env.dir_config || path.join(root, 'data/config'),
lifecycle,
});
console.log('更新及启动检查成功...');
}),
);
}
module.exports = {
releaseBranch,
verifyUpgrade,
withUpgradeLock,
stageUpgrade,
selectedUpgrade,
replaceAndReload,
reloadSystem,
waitForHealth,
runProcess,
withLifecycleHooks,
main,
};
if (require.main === module)
main().catch((error) => {
console.error(`更新失败: ${error.message}`);
process.exitCode = 1;
});
+28 -52
View File
@@ -53,12 +53,23 @@ export async function request(
// Check HTTP status before decoding (proxies may return HTML on denial).
if (!response.ok) {
await response.body?.cancel();
rejectResponse(
response.status,
'HTTP',
method,
resource,
credentialsHint,
const suffix =
method !== 'GET' && response.status >= 500
? translate(
process.env,
' 执行结果可能未知,请先检查%s状态再考虑重试。',
resource,
)
: '';
fail(
translate(
process.env,
'API 请求被拒绝(HTTP %s),请检查%s。%s',
response.status,
credentialsHint,
suffix,
),
[401, 403].includes(response.status) ? 3 : 1,
);
}
if (
@@ -89,63 +100,28 @@ export async function request(
),
);
}
if (
endpoint.split('?')[0] === 'user/login' &&
isRecord(result) &&
result.code === 420
)
fail(
translate(process.env, '需要双因素验证,请调用 user two-factor-login。'),
3,
);
if (endpoint.split('?')[0] === 'user/login' && isRecord(result) && result.code === 420)
fail(translate(process.env, '需要双因素验证,请调用 user two-factor-login。'), 3);
if (!isRecord(result) || result.code !== 200) {
const status =
isRecord(result) && typeof result.code === 'number'
? result.code
: undefined;
rejectResponse(status, 'code', method, resource, credentialsHint);
fail(
translate(
process.env,
'API 请求被拒绝(code %s),请检查%s。',
status ?? translate(process.env, '未知'),
credentialsHint,
),
status === 401 || status === 403 ? 3 : 1,
);
}
if (options.output)
fail(translate(process.env, '接口返回 JSON,未保存下载文件。'));
return result;
}
function rejectResponse(
status: number | undefined,
kind: 'HTTP' | 'code',
method: string,
resource: string,
credentialsHint: string,
): never {
const denied = status === 401 || status === 403;
const reason = denied
? translate(process.env, '请检查%s。', credentialsHint)
: status === 409
? translate(process.env, '资源冲突,请检查重复的名称或值。')
: status !== undefined && status >= 500
? translate(process.env, '服务端错误,请检查服务状态和日志。')
: translate(process.env, '请求未成功,请检查请求参数和服务日志。');
const suffix =
method !== 'GET' && (status === undefined || status >= 500)
? translate(
process.env,
' 执行结果可能未知,请先检查%s状态再考虑重试。',
resource,
)
: '';
fail(
translate(
process.env,
'API 请求失败(%s %s)。%s%s',
kind,
status ?? translate(process.env, '未知'),
reason,
suffix,
),
denied ? 3 : 1,
);
}
export async function authenticate(config: Credentials): Promise<Session> {
const query = new URLSearchParams({
client_id: config.clientId,
+1 -1
View File
@@ -241,7 +241,7 @@ export async function openCommand(
body,
authenticated: !op.anonymous,
output: values.output as string | undefined,
text: ['system/log', 'system/config/node-mirror'].includes(op.path),
text: op.path === 'system/log',
timeoutMs: values.timeout ? Number(values.timeout) * 1000 : undefined,
});
const data =
-6
View File
@@ -151,12 +151,6 @@ export const english: Record<string, string> = {
应用凭据: 'application credentials',
'应用凭据和 %s 权限': 'application credentials and %s permission',
未知: 'unknown',
'请检查%s。': 'Check %s.',
'资源冲突,请检查重复的名称或值。': 'Resource conflict; check for duplicate names or values.',
'服务端错误,请检查服务状态和日志。': 'Server error; check service status and logs.',
'请求未成功,请检查请求参数和服务日志。': 'Request unsuccessful; check request parameters and service logs.',
'API 请求失败(%s %s)。%s%s': 'API request failed (%s %s). %s%s',
' 执行结果可能未知,请先检查%s状态再考虑重试。':
' Execution outcome may be unknown; check %s status before retrying.',
'API 请求被拒绝(HTTP %s),请检查%s。%s':
+8 -9
View File
@@ -7,7 +7,7 @@ const { execFileSync, spawnSync } = require('node:child_process');
const { createContext, sourceEnvironment } = require('../../dist/internal/runtime/context');
const { executeTask } = require('../../dist/internal/execution/taskRunner');
test('Shell and CLI agree on isolated script state and account modes', async (t) => {
test('unmodified legacy Shell and TS agree on shell hook state and account modes', async (t) => {
const root = await fs.realpath(
await fs.mkdtemp(path.join(os.tmpdir(), 'ql-differential-')),
);
@@ -31,7 +31,7 @@ test('Shell and CLI agree on isolated script state and account modes', async (t)
path.join(root, 'static/build'),
])
await fs.mkdir(directory, { recursive: true });
for (const name of ['task.sh', 'task-timeout.sh', 'otask.sh', 'share.sh', 'api.sh', 'env.sh'])
for (const name of ['task.sh', 'otask.sh', 'share.sh', 'api.sh', 'env.sh'])
await fs.copyFile(
path.resolve(__dirname, '../../../shell', name),
path.join(context.paths.dir_shell, name),
@@ -110,13 +110,13 @@ test('Shell and CLI agree on isolated script state and account modes', async (t)
async () => {
const cases = [
{ name: 'return', script: 'return 7', code: 7, legacy: 0, after: true },
{ name: 'exit', script: 'exit 7', code: 7, legacy: 0, after: true },
{ name: 'exit', script: 'exit 7', code: 7, legacy: 7, after: false },
{
name: 'errexit',
script: 'set -e; false',
code: 1,
legacy: 0,
after: true,
legacy: 1,
after: false,
},
{
name: 'trap-return',
@@ -130,8 +130,8 @@ test('Shell and CLI agree on isolated script state and account modes', async (t)
name: 'trap-exit',
script: 'trap \'printf "exit-trap\\n" >> "$TRACE"\' EXIT; exit 7',
code: 7,
legacy: 0,
after: true,
legacy: 7,
after: false,
trap: true,
},
];
@@ -174,8 +174,8 @@ test('Shell and CLI agree on isolated script state and account modes', async (t)
assert.equal(result.exitCode, item.code, item.name);
const expected = [
'before',
...(item.trap ? ['exit-trap'] : []),
...(item.after ? [`after:${item.code}`] : []),
...(item.trap ? ['exit-trap'] : []),
];
for (const file of [legacyTrace, modernTrace])
assert.deepEqual(
@@ -355,7 +355,6 @@ t() { :; }
sleep() { printf DELAYED; }
enter_script_workdir() { :; }
run_else() { :; }
run_task_command() { "$@"; }
which_program=true
main "$@"
`,
@@ -10,7 +10,7 @@ test('legacy Shell and TS resolve executable paths after selecting the working d
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
for (const dir of ['shell/preload', 'shell/lang', 'data/config', 'data/scripts/bin', 'data/scripts/custom', 'data/log', 'static/build', 'bin'])
fs.mkdirSync(path.join(root, dir), { recursive: true });
for (const name of ['task.sh', 'task-timeout.sh', 'otask.sh', 'share.sh', 'api.sh', 'env.sh'])
for (const name of ['task.sh', 'otask.sh', 'share.sh', 'api.sh', 'env.sh'])
fs.copyFileSync(path.resolve(__dirname, '../../../shell', name), path.join(root, 'shell', name));
for (const name of ['zh.sh', 'en.sh'])
fs.copyFileSync(path.resolve(__dirname, '../../../shell/lang', name), path.join(root, 'shell/lang', name));
+1 -1
View File
@@ -10,7 +10,7 @@ test('legacy Shell and TS preserve task stdin in each execution mode', t => {
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
for (const dir of ['shell/preload', 'shell/lang', 'data/config', 'data/scripts', 'data/log', 'static/build', 'bin'])
fs.mkdirSync(path.join(root, dir), { recursive: true });
for (const name of ['task.sh', 'task-timeout.sh', 'otask.sh', 'share.sh', 'api.sh', 'env.sh'])
for (const name of ['task.sh', 'otask.sh', 'share.sh', 'api.sh', 'env.sh'])
fs.copyFileSync(path.resolve(__dirname, '../../../shell', name), path.join(root, 'shell', name));
for (const name of ['zh.sh', 'en.sh'])
fs.copyFileSync(path.resolve(__dirname, '../../../shell/lang', name), path.join(root, 'shell/lang', name));
+4 -4
View File
@@ -445,7 +445,7 @@ test('hook exports reach scripts and after hooks share shell state with inline c
assert.match(log, /hook:before:inline:after:7/);
});
test('shell tasks inherit before-hook state without leaking script state into after hooks', async (t) => {
test('ordinary shell tasks share unexported variables and functions across before script and after hooks', async (t) => {
const { executeTask } = require('../../dist/internal/execution/taskRunner');
const root = sandbox(t);
const context = createContext(
@@ -464,7 +464,7 @@ test('shell tasks inherit before-hook state without leaking script state into af
);
fs.writeFileSync(
context.paths.file_task_after,
'if declare -F script_function >/dev/null; then exit 99; fi\nprintf "after:%s:%s:%s\\n" "$before_value" "${script_value-unset}" "$_task_exit_code"\n',
'script_function\nprintf "after:%s:%s:%s\\n" "$before_value" "$script_value" "$_task_exit_code"\n',
);
const result = await executeTask(context, {
argv: ['shared.sh'],
@@ -478,7 +478,7 @@ test('shell tasks inherit before-hook state without leaking script state into af
);
assert.match(
log,
/before:private\narg:space value\nafter:private:unset:7/,
/before:private\narg:space value\nscript:changed\nafter:private:changed:7/,
);
});
@@ -681,7 +681,7 @@ test('designated shell accounts retain hook functions and script state in a shar
path.join(context.paths.dir_log, result.logPath),
'utf8',
);
assert.match(log, /selected:third&first\nafter:third&first::4/);
assert.match(log, /selected:third&first\nafter:third&first:from_script:4/);
});
test('lifecycle reports share execution identity and statistics survive a rejected final status', async (t) => {
+23 -51
View File
@@ -15,17 +15,8 @@ const {
reloadPanel,
} = require('../../dist/internal/maintenance/upgrade');
const { parse } = require('../helpers/commands.cjs');
const {
writeManifest,
sourceCommit,
} = require('../../../test/helpers/upgrade-fixture.cjs');
async function fixture(t) {
t.mock.method(
require('../../dist/internal/maintenance/upgradeArtifacts.cjs'),
'waitForHealth',
async () => {},
);
const root = await fs.realpath(
await fs.mkdtemp(path.join(os.tmpdir(), 'ql-operator-')),
);
@@ -128,27 +119,14 @@ test('PM2 restarts retain configured ports without mutating legacy environment',
ctx.env.CALLS = path.join(ctx.root, 'ports.jsonl');
ctx.env.BACK_PORT = '5600';
ctx.env.GRPC_PORT = '5400';
await stub(
ctx,
'pm2',
'require("node:fs").appendFileSync(process.env.CALLS, JSON.stringify([process.env.BACK_PORT,process.env.GRPC_PORT])+"\\n");',
);
await stub(ctx, 'pm2', 'require("node:fs").appendFileSync(process.env.CALLS, JSON.stringify([process.env.BACK_PORT,process.env.GRPC_PORT])+"\\n");');
await startPanel(ctx);
ctx.env.QlPort = '5799';
ctx.env.QlGrpcPort = '5599';
await startPanel(ctx);
assert.deepEqual(
(await fs.readFile(ctx.env.CALLS, 'utf8'))
.trim()
.split('\n')
.map(JSON.parse),
[
['5700', '5500'],
['5700', '5500'],
['5799', '5599'],
['5799', '5599'],
],
);
assert.deepEqual((await fs.readFile(ctx.env.CALLS, 'utf8')).trim().split('\n').map(JSON.parse), [
['5700', '5500'], ['5700', '5500'], ['5799', '5599'], ['5799', '5599'],
]);
assert.equal(ctx.env.BACK_PORT, '5600');
assert.equal(ctx.env.GRPC_PORT, '5400');
});
@@ -221,41 +199,24 @@ test('upgrade staging extracts both archives before publishing readiness and rej
const payloads = path.join(ctx.root, 'payloads');
const archives = path.join(ctx.root, 'archives');
await fs.mkdir(archives);
await fs.mkdir(path.join(payloads, `qinglong-${sourceCommit}`), {
recursive: true,
});
await fs.mkdir(path.join(payloads, 'qinglong-master'), { recursive: true });
await fs.mkdir(path.join(payloads, 'qinglong-static-master/build'), {
recursive: true,
});
const manifest = '{"name":"fixture"}';
await fs.writeFile(path.join(ctx.root, 'package.json'), manifest);
await fs.writeFile(
path.join(payloads, `qinglong-${sourceCommit}/package.json`),
path.join(payloads, 'qinglong-master/package.json'),
manifest,
);
await fs.writeFile(
path.join(payloads, 'qinglong-static-master/build/app.js'),
'console.log("fixture")',
);
await fs.mkdir(path.join(payloads, `qinglong-${sourceCommit}/sample`));
await fs.writeFile(
path.join(payloads, `qinglong-${sourceCommit}/sample/config.sample.sh`),
'new sample',
);
writeManifest(
path.join(payloads, `qinglong-${sourceCommit}`),
path.join(payloads, 'qinglong-static-master'),
);
await fs.writeFile(path.join(ctx.root, 'pnpm-lock.yaml'), 'fixture lock\n');
await fs.mkdir(path.join(ctx.root, 'node_modules'));
for (const repo of ['qinglong', 'qinglong-static'])
execFileSync(
'/usr/bin/zip',
[
'-qry',
path.join(archives, `${repo}.zip`),
`${repo}-${repo === 'qinglong' ? sourceCommit : 'master'}`,
],
['-qry', path.join(archives, `${repo}.zip`), `${repo}-master`],
{ cwd: payloads },
);
ctx.env.ARCHIVES = archives;
@@ -264,6 +225,18 @@ test('upgrade staging extracts both archives before publishing readiness and rej
'curl',
'const fs=require("node:fs"),p=require("node:path"),args=process.argv.slice(2),out=args[args.indexOf("--output")+1]; fs.copyFileSync(p.join(process.env.ARCHIVES,p.basename(out)),out);',
);
await fs.mkdir(path.join(payloads, 'qinglong-master/sample'), {
recursive: true,
});
await fs.writeFile(
path.join(payloads, 'qinglong-master/sample/config.sample.sh'),
'new sample',
);
execFileSync(
'/usr/bin/zip',
['-qry', path.join(archives, 'qinglong.zip'), 'qinglong-master'],
{ cwd: payloads },
);
const staged = await stageUpgrade(ctx, 'github');
assert.equal(
await fs.readFile(path.join(staged.source, 'package.json'), 'utf8'),
@@ -276,15 +249,15 @@ test('upgrade staging extracts both archives before publishing readiness and rej
'utf8',
),
),
{ ...staged, sourceCommit, dependenciesChanged: false },
staged,
);
await fs.symlink(
'../../outside',
path.join(payloads, `qinglong-${sourceCommit}/escape`),
path.join(payloads, 'qinglong-master/escape'),
);
execFileSync(
'/usr/bin/zip',
['-qry', path.join(archives, 'qinglong.zip'), `qinglong-${sourceCommit}`],
['-qry', path.join(archives, 'qinglong.zip'), 'qinglong-master'],
{ cwd: payloads },
);
await assert.rejects(stageUpgrade(ctx, 'github'), /symlinks/);
@@ -755,7 +728,7 @@ test('invalid or incomplete staged pointers cannot stop the installed panel', as
reloadPanel(ctx, 'system'),
(error) =>
error.code === 'ENOENT' &&
error.path === path.join(ctx.paths.dir_tmp, 'upgrade-ready-master.json'),
error.path === path.join(ctx.paths.dir_tmp, 'qinglong-master'),
);
});
@@ -810,7 +783,6 @@ test('system upgrade preserves installed dotenv bytes and permissions even when
);
await fs.writeFile(path.join(staticRoot, 'build/app.js'), '');
await fs.writeFile(path.join(source, '.env'), 'PORT=9999\nREPLACE_ME=true\n');
writeManifest(source, staticRoot);
const target = path.join(ctx.root, '.env');
const original = 'PORT=5700\nPRIVATE_VALUE=preserved\n';
await fs.writeFile(target, original, { mode: 0o600 });
+2 -3
View File
@@ -27,14 +27,13 @@ test('shared Shell lifecycle scopes timeouts to the task and runs cleanup exactl
{ name: 'failure', task: 'return 7', code: 7 },
{ name: 'function timeout preserves state', task: 'STATE=changed; sleep 10', code: 124, timeout: true, state: 'changed' },
{ name: 'external timeout', args: ['sleep', '10'], code: 124, timeout: true },
{ name: 'Shell script timeout preserves parent hook state', task: 'STATE=changed; sleep 10', shell: true, code: 124, timeout: true },
{ name: 'Shell script timeout preserves state', task: 'STATE=changed; sleep 10', shell: true, code: 124, timeout: true, state: 'changed' },
{ name: 'slow before is outside task timeout', before: 'sleep 0.2', task: 'return 0', code: 0 },
{ name: 'slow after is outside task timeout', after: 'sleep 0.2', task: 'return 0', code: 0 },
{ name: 'background child cannot hold pipes after timeout cleanup', task: '(trap "" INT; sleep 30) & sleep 30', after: 'sleep 0.3', code: 124, timeout: true },
{ name: 'script may redirect fd 3', shell: true, task: 'exec 3> "$TRACE.fd"; return 0', after: 'sleep 0.3', code: 0 },
{ name: 'before may close fd 3 and redirect fd 9', before: 'exec 3>&- 9> "$TRACE.lock"', task: 'return 0', after: 'sleep 0.3', code: 0 },
{ name: 'timeout cleanup survives script fd 3 redirection', shell: true, task: 'exec 3> "$TRACE.fd"; STATE=changed; sleep 30', after: 'sleep 0.3', code: 124, timeout: true },
{ name: 'Shell exit preserves user trap and runs after once', shell: true, task: 'trap \'printf "exit\\n" >> "$TRACE"\' EXIT; exit 9', code: 9, userExit: true },
{ name: 'timeout cleanup survives script fd 3 redirection', shell: true, task: 'exec 3> "$TRACE.fd"; STATE=changed; sleep 30', after: 'sleep 0.3', code: 124, timeout: true, state: 'changed' },
{ name: 'explicit exit preserves user trap and skips after', task: 'trap \'printf "exit\\n" >> "$TRACE"\' EXIT; exit 9', code: 9, noAfter: true, userExit: true },
{ name: 'user EXIT trap survives timeout with parent cleanup', task: 'trap \'printf "exit\\n" >> "$TRACE"\' EXIT; sleep 10', code: 124, timeout: true, userExit: true },
{ name: 'forced termination falls back to cleanup', task: 'trap "" INT; sleep 30', code: 124, timeout: true },
@@ -6,16 +6,10 @@ const os = require('node:os');
const { createContext } = require('../../dist/internal/runtime/context');
const { reloadPanel } = require('../../dist/internal/maintenance/upgrade');
const operator = require('../../dist/internal/maintenance/operator');
const { writeManifest } = require('../../../test/helpers/upgrade-fixture.cjs');
const selectedLoader =
'// QL_CLI_ROOT dist/local/entrypoints.js dist/local/cronEntrypoint.js\n';
async function fixture(t) {
t.mock.method(
require('../../dist/internal/maintenance/upgradeArtifacts.cjs'),
'waitForHealth',
async () => {},
);
const base = await fs.mkdtemp(
path.join(os.tmpdir(), 'ql-upgrade-selection-'),
);
@@ -49,7 +43,6 @@ async function fixture(t) {
await fs.writeFile(incoming, '// original released loader');
await fs.writeFile(path.join(ctx.paths.dir_static, 'build/app.js'), 'old');
await fs.writeFile(path.join(staticRoot, 'build/app.js'), 'new');
writeManifest(source, staticRoot);
return { ctx, source, staticRoot, loader, incoming };
}
@@ -111,33 +104,3 @@ test('unsupported target and unrecognized loader fail before service stop', asyn
assert.equal(stopped, false);
assert.equal(await fs.readFile(f.loader, 'utf8'), 'unrecognized');
});
test('successful process start with failed health check rolls back the installed version', async (t) => {
const f = await fixture(t);
t.mock.method(operator, 'stopPanel', async () => {});
t.mock.method(operator, 'startPanel', async () => ({ manager: 'fixture' }));
let probes = 0;
t.mock.method(
require('../../dist/internal/maintenance/upgradeArtifacts.cjs'),
'waitForHealth',
async () => {
if (++probes === 1) throw new Error('health check failed');
assert.equal(
await fs.readFile(
path.join(f.ctx.paths.dir_static, 'build/app.js'),
'utf8',
),
'old',
);
},
);
await assert.rejects(
reloadPanel(f.ctx, 'system', { source: f.source, static: f.staticRoot }),
/health check failed/,
);
assert.equal(probes, 2);
assert.equal(
await fs.readFile(path.join(f.ctx.root, 'package.json'), 'utf8'),
'{"version":"2.19.0"}',
);
});
+1 -52
View File
@@ -54,7 +54,7 @@ test('API diagnostics preserve resource scope, uncertainty, codes and secret sup
language === 'en' ? /outcome.*unknown/ : /结果.*未知/,
);
}
if (failure === 'permission' || failure === 'api') {
if (failure !== 'network') {
assert.ok(
error.message.includes(
endpoint.startsWith('subscriptions')
@@ -89,54 +89,3 @@ test('API diagnostics preserve resource scope, uncertainty, codes and secret sup
);
}
});
test('HTTP and body errors distinguish conflicts, authentication and server failures without leaking bodies', async (t) => {
const previous = process.env.QL_LANG;
t.after(() => {
if (previous === undefined) delete process.env.QL_LANG;
else process.env.QL_LANG = previous;
});
for (const language of ['en', 'zh']) {
process.env.QL_LANG = language;
for (const kind of ['HTTP', 'code'])
for (const status of [400, 401, 403, 409, 500, 503])
for (const method of ['GET', 'PUT']) {
t.mock.method(
global,
'fetch',
async () =>
new Response(
JSON.stringify({ code: status, message: 'secret-value' }),
{ status: kind === 'HTTP' ? status : 200 },
),
);
await assert.rejects(
request(
{ url: 'https://fixture.invalid', token: 'secret-token' },
'envs',
{ method },
),
(error) => {
assert.equal(error.exitCode, [401, 403].includes(status) ? 3 : 1);
assert.match(error.message, new RegExp(`${kind} ${status}`));
assert.doesNotMatch(error.message, /secret-|fixture/);
if ([401, 403].includes(status))
assert.match(error.message, /credentials|凭据/);
else
assert.doesNotMatch(
error.message,
/credentials|permission|凭据|权限/,
);
if (status === 409) assert.match(error.message, /conflict|冲突/);
if (status >= 500)
assert.match(error.message, /Server error|服务端错误/);
if (status >= 500 && method === 'PUT')
assert.match(error.message, /unknown|未知/);
else assert.doesNotMatch(error.message, /unknown|未知/);
return true;
},
);
t.mock.restoreAll();
}
}
});
-26
View File
@@ -211,29 +211,3 @@ test('owner two-factor challenge remains actionable without exposing server mess
assert.equal(result.code, 3); assert.match(result.err, /two-factor-login/); assert.doesNotMatch(result.err, /SERVER-SECRET/);
assert.equal(f.requests.length, 1); assert.equal(f.requests[0].headers.authorization, undefined);
});
test('node mirror configuration accepts the real empty response for named and raw API commands', async t => {
const f = await fixture(t);
f.reply((_req, res) => { res.writeHead(200, { 'content-type': 'application/octet-stream' }); res.end(); });
for (const args of [
['system', 'config-node-mirror'],
['api', 'request', 'PUT', '/open/system/config/node-mirror'],
]) {
const reply = success(await f.run([...args, '--data', '{"nodeMirror":"https://registry.npmmirror.com"}']));
assert.equal(reply.code, 200);
assert.equal(f.requests.at(-1).method, 'PUT');
assert.equal(f.requests.at(-1).url, '/panel/open/system/config/node-mirror');
}
assert.equal(f.requests.length, 2, 'configuration writes must not be retried');
});
test('empty node mirror responses still reject authentication and server failures without retries', async t => {
const f = await fixture(t);
for (const status of [401, 403, 500]) {
f.reply((_req, res) => { res.writeHead(status, { 'content-type': 'application/octet-stream' }); res.end(); });
const result = await f.run(['system', 'config-node-mirror', '--data', '{}']);
assert.notEqual(result.code, 0);
assert.equal(result.out, '');
}
assert.equal(f.requests.length, 3);
});
-49
View File
@@ -1,49 +0,0 @@
# Node cron 调度器迁移
Node 调度路径使用固定版本 `node-cron@4.6.0`,普通任务(含额外规则)、订阅和系统 cron 共用 `back/shared/cronScheduler.ts`。启动/一次性任务、间隔任务、脚本并发限制和进程执行链路保持原有实现。系统模式只将语义一致的普通五字段规则交给 crond。
移除 `node-schedule`、其类型包以及兼容别名 `cron-parser-v4`。保留 `cron-parser@5` 做旧表达式的校验与标准化;计时器和日历推进由 node-cron 管理。
## 兼容行为
- 保留旧版 1–6 字段补全、五种宏、月份/星期名称、数字起点步长、L 和 #。另外明确新增 `@annually`(等同 `@yearly`)、`@midnight`(等同 `@daily`)、`@minutely`(每分钟第 0 秒)。宏按进程时区解释。`?` 仅允许日期和星期字段;裸 `/N`、H 和其他未列出的宏仍拒绝。
- 日期与星期同时指定时保留 OR 语义,不采用 node-cron 默认的 AND。两个原生任务分别处理日期和星期,并按计划时间过滤交集,避免重复执行;保留旧解析器的月份全日期范围及全局 nth-week 限制。
- 原生任务先暂停创建并验证,再替换旧任务。构造失败时销毁已准备的实例并保留旧快照。
- 进程内迟到的触发通过 `execution:missed` 加入既有执行队列,记录补执行日志。并发和单任务积压上限仍由青龙队列控制;这不提供进程退出期间的持久化补跑或分布式 exactly-once 保证。
- 标准化结果使用上限 512 条的只读缓存,仅缓存语法,不缓存运行实例或下次执行时间。避免大批相同表达式重复解析导致恢复 RPC 超时。
- 回调异常记录日志,后续调度继续;取消时调用原生 `destroy()` 释放计时器和注册表。
- 持久化无效任务在恢复时跳过并记录日志,无效订阅不会产生未处理的启动拒绝。API、导入和执行使用同一套表达式校验。
## 复审修复
- 移除没有生产调用的 `nextInvocation()` 接口,避免对 OR/# 等过滤规则报告不会实际执行的原生日期。回归测试直接观察实际执行和原生实例状态。
- 批量注册 RPC 时限为 5 秒加每条规则 5 毫秒,计入额外规则并封顶 120 秒;保留超时后的恢复机制,不立即重放结果不确定的写入。
- Alpine/系统模式下,数字起点步长(如 `0/5`)、星期包含 `7`、日期步长、日期和星期同时受限的表达式转入 Node 路径。BusyBox 的这些解析或通配语义与旧 Node 解析器不同,不能仅因是纯数字五字段就交给 crond。
- 这些规则在 `crontab.list` 中仍保留为注释,避免同时被系统和 Node 执行。其他可移植规则仍使用系统 crond。
## 夏令时差异
采用 node-cron 的当地时间语义:春季跳时不存在的时间跳过,秋季重复小时不再执行第二次。旧 cron-parser 对部分春季时间会顺延,并可能重复秋季高频任务。使用有夏令时的时区时需要考虑该行为变化;Asia/Shanghai 不受影响。回归测试固定 America/New_York 的 2026 年转换日期。
## 验证
`npm test` 和 `npm run build:back`。
- `test/fixtures/legacy-cron.json` 是迁移前使用 node-schedule 2.1.1 / cron-parser 4.9.0 保存的快照:269 条语法判定,18 组日历序列,每组 10 次执行。
- 真正的 node-cron 实例配合虚拟时钟验证日期/星期 OR、交集去重、闰年、L/#、8 秒迟到补执行、取消释放、回调拒绝和夏令时。
- 注册构造失败回滚、无效订阅隔离、空表达式立即运行,以及现有调度恢复/503/子进程测试覆盖接入流程。
- 隔离的正式 Debian 镜像使用实际编译产物验证 10,000 条有效任务和 2 条错误任务的恢复、300 个同时到期的 Python 脚本、并发上限 16 及 HTTP 健康检查。具体结果见本次测试报告;这是生产镜像隔离验证,不是线上长期运行证明。
### 本次验收结果(2026-09-27)
完整回归 260 通过、3 项原有跳过,后端编译成功。Node 20 生产运行时的 21 项兼容测试通过。隔离镜像中 300/300 脚本完成,无重复或遗漏,最大并发 16,270 个成功及 30 个预设失败均落库;完成耗时 52.36 秒。恢复后健康检查 1,870/1,870 成功,任务列表 38/38 成功。重启恢复阶段另有 50 次健康检查失败。
初版重复解析曾导致恢复超过 5 秒 RPC 时限,加入有上限的语法缓存后重跑通过。本次 10,000 个任务共享两个表达式,不能视为一万种复杂表达式的容量保证;实际脚本耗时也不代表纯调度性能。
新增宏在 UTC 和 Asia/Shanghai 下验证连续两次实际触发及不提前触发;原有语法快照仅对这三个新增宏允许预期变化。
### 复审后验收
完整回归 267 通过、3 项原有跳过;Node 20 的 10 项修复回归及 Alpine 镜像的 24 项兼容/宏测试通过。
额外使用 9,700 个不同表达式和 300 个同时到期任务验证恢复(另有 2 个无效任务),不再局限于共享两个表达式。300/300 脚本完成,无重复或遗漏,最大并发 16,270 个退出码 0、30 个预设退出码 3 均落库。恢复后健康检查 1,650/1,650 成功、任务列表 33/33 成功;启动恢复阶段另有 260 次健康检查失败,整批脚本耗时 60.27 秒。较大快照恢复仍需等待,不承诺启动阶段始终可用或任意规模都能在时限内恢复。
-78
View File
@@ -1,78 +0,0 @@
# Task startup and notification loading
`task` performs shell configuration, status reporting and dependency-path discovery
before invoking the selected runtime. These costs are separate from cron callback
latency and time spent waiting for an execution slot.
## Dependency-path cache
Node scheduling usually starts in `/ql`, while Alpine crond starts in `/root`.
The pnpm discovery key includes the working directory, npm/pnpm configuration,
relevant environment and executable metadata. Each key now has its own cached
record, so alternating scheduler environments do not evict each other.
Records expire after 60 seconds. Refreshes share a persistent per-user lock;
records older than an hour are removed on refresh. Warm lookups do not scan the
cache directory. The lock is never removed while another process might use it.
The opt-out `QL_NODE_PATH_CACHE=0`, failed-discovery fallback, and package resolution
order remain available. This caches the dependency directory, not its contents.
## Language startup paths
| Invocation | Environment and before hooks | Automatic notification loading |
| --- | --- | --- |
| `task script.py` / `.pyc` | Python preload imports generated environment, runs the shell/command before hooks, imports `task_before.py`, then applies account selection | `QLAPI.notify(...)` imports `__ql_notify__` on first call |
| `task script.js` / `.mjs` / `.ts` | Node preload imports generated environment, runs shell/command before hooks, requires `task_before.js`, then applies account selection | `QLAPI.notify(...)` requires `__ql_notify__.js` on first call |
| `task script.sh` | Shell sources the generated environment and runs shell before hooks | No automatic Python/Node notification module |
| Explicit interpreter or other command, e.g. `task python3 script.py` | Retains generated environment and shell before-hook exports; it does not automatically install the Python/Node preload | No automatic notification module unless configured by the caller |
TypeScript uses `ts-node-transpile-only`; MJS uses the same Node preload plus the
ESM loader. There are no separate built-in Ruby, Go, Java or PHP preload modules.
All these task paths still pass through shared dependency-path setup: non-JS
scripts and their hooks may themselves call Node or use the QLAPI bridge.
The before hooks are deliberately still synchronous: they can export environment
variables or perform work required by the user's script. Python's preload still
starts a child shell/Python to capture that environment; Node's preload starts a
child shell/Node. Node also still loads its gRPC client eagerly. These are remaining
startup costs, not work performed by the cron library, and are not removed by this
change.
Shell scripts run in a child shell that inherits the wrapper's environment and
before-hook functions. This lets the wrapper run its after hooks and report the
exit status even when a script calls `exit`, replaces itself with `exec`, or fails
under `set -e`. The script's own EXIT trap still runs in the child. Changes made
inside the script (variables, working directory, functions and shell options)
do not propagate back to the wrapper or its after hooks.
## Lazy notifications
Scripts keep calling `QLAPI.notify` with the same arguments. When no notification
is sent, the notification provider and its channel dependencies are never imported
by the preload. First use loads the provider; subsequent uses reuse the standard
Python/Node module cache. JS forwards the receiver and return value, including
promises, and Python forwards positional and keyword arguments. Import/send errors
surface at the notification call rather than preventing unrelated scripts from
obtaining QLAPI during startup. A failed import can be retried after its cause is
resolved.
Provider initialization, including provider configuration reads, now happens at
first use. A script's explicit `import __ql_notify__`, `require(...)`, or independent
notification helper still loads that module immediately.
## Task timeouts
`task -m DURATION ...` also applies to sourced Shell scripts. The timeout worker
inherits before-hook variables, arrays and functions without serializing them.
At the deadline, the worker's process group receives TERM, followed by KILL after
0.5 seconds, so ordinary child processes cannot continue after the timeout.
Explicit interpreters and other commands use the same mechanism. Processes that
intentionally create a separate session/process group are outside this boundary.
The wrapper records status 124 and runs its after hook; concurrent account mode
retains its existing aggregate status behavior. Zero disables the timeout.
Cron executions waiting for a concurrency slot retain their schedule revision.
Disabling, deleting or replacing that revision prevents its queued executions
from starting when a slot becomes available. Already running executions retain
the existing stop behavior. Invalidated callbacks do not count against a new
revision's limit of five pending/running executions.
-33
View File
@@ -1,33 +0,0 @@
# 青龙 2.23.0
## 新增功能
- 脚本历史版本:查看、对比和恢复历史内容;每个脚本保留最多 20 个版本,支持不超过 1 MiB 的 UTF-8 文本脚本。恢复前校验当前内容哈希,发生并发修改时返回冲突,避免覆盖新内容。
- 独立远程 CLI 0.1.2 增加脚本历史查看、对比和恢复命令,并完善资源冲突及认证错误反馈。最低要求仍为 Node.js 22.12。
## 修复与优化
- 普通任务、额外规则和订阅的 Node 调度路径迁移到 node-cron。保留旧规则标准化、日期/星期 OR、L/# 等兼容行为,补充 `@annually`、`@midnight`、`@minutely`。
- 禁用、删除或修改任务后,旧规则对应的排队执行不会再启动。完善任务进程超时、停止回收、Shell before/after 环境传递和退出状态。
- 修复系统令牌未主动续期和认证缓存并发更新:定时生成与启动后重新计时会主动续期,避免长期运行后失效。Shell API 每次请求检查令牌,遇到 401 时刷新并重试一次;其他错误不自动重试。
- 降低任务启动开销;JS/Python 的 QLAPI 通知依赖在首次通知调用时加载,减少不发送通知的任务启动成本。
- 加固环境变量名和字面值、订阅参数、路径访问及 SSH 配置输入;保留合法的中文及分支别名兼容。
- 修复页面升级、`ql update` 和静态资源的版本同步,完善升级暂存校验及应用、完整备份与恢复流程。
- 修复 Debian 上的依赖安装、通知和调度回归,补齐前端缺失的中英文翻译,改进 Gitee/GitLab 静态资源镜像同步检查。
- 最小升级 compression、multer、@grpc/grpc-js、proxy-addr、websocket-driver、sequelize,修补现有 HTTP、上传、代理信任、WebSocket、gRPC 和 JSON 查询路径相关风险。完整依赖治理、Node/pnpm 迁移及工具打包调整延期。
## 升级说明
升级前停止面板与任务,备份完整持久化数据目录,包括数据库、脚本、配置、日志、依赖缓存及已有的 `script-history` 目录,并记录原镜像摘要和启动配置。使用面板备份接口时,确认选择了脚本、配置和日志;仅备份数据库不能恢复全部数据。
升级后试运行代表性的 JavaScript、Python、Shell 任务,检查订阅、环境变量、依赖安装以及通知。存在夏令时的时区采用 node-cron 行为:跳过春季不存在的当地时间,秋季重复小时不重复执行;Asia/Shanghai 不受影响。调度器不提供进程退出期间的持久化补跑保证。
任务超时会终止普通子进程所在的进程组并记录退出状态 124;主动创建独立会话/进程组的程序不在该边界内。并发账号执行模式保留既有汇总退出状态。
本次沿用现有 Dockerfile 和 pnpm 8.3.1。默认 Debian 镜像使用 Debian 12 与 Node.js 20,默认 Alpine 镜像的 Node.js 由 APK 提供;默认 Python 为 3.11。Node 20 与 Node 18 构建运行时已结束上游维护,相关迁移延期,功能验证通过不代表全部依赖风险清零。
2.23.0 是 Python 3.10 兼容镜像的最后一个版本。`2.23.0-python3.10` 和 `2.23.0-debian-python3.10` 发布后停止维护,不再提供后续更新;请验证脚本和依赖后迁移至默认 Python 3.11 镜像。
主包 `@whyour/qinglong` 与独立远程包 `@whyour/qinglong-cli` 的 `ql` 命令用途不同。在已有面板的主机上使用独立 npm prefix 或 `npm exec --package=@whyour/qinglong-cli -- ql ...`,避免覆盖面板内置入口。
发布时间:2026-10-11 12:27:02(Asia/Shanghai)。本版本已完成四类镜像验证及独立 dev 环境超过两小时的持续功能验收。
+1 -1
View File
@@ -31,7 +31,7 @@
文件保存复用现有锁路径,同时以独立跨进程目录锁串行化历史写入和配额清理;`finally` 释放锁。先准备并同步脚本暂存文件,保留文件权限和所有者;随后写并同步历史暂存文件、原子提交历史,再原子替换脚本。任一步准备失败不会截断原脚本;最后替换脚本失败时还原此前历史;成功后才清理配额。超限确认保存不创建新历史包,使用同样的脚本暂存和替换方式,并流式计算旧文件哈希。外部进程不遵守该锁时,不承诺与其并发写入的事务一致性。
恢复请求携带预览时的当前内容哈希,锁内检查后才写入;文件变化则返回 409。历史读写与脚本读取采用相同的路径和符号链接边界检查。面板配置目录中的敏感文件仍受黑名单保护;脚本目录中的 auth.json、token.json 等普通脚本名称可以正常保存和留档。
恢复请求携带预览时的当前内容哈希,锁内检查后才写入;文件变化则返回 409。历史读写与脚本读取采用相同的路径、黑名单和符号链接边界检查。
## 压缩样本
+9 -9
View File
@@ -1,7 +1,7 @@
{
"name": "@whyour/qinglong",
"packageManager": "pnpm@8.3.1",
"version": "2.23.0",
"version": "2.22.0",
"description": "Timed task management platform supporting Python3, JavaScript, Shell, Typescript",
"repository": {
"type": "git",
@@ -74,24 +74,23 @@
"overrides": {
"sqlite3": "npm:@whyour/sqlite3@1.1.2",
"@codemirror/state": "6.5.4",
"@codemirror/view": "6.39.16",
"proxy-addr": "2.0.8",
"websocket-driver": "0.7.5"
"@codemirror/view": "6.39.16"
}
},
"dependencies": {
"@ant-design/plots": "^2.6.8",
"@bufbuild/protobuf": "^2.10.0",
"@grpc/grpc-js": "1.14.5",
"@grpc/grpc-js": "^1.14.0",
"@grpc/proto-loader": "^0.8.0",
"@keyv/sqlite": "^4.0.1",
"@otplib/preset-default": "^12.0.1",
"body-parser": "^1.20.3",
"celebrate": "^15.0.3",
"chokidar": "^4.0.1",
"compression": "1.8.2",
"compression": "^1.7.4",
"cors": "^2.8.5",
"cron-parser": "^5.4.0",
"cron-parser-v4": "npm:cron-parser@4.9.0",
"cross-spawn": "^7.0.6",
"dayjs": "^1.11.13",
"dotenv": "^16.4.6",
@@ -108,15 +107,15 @@
"jsonwebtoken": "^9.0.2",
"keyv": "^5.2.3",
"lodash": "^4.17.21",
"multer": "2.3.0",
"node-cron": "4.6.0",
"multer": "2.1.1",
"node-schedule": "^2.1.0",
"nodemailer": "^8.0.1",
"p-queue-cjs": "7.3.4",
"proper-lockfile": "^4.1.2",
"ps-tree": "^1.2.0",
"reflect-metadata": "^0.2.2",
"request-ip": "3.3.0",
"sequelize": "6.37.8",
"sequelize": "^6.37.5",
"sockjs": "^0.3.24",
"sqlite3": "npm:@whyour/sqlite3@1.1.2",
"toad-scheduler": "^3.0.1",
@@ -146,6 +145,7 @@
"@types/lodash": "^4.14.185",
"@types/multer": "^1.4.7",
"@types/node": "^17.0.21",
"@types/node-schedule": "^1.3.2",
"@types/nodemailer": "^6.4.4",
"@types/proper-lockfile": "^4.1.4",
"@types/ps-tree": "^1.1.6",
+73 -41
View File
@@ -4,8 +4,6 @@ overrides:
sqlite3: npm:@whyour/sqlite3@1.1.2
'@codemirror/state': 6.5.4
'@codemirror/view': 6.39.16
proxy-addr: 2.0.8
websocket-driver: 0.7.5
dependencies:
'@ant-design/plots':
@@ -15,8 +13,8 @@ dependencies:
specifier: ^2.10.0
version: 2.10.0
'@grpc/grpc-js':
specifier: 1.14.5
version: 1.14.5
specifier: ^1.14.0
version: 1.14.0
'@grpc/proto-loader':
specifier: ^0.8.0
version: 0.8.0
@@ -36,14 +34,17 @@ dependencies:
specifier: ^4.0.1
version: 4.0.1
compression:
specifier: 1.8.2
version: 1.8.2
specifier: ^1.7.4
version: 1.7.5
cors:
specifier: ^2.8.5
version: 2.8.5
cron-parser:
specifier: ^5.4.0
version: 5.4.0
cron-parser-v4:
specifier: npm:cron-parser@4.9.0
version: /cron-parser@4.9.0
cross-spawn:
specifier: ^7.0.6
version: 7.0.6
@@ -93,11 +94,11 @@ dependencies:
specifier: ^4.17.21
version: 4.17.21
multer:
specifier: 2.3.0
version: 2.3.0
node-cron:
specifier: 4.6.0
version: 4.6.0
specifier: 2.1.1
version: 2.1.1
node-schedule:
specifier: ^2.1.0
version: 2.1.1
nodemailer:
specifier: ^8.0.1
version: 8.0.1
@@ -117,8 +118,8 @@ dependencies:
specifier: 3.3.0
version: 3.3.0
sequelize:
specifier: 6.37.8
version: 6.37.8(@whyour/sqlite3@1.1.2)
specifier: ^6.37.5
version: 6.37.5(@whyour/sqlite3@1.1.2)
sockjs:
specifier: ^0.3.24
version: 0.3.24
@@ -202,6 +203,9 @@ devDependencies:
'@types/node':
specifier: ^17.0.21
version: 17.0.45
'@types/node-schedule':
specifier: ^1.3.2
version: 1.3.2
'@types/nodemailer':
specifier: ^6.4.4
version: 6.4.17
@@ -2964,8 +2968,8 @@ packages:
deprecated: the package is rather renamed to @formatjs/ecma-abstract with some changes in functionality (primarily selectUnit is removed and we don't plan to make any further changes to this package
dev: true
/@grpc/grpc-js@1.14.5:
resolution: {integrity: sha512-7VZM+SVdEcUUqSQeNI3zM8Qs/BhQKZndPo2h5VkYkAM8Iz0wJIa8mKV5ekQGqG8UUsnkQ0NMxIxwkIHYvj0qOw==}
/@grpc/grpc-js@1.14.0:
resolution: {integrity: sha512-N8Jx6PaYzcTRNzirReJCtADVoq4z7+1KQ4E70jTg/koQiMoUSN1kbNjPOqpPbhMFhfU1/l7ixspPl8dNY+FoUg==}
engines: {node: '>=12.10.0'}
dependencies:
'@grpc/proto-loader': 0.8.0
@@ -4512,6 +4516,12 @@ packages:
'@types/express': 4.17.21
dev: true
/@types/node-schedule@1.3.2:
resolution: {integrity: sha512-Y0CqdAr+lCpArT8CJJjJq4U2v8Bb5e7ru2nV/NhDdaptCMCRdOL3Y7tAhen39HluQMaIKWvPbDuiFBUQpg7Srw==}
dependencies:
'@types/node': 17.0.45
dev: true
/@types/node@17.0.45:
resolution: {integrity: sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==}
@@ -4942,7 +4952,7 @@ packages:
'@umijs/bundler-utils': 4.4.4
'@umijs/mako': 0.11.1
chalk: 4.1.2
compression: 1.8.2
compression: 1.7.5
connect-history-api-fallback: 2.0.0
cors: 2.8.5
express: 4.21.2
@@ -6830,16 +6840,15 @@ packages:
serialize-javascript: 6.0.2
dev: true
/compression@1.8.2:
resolution: {integrity: sha512-o8vI5RE5A6EVVOd9o41jKp41aJom+QTEO/Bx8MYNjexMo/Bv2WOjUfZr+aL0WnYSgymUy6zeguqLTsIhV0gMvQ==}
/compression@1.7.5:
resolution: {integrity: sha512-bQJ0YRck5ak3LgtnpKkiabX5pNF7tMUh1BSy2ZBOTh0Dim0BUu6aPPwByIns6/A5Prh8PufSPerMDUklpzes2Q==}
engines: {node: '>= 0.8.0'}
dependencies:
bytes: 3.1.2
compressible: 2.0.18
debug: 2.6.9
destroy: 1.2.0
negotiator: 0.6.4
on-headers: 1.1.0
on-headers: 1.0.2
safe-buffer: 5.2.1
vary: 1.1.2
transitivePeerDependencies:
@@ -7037,6 +7046,13 @@ packages:
resolution: {integrity: sha512-VQ2MBenTq1fWZUH9DJNGti7kKv6EeAuYr3cLwxUWhIu1baTaXh4Ib5W2CqHVqib4/MqbYGJqiL3Zb8GJZr3l4g==}
dev: true
/cron-parser@4.9.0:
resolution: {integrity: sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==}
engines: {node: '>=12.0.0'}
dependencies:
luxon: 3.5.0
dev: false
/cron-parser@5.4.0:
resolution: {integrity: sha512-HxYB8vTvnQFx4dLsZpGRa0uHp6X3qIzS3ZJgJ9v6l/5TJMgeWQbLkR5yiJ5hOxGbc9+jCADDnydIe15ReLZnJA==}
engines: {node: '>=18'}
@@ -7734,7 +7750,6 @@ packages:
/dottie@2.0.6:
resolution: {integrity: sha512-iGCHkfUc5kFekGiqhe8B/mdaurD+lakO9txNnTvKtA6PISrw86LgqHvRzWYPyoE2Ph5aMIrCw9/uko6XHTKCwA==}
deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.
dev: false
/dprint-node@1.0.8:
@@ -8531,7 +8546,7 @@ packages:
on-finished: 2.4.1
parseurl: 1.3.3
path-to-regexp: 0.1.12
proxy-addr: 2.0.8
proxy-addr: 2.0.7
qs: 6.13.0
range-parser: 1.2.1
safe-buffer: 5.2.1
@@ -8615,7 +8630,7 @@ packages:
resolution: {integrity: sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g==}
engines: {node: '>=0.8.0'}
dependencies:
websocket-driver: 0.7.5
websocket-driver: 0.7.4
/fb-watchman@2.0.2:
resolution: {integrity: sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==}
@@ -10430,6 +10445,10 @@ packages:
triple-beam: 1.4.1
dev: false
/long-timeout@0.1.1:
resolution: {integrity: sha512-BFRuQUqc7x2NWxfJBCyUrN8iYUYznzL9JROmRz1gZ6KlOIgmoD+njPVbb+VNn2nGMKggMsK79iUNErillsrx7w==}
dev: false
/long@5.2.3:
resolution: {integrity: sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==}
dev: false
@@ -10463,6 +10482,11 @@ packages:
yallist: 4.0.0
dev: true
/luxon@3.5.0:
resolution: {integrity: sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==}
engines: {node: '>=12'}
dev: false
/luxon@3.7.2:
resolution: {integrity: sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==}
engines: {node: '>=12'}
@@ -10755,8 +10779,8 @@ packages:
/ms@2.1.3:
resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
/multer@2.3.0:
resolution: {integrity: sha512-cjNbm3sttszgZeGfJR124D+jFEfkXCVAsoPBmFn9X7UxmDSFHWqE2CoEj0vrmSpuAFnqWR1Szcm9QTsiHr60Xw==}
/multer@2.1.1:
resolution: {integrity: sha512-mo+QTzKlx8R7E5ylSXxWzGoXoZbOsRMpyitcht8By2KHvMbf3tjwosZ/Mu/XYU6UuJ3VZnODIrak5ZrPiPyB6A==}
engines: {node: '>= 10.16.0'}
dependencies:
append-field: 1.0.0
@@ -10825,11 +10849,6 @@ packages:
resolution: {integrity: sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ==}
dev: false
/node-cron@4.6.0:
resolution: {integrity: sha512-Si/bzYiKRHOB8/a99T2+SDGN582ONDMSTlJr5oCkT6GtnqPjZ2s10eoQRYkW9ZHwjVxONL+W8Fb+qR0AHMQsdg==}
engines: {node: '>=20'}
dev: false
/node-domexception@1.0.0:
resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
engines: {node: '>=10.5.0'}
@@ -10944,6 +10963,15 @@ packages:
resolution: {integrity: sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g==}
dev: true
/node-schedule@2.1.1:
resolution: {integrity: sha512-OXdegQq03OmXEjt2hZP33W2YPs/E5BcFQks46+G2gAxs4gHOIVD1u7EqlYLYSKsaIpyKCK9Gbk0ta1/gjRSMRQ==}
engines: {node: '>=6'}
dependencies:
cron-parser: 4.9.0
long-timeout: 0.1.1
sorted-array-functions: 1.3.0
dev: false
/nodemailer@8.0.1:
resolution: {integrity: sha512-5kcldIXmaEjZcHR6F28IKGSgpmZHaF1IXLWFTG+Xh3S+Cce4MiakLtWY+PlBU69fLbRa8HlaGIrC/QolUpHkhg==}
engines: {node: '>=6.0.0'}
@@ -11144,8 +11172,8 @@ packages:
dependencies:
ee-first: 1.1.1
/on-headers@1.1.0:
resolution: {integrity: sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==}
/on-headers@1.0.2:
resolution: {integrity: sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==}
engines: {node: '>= 0.8'}
/once@1.4.0:
@@ -12181,8 +12209,8 @@ packages:
long: 5.2.3
dev: false
/proxy-addr@2.0.8:
resolution: {integrity: sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ==}
/proxy-addr@2.0.7:
resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==}
engines: {node: '>= 0.10'}
dependencies:
forwarded: 0.2.0
@@ -13914,8 +13942,8 @@ packages:
engines: {node: '>= 10.0.0'}
dev: false
/sequelize@6.37.8(@whyour/sqlite3@1.1.2):
resolution: {integrity: sha512-HJ0IQFqcTsTiqbEgiuioYFMSD00TP6Cz7zoTti+zVVBwVe9fEhev9cH6WnM3XU31+ABS356durAb99ZuOthnKw==}
/sequelize@6.37.5(@whyour/sqlite3@1.1.2):
resolution: {integrity: sha512-10WA4poUb3XWnUROThqL2Apq9C2NhyV1xHPMZuybNMCucDsbbFuKg51jhmyvvAUyUqCiimwTZamc3AHhMoBr2Q==}
engines: {node: '>=10.0.0'}
peerDependencies:
ibm_db: '*'
@@ -13949,7 +13977,7 @@ packages:
dependencies:
'@types/debug': 4.1.12
'@types/validator': 13.12.2
debug: 4.4.3
debug: 4.3.7(supports-color@5.5.0)
dottie: 2.0.6
inflection: 1.13.4
lodash: 4.17.21
@@ -13957,7 +13985,7 @@ packages:
moment-timezone: 0.5.46
pg-connection-string: 2.7.0
retry-as-promised: 7.0.4
semver: 7.7.4
semver: 7.6.3
sequelize-pool: 7.1.0
sqlite3: /@whyour/sqlite3@1.1.2
toposort-class: 1.0.1
@@ -14127,7 +14155,7 @@ packages:
dependencies:
faye-websocket: 0.11.4
uuid: 8.3.2
websocket-driver: 0.7.5
websocket-driver: 0.7.4
dev: false
/sonic-boom@2.8.0:
@@ -14152,6 +14180,10 @@ packages:
sort-object-keys: 1.1.3
dev: true
/sorted-array-functions@1.3.0:
resolution: {integrity: sha512-2sqgzeFlid6N4Z2fUQ1cvFmTOLRi/sEDzSQ0OKYchqgoPmQBVyM3959qYx3fpS6Esef80KjmpgPeEr028dP3OA==}
dev: false
/source-map-js@1.2.1:
resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==}
engines: {node: '>=0.10.0'}
@@ -15532,8 +15564,8 @@ packages:
javascript-stringify: 2.1.0
dev: true
/websocket-driver@0.7.5:
resolution: {integrity: sha512-ZL2+3c7kMBdIRCMz6l8jQMHyGVxj+UL+xVk74Ombiciboca8rHa15L86B19E5oh1pL9Ii/uj54gtsIrZGMo6zA==}
/websocket-driver@0.7.4:
resolution: {integrity: sha512-b17KeDIQVjvb0ssuSDF2cYXSg2iztliJ4B9WdsuB6J952qCPKmnVq4DyW5motImXHDC1cBT/1UezrJVsKw5zjg==}
engines: {node: '>=0.8.0'}
dependencies:
http-parser-js: 0.5.8
-34
View File
@@ -1,34 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
destination=$1
ref=$2
ref_type=$3
git check-ref-format "refs/heads/$ref"
script_dir=$(cd "$(dirname "$0")" && pwd)
# Publish only a complete artifact from the checked-out source commit.
node "$script_dir/../docker/verify-build.cjs"
if [[ "$ref_type" == 'tag' ]]; then
version=$(awk '$1 == "version:" { print $2; exit }' version.yaml)
if [[ "$ref" != "v$version" ]]; then
echo "::error::Release tag $ref does not match version.yaml ($version)"
exit 1
fi
elif [[ "$ref_type" != 'branch' ]]; then
echo "::error::Unsupported ref type: $ref_type"
exit 1
fi
workspace=$(mktemp -d)
trap 'rm -rf "$workspace"' EXIT
cp -a static/. "$workspace/"
git init -b "$ref" "$workspace"
git -C "$workspace" config user.name 'github-actions[bot]'
git -C "$workspace" config user.email 'github-actions[bot]@users.noreply.github.com'
git -C "$workspace" add -f .
git -C "$workspace" commit --allow-empty -m "Build static from $(git rev-parse HEAD)"
refs=("refs/heads/$ref:refs/heads/$ref")
if [[ "$ref_type" == 'tag' ]]; then
# Keep version branches for existing archive URLs and publish real Git tags.
git -C "$workspace" tag "$ref"
refs+=("refs/tags/$ref:refs/tags/$ref")
fi
git -C "$workspace" push --force --quiet "$destination" "${refs[@]}"
-20
View File
@@ -1,20 +0,0 @@
#!/usr/bin/env bash
# Static branches are orphan snapshots. Fetch complete commits for the branch
# that was built: a shallow/default-branch clone is rejected by mirror servers.
set -euo pipefail
destination=$1
branch=$2
ref_type=${3:-branch}
git check-ref-format "refs/heads/$branch"
workspace=$(mktemp -d)
trap 'rm -rf "$workspace"' EXIT
git init --bare "$workspace"
fetch_refs=("+refs/heads/$branch:refs/heads/$branch")
push_refs=("refs/heads/$branch:refs/heads/$branch")
if [[ "$ref_type" == 'tag' ]]; then
fetch_refs+=("+refs/tags/$branch:refs/tags/$branch")
push_refs+=("refs/tags/$branch:refs/tags/$branch")
fi
git -C "$workspace" fetch --no-tags "${QL_STATIC_SOURCE:-https://github.com/whyour/qinglong-static.git}" \
"${fetch_refs[@]}"
git -C "$workspace" push --force "$destination" "${push_refs[@]}"
-214
View File
@@ -1,214 +0,0 @@
#!/usr/bin/env bash
# Observe Pull mirrors; optionally notify them through a small GitHub push.
set -euo pipefail
if [[ $# -lt 3 || $# -gt 4 ]]; then
echo 'Usage: wait-for-mirror.sh SOURCE MIRROR REF [branch|tag]' >&2
exit 2
fi
source=$1
mirror=$2
ref=$3
ref_type=${4:-branch}
wait_seconds=${QL_MIRROR_WAIT_SECONDS:-600}
poll_interval=${QL_MIRROR_POLL_INTERVAL:-30}
query_timeout=${QL_MIRROR_QUERY_TIMEOUT:-20}
include_version_branch=${QL_MIRROR_INCLUDE_VERSION_BRANCH:-false}
notify_repo=${QL_MIRROR_NOTIFY_REPO:-}
retry_after=${QL_MIRROR_RETRY_AFTER_SECONDS:-310}
notify_on_start=${QL_MIRROR_NOTIFY_ON_START:-true}
timings=("$wait_seconds" "$poll_interval" "$query_timeout")
if [[ -n "$notify_repo" ]]; then timings+=("$retry_after"); fi
for value in "${timings[@]}"; do
if [[ ! "$value" =~ ^[1-9][0-9]{0,4}$ ]] || (( value > 86400 )); then
echo 'Mirror timing values must be integers between 1 and 86400 seconds' >&2
exit 2
fi
done
case "$ref_type" in
branch) refs=("refs/heads/$ref") ;;
tag)
refs=("refs/tags/$ref")
if [[ "$include_version_branch" == true ]]; then
refs+=("refs/heads/$ref")
fi
;;
*) echo 'Mirror ref type must be branch or tag' >&2; exit 2 ;;
esac
if [[ "$include_version_branch" != true && "$include_version_branch" != false ]]; then
echo 'QL_MIRROR_INCLUDE_VERSION_BRANCH must be true or false' >&2
exit 2
fi
if [[ "$notify_on_start" != true && "$notify_on_start" != false ]]; then
echo 'QL_MIRROR_NOTIFY_ON_START must be true or false' >&2
exit 2
fi
for requested_ref in "${refs[@]}"; do
git check-ref-format "$requested_ref"
done
command -v timeout >/dev/null || { echo 'GNU timeout is required' >&2; exit 2; }
if [[ -n "$notify_repo" ]]; then
if [[ ! "$notify_repo" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]; then
echo 'QL_MIRROR_NOTIFY_REPO must be a GitHub owner/repository' >&2
exit 2
fi
command -v gh >/dev/null || { echo 'GitHub CLI is required for notifications' >&2; exit 2; }
fi
export GIT_TERMINAL_PROMPT=0
polling=false
notification_command() {
local budget=$((notification_deadline - SECONDS))
if (( budget <= 0 )); then return 1; fi
if (( budget > 10 )); then budget=10; fi
timeout --signal=KILL "${budget}s" "$@"
}
notify_static_mirror() {
local notification_deadline=$((SECONDS + 30))
if [[ "$polling" == true ]] && (( notification_deadline > wait_seconds )); then
notification_deadline=$wait_seconds
fi
local trigger_ref=refs/heads/gitee-sync-trigger
local attempt remote_refs source_sha parent tree commit delay remaining
local commit_args
if ! remote_refs=$(notification_command git ls-remote --refs "$source" "${refs[0]}"); then
echo '::warning::Could not read the published source; continuing mirror verification'
return 1
fi
source_sha=${remote_refs%%$'\t'*}
if [[ ! "$source_sha" =~ ^[0-9a-f]{40}$ ]]; then
echo "::error::Published source ref ${refs[0]} does not exist"
return 2
fi
for attempt in 1 2 3; do
if (( attempt > 1 )); then
delay=$((attempt * 2 - 2))
remaining=$((notification_deadline - SECONDS))
if (( remaining <= delay )); then break; fi
sleep "$delay"
fi
# Refresh the parent on every attempt. REST avoids the Git push endpoint's
# transient 500s, and non-forced updates preserve concurrent notifications.
if parent=$(notification_command gh api "repos/$notify_repo/git/ref/heads/gitee-sync-trigger" --jq '.object.sha' 2>&1); then
if ! tree=$(notification_command gh api "repos/$notify_repo/git/commits/$parent" --jq '.tree.sha'); then
echo "::warning::Could not read notification history (attempt $attempt)"
continue
fi
elif [[ "$parent" == *'(HTTP 404)'* ]]; then
parent=''
# A new notification branch starts with one tiny marker, never artifacts.
# Later commits reuse this tree and have no changed files in the webhook.
if ! tree=$(printf '%s' '{"tree":[{"path":".gitee-sync-trigger","mode":"100644","type":"blob","content":"Gitee Pull mirror notifications.\n"}]}' |
notification_command gh api --method POST "repos/$notify_repo/git/trees" --input - --jq '.sha'); then
echo "::warning::Could not create notification tree (attempt $attempt)"
continue
fi
else
echo "::warning::Could not read notification ref (attempt $attempt)"
continue
fi
commit_args=(-f "tree=$tree" -f "message=Trigger Gitee mirror for ${refs[0]} at $source_sha")
if [[ -n "$parent" ]]; then commit_args+=(-f "parents[]=$parent"); fi
if ! commit=$(notification_command gh api --method POST "repos/$notify_repo/git/commits" "${commit_args[@]}" --jq '.sha'); then
echo "::warning::Could not create notification commit (attempt $attempt)"
continue
fi
if [[ -n "$parent" ]]; then
if notification_command gh api --method PATCH "repos/$notify_repo/git/refs/heads/gitee-sync-trigger" -f "sha=$commit" -F force=false --silent; then
echo '::notice::Sent a small GitHub push event via REST to trigger the Gitee Pull mirror'
return 0
fi
elif notification_command gh api --method POST "repos/$notify_repo/git/refs" -f "ref=$trigger_ref" -f "sha=$commit" --silent; then
echo '::notice::Sent a small GitHub push event via REST to trigger the Gitee Pull mirror'
return 0
fi
echo "::warning::Mirror notification update failed (attempt $attempt)"
done
echo '::warning::Could not send the mirror notification; continuing mirror verification'
return 1
}
if [[ -n "$notify_repo" && "$notify_on_start" == true ]]; then
if notify_static_mirror; then :; elif [[ $? -eq 2 ]]; then exit 1; fi
fi
SECONDS=0
polling=true
attempt=0
retried=false
last_source='(not read)'
last_mirror='(not read)'
while (( SECONDS < wait_seconds )); do
attempt=$((attempt + 1))
outputs=()
ready=true
# Re-read the source after the mirror so a moving source cannot pass with
# the previous SHA. --refs compares annotated tag objects as well as branches.
urls=("$source" "$mirror" "$source")
labels=('source' 'mirror' 'source confirmation')
for index in 0 1 2; do
remaining=$((wait_seconds - SECONDS))
if (( remaining <= 0 )); then
ready=false
break
fi
budget=$query_timeout
if (( budget > remaining )); then budget=$remaining; fi
if ! output=$(timeout --signal=KILL "${budget}s" git ls-remote --refs \
"${urls[$index]}" "${refs[@]}" | LC_ALL=C sort); then
echo "::warning::Could not read ${labels[$index]} refs on attempt $attempt; retrying"
ready=false
break
fi
if (( index == 0 )); then last_source=$output; fi
if (( index == 1 )); then last_mirror=$output; fi
for requested_ref in "${refs[@]}"; do
found=false
while IFS=$'\t' read -r sha returned_ref; do
if [[ "$returned_ref" == "$requested_ref" && "$sha" =~ ^[0-9a-f]{40}([0-9a-f]{24})?$ ]]; then
found=true
fi
done <<< "$output"
if [[ "$found" == false ]]; then
echo "::notice::${labels[$index]} is missing $requested_ref; waiting"
ready=false
break
fi
done
if [[ "$ready" == false ]]; then break; fi
outputs+=("$output")
# Only confirm a source when the initial source and mirror reads match.
if (( index == 1 )) && [[ "${outputs[0]}" != "${outputs[1]}" ]]; then
ready=false
break
fi
done
if [[ "$ready" == true && "${outputs[0]}" == "${outputs[2]}" ]]; then
echo "::notice::Mirror matches the latest source refs after ${SECONDS}s"
printf '%s\n' "${outputs[2]}"
exit 0
fi
echo "Waiting for mirror (attempt $attempt, ${SECONDS}/${wait_seconds}s)"
remaining=$((wait_seconds - SECONDS))
if (( remaining <= 0 )); then break; fi
# Gitee requires five minutes between sync requests. Retry once after that
# interval without restarting the polling deadline.
if [[ -n "$notify_repo" && "$retried" == false ]] && (( SECONDS >= retry_after )); then
echo '::notice::Retrying the GitHub notification after the Gitee sync interval'
if notify_static_mirror; then :; elif [[ $? -eq 2 ]]; then exit 1; fi
retried=true
continue
fi
delay=$poll_interval
if (( delay > remaining )); then delay=$remaining; fi
if [[ -n "$notify_repo" && "$retried" == false ]] && (( SECONDS < retry_after && delay > retry_after - SECONDS )); then
delay=$((retry_after - SECONDS))
fi
sleep "$delay"
done
echo "::error::Mirror did not match the latest source refs within ${wait_seconds}s"
printf 'Last source refs:\n%s\nLast mirror refs:\n%s\n' "$last_source" "$last_mirror"
exit 1
+70 -56
View File
@@ -26,28 +26,6 @@ get_token() {
create_token
}
# Check the cached token before every authenticated API request. A rejected
# token is refreshed and retried once; other failures are never retried.
# Arguments are curl options, including the URL, method and optional body.
ql_api_request() {
local api=""
local attempt
get_token || return $?
for attempt in 1 2; do
api=$(curl -s --noproxy "*" \
-H "Authorization: Bearer ${__ql_token__}" \
-H "Content-Type: application/json;charset=UTF-8" \
"$@" \
--compressed)
if [[ "$attempt" == 2 || "$api" == '{"code":200}' ]] ||
[[ "$(jq -r '.code' <<< "$api" 2>/dev/null)" != 401 ]]; then
break
fi
create_token || return $?
done
printf '%s' "$api"
}
# Read the status response once; preserve multiline error messages and the
# legacy code/message variables used by the shell API callers.
ql_parse_status_response() {
@@ -85,10 +63,13 @@ add_cron_api() {
sub_id="null"
fi
local api=$(ql_api_request \
"http://localhost:${ql_port}/open/crons?t=$currentTimeStamp" \
--data-raw "{\"name\":\"${name//\"/\\\"}\",\"command\":\"${command//\"/\\\"}\",\"schedule\":\"$schedule\",\"sub_id\":$sub_id}" \
-X POST)
local api=$(
curl -s --noproxy "*" "http://localhost:${ql_port}/open/crons?t=$currentTimeStamp" \
-H "Authorization: Bearer ${__ql_token__}" \
-H "Content-Type: application/json;charset=UTF-8" \
--data-raw "{\"name\":\"${name//\"/\\\"}\",\"command\":\"${command//\"/\\\"}\",\"schedule\":\"$schedule\",\"sub_id\":$sub_id}" \
--compressed
)
code=$(echo "$api" | jq -r .code)
message=$(echo "$api" | jq -r .message)
if [[ $code == 200 ]]; then
@@ -112,10 +93,14 @@ update_cron_api() {
local id="$4"
fi
local api=$(ql_api_request \
"http://localhost:${ql_port}/open/crons?t=$currentTimeStamp" \
-X PUT \
--data-raw "{\"name\":\"${name//\"/\\\"}\",\"command\":\"${command//\"/\\\"}\",\"schedule\":\"$schedule\",\"id\":\"$id\"}")
local api=$(
curl -s --noproxy "*" "http://localhost:${ql_port}/open/crons?t=$currentTimeStamp" \
-X 'PUT' \
-H "Authorization: Bearer ${__ql_token__}" \
-H "Content-Type: application/json;charset=UTF-8" \
--data-raw "{\"name\":\"${name//\"/\\\"}\",\"command\":\"${command//\"/\\\"}\",\"schedule\":\"$schedule\",\"id\":\"$id\"}" \
--compressed
)
code=$(echo "$api" | jq -r .code)
message=$(echo "$api" | jq -r .message)
if [[ $code == 200 ]]; then
@@ -135,10 +120,14 @@ update_cron_command_api() {
local id="$2"
fi
local api=$(ql_api_request \
"http://localhost:${ql_port}/open/crons?t=$currentTimeStamp" \
-X PUT \
--data-raw "{\"command\":\"${command//\"/\\\"}\",\"id\":\"$id\"}")
local api=$(
curl -s --noproxy "*" "http://localhost:${ql_port}/open/crons?t=$currentTimeStamp" \
-X 'PUT' \
-H "Authorization: Bearer ${__ql_token__}" \
-H "Content-Type: application/json;charset=UTF-8" \
--data-raw "{\"command\":\"${command//\"/\\\"}\",\"id\":\"$id\"}" \
--compressed
)
code=$(echo "$api" | jq -r .code)
message=$(echo "$api" | jq -r .message)
if [[ $code == 200 ]]; then
@@ -151,10 +140,14 @@ update_cron_command_api() {
del_cron_api() {
local ids="$1"
local currentTimeStamp=$(date +%s)
local api=$(ql_api_request \
"http://localhost:${ql_port}/open/crons?t=$currentTimeStamp" \
-X DELETE \
--data-raw "[$ids]")
local api=$(
curl -s --noproxy "*" "http://localhost:${ql_port}/open/crons?t=$currentTimeStamp" \
-X 'DELETE' \
-H "Authorization: Bearer ${__ql_token__}" \
-H "Content-Type: application/json;charset=UTF-8" \
--data-raw "[$ids]" \
--compressed
)
code=$(echo "$api" | jq -r .code)
message=$(echo "$api" | jq -r .message)
if [[ $code == 200 ]]; then
@@ -178,10 +171,14 @@ update_cron() {
dataRaw="${dataRaw},\"exit_code\":$exitCode"
fi
dataRaw="${dataRaw}}"
local api=$(ql_api_request \
"http://localhost:${ql_port}/open/crons/status?t=$currentTimeStamp" \
-X PUT \
--data-raw "$dataRaw")
local api=$(
curl -s --noproxy "*" "http://localhost:${ql_port}/open/crons/status?t=$currentTimeStamp" \
-X 'PUT' \
-H "Authorization: Bearer ${__ql_token__}" \
-H "Content-Type: application/json;charset=UTF-8" \
--data-raw "$dataRaw" \
--compressed
)
ql_parse_status_response "$api" || true
if [[ $code != 200 ]]; then
if [[ ! $message ]]; then
@@ -195,10 +192,14 @@ notify_api() {
local title="$1"
local content="$2"
local currentTimeStamp=$(date +%s)
local api=$(ql_api_request \
"http://localhost:${ql_port}/open/system/notify?t=$currentTimeStamp" \
-X PUT \
--data-raw "{\"title\":\"${title//\"/\\\"}\",\"content\":\"${content//\"/\\\"}\"}")
local api=$(
curl -s --noproxy "*" "http://localhost:${ql_port}/open/system/notify?t=$currentTimeStamp" \
-X 'PUT' \
-H "Authorization: Bearer ${__ql_token__}" \
-H "Content-Type: application/json;charset=UTF-8" \
--data-raw "{\"title\":\"${title//\"/\\\"}\",\"content\":\"${content//\"/\\\"}\"}" \
--compressed
)
code=$(echo "$api" | jq -r .code)
message=$(echo "$api" | jq -r .message)
if [[ $code == 200 ]]; then
@@ -211,9 +212,12 @@ notify_api() {
find_cron_api() {
local params="$1"
local currentTimeStamp=$(date +%s)
local api=$(ql_api_request \
"http://localhost:${ql_port}/open/crons/detail?$params&t=$currentTimeStamp" \
-X GET)
local api=$(
curl -s --noproxy "*" "http://localhost:${ql_port}/open/crons/detail?$params&t=$currentTimeStamp" \
-H "Authorization: Bearer ${__ql_token__}" \
-H "Content-Type: application/json;charset=UTF-8" \
--compressed
)
data=$(echo "$api" | jq -r .data)
if [[ $data == 'null' ]]; then
echo -e ""
@@ -227,10 +231,14 @@ update_auth_config() {
local body="$1"
local tip="$2"
local currentTimeStamp=$(date +%s)
local api=$(ql_api_request \
"http://localhost:${ql_port}/open/system/auth/reset?t=$currentTimeStamp" \
-X PUT \
--data-raw "{$body}")
local api=$(
curl -s --noproxy "*" "http://localhost:${ql_port}/open/system/auth/reset?t=$currentTimeStamp" \
-X 'PUT' \
-H "Authorization: Bearer ${__ql_token__}" \
-H "Content-Type: application/json;charset=UTF-8" \
--data-raw "{$body}" \
--compressed
)
code=$(echo "$api" | jq -r .code)
message=$(echo "$api" | jq -r .message)
if [[ $code == 200 ]]; then
@@ -246,10 +254,14 @@ record_cron_stat() {
local elapsed="${3:-0}"
[[ $ref_id ]] && [[ $ref_id -gt 0 ]] 2>/dev/null || return
local api=$(ql_api_request \
"http://localhost:${ql_port:-5700}/open/dashboard/record" \
local api=$(
curl -s --noproxy "*" "http://localhost:${ql_port:-5700}/open/dashboard/record" \
-X POST \
--data-raw "{\"ref_id\":$ref_id,\"code\":$exit_code,\"elapsed\":$elapsed}")
-H "Authorization: Bearer ${__ql_token__}" \
-H "Content-Type: application/json;charset=UTF-8" \
--data-raw "{\"ref_id\":$ref_id,\"code\":$exit_code,\"elapsed\":$elapsed}" \
--compressed
)
ql_parse_status_response "$api" || true
if [[ $code != 200 ]]; then
if [[ ! $message ]]; then
@@ -258,3 +270,5 @@ record_cron_stat() {
echo -e "${message}"
fi
}
get_token
+2 -10
View File
@@ -65,9 +65,7 @@ ql_read_node_path_cache() {
# the caller's descriptors. A crashed refresher releases the kernel lock.
ql_refresh_node_global_path() (
local cache="$1" key="$2" now result previous_umask
# Keep one persistent lock per owner. Removing per-key lock files could
# split concurrent waiters across different inodes during cache cleanup.
local lock="${dir_tmp}/pnpm-root-${EUID}.lock"
local lock="${cache}.lock"
previous_umask=$(umask)
umask 077
if type -P flock &>/dev/null && mkdir -p -- "$dir_tmp" 2>/dev/null; then
@@ -95,10 +93,6 @@ ql_refresh_node_global_path() (
# Private lock creation must not change pnpm's inherited creation mask.
umask "$previous_umask"
# Warm lookups never scan the directory. Retire old environment records on
# refresh, while preserving the shared lock and unrelated cache files.
find "$dir_tmp" -maxdepth 1 -type f -user "$EUID" \
-name "pnpm-root-${EUID}-*.cache" -mmin +60 -delete 2>/dev/null || true
now=${EPOCHSECONDS:-$(date +%s)}
result=$(pnpm root -g 9>&- 2>/dev/null) || return $?
# Never cache failed, empty, multiline or non-absolute answers.
@@ -125,10 +119,8 @@ ql_get_node_global_path() {
fi
local key cache
cache="${dir_tmp}/pnpm-root-${EUID}.cache"
key=$(ql_node_path_cache_key) || { pnpm root -g 2>/dev/null; return $?; }
# Node and crond may have different working directories/configuration.
# Preserve both discoveries instead of evicting each other on every run.
cache="${dir_tmp}/pnpm-root-${EUID}-${key// /-}.cache"
if ql_read_node_path_cache "$cache" "$key"; then
return 0
fi
+9 -7
View File
@@ -84,9 +84,7 @@ env_str_to_array() {
}
clear_non_sh_env() {
# Only automatic Node/Python preloads reconstruct the generated environment.
# Explicit interpreters and other commands must keep the shell-loaded values.
if [[ $isJsOrPythonFile == 'true' ]]; then
if [[ $file_param != *.sh ]]; then
clear_env
fi
}
@@ -164,7 +162,7 @@ run_normal() {
if [[ $isJsOrPythonFile == 'false' ]]; then
clear_non_sh_env
fi
run_task_command $which_program $file_param "${script_params[@]}"
$timeoutCmd $which_program $file_param "${script_params[@]}"
}
handle_env_split() {
@@ -203,7 +201,7 @@ run_concurrent() {
export "${env_param}=${array[$i - 1]}"
clear_non_sh_env
fi
eval envParam="${env_param}" numParam="${i}" run_task_command $which_program $file_param "${script_params[@]}" &>$single_log_path &
eval envParam="${env_param}" numParam="${i}" $timeoutCmd $which_program $file_param "${script_params[@]}" &>$single_log_path &
done
wait
@@ -243,7 +241,7 @@ run_designated() {
enter_script_workdir
envParam="${env_param}" numParam="${num_param}" run_task_command $which_program $file_param "${script_params[@]}"
envParam="${env_param}" numParam="${num_param}" $timeoutCmd $which_program $file_param "${script_params[@]}"
}
## 运行其他命令
@@ -299,7 +297,7 @@ run_else() {
fi
clear_non_sh_env
run_task_command $which_program $file_param "$@"
$timeoutCmd $which_program $file_param "$@"
}
check_file() {
@@ -336,6 +334,10 @@ check_nounset() {
main() {
if [[ $1 == *.js ]] || [[ $1 == *.mjs ]] || [[ $1 == *.py ]] || [[ $1 == *.pyc ]] || [[ $1 == *.sh ]] || [[ $1 == *.ts ]]; then
if [[ $1 == *.sh ]]; then
timeoutCmd=""
fi
case $# in
1)
run_normal "$1"
+2 -6
View File
@@ -159,13 +159,9 @@ try {
run();
const { sendNotify } = require('./__ql_notify__.js');
global.QLAPI = {
notify(...args) {
// Keep notification dependencies out of ordinary script startup.
// require caches the module after the first notification.
const { sendNotify } = require('./__ql_notify__.js');
return sendNotify.apply(this, args);
},
notify: sendNotify,
...client,
};
} catch (error) {
+2 -4
View File
@@ -129,12 +129,10 @@ try:
run()
from __ql_notify__ import send
class BaseApi(Client):
def notify(self, *args, **kwargs):
# Most scripts never send a notification. Python caches this
# module after the first call, including its channel dependencies.
from __ql_notify__ import send
return send(*args, **kwargs)
QLAPI = BaseApi()
-63
View File
@@ -1,63 +0,0 @@
#!/usr/bin/env bash
# Fork from the current Bash state: an external `bash -c` would lose private
# before-hook variables, arrays and functions used by sourced Shell scripts.
run_task_command() {
local timeout_seconds
if [[ -z ${command_timeout_time:-} ]]; then
"$@"
return $?
fi
timeout_seconds=$(awk -v value="$command_timeout_time" 'BEGIN {
if (value !~ /^([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[smhd]?$/) exit 1
unit = substr(value, length(value), 1)
factor = unit == "d" ? 86400 : unit == "h" ? 3600 : unit == "m" ? 60 : 1
printf "%.9f", (value + 0) * factor
}') || { printf 'Invalid task timeout: %s\n' "$command_timeout_time" >&2; return 125; }
if [[ $timeout_seconds == '0.000000000' ]]; then
"$@"
return $?
fi
(
timeout_dir=$(mktemp -d "${TMPDIR:-/tmp}/ql-task-timeout.XXXXXXXX") || exit 125
# Both children get private process groups. The timer and the wrapper must
# survive terminating the task, and descendants must not outlive a timeout.
set -m
( set +m; "$@" ) <&0 &
timeout_worker=$!
(
set +m
: > "$timeout_dir/ready"
sleep "$timeout_seconds" || exit 125
: > "$timeout_dir/expired"
kill -TERM -- "-$timeout_worker" 2>/dev/null || :
sleep 0.5
kill -KILL -- "-$timeout_worker" 2>/dev/null || :
) &
timeout_watcher=$!
set +m
trap 'kill -KILL -- "-$timeout_watcher" 2>/dev/null || :
if kill -TERM -- "-$timeout_worker" 2>/dev/null; then sleep 0.1; fi
kill -KILL -- "-$timeout_worker" 2>/dev/null || :
wait "$timeout_worker" 2>/dev/null || :
wait "$timeout_watcher" 2>/dev/null || :
rm -rf -- "$timeout_dir"' EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
trap 'exit 129' HUP
trap 'exit 131' QUIT
# Do not clean up a timer whose process group has not initialized yet.
while [[ ! -f "$timeout_dir/ready" ]]; do
kill -0 "$timeout_watcher" 2>/dev/null || exit 125
sleep 0.001
done
wait "$timeout_worker"
timeout_code=$?
if [[ -f "$timeout_dir/expired" ]]; then
# Let the timer finish escalation even if the task leader exited first.
wait "$timeout_watcher" 2>/dev/null || :
exit 124
fi
exit "$timeout_code"
)
}
+6 -8
View File
@@ -3,7 +3,6 @@
dir_shell=$QL_DIR/shell
. $dir_shell/share.sh
. $dir_shell/api.sh
. $dir_shell/task-timeout.sh
trap 'single_hanle SIGINT' INT
trap 'single_hanle SIGTERM' TERM
@@ -19,12 +18,6 @@ single_hanle() {
exit 1
}
run_shell_script() {
# Preserve inherited variables/functions, but contain exit, exec and shell
# options so the task wrapper can run after hooks and report completion.
( . "$@" )
}
define_program() {
local file_param=$1
if [[ $file_param == *.js ]] || [[ $file_param == *.mjs ]]; then
@@ -32,7 +25,7 @@ define_program() {
elif [[ $file_param == *.py ]] || [[ $file_param == *.pyc ]]; then
which_program="python3"
elif [[ $file_param == *.sh ]]; then
which_program="run_shell_script"
which_program="."
elif [[ $file_param == *.ts ]]; then
which_program="ts-node-transpile-only"
else
@@ -113,6 +106,11 @@ format_params() {
mtime_format="%Y-%m-%d %H:%M:%S.%3N"
fi
timeoutCmd=""
if [[ $command_timeout_time ]]; then
if type timeout &>/dev/null; then
timeoutCmd="timeout --foreground -s 2 -k 10s $command_timeout_time "
fi
fi
# params=$(echo "$@" | sed -E 's/([^ ])&([^ ])/\1\\\&\2/g')
# 分割 task 内置参数和脚本参数
+98 -43
View File
@@ -283,44 +283,98 @@ usage() {
}
reload_qinglong() {
local reload_target="${1}"
if [[ "$reload_target" == 'system' ]]; then
QlPort="${ql_port:-${QlPort:-5700}}" QlBaseUrl="${ql_base_url:-${QlBaseUrl:-/}}" node "$dir_shell/upgrade.cjs" reload
return $?
fi
# Data import and a plain service restart retain their existing behavior.
echo -e "[reload_qinglong] deleting Triggered at $(date)" >>"${dir_log}/reload.log"
echo -e "[reload_qinglong] deleting Triggered at $(date)" >>${dir_log}/reload.log
sleep 3
delete_pm2
if [[ "$reload_target" == 'data' ]]; then
rm -rf "${dir_data:?}"/*
mv -f "$dir_tmp"/data/* "$dir_data"/ || return $?
echo -e "[reload_qinglong] deleted Triggered at $(date)" >>${dir_log}/reload.log
local reload_target="${1}"
local primary_branch="master"
if [[ "${QL_BRANCH}" == "develop" ]] || [[ "${QL_BRANCH}" == "debian" ]] || [[ "${QL_BRANCH}" == "debian-dev" ]]; then
primary_branch="${QL_BRANCH}"
fi
if [[ "$reload_target" == 'system' ]]; then
rm -rf ${dir_root}/back ${dir_root}/cli ${dir_root}/docker ${dir_root}/sample ${dir_root}/shell ${dir_root}/src
mv -f ${dir_tmp}/qinglong-${primary_branch}/* ${dir_root}/
rm -rf $dir_static/*
mv -f ${dir_tmp}/qinglong-static-${primary_branch}/* ${dir_static}/
cp -f $file_config_sample $dir_config/config.sample.sh
fi
if [[ "$reload_target" == 'data' ]]; then
rm -rf ${dir_data}/*
mv -f ${dir_tmp}/data/* ${dir_data}/
fi
echo -e "[reload_qinglong] starting Triggered at $(date)" >>${dir_log}/reload.log
reload_pm2
echo -e "[reload_qinglong] started Triggered at $(date)\n" >>${dir_log}/reload.log
}
## 更新 qinglong
update_qinglong() {
rm -rf ${dir_tmp}/*
local mirror="gitee"
if curl --fail --silent --noproxy "*" --max-time 2 --head "https://github.com" >/dev/null; then
local downloadQLUrl="https://gitee.com/whyour/qinglong/repository/archive"
local downloadStaticUrl="https://gitee.com/whyour/qinglong-static/repository/archive"
local githubStatus=$(curl -s --noproxy "*" -m 2 -IL "https://google.com" | grep 200)
if [[ ! -z $githubStatus ]]; then
mirror="github"
downloadQLUrl="https://github.com/whyour/qinglong/archive/refs/heads"
downloadStaticUrl="https://github.com/whyour/qinglong-static/archive/refs/heads"
fi
t '使用 %s 源更新...\n' "${mirror}"
local primary_branch="master"
if [[ "${QL_BRANCH}" == "develop" ]] || [[ "${QL_BRANCH}" == "debian" ]] || [[ "${QL_BRANCH}" == "debian-dev" ]]; then
primary_branch="${QL_BRANCH}"
fi
wget -cqO "${dir_tmp}/ql.zip" "${downloadQLUrl}/${primary_branch}.zip"
exit_status=$?
if [[ $exit_status -eq 0 ]]; then
t '更新青龙源文件成功...\n'
unzip -oq ${dir_tmp}/ql.zip -d ${dir_tmp}
update_qinglong_static
else
t '更新青龙源文件失败,请检查网络...\n'
fi
t '使用 %s 源更新...\n' "$mirror"
QlPort="${ql_port:-${QlPort:-5700}}" QlBaseUrl="${ql_base_url:-${QlBaseUrl:-/}}" node "$dir_shell/upgrade.cjs" update "$mirror" "$needRestart"
}
# Preserve the updater's status through both tee and the final timing output.
run_update_logged() (
set -o pipefail
if [[ "$real_time" == 'true' ]]; then
"$@"
elif [[ "$no_tee" == 'true' ]]; then
"$@" >>"$file_path" 2>&1
update_qinglong_static() {
wget -cqO "${dir_tmp}/static.zip" "${downloadStaticUrl}/${primary_branch}.zip"
exit_status=$?
if [[ $exit_status -eq 0 ]]; then
t '更新青龙静态资源成功...\n'
unzip -oq ${dir_tmp}/static.zip -d ${dir_tmp}
check_update_dep
else
"$@" 2>&1 | tee -a "$file_path"
t '更新青龙静态资源失败,请检查网络...\n'
fi
)
}
check_update_dep() {
t '\n开始检测依赖...\n'
if [[ $(diff $dir_root/package.json ${dir_tmp}/qinglong-${primary_branch}/package.json) ]]; then
npm_install_2 "${dir_tmp}/qinglong-${primary_branch}"
fi
if [[ $exit_status -eq 0 ]]; then
t '\n依赖检测安装成功...\n'
t '更新包下载成功...\n'
if [[ "$needRestart" == 'true' ]]; then
reload_qinglong "system"
fi
else
t '\n依赖检测安装失败,请检查网络...\n'
fi
}
## 对比脚本
diff_scripts() {
@@ -375,19 +429,23 @@ gen_list_repo() {
cd ${repo_path}
local extension_string="${6:-$file_extensions}"
extension_string="${extension_string//|/ }"
extension_string="${extension_string//$'\n'/ }"
local -a extensions=() find_args=()
IFS=$' \t\n' read -r -a extensions <<< "$extension_string"
local extension
for extension in "${extensions[@]}"; do
if [[ ${#find_args[@]} -gt 0 ]]; then
find_args+=(-o)
local cmd="find ."
local index=0
if [[ $6 ]]; then
file_extensions="$6"
if [[ $file_extensions =~ "|" ]]; then
file_extensions=$(echo $file_extensions | sed 's/|/ /g')
fi
find_args+=(-name "*.${extension}")
fi
for extension in $file_extensions; do
if [[ $index -eq 0 ]]; then
cmd="${cmd} -name \"*.${extension}\""
else
cmd="${cmd} -o -name \"*.${extension}\""
fi
let index+=1
done
files=$(find . "${find_args[@]}" | sed 's/^..//')
files=$(eval $cmd | sed 's/^..//')
if [[ $path ]]; then
files=$(echo "$files" | egrep "$path")
fi
@@ -400,7 +458,7 @@ gen_list_repo() {
if [[ $dependence ]]; then
cd ${repo_path}
results=$(find . "${find_args[@]}" | sed 's/^..//' | egrep "$dependence")
results=$(eval $cmd | sed 's/^..//' | egrep "$dependence")
for _file in ${results}; do
file_path=$(dirname $_file)
make_dir "${dir_scripts}/${uniq_path}/${file_path}"
@@ -471,7 +529,7 @@ main() {
if [[ "$p1" == "log" ]]; then
show_service_logs "${@:2}"
return 0
return $?
fi
local log_dir="${p1}"
@@ -500,17 +558,14 @@ main() {
[[ $ID ]] && update_cron "\"$ID\"" "0" "$$" "$log_path" "$begin_timestamp"
local command_status=0
case $p1 in
update)
fix_config
local needRestart=${p2:-"true"}
run_update_logged update_qinglong
command_status=$?
eval update_qinglong $cmd
;;
reload)
run_update_logged reload_qinglong "$p2"
command_status=$?
eval reload_qinglong "$p2" $cmd
;;
extra)
eval run_extra_shell $cmd
@@ -569,9 +624,9 @@ main() {
if [[ "$p1" != "repo" ]] && [[ "$p1" != "raw" ]]; then
eval echo -e "\\\n\#\# 执行结束... $end_time 耗时 $diff_time 秒     " $cmd
fi
return "$command_status"
}
import_config "$@"
main "$@"
exit $?
exit 0
-8
View File
@@ -1,8 +0,0 @@
// The image updater and local CLI share one implementation and build contract.
const upgrade = require('../cli/src/internal/maintenance/upgradeArtifacts.cjs');
module.exports = upgrade;
if (require.main === module)
upgrade.main().catch((error) => {
console.error(`更新失败: ${error.message}`);
process.exitCode = 1;
});
-6
View File
@@ -19,9 +19,6 @@
"Minimum is 4": "Minimum is 4",
"NPM 镜像源": "NPM Mirror Source",
"Node 软件包镜像源": "Node Software Package Mirror Source",
"OpeniLink Hub地址,默认为 https://hub.openilink.com,自建Hub时填写自己的地址": "OpeniLink Hub URL, defaults to https://hub.openilink.com. For a self-hosted Hub, enter your own URL.",
"OpeniLink的app_token,在OpeniLink Hub后台安装App后获取,参考 https://openilink.com/docs/hub/apps": "OpeniLink app_token. Install an app in the OpeniLink Hub dashboard to obtain it. See https://openilink.com/docs/hub/apps",
"OpeniLink的context_token,用于标识消息会话上下文,可从消息事件中获取": "OpeniLink context_token identifies the message conversation context and can be obtained from message events.",
"PushDeer的Key,https://github.com/easychen/pushdeer": "PushDeer Key, https://github.com/easychen/pushdeer",
"PushDeer的自架API endpoint,默认是 https://api2.pushdeer.com/message/push": "PushDeer's self-hosted API endpoint, default is https://api2.pushdeer.com/message/push",
"PushMe的Key,https://push.i-i.me/": "PushMe key, https://push.i-i.me/",
@@ -38,7 +35,6 @@
"alpine linux 镜像源": "Alpine Linux Mirror Source",
"chat的token码": "Chat token code",
"corpid、corpsecret、touser(注:多个成员ID使用|隔开)、agentid、消息类型(选填,不填默认文本消息类型) 注意用,号隔开(英文输入法的逗号),例如:wwcfrs,B-76WERQ,qinglong,1000001,2COat": "corpid, corpsecret, touser (note: separate multiple member IDs with |), agentid, message type (optional, defaults to text message type) – separated by commas (`,`), e.g., wwcfrs, B-76WERQ, qinglong, 1000001, 2COat",
"debian linux 镜像源": "Debian Linux Mirror Source",
"docker安装在持久化config目录下的chronocat.yml文件可找到": "The docker installation can be found in the persistence config directory in the chronocat.yml file",
"gotify的url地址,例如 https://push.example.de:8080": "gotify URL address, e.g., https://push.example.de:8080",
"gotify的消息应用token码": "gotify message application token code",
@@ -355,7 +351,6 @@
"查看日志": "View Log",
"标签": "Tags",
"标签统计": "Label Stats",
"核心": "cores",
"检查更新": "Check for Updates",
"检查更新中...": "Checking for updates...",
"次数": "Runs",
@@ -589,7 +584,6 @@
"邮箱": "Email",
"邮箱地址": "Email Address",
"邮箱服务名称,比如126、163、Gmail、QQ等,支持列表https://github.com/nodemailer/nodemailer/blob/master/lib/well-known/services.json": "Email service name, e.g., 126, 163, Gmail, QQ, etc. Supported list: https://github.com/nodemailer/nodemailer/blob/master/lib/well-known/services.json",
"邮箱认证地址": "Email address for SMTP authentication",
"配置文件": "Configuration Files",
"重启": "Restart",
"重命名": "Rename",
-6
View File
@@ -18,9 +18,6 @@
"Linux 软件包镜像源": "Linux 软件包镜像源",
"NPM 镜像源": "NPM 镜像源",
"Node 软件包镜像源": "Node 软件包镜像源",
"OpeniLink Hub地址,默认为 https://hub.openilink.com,自建Hub时填写自己的地址": "OpeniLink Hub地址,默认为 https://hub.openilink.com,自建Hub时填写自己的地址",
"OpeniLink的app_token,在OpeniLink Hub后台安装App后获取,参考 https://openilink.com/docs/hub/apps": "OpeniLink的app_token,在OpeniLink Hub后台安装App后获取,参考 https://openilink.com/docs/hub/apps",
"OpeniLink的context_token,用于标识消息会话上下文,可从消息事件中获取": "OpeniLink的context_token,用于标识消息会话上下文,可从消息事件中获取",
"PushDeer的Key,https://github.com/easychen/pushdeer": "PushDeer的Key,https://github.com/easychen/pushdeer",
"PushDeer的自架API endpoint,默认是 https://api2.pushdeer.com/message/push": "PushDeer的自架API endpoint,默认是 https://api2.pushdeer.com/message/push",
"PushMe的Key,https://push.i-i.me/": "PushMe的Key,https://push.i-i.me/",
@@ -37,7 +34,6 @@
"alpine linux 镜像源": "alpine linux 镜像源",
"chat的token码": "chat的token码",
"corpid、corpsecret、touser(注:多个成员ID使用|隔开)、agentid、消息类型(选填,不填默认文本消息类型) 注意用,号隔开(英文输入法的逗号),例如:wwcfrs,B-76WERQ,qinglong,1000001,2COat": "corpid,corpsecret,touser(注:多个成员ID使用|隔开),agentid,消息类型(选填,不填默认文本消息类型) 注意用,号隔开(英文输入法的逗号),例如:wwcfrs,B-76WERQ,qinglong,1000001,2COat",
"debian linux 镜像源": "debian linux 镜像源",
"docker安装在持久化config目录下的chronocat.yml文件可找到": "docker安装在持久化config目录下的chronocat.yml文件可找到",
"gotify的url地址,例如 https://push.example.de:8080": "gotify的url地址,例如 https://push.example.de:8080",
"gotify的消息应用token码": "gotify的消息应用token码",
@@ -355,7 +351,6 @@
"查看日志": "查看日志",
"标签": "标签",
"标签统计": "标签统计",
"核心": "核心",
"检查更新": "检查更新",
"检查更新中...": "检查更新中...",
"次数": "次数",
@@ -589,7 +584,6 @@
"邮箱": "邮箱",
"邮箱地址": "邮箱地址",
"邮箱服务名称,比如126、163、Gmail、QQ等,支持列表https://github.com/nodemailer/nodemailer/blob/master/lib/well-known/services.json": "邮箱服务名称,比如126、163、Gmail、QQ等,支持列表https://github.com/nodemailer/nodemailer/blob/master/lib/well-known/services.json",
"邮箱认证地址": "邮箱认证地址",
"配置文件": "配置文件",
"重启": "重启",
"重命名": "重命名",
+17 -24
View File
@@ -25,16 +25,13 @@ const Diff = () => {
const [language, setLanguage] = useState<string>('shell');
const getConfig = () => {
const url = current.startsWith('data/scripts/')
? `${config.apiPrefix}scripts/detail?file=${encodeURIComponent(
current.slice('data/scripts/'.length),
)}`
: `${config.apiPrefix}configs/detail?path=${encodeURIComponent(current)}`;
request.get(url).then(({ code, data }) => {
if (code === 200) {
setCurrentValue(data);
}
});
request
.get(`${config.apiPrefix}configs/detail?path=${encodeURIComponent(current)}`)
.then(({ code, data }) => {
if (code === 200) {
setCurrentValue(data);
}
});
};
const getSample = () => {
@@ -52,20 +49,16 @@ const Diff = () => {
? editorRef.current.getModel().modified.getValue().replace(/\r\n/g, '\n')
: currentValue;
const saving = current.startsWith('data/scripts/')
? request.post(`${config.apiPrefix}scripts`, {
content,
filename: current.slice('data/scripts/'.length),
})
: request.post(`${config.apiPrefix}configs/save`, {
content,
name: current,
});
saving.then(({ code }) => {
if (code === 200) {
message.success(intl.get('保存成功'));
}
});
request
.post(`${config.apiPrefix}configs/save`, {
content,
name: current,
})
.then(({ code, data }) => {
if (code === 200) {
message.success(intl.get('保存成功'));
}
});
};
const getFiles = () => {
+1 -14
View File
@@ -3,10 +3,6 @@
align-items: center;
justify-content: space-between;
gap: 16px;
:global(.ant-typography) {
min-width: 0;
overflow-wrap: anywhere;
}
:global(.ant-btn) {
flex-shrink: 0;
}
@@ -25,16 +21,7 @@
}
@media (max-width: 600px) {
.footer {
flex-direction: column;
align-items: stretch;
gap: 8px;
:global(.ant-btn) {
width: 100%;
height: auto;
min-height: 32px;
white-space: normal;
overflow-wrap: anywhere;
}
align-items: flex-end;
}
.labels {
gap: 8px;
-416
View File
@@ -1,416 +0,0 @@
const assert = require('node:assert/strict');
const test = require('node:test');
const fs = require('node:fs');
const path = require('node:path');
const os = require('node:os');
const vm = require('node:vm');
const { execFileSync } = require('node:child_process');
const load = require('../helpers/load-security-module.cjs');
const decorators = { Service: () => (x) => x, Inject: () => () => {} };
const logger = { info() {}, warn() {}, error() {} };
const translate = { t: (x) => x, tf: (x) => x };
function temporary(t) {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-advisory-'));
t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
return dir;
}
test('environment values remain literal when the generated JavaScript loads', async () => {
const output = {};
const Env = load(path.resolve('back/services/env.ts'), {
typedi: decorators,
'../config': { envFile: 'sh', jsEnvFile: 'js', pyEnvFile: 'py' },
'../data/env': { EnvStatus: { normal: 0 } },
'../data': {},
'../shared/utils': {
writeFileWithLock: async (file, text) => (output[file] = text),
},
}).default;
const service = new Env(logger);
const values = [
'${globalThis.injected = true}',
'quotes\'"` \\ newline\n中文\u2028',
'${1 + 2}',
];
service.envs = async () =>
values.map((value, i) => ({ name: `VALUE_${i}`, value }));
await service.set_envs();
const context = { process: { env: {} } };
vm.runInNewContext(output.js, context);
assert.equal(context.injected, undefined);
values.forEach((value, i) =>
assert.equal(context.process.env[`VALUE_${i}`], value),
);
});
test('subscription shell parameters round trip without expansion or command substitution', () => {
const { formatCommand } = load(path.resolve('back/config/subscription.ts'));
const payload =
'a $(printf INJECTED) `printf INJECTED` " ; echo BAD; # \' 中文\nnext';
for (const type of ['file', 'repo']) {
const fields =
type === 'file'
? ['url', 'proxy']
: [
'url',
'whitelist',
'blacklist',
'dependences',
'branch',
'extensions',
'proxy',
];
for (const field of fields) {
const doc = {
id: 12,
type,
url: 'https://example.invalid/repo',
[field]: payload,
};
const command = formatCommand(doc);
const stdout = execFileSync('/bin/bash', [
'-c',
`ql() { printf '%s\\0' "$@"; }; ${command}`,
]);
const args = stdout.toString().split('\0').slice(0, -1);
assert.equal(
args[fields.indexOf(field) + 1],
payload,
`${type}:${field}`,
);
assert.equal(args[0], type === 'file' ? 'raw' : 'repo');
assert.equal(args.length, fields.length + 3);
}
}
});
test('SSH alias traversal and symlinks cannot write or remove outside files', async (t) => {
const base = temporary(t);
const root = path.join(base, 'ssh');
fs.mkdirSync(root);
const outside = path.join(base, 'outside');
fs.writeFileSync(outside, 'keep');
fs.symlinkSync(outside, path.join(root, 'link'));
const Ssh = load(path.resolve('back/services/sshKey.ts'), {
typedi: decorators,
'../config': { sshdPath: root },
'../config/util': {
rmPath: (p) => fs.promises.rm(p, { recursive: true, force: true }),
},
'../shared/utils': {
writeFileWithLock: (p, value) => fs.promises.writeFile(p, value),
},
}).default;
const service = Object.assign(Object.create(Ssh.prototype), {
sshPath: root,
logger,
});
for (const alias of [
'../outside',
'..',
'.',
'/tmp/outside',
'link',
'x\nHost *',
]) {
await service.generatePrivateKeyFile(alias, 'overwrite');
await service.removePrivateKeyFile(alias);
await assert.rejects(service.generateSingleSshConfig(alias, 'github.com'));
assert.equal(fs.readFileSync(outside, 'utf8'), 'keep');
}
await service.generatePrivateKeyFile('仓库-main', 'private-key');
await service.generateSingleSshConfig(
'仓库-main',
'github.com',
'127.0.0.1:1080',
);
assert.equal(
fs.readFileSync(path.join(root, '仓库-main'), 'utf8').trim(),
'private-key',
);
await service.generatePrivateKeyFile('~global_default', 'global-key');
await service.generateGlobalSshConfig('~global_default');
assert.match(
fs.readFileSync(path.join(root, '~global_default.config'), 'utf8'),
/Host \*/,
);
await assert.rejects(
service.generateSingleSshConfig('safe', 'host\nProxyCommand bad'),
);
await assert.rejects(
service.generateSingleSshConfig('safe', 'host', '$(printf BAD)'),
);
});
function cronService(config, updates = []) {
const Cron = load(path.resolve('back/services/cron.ts'), {
typedi: decorators,
'../config': config,
'../data/cron': {
CrontabStatus: { running: 0, idle: 1, queued: 3 },
CrontabModel: { update: async (v) => updates.push(v) },
},
'../data/runningInstance': { RunningInstanceModel: {}, InstanceStatus: {} },
'../config/util': { fileExist: async (p) => fs.existsSync(p) },
'../config/const': {},
'../schedule/client': {},
'../shared/pLimit': {},
'../shared/utils': {},
'../shared/i18n': translate,
'../shared/logStreamManager': {},
'../shared/cronSchedule': {},
}).default;
return new Cron(logger);
}
function subscriptionService(config, model = {}) {
const Subscription = load(path.resolve('back/services/subscription.ts'), {
typedi: decorators,
'../config': config,
'../data/subscription': {
SubscriptionModel: model,
SubscriptionStatus: {},
},
'../data/cron': {},
'../config/util': { fileExist: async (p) => fs.existsSync(p) },
'../config/const': {},
'../shared/i18n': translate,
'../shared/pLimit': {},
'../shared/logStreamManager': {},
'./schedule': {},
'./sock': {},
'./sshKey': {},
'./cron': {},
}).default;
return new Subscription(logger, {}, {}, {}, {});
}
test('stored cron/subscription paths cannot read or enumerate outside the log root', async (t) => {
const base = temporary(t);
const root = path.join(base, 'log');
fs.mkdirSync(root);
fs.mkdirSync(path.join(base, 'outside'));
const secret = path.join(base, 'outside', 'secret');
fs.writeFileSync(secret, 'SECRET');
fs.symlinkSync(path.dirname(secret), path.join(root, 'escape'));
fs.writeFileSync(path.join(root, 'normal.log'), 'NORMAL');
const updates = [];
const cron = cronService({ logPath: root }, updates);
const subscription = subscriptionService(
{ logPath: root },
{ update: async (v) => updates.push(v) },
);
for (const log_path of [secret, '../outside/secret', 'escape/secret']) {
cron.getDb = subscription.getDb = async () => ({ log_path, status: 1 });
await assert.rejects(cron.log(1), /outside the log directory/);
await assert.rejects(cron.logs(1), /outside the log directory/);
await assert.rejects(subscription.logs(1), /outside the log directory/);
await assert.rejects(
cron.status({ ids: [1], status: 0, log_path }),
/outside the log directory/,
);
await assert.rejects(
subscription.status({ ids: [1], status: 0, log_path }),
/outside the log directory/,
);
}
assert.equal(updates.length, 0);
cron.getDb = subscription.getDb = async () => ({
log_path: 'normal.log',
status: 1,
});
assert.equal((await cron.log(1)).content, 'NORMAL');
assert.ok((await cron.logs(1)).some((x) => x.filename === 'normal.log'));
assert.ok(
(await subscription.logs(1)).some((x) => x.filename === 'normal.log'),
);
});
test('subscription aliases are checked before persistence and forced deletion', async (t) => {
const base = temporary(t);
for (const dir of ['scripts', 'repo']) fs.mkdirSync(path.join(base, dir));
let writes = 0;
const service = subscriptionService(
{
scriptPath: path.join(base, 'scripts'),
repoPath: path.join(base, 'repo'),
},
{
findAll: async () => [{ alias: '../victim' }],
destroy: async () => writes++,
},
);
service.insert = service.updateDb = async () => writes++;
await assert.rejects(
service.create({ alias: '../victim' }),
/Invalid subscription alias/,
);
await assert.rejects(
service.update({ alias: '../victim' }),
/Invalid subscription alias/,
);
await assert.rejects(
service.remove([1], { force: true }),
/Invalid subscription alias/,
);
assert.equal(writes, 0);
});
function systemService(config, overrides = {}) {
const System = load(path.resolve('back/services/system.ts'), {
typedi: decorators,
'../config': config,
'../config/const': {},
'../data/dependence': {
DependenceModel: { findAll: async () => [] },
DependenceTypes: {},
DependenceStatus: {},
},
'../data/system': {},
'../shared/pLimit': {},
'../schedule/client': {},
'../shared/i18n': { ...translate, setLang() {} },
'./notify': {},
'./schedule': {},
'./sock': {},
'../config/util': {},
...overrides,
}).default;
return new System(logger, {}, {});
}
test('backup rejects unrecognized paths before spawning; valid export uses argv without a shell', async (t) => {
const base = temporary(t);
const calls = [];
const service = systemService(
{
dataPath: path.join(base, 'data'),
dataTgzFile: path.join(base, 'backup file.tgz'),
},
{
child_process: {
execFile: (...args) => {
calls.push(args.slice(0, -1));
args.at(-1)(null, '', '');
},
},
'../config/util': {
promiseExec: () => assert.fail('must not invoke shell'),
},
},
);
let downloaded = false;
const res = {
status() {
return this;
},
send(body) {
return body;
},
download() {
downloaded = true;
},
};
for (const entry of [
'$(printf BAD)',
'../outside',
'/etc',
'scripts;echo BAD',
'--checkpoint-action=exec=bad',
]) {
assert.equal((await service.exportData(res, [entry])).code, 400);
}
assert.equal(calls.length, 0);
await service.exportData(res, ['base', 'scripts', 'config', 'scripts']);
assert.equal(calls.length, 1);
assert.equal(calls[0][0], 'tar');
assert.deepEqual(calls[0][1].slice(2), [
'--',
'data/db',
'data/upload',
'data/scripts',
'data/config',
]);
assert.equal(calls[0][2].shell, undefined);
assert.equal(downloaded, true);
});
test('proxy, language and mirror settings stay literal at their shell boundaries', async (t) => {
const base = temporary(t);
const commands = [];
const service = systemService(
{
dependenceProxyFile: path.join(base, 'proxy.sh'),
langEnvFile: path.join(base, 'lang.sh'),
},
{
'../config/util': {
promiseExec: async (command) => commands.push(command),
},
},
);
service.getSystemConfig = async () => ({ info: {} });
service.updateAuthDb = async () => {};
service.scheduleService = { runTask: (command) => commands.push(command) };
const value = 'x\'$(printf BAD)`printf BAD`"; echo BAD; #';
await service.updateDependenceProxy({ dependenceProxy: value });
await service.updateLanguage({ lang: value });
const stdout = execFileSync('/bin/bash', [
'-c',
'source "$1"; source "$2"; printf "%s\\0%s" "$http_proxy" "$QL_LANG"',
'bash',
path.join(base, 'proxy.sh'),
path.join(base, 'lang.sh'),
]).toString();
assert.equal(stdout, value + '\0' + value);
await service.updatePythonMirror({ pythonMirror: value });
await service.updateNodeMirror({ nodeMirror: value });
for (const command of commands) {
const stdout = execFileSync('/bin/bash', [
'-c',
`pip3() { printf '%s\\0' "$@"; }; pnpm() { printf '%s\\0' "$@"; }; ${command}`,
]).toString();
assert.equal(stdout.split('\0').at(-2), value);
}
assert.equal((await service.reloadSystem('$(printf BAD)')).code, 400);
});
test('the ignored-log sentinel remains supported without exposing device files', async (t) => {
const root = temporary(t);
const cron = cronService({ logPath: root });
cron.getDb = async () => ({ log_path: '/dev/null', log_name: '/dev/null' });
assert.equal((await cron.log(1)).status, 'ignored');
assert.deepEqual(await cron.logs(1), []);
await cron.status({ ids: [], status: 0, log_path: '/dev/null' });
await assert.rejects(
cron.status({ ids: [], status: 0, log_path: '/dev/zero' }),
);
});
test('normal backup creates an archive containing the selected data directories', async (t) => {
const base = temporary(t);
const dataPath = path.join(base, 'data');
for (const name of ['db', 'upload', 'scripts'])
fs.mkdirSync(path.join(dataPath, name), { recursive: true });
fs.writeFileSync(path.join(dataPath, 'scripts/example.js'), 'literal script');
const dataTgzFile = path.join(base, 'backup with spaces.tgz');
const service = systemService({ dataPath, dataTgzFile });
let downloaded;
await service.exportData(
{
download(file) {
downloaded = file;
},
send(error) {
assert.fail(JSON.stringify(error));
},
},
['scripts'],
);
assert.equal(downloaded, dataTgzFile);
const content = execFileSync('tar', [
'-xzOf',
dataTgzFile,
'data/scripts/example.js',
]).toString();
assert.equal(content, 'literal script');
});
-189
View File
@@ -1,189 +0,0 @@
const assert = require('node:assert/strict');
const test = require('node:test');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { execFileSync } = require('node:child_process');
const express = require('express');
const load = require('../helpers/load-security-module.cjs');
test('Python environment generation escapes backslashes and triple quotes together', async () => {
const output = {};
const Env = load(path.resolve('back/services/env.ts'), {
typedi: { Service: () => (x) => x, Inject: () => () => {} },
'../config': { envFile: 'sh', jsEnvFile: 'js', pyEnvFile: 'py' },
'../data/env': { EnvStatus: { normal: 0 } },
'../data': {},
'../shared/utils': {
writeFileWithLock: async (file, text) => (output[file] = text),
},
}).default;
const values = [
"\\'''; injected = True; #",
"'''",
'\\\\\\',
'quotes\'"`\n中文\u2028',
'${1+2}',
];
const service = new Env({});
service.envs = async () =>
values.map((value, i) => ({ name: `VALUE_${i}`, value }));
await service.set_envs();
const result = execFileSync(
'python3',
[
'-c',
output.py +
'\nimport json\nprint(json.dumps([os.environ["VALUE_"+str(i)] for i in range(5)]))\nassert "injected" not in globals()',
],
{ encoding: 'utf8' },
);
assert.deepEqual(JSON.parse(result), values);
});
test('cron lookup treats regex and shell metacharacters as literal command text', () => {
const { findCronId } = load(path.resolve('back/shared/cronCommand.ts'));
const command = 'task file[1].js --value "$(touch ignored)"';
const crontab =
`* * * * * real_time=false no_tee=true ID=17 ${command}\n` +
'* * * * * real_time=false no_tee=true ID=18 task other.js\n';
assert.equal(findCronId(crontab, command), '17');
assert.equal(findCronId(crontab, 'task file1.js'), '');
assert.equal(findCronId(crontab, '.*'), '');
assert.equal(findCronId(crontab, 'task other.js'), '18');
assert.equal(findCronId('', command), '');
assert.equal(findCronId(crontab, ''), '');
});
test('command route does not execute input while looking up the cron id', async (t) => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-command-'));
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
const marker = path.join(root, 'unexpected');
const command = `task demo.js "$(touch ${marker})"`;
const crontabFile = path.join(root, 'crontab');
fs.writeFileSync(
crontabFile,
`* * * * * real_time=false no_tee=true ID=42 ${command}\n`,
);
const seen = [];
const app = express.Router();
load(path.resolve('back/api/system.ts'), {
'../config': { crontabFile, tmpPath: root },
'../services/system': {},
'../services/user': {},
'../shared/i18n': { t: (x) => x },
'../shared/logStreamManager': { logStreamManager: {} },
typedi: {
Container: { get: () => ({ run: async (value) => seen.push(value) }) },
},
'../config/util': {
getUniqPath: async (_command, id) => {
assert.equal(id, '42');
return 'test';
},
promiseExec: async (cmd) => {
return execFileSync('/bin/sh', ['-c', cmd], { encoding: 'utf8' });
},
},
}).default(app);
const router = app.stack.find((x) => x.name === 'router').handle;
const route = router.stack.find(
(x) => x.route?.path === '/command-run',
).route;
await route.stack
.at(-1)
.handle({ body: { command } }, { setHeader() {} }, (e) => {
throw e;
});
assert.equal(fs.existsSync(marker), false);
assert.equal(seen.length, 1);
assert.equal(seen[0].command, command);
});
test('command execution is limited before running across both API mounts', async (t) => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-command-limit-'));
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
let runs = 0;
const app = express();
app.use(express.json());
const router = express.Router();
load(path.resolve('back/api/system.ts'), {
'../config': { crontabFile: path.join(root, 'missing'), tmpPath: root },
'../services/system': {},
'../services/user': {},
'../shared/i18n': { t: (x) => x },
'../shared/logStreamManager': {
logStreamManager: { closeStream: async () => {} },
},
'../config/util': {
getUniqPath: async () => 'test',
handleLogPath: async () => path.join(root, 'unused.log'),
},
typedi: {
Container: {
get: () => ({
run: async (value, callbacks) => {
runs++;
await callbacks.onEnd();
},
}),
},
},
}).default(router);
app.use('/api', router);
app.use('/open', router);
const server = await new Promise((resolve) => {
const s = app.listen(0, '127.0.0.1', () => resolve(s));
});
t.after(() => new Promise((resolve) => server.close(resolve)));
const base = `http://127.0.0.1:${server.address().port}`;
for (let i = 0; i < 61; i++) {
const response = await fetch(
`${base}/${i % 2 ? 'api' : 'open'}/system/command-run`,
{
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ command: 'echo safe' }),
},
);
await response.text();
assert.equal(response.status, i < 60 ? 200 : 429, `request ${i + 1}`);
}
assert.equal(runs, 60);
});
test('anonymous token exchange is limited per source across both API mounts', async (t) => {
let attempts = 0;
const app = express();
const router = express.Router();
load(path.resolve('back/api/open.ts'), {
'../services/open': {},
typedi: {
Container: {
get: () => ({
authToken: async () => {
attempts++;
return { code: 401 };
},
}),
},
},
}).default(router);
app.use('/api', router);
app.use('/open', router);
const server = await new Promise((resolve) => {
const s = app.listen(0, '127.0.0.1', () => resolve(s));
});
t.after(() => new Promise((resolve) => server.close(resolve)));
const base = `http://127.0.0.1:${server.address().port}`;
for (let i = 0; i < 101; i++) {
const response = await fetch(
`${base}/${
i % 2 ? 'api' : 'open'
}/auth/token?client_id=id&client_secret=wrong`,
);
await response.text();
assert.equal(response.status, i < 100 ? 200 : 429, `request ${i + 1}`);
}
assert.equal(attempts, 100);
});
-51
View File
@@ -1,51 +0,0 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { createCronJob } = require('../../back/shared/cronScheduler');
const { isValidCronSchedule } = require('../../back/shared/cronSchedule');
// node-cron caches timezone formatters; model each deployment in a fresh process.
if (!process.env.QL_MACRO_TEST_TZ) {
for (const tz of ['UTC', 'Asia/Shanghai']) {
test(`macro execution in ${tz}`, () => {
const result = require('node:child_process').spawnSync(process.execPath,
['-r', 'ts-node/register/transpile-only', '--test', __filename],
{ env: { ...process.env, TZ: tz, QL_MACRO_TEST_TZ: tz }, encoding: 'utf8' });
assert.equal(result.status, 0, result.stdout + result.stderr);
});
}
}
for (const tz of process.env.QL_MACRO_TEST_TZ ? [process.env.QL_MACRO_TEST_TZ] : []) {
for (const [macro, start, next] of [
['@annually', [2026, 11, 31, 23, 59, 59], [[2027, 0, 1, 0, 0, 0], [2028, 0, 1, 0, 0, 0]]],
['@midnight', [2026, 8, 27, 23, 59, 59], [[2026, 8, 28, 0, 0, 0], [2026, 8, 29, 0, 0, 0]]],
['@minutely', [2026, 8, 27, 12, 34, 59], [[2026, 8, 27, 12, 35, 0], [2026, 8, 27, 12, 36, 0]]],
]) {
test(`${macro} fires at the expected local times in ${tz}`, async (t) => {
const previousTZ = process.env.TZ;
process.env.TZ = tz;
t.after(() => { if (previousTZ === undefined) delete process.env.TZ; else process.env.TZ = previousTZ; });
t.mock.timers.enable({ apis: ['Date', 'setTimeout', 'setInterval'], now: new Date(...start) });
const expected = next.map((parts) => new Date(...parts).getTime());
const actual = [];
assert.equal(isValidCronSchedule(` ${macro} `), true);
const job = createCronJob(macro, (date) => actual.push(date.getTime()), {
name: macro, logger: { warn() {}, error() {} },
});
t.after(() => job.cancel());
for (const time of expected) {
t.mock.timers.tick(time - Date.now() - 1);
await new Promise(setImmediate);
assert.equal(actual.includes(time), false, 'must not fire early');
t.mock.timers.tick(1);
await new Promise(setImmediate);
}
assert.deepEqual(actual, expected);
});
}
}
test('new aliases do not enable unknown macros or raw parser extensions', () => {
for (const schedule of ['@annuallyx', '@midnightx', '@minutelyx', '@secondly', '@reboot', 'H * * * *', '0 /5 * * * *']) {
assert.equal(isValidCronSchedule(schedule), false, schedule);
}
});
+35 -63
View File
@@ -1,69 +1,41 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const nodeSchedule = require('node-schedule');
const { isValidCronSchedule } = require('../../back/shared/cronSchedule');
const { createCronJob } = require('../../back/shared/cronScheduler');
const legacy = require('../fixtures/legacy-cron.json');
const logger = { warn() {}, error() {} };
const flush = () => new Promise(setImmediate);
test('validation preserves legacy syntax except the explicitly added macros', () => {
const addedMacros = new Set(['@annually', '@midnight', '@minutely']);
assert.ok(legacy.validation.length > 250);
for (const { schedule, accepted } of legacy.validation) {
assert.equal(isValidCronSchedule(schedule), accepted || addedMacros.has(schedule), schedule);
}
});
for (const sample of legacy.times) {
test(`calendar compatibility: ${sample.schedule} from ${sample.currentDate}`, async (t) => {
const previousTZ = process.env.TZ;
process.env.TZ = 'UTC';
t.after(() => { if (previousTZ === undefined) delete process.env.TZ; else process.env.TZ = previousTZ; });
t.mock.timers.enable({ apis: ['Date', 'setTimeout', 'setInterval'], now: new Date(sample.currentDate) });
const actual = [];
const job = createCronJob(sample.schedule, (date) => actual.push(date.toISOString()), { name: 'calendar', logger });
t.after(() => job.cancel());
for (const expected of sample.next) {
t.mock.timers.tick(new Date(expected).getTime() - Date.now());
await flush();
test('validation agrees with the actual scheduler across legacy cron syntax', () => {
const candidates = new Set([
'*', '0', '?', '0 0', '0 0 *', '0 0 * *',
'@yearly', '@annually', '@monthly', '@weekly', '@daily', '@midnight',
'@hourly', '@secondly', '@minutely', '@weekdays', '@weekends', '@reboot',
'0 0 1 1 * 2027', '0 0 0 L * *', '0 0 0 * * 5L',
'0 0 0 * * MON#2', '0 0 12 LW * *', '0 0 12 15W * *',
'0 0 0 ? * MON', '0 0/30 * * * ?', '0 /5 * * * *',
]);
const fields = [
['*', '?', '/5', '*/5', '0/5', '0', '01', '1-5', '1,3', '5-1', 'H', 'H/5', 'H(0-10)', 'L'],
['*', '?', '/5', '*/5', '0/5', '0', '01', '1-5', '1,3', '5-1', 'H', 'H/5', 'H(0-10)', 'L'],
['*', '?', '/2', '*/2', '0/2', '0', '01', '1-5', '1,3', '23-2', 'H', 'L'],
['*', '?', '/2', '*/2', '1/2', '1', '01', '1-5', '1,3', 'L', 'L-1', 'LW', '15W', 'H'],
['*', '?', '/2', '*/2', '1/2', '1', '01', '1-5', '1,3', 'JAN', 'jan', 'JAN-MAR', 'DEC-FEB', 'H'],
['*', '?', '/2', '*/2', '0/2', '0', '7', '01', '1-5', '1,3', 'MON', 'mon', 'MON-FRI', 'FRI-MON', '5L', 'L', 'MON#2', '1#5', 'H'],
];
fields.forEach((options, index) => options.forEach((field) => {
const values = ['0', '0', '0', '*', '*', '*'];
values[index] = field;
candidates.add(values.join(' '));
}));
for (const value of [...candidates]) {
if (value.split(' ').length === 6) {
candidates.add(` ${value} `);
candidates.add(value.split(' ').slice(1).join(' '));
}
assert.deepEqual(actual, sample.next);
});
}
test('late callbacks catch up exactly once and cancellation destroys native tasks', async (t) => {
t.mock.timers.enable({ apis: ['Date', 'setTimeout', 'setInterval'], now: new Date('2026-09-27T00:00:00Z') });
const native = require('node-cron');
const before = native.getTasks().size;
const dates = [], warnings = [];
const job = createCronJob('* * * * * *', (date) => dates.push(date.getTime()), {
name: 'late', logger: { warn: (...args) => warnings.push(args), error() {} },
});
t.after(() => job.cancel());
t.mock.timers.tick(8000);
await flush();
assert.equal(dates.length, 8);
assert.equal(new Set(dates).size, 8);
assert.ok(warnings.length > 0);
job.cancel();
t.mock.timers.tick(5000);
await flush();
assert.equal(dates.length, 8);
assert.equal(native.getTasks().size, before);
});
test('callback rejection is logged and future executions continue', async (t) => {
t.mock.timers.enable({ apis: ['Date', 'setTimeout', 'setInterval'], now: new Date('2026-09-27T00:00:00Z') });
const errors = [];
let count = 0;
const job = createCronJob('* * * * * *', async () => { count++; throw Error('callback failed'); }, {
name: 'failure', logger: { warn() {}, error: (...args) => errors.push(args) },
});
t.after(() => job.cancel());
t.mock.timers.tick(1000);
await flush();
t.mock.timers.tick(1000);
await flush();
assert.equal(count, 2);
assert.equal(errors.length, 2);
}
for (const value of candidates) {
const job = nodeSchedule.scheduleJob(value, () => {});
const accepted = Boolean(job);
job?.cancel();
assert.equal(isValidCronSchedule(value), accepted, value);
}
assert.ok(candidates.size > 250);
});
+1 -9
View File
@@ -25,14 +25,6 @@ test('system and node modes route extended cron once and keep portable rules in
const rows = [
{ id: 'plain', schedule: '*/5 0-23 * * 1,3' },
{ id: 'spaces', schedule: ' 0\t0 * * * ' },
{ id: 'numeric-step', schedule: '0/5 * * * *' },
{ id: 'sunday-seven', schedule: '0 0 * * 7' },
{ id: 'day-step', schedule: '0 0 */2 * *' },
{ id: 'restricted-days', schedule: '0 0 1-31 * 1' },
{ id: 'restricted-week', schedule: '0 0 1 * 0-6' },
{ id: 'annually', schedule: '@annually' },
{ id: 'midnight', schedule: '@midnight' },
{ id: 'minutely', schedule: '@minutely' },
{ id: 'seconds', schedule: '0 0 * * * *' },
{ id: 'question', schedule: '0 0 * * ?' },
{ id: 'last', schedule: '0 0 L * *' },
@@ -51,7 +43,7 @@ test('system and node modes route extended cron once and keep portable rules in
}
await service.setCrontab({ data: rows, total: rows.length });
assert.equal(installs, mode === 'system' ? 1 : 0);
for (const id of ['numeric-step', 'sunday-seven', 'day-step', 'restricted-days', 'restricted-week', 'annually', 'midnight', 'minutely', 'seconds', 'question', 'last', 'nth', 'named', 'macro', 'short', 'extra', 'once', 'boot']) {
for (const id of ['seconds', 'question', 'last', 'nth', 'named', 'macro', 'short', 'extra', 'once', 'boot']) {
const line = file.split('\n').find((line) => line.endsWith(`task ${id}.js`));
assert.ok(line.startsWith('# '), `${mode}: ${id} cannot also run from system crontab`);
}
-235
View File
@@ -1,235 +0,0 @@
const assert = require('node:assert/strict');
const test = require('node:test');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const express = require('express');
const load = require('../helpers/load-security-module.cjs');
function fixture(t, api = 'script') {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-draft-path-'));
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
for (const dir of [
'scripts',
'config/grpc',
'log',
'tmp',
'bak',
'outside',
]) {
fs.mkdirSync(path.join(root, dir), { recursive: true });
}
const config = { blackFileList: ['auth.json', 'token.json', 'grpc'] };
for (const [key, dir] of Object.entries({
scriptPath: 'scripts',
configPath: 'config',
logPath: 'log',
tmpPath: 'tmp',
bakPath: 'bak',
}))
config[key] = path.join(root, dir);
config.writePathList = [config.scriptPath, config.configPath];
const stopped = [];
const removed = [];
const service = {
stopScript: async (...args) => {
stopped.push(args);
return { code: 200 };
},
};
const app = express.Router();
load(path.resolve(`back/api/${api}.ts`), {
'../config': config,
'../config/const': {},
'../services/script': {},
'../services/scriptHistory': {},
'../services/config': {},
'../shared/i18n': { t: (x) => x },
typedi: { Container: { get: () => service } },
'../shared/utils': {
writeFileWithLock: (p, s) => fs.promises.writeFile(p, s),
},
'../config/util': {
fileExist: async (p) => fs.existsSync(p),
rmPath: async (p) => {
removed.push(p);
await fs.promises.rm(p, { recursive: true, force: true });
},
},
}).default(app);
const router = app.stack.find((x) => x.name === 'router').handle;
const invoke = async (method, url, body) => {
const route = router.stack.find(
(x) => x.route?.path === url && x.route.methods[method],
).route;
let result;
await route.stack
.at(-1)
.handle({ body }, { send: (x) => (result = x) }, (e) => {
throw e;
});
return result;
};
return { root, config, stopped, removed, invoke };
}
test('script creation only creates the validated destination parent', async (t) => {
const { root, config, invoke } = fixture(t);
const outside = path.join(root, 'outside', 'unrequested');
const result = await invoke('post', '/', {
path: `${config.scriptPath}/../outside/unrequested`,
filename: '../../scripts/safe.txt',
content: 'safe',
});
assert.equal(result.code, 200);
assert.equal(
fs.readFileSync(path.join(config.scriptPath, 'safe.txt'), 'utf8'),
'safe',
);
assert.equal(fs.existsSync(outside), false);
assert.equal(
(
await invoke('post', '/', {
path: `${config.scriptPath}/../../escape`,
filename: 'poc.txt',
content: 'blocked',
})
).code,
403,
);
});
test('script creation supports root and nested parents in both writable directories', async (t) => {
const { config, invoke } = fixture(t);
for (const root of [config.scriptPath, config.configPath]) {
for (const filename of ['ordinary.js', 'nested/deeper/script.js']) {
assert.equal(
(await invoke('post', '/', {
path: root,
filename,
content: 'safe',
})).code,
200,
);
assert.equal(fs.readFileSync(path.join(root, filename), 'utf8'), 'safe');
}
}
});
test('script parent creation rejects sibling prefixes and external symlinks', async (t) => {
const { root, config, invoke } = fixture(t);
const sibling = config.scriptPath + '-sibling';
const outside = path.join(root, 'outside');
fs.mkdirSync(sibling);
fs.symlinkSync(outside, path.join(config.scriptPath, 'escape'));
for (const parent of [sibling, path.join(config.scriptPath, 'escape')]) {
assert.equal(
(await invoke('post', '/', {
path: parent,
filename: 'nested/poc.js',
content: 'blocked',
})).code,
403,
);
}
assert.equal(fs.existsSync(path.join(sibling, 'nested')), false);
assert.equal(fs.existsSync(path.join(outside, 'nested')), false);
});
test('stopping scripts rejects logs outside the log root before any side effect', async (t) => {
const { root, config, stopped, invoke } = fixture(t);
const callbacks = [];
t.mock.method(global, 'setTimeout', (fn) => {
callbacks.push(fn);
return {};
});
fs.mkdirSync(path.join(config.configPath, 'victim.swap'));
fs.mkdirSync(path.join(config.scriptPath, 'linked'));
fs.symlinkSync(
path.join(root, 'outside'),
path.join(config.logPath, 'linked'),
);
for (const dir of ['../config', 'linked']) {
assert.equal(
(await invoke('put', '/stop', { filename: 'victim.js', path: dir })).code,
403,
);
}
assert.equal(stopped.length, 0);
assert.equal(callbacks.length, 0);
});
test('log cleanup revalidates delayed paths and still removes ordinary debug logs', async (t) => {
const { root, config, removed, stopped, invoke } = fixture(t);
const callbacks = [];
t.mock.method(global, 'setTimeout', (fn) => {
callbacks.push(fn);
return {};
});
fs.mkdirSync(path.join(config.scriptPath, 'nested'));
fs.mkdirSync(path.join(config.logPath, 'nested'));
fs.mkdirSync(path.join(root, 'outside', 'victim.swap'));
fs.mkdirSync(path.join(config.logPath, 'normal.swap'));
assert.equal(
(await invoke('put', '/stop', { filename: 'normal.js' })).code,
200,
);
assert.equal(
(await invoke('put', '/stop', { filename: 'victim.js', path: 'nested' }))
.code,
200,
);
fs.rmdirSync(path.join(config.logPath, 'nested'));
fs.symlinkSync(
path.join(root, 'outside'),
path.join(config.logPath, 'nested'),
);
callbacks.forEach((fn) => fn());
assert.equal(stopped.length, 2);
assert.deepEqual(removed, [path.join(config.logPath, 'normal.swap')]);
assert.equal(fs.existsSync(path.join(root, 'outside', 'victim.swap')), true);
});
test('draft configuration traversal and normalized secret names cannot write files', async (t) => {
const { config, invoke } = fixture(t, 'config');
fs.writeFileSync(path.join(config.configPath, 'auth.json'), 'secret');
fs.symlinkSync(
path.join(config.configPath, 'auth.json'),
path.join(config.configPath, 'alias'),
);
for (const name of [
'auth.json',
'sub/../auth.json',
'sub/../token.json',
'grpc/client.key',
'../../shell/preload/env.sh',
'../scripts/poc.js',
'data/scripts/../config/auth.json',
'alias',
]) {
assert.equal(
(await invoke('post', '/save', { name, content: 'changed' })).code,
403,
name,
);
}
assert.equal(
fs.readFileSync(path.join(config.configPath, 'auth.json'), 'utf8'),
'secret',
);
assert.equal(
(await invoke('post', '/save', { name: 'config.sh', content: 'normal' }))
.code,
200,
);
assert.equal(
(
await invoke('post', '/save', {
name: 'data/scripts/normal.js',
content: 'normal',
})
).code,
403,
);
assert.equal(fs.existsSync(path.join(config.scriptPath, 'normal.js')), false);
});
-91
View File
@@ -1,91 +0,0 @@
const assert = require('node:assert/strict');
const test = require('node:test');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const express = require('express');
const load = require('../helpers/load-security-module.cjs');
test('environment import never writes upload filenames, including rejected files', async (t) => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-env-import-'));
fs.writeFileSync(path.join(root, 'sendNotify.js'), 'ORIGINAL');
const created = [];
const Env = class {};
const app = express();
load(path.resolve('back/api/env.ts'), {
'../config': { scriptPath: root },
'../config/util': {
safeJSONParse: (value) => {
try {
return JSON.parse(value);
} catch {
return {};
}
},
},
'../services/env': Env,
'../shared/i18n': { t: (s) => s },
typedi: {
Container: {
get: () => ({
create: async (values) => {
created.push(values);
return values;
},
}),
},
},
}).default(app);
app.use((err, _req, res, _next) =>
res.status(400).json({ code: 400, message: err.message }),
);
const server = app.listen(0, '127.0.0.1');
t.after(async () => {
await new Promise((resolve) => server.close(resolve));
fs.rmSync(root, { recursive: true, force: true });
});
await new Promise((resolve, reject) => {
server.on('listening', resolve);
server.on('error', reject);
});
const upload = async (content, filename = 'sendNotify.js') => {
const form = new FormData();
form.append('env', new Blob([content]), filename);
const response = await fetch(
`http://127.0.0.1:${server.address().port}/envs/upload`,
{ method: 'POST', body: form },
);
return response.json();
};
for (const payload of [
'globalThis.injected=true;',
'null',
'[]',
'[{"name":"BAD;CODE","value":"x"}]',
'[{"name":"X","value":{}}]',
]) {
assert.equal((await upload(payload)).code, 400);
assert.equal(
fs.readFileSync(path.join(root, 'sendNotify.js'), 'utf8'),
'ORIGINAL',
);
}
assert.equal((await upload('x'.repeat(5 * 1024 * 1024 + 1))).code, 400);
assert.equal(created.length, 0);
assert.equal(
(
await upload(
'[{"name":"TOKEN","value":"literal ${value}","remarks":"ok"}]',
'../sendNotify.js',
)
).code,
200,
);
assert.equal(created.length, 1);
assert.equal(created[0][0].value, 'literal ${value}');
assert.deepEqual(fs.readdirSync(root), ['sendNotify.js']);
assert.equal(
fs.readFileSync(path.join(root, 'sendNotify.js'), 'utf8'),
'ORIGINAL',
);
});
+1 -1
View File
@@ -36,7 +36,7 @@ test(
},
'../loaders/logger': logger,
'../data/cron': {
CrontabModel: { findOne: async () => ({ isDisabled: 0 }) },
CrontabModel: { findOne: async () => null },
CrontabStatus: {},
},
'../data/runningInstance': {

Some files were not shown because too many files have changed in this diff Show More