diff --git a/.env.example b/.env.example index c0b29702..5bc5eb04 100644 --- a/.env.example +++ b/.env.example @@ -21,3 +21,6 @@ JWT_EXPIRES_IN= QINIU_AK= QINIU_SK= QINIU_SCOPE= + +# Python 虚拟环境路径,留空则使用系统 Python +# PYTHON_VENV_DIR=./.venv diff --git a/README-NODE.md b/README-NODE.md new file mode 100644 index 00000000..0671dddc --- /dev/null +++ b/README-NODE.md @@ -0,0 +1,10 @@ +# Node.js 原生部署与 Python venv + +设置 `QL_DIR` 为安装目录,`QL_DATA_DIR` 为以 `/data` 结尾的绝对持久化数据路径。 +如需隔离 Python,请在启动进程的环境中显式设置 `PYTHON_VENV_DIR`(绝对路径或相对 `QL_DIR` 的路径),例如 `export PYTHON_VENV_DIR=/srv/qinglong/.venv`。未设置时保留现有 Python prefix 行为。 + +- `qinglong`(`shell/start.sh`)安装系统依赖和 Node 工具,并安装 Python requests,然后调用公共启动脚本。 +- `npm run start:simple`(`shell/start-simplify.sh`)使用已准备的 Node、pm2、nginx、jq 等依赖,不安装系统包、全局 npm 包或 requests;显式指定的 venv 不存在时仍会创建。请先在 venv 中安装 requests 和任务所需依赖。 +- 两种启动方式均保留已有 `.env`;使用 venv 时清除继承的 `PYTHONPATH`/`PYTHONUSERBASE`,依赖查询、安装和卸载使用该 venv 的 Python。 + +该模式需具备现有启动流程所要求的 nginx 配置和 pm2 服务权限。生产环境设置变量应通过 systemd 的 `Environment` 或相应进程配置完成。已有系统 Python 依赖不自动迁移至 venv,需在 venv 中重新安装。 diff --git a/back/config/const.ts b/back/config/const.ts index 3179af5e..459cf0fd 100644 --- a/back/config/const.ts +++ b/back/config/const.ts @@ -27,6 +27,7 @@ export const SAMPLE_FILES = [ ]; export const PYTHON_INSTALL_DIR = process.env.PYTHON_HOME; +export const PYTHON_VENV_DIR = process.env.PYTHON_VENV_DIR; export const NotificationModeStringMap = { 0: 'gotify', diff --git a/back/config/util.ts b/back/config/util.ts index 88d591c7..a1372381 100644 --- a/back/config/util.ts +++ b/back/config/util.ts @@ -5,15 +5,20 @@ import psTreeFun from 'ps-tree'; import { promisify } from 'util'; import { load } from 'js-yaml'; import config from './index'; -import { PYTHON_INSTALL_DIR, TASK_COMMAND } from './const'; +import { PYTHON_INSTALL_DIR, PYTHON_VENV_DIR, TASK_COMMAND } from './const'; import Logger from '../loaders/logger'; import { writeFileWithLock } from '../shared/utils'; import { DependenceTypes } from '../data/dependence'; import { FormData } from 'undici'; import os from 'os'; import { maybeSudo, isInContainer } from './container'; +import { shellQuote } from '../shared/shellQuote'; import { resolveFileAccess } from '../shared/fileAccess'; -import { mirrorWriteCommand, normalizeMirror, rewriteMirrorSources } from '../shared/linuxMirror'; +import { + mirrorWriteCommand, + normalizeMirror, + rewriteMirrorSources, +} from '../shared/linuxMirror'; export * from './share'; @@ -671,7 +676,9 @@ export async function setSystemTimezone(timezone: string): Promise { throw new Error('Invalid timezone'); } - await promiseExec(maybeSudo(`ln -sf /usr/share/zoneinfo/${timezone} /etc/localtime`)); + await promiseExec( + maybeSudo(`ln -sf /usr/share/zoneinfo/${timezone} /etc/localtime`), + ); await promiseExec(`echo "${timezone}" | ${maybeSudo('tee /etc/timezone')}`); return true; @@ -685,7 +692,13 @@ export function getGetCommand(type: DependenceTypes, name: string): string { const baseCommands = { [DependenceTypes.nodejs]: `pnpm ls -g | grep "${name}" | head -1`, [DependenceTypes.python3]: ` - python3 -c "exec(''' + ${ + PYTHON_VENV_DIR + ? shellQuote( + path.resolve(config.rootPath, PYTHON_VENV_DIR, 'bin/python3'), + ) + : 'python3' + } -c "exec(''' name='${name}' try: from importlib.metadata import version @@ -696,9 +709,10 @@ except: spec=u.find_spec(name) print(name if spec else '') ''')"`, - [DependenceTypes.linux]: getOsTypeSync() === 'Alpine' - ? `apk info -es ${name}` - : maybeSudo(`dpkg-query -s ${name}`), + [DependenceTypes.linux]: + getOsTypeSync() === 'Alpine' + ? `apk info -es ${name}` + : maybeSudo(`dpkg-query -s ${name}`), }; return baseCommands[type]; @@ -707,16 +721,28 @@ except: export function getInstallCommand(type: DependenceTypes, name: string): string { const baseCommands = { [DependenceTypes.nodejs]: 'pnpm add -g', - [DependenceTypes.python3]: - 'pip3 install --disable-pip-version-check --root-user-action=ignore', - [DependenceTypes.linux]: getOsTypeSync() === 'Alpine' - ? 'apk add --no-check-certificate' - : maybeSudo('apt-get install -y'), + [DependenceTypes.python3]: `${ + PYTHON_VENV_DIR + ? shellQuote( + path.resolve(config.rootPath, PYTHON_VENV_DIR, 'bin/python3'), + ) + ' -m pip' + : 'pip3' + } install --disable-pip-version-check${ + PYTHON_VENV_DIR ? '' : ' --root-user-action=ignore' + }`, + [DependenceTypes.linux]: + getOsTypeSync() === 'Alpine' + ? 'apk add --no-check-certificate' + : maybeSudo('apt-get install -y'), }; let command = baseCommands[type]; - if (type === DependenceTypes.python3 && PYTHON_INSTALL_DIR) { + if ( + type === DependenceTypes.python3 && + PYTHON_INSTALL_DIR && + !PYTHON_VENV_DIR + ) { command = `${command} --prefix=${PYTHON_INSTALL_DIR}`; } @@ -729,11 +755,17 @@ export function getUninstallCommand( ): string { const baseCommands = { [DependenceTypes.nodejs]: 'pnpm remove -g', - [DependenceTypes.python3]: - 'pip3 uninstall --disable-pip-version-check --root-user-action=ignore -y', - [DependenceTypes.linux]: getOsTypeSync() === 'Alpine' - ? 'apk del' - : maybeSudo('apt-get remove -y'), + [DependenceTypes.python3]: `${ + PYTHON_VENV_DIR + ? shellQuote( + path.resolve(config.rootPath, PYTHON_VENV_DIR, 'bin/python3'), + ) + ' -m pip' + : 'pip3' + } uninstall --disable-pip-version-check${ + PYTHON_VENV_DIR ? '' : ' --root-user-action=ignore' + } -y`, + [DependenceTypes.linux]: + getOsTypeSync() === 'Alpine' ? 'apk del' : maybeSudo('apt-get remove -y'), }; return `${baseCommands[type]} ${name.trim()}`; diff --git a/package.json b/package.json index afa6a13a..e36e566e 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,8 @@ "panel": "npm run build:back && node static/build/app.js", "gen:proto": "protoc --experimental_allow_proto3_optional --plugin=./node_modules/.bin/protoc-gen-ts_proto ./back/protos/*.proto --ts_proto_out=./ --ts_proto_opt=outputServices=grpc-js,env=node,esModuleInterop=true,snakeToCamel=false", "prettier": "prettier --write '**/*.{js,jsx,tsx,ts,less,md,json}'", - "postinstall": "max setup 2>/dev/null || true" + "postinstall": "max setup 2>/dev/null || true", + "start:simple": "bash shell/start-simplify.sh" }, "gitHooks": { "pre-commit": "lint-staged" diff --git a/shell/start-simplify.sh b/shell/start-simplify.sh new file mode 100644 index 00000000..f016ada5 --- /dev/null +++ b/shell/start-simplify.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +set -e + +: "${QL_DIR:?Please set QL_DIR}" +: "${QL_DATA_DIR:?Please set QL_DATA_DIR}" +[[ "$QL_DIR" = /* && "$QL_DATA_DIR" = /*/data ]] || { echo "QL_DIR and QL_DATA_DIR must be absolute paths (data directory ends in /data)" >&2; exit 1; } +command="$1" + +export PNPM_HOME="${PNPM_HOME:-${QL_DATA_DIR}/dep_cache/node}" +if [[ -n "${PYTHON_VENV_DIR:-}" ]]; then + [[ "$PYTHON_VENV_DIR" = /* ]] || PYTHON_VENV_DIR="${QL_DIR}/${PYTHON_VENV_DIR}" + export PYTHON_VENV_DIR + if [[ ! -x "$PYTHON_VENV_DIR/bin/python3" ]]; then + python3 -m venv "$PYTHON_VENV_DIR" + fi + [[ -x "$PYTHON_VENV_DIR/bin/python3" ]] || { echo "Invalid Python venv" >&2; exit 1; } + unset PYTHON_HOME PYTHONUSERBASE PYTHONPATH + export PATH="${PYTHON_VENV_DIR}/bin:${PATH}:${PNPM_HOME}" + export PIP_CACHE_DIR="${PYTHON_VENV_DIR}/pip" +else + export PYTHON_SHORT_VERSION=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') + export PYTHON_HOME="${PYTHON_HOME:-${QL_DATA_DIR}/dep_cache/python3}" + export PYTHONUSERBASE="$PYTHON_HOME" + export PATH="${PATH}:${PNPM_HOME}:${PYTHON_HOME}/bin" + export PIP_CACHE_DIR="${PYTHON_HOME}/pip" + export PYTHONPATH="${PYTHON_HOME}:${PYTHON_HOME}/lib/python${PYTHON_SHORT_VERSION}:${PYTHON_HOME}/lib/python${PYTHON_SHORT_VERSION}/site-packages" +fi +export NODE_PATH="${NODE_PATH:-/usr/local/bin:/usr/local/lib/node_modules:${PNPM_HOME}/global/5/node_modules}" +mkdir -p "$PIP_CACHE_DIR" + +cd "${QL_DIR}" +if [[ ! -e .env ]]; then + # Preserve user configuration, including a file created concurrently. + (umask 077; set -C; cat .env.example > .env) +fi +chmod u+x "${QL_DIR}"/shell/*.sh + +. "${QL_DIR}/shell/share.sh" +. "${QL_DIR}/shell/env.sh" + +log_with_style() { + local level="$1" + local message="$2" + local timestamp=$(date '+%Y-%m-%d %H:%M:%S') + + printf "\n[%s] [%7s] %s\n" "${timestamp}" "${level}" "${message}" +} + +log_with_style "INFO" "🚀 1. 检测配置文件..." +import_config "$@" +make_dir /etc/nginx/conf.d +make_dir /run/nginx +fix_config + +pm2 l &>/dev/null + +log_with_style "INFO" "🔄 2. 启动 nginx..." +nginx -s reload 2>/dev/null || nginx -c /etc/nginx/nginx.conf + +log_with_style "INFO" "⚙️ 3. 启动 pm2 服务..." +reload_pm2 + +if [[ $command != "reload" ]]; then + if [[ $AutoStartBot == true ]]; then + log_with_style "INFO" "🤖 4. 启动 bot..." + nohup ql bot >$dir_log/bot.log 2>&1 & + fi + + if [[ $EnableExtraShell == true ]]; then + log_with_style "INFO" "🛠️ 5. 执行自定义脚本..." + nohup ql extra >$dir_log/extra.log 2>&1 & + fi + + pm2 startup + pm2 save +fi + +log_with_style "SUCCESS" "🎉 启动成功!" diff --git a/shell/start.sh b/shell/start.sh index 657f7aff..5553c359 100644 --- a/shell/start.sh +++ b/shell/start.sh @@ -77,65 +77,13 @@ if [[ $command != "reload" ]]; then npm install -g pnpm@8.3.1 pm2 ts-node typescript@5 fi -export PYTHON_SHORT_VERSION=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') -export PNPM_HOME=${QL_DIR}/data/dep_cache/node -export PYTHON_HOME=${QL_DIR}/data/dep_cache/python3 -export PYTHONUSERBASE=${QL_DIR}/data/dep_cache/python3 - -export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PNPM_HOME}:${PYTHON_HOME}/bin -export NODE_PATH=/usr/local/bin:/usr/local/lib/node_modules:${PNPM_HOME}/global/5/node_modules -export PIP_CACHE_DIR=${PYTHON_HOME}/pip -export PYTHONPATH=${PYTHON_HOME}:${PYTHON_HOME}/lib/python${PYTHON_SHORT_VERSION}:${PYTHON_HOME}/lib/python${PYTHON_SHORT_VERSION}/site-packages - -if [[ $command != "reload" ]]; then - pip3 install --prefix ${PYTHON_HOME} requests -fi - -cd "${QL_DIR}" -if [[ ! -e .env ]]; then - # Preserve user configuration, including a file created concurrently. - (umask 077; set -C; cat .env.example > .env) -fi -chmod 777 ${QL_DIR}/shell/*.sh - -. ${QL_DIR}/shell/share.sh -. ${QL_DIR}/shell/env.sh - -log_with_style() { - local level="$1" - local message="$2" - local timestamp=$(date '+%Y-%m-%d %H:%M:%S') - - printf "\n[%s] [%7s] %s\n" "${timestamp}" "${level}" "${message}" -} - -log_with_style "INFO" "🚀 1. 检测配置文件..." -import_config "$@" -make_dir /etc/nginx/conf.d -make_dir /run/nginx -fix_config - -pm2 l &>/dev/null - -log_with_style "INFO" "🔄 2. 启动 nginx..." -nginx -s reload 2>/dev/null || nginx -c /etc/nginx/nginx.conf - -log_with_style "INFO" "⚙️ 3. 启动 pm2 服务..." -reload_pm2 - -if [[ $command != "reload" ]]; then - if [[ $AutoStartBot == true ]]; then - log_with_style "INFO" "🤖 4. 启动 bot..." - nohup ql bot >$dir_log/bot.log 2>&1 & +if [[ "$command" != "reload" ]]; then + if [[ -n "${PYTHON_VENV_DIR:-}" ]]; then + [[ "$PYTHON_VENV_DIR" = /* ]] || export PYTHON_VENV_DIR="${QL_DIR}/${PYTHON_VENV_DIR}" + [[ -x "$PYTHON_VENV_DIR/bin/python3" ]] || python3 -m venv "$PYTHON_VENV_DIR" + "$PYTHON_VENV_DIR/bin/python3" -m pip install requests + else + pip3 install --prefix "${PYTHON_HOME:-${QL_DATA_DIR}/dep_cache/python3}" requests fi - - if [[ $EnableExtraShell == true ]]; then - log_with_style "INFO" "🛠️ 5. 执行自定义脚本..." - nohup ql extra >$dir_log/extra.log 2>&1 & - fi - - pm2 startup - pm2 save fi - -log_with_style "SUCCESS" "🎉 启动成功!" +exec /bin/bash "${QL_DIR}/shell/start-simplify.sh" "$@" diff --git a/test/back/python-venv.test.cjs b/test/back/python-venv.test.cjs new file mode 100644 index 00000000..0b7439b6 --- /dev/null +++ b/test/back/python-venv.test.cjs @@ -0,0 +1,69 @@ +const test = require('node:test'), + assert = require('node:assert/strict'), + fs = require('node:fs'), + os = require('node:os'), + path = require('node:path'), + { execFileSync } = require('node:child_process'); +const load = require('../helpers/load-security-module.cjs'); +test('venv installs, uninstalls and probes use the selected Python, including spaces', () => { + const methods = load('back/config/util.ts', { + './index': { rootPath: '/application' }, + './const': { + PYTHON_VENV_DIR: 'virtual env', + PYTHON_INSTALL_DIR: '/legacy', + }, + '../loaders/logger': {}, + '../shared/utils': {}, + '../data/dependence': { + DependenceTypes: { nodejs: 0, python3: 1, linux: 2 }, + }, + }); + for (const name of [ + 'getInstallCommand', + 'getUninstallCommand', + 'getGetCommand', + ]) { + const cmd = methods[name](1, 'requests'); + assert.match(cmd, /'\/application\/virtual env\/bin\/python3'/); + assert.doesNotMatch(cmd, /--prefix=/); + } +}); +test('venv setup keeps Python packages isolated and retains existing startup configuration', (t) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-venv-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + const ql = path.join(dir, 'application with spaces'); + fs.mkdirSync(ql); + const venv = path.join(dir, 'venv with spaces'); + execFileSync('python3', ['-m', 'venv', venv]); + const source = fs.readFileSync('shell/start-simplify.sh', 'utf8'); + const snippet = source.slice( + source.indexOf('export PNPM_HOME='), + source.indexOf('\ncd "${QL_DIR}"'), + ); + const result = execFileSync( + '/bin/bash', + [ + '-c', + snippet + + '\npython3 -c "import sys; print(sys.prefix); print(sys.base_prefix)"', + ], + { + env: { + ...process.env, + QL_DIR: ql, + QL_DATA_DIR: path.join(dir, 'data'), + PYTHON_VENV_DIR: venv, + PYTHONPATH: '/legacy/modules', + PYTHONUSERBASE: '/legacy', + PATH: process.env.PATH, + }, + encoding: 'utf8', + }, + ) + .trim() + .split('\n'); + assert.equal(fs.realpathSync(result[0]), fs.realpathSync(venv)); + assert.notEqual(result[1], venv); + assert.match(source, /set -C; cat .env.example > .env/); + assert.doesNotMatch(source, /npm install|pip3 install|chmod 777/); +}); diff --git a/test/back/start-env-preservation.test.cjs b/test/back/start-env-preservation.test.cjs index 0b3bcc57..4ac10be2 100644 --- a/test/back/start-env-preservation.test.cjs +++ b/test/back/start-env-preservation.test.cjs @@ -7,8 +7,8 @@ const { spawnSync } = require('node:child_process'); // Run the actual startup prefix, stopping before service initialization. const source = fs - .readFileSync('shell/start.sh', 'utf8') - .split('. ${QL_DIR}/shell/share.sh')[0]; + .readFileSync('shell/start-simplify.sh', 'utf8') + .split('. "${QL_DIR}/shell/share.sh"')[0]; for (const command of ['start', 'reload']) { for (const existing of [false, true]) { test(`${command} ${