feat(python): support isolated venv environments for native deployment

2026-10-16 22:35:32 +08:00 · 2026-10-14 10:19:50 +08:00
parent 6469792546 db88d037db
commit 6532775a25
9 changed files with 223 additions and 81 deletions
@@ -21,3 +21,6 @@ JWT_EXPIRES_IN=
 QINIU_AK=
 QINIU_SK=
 QINIU_SCOPE=
+
+# Python 虚拟环境路径，留空则使用系统 Python
+# PYTHON_VENV_DIR=./.venv
@@ -0,0 +1,10 @@
+# Node.js 原生部署与 Python venv
+
+设置 `QL_DIR` 为安装目录，`QL_DATA_DIR` 为以 `/data` 结尾的绝对持久化数据路径。
+如需隔离 Python，请在启动进程的环境中显式设置 `PYTHON_VENV_DIR`（绝对路径或相对 `QL_DIR` 的路径），例如 `export PYTHON_VENV_DIR=/srv/qinglong/.venv`。未设置时保留现有 Python prefix 行为。
+
+- `qinglong`（`shell/start.sh`）安装系统依赖和 Node 工具，并安装 Python requests，然后调用公共启动脚本。
+- `npm run start:simple`（`shell/start-simplify.sh`）使用已准备的 Node、pm2、nginx、jq 等依赖，不安装系统包、全局 npm 包或 requests；显式指定的 venv 不存在时仍会创建。请先在 venv 中安装 requests 和任务所需依赖。
+- 两种启动方式均保留已有 `.env`；使用 venv 时清除继承的 `PYTHONPATH`/`PYTHONUSERBASE`，依赖查询、安装和卸载使用该 venv 的 Python。
+
+该模式需具备现有启动流程所要求的 nginx 配置和 pm2 服务权限。生产环境设置变量应通过 systemd 的 `Environment` 或相应进程配置完成。已有系统 Python 依赖不自动迁移至 venv，需在 venv 中重新安装。
@@ -27,6 +27,7 @@ export const SAMPLE_FILES = [
 ];

 export const PYTHON_INSTALL_DIR = process.env.PYTHON_HOME;
+export const PYTHON_VENV_DIR = process.env.PYTHON_VENV_DIR;

 export const NotificationModeStringMap = {
  0: 'gotify',
@@ -5,15 +5,20 @@ import psTreeFun from 'ps-tree';
 import { promisify } from 'util';
 import { load } from 'js-yaml';
 import config from './index';
-import { PYTHON_INSTALL_DIR, TASK_COMMAND } from './const';
+import { PYTHON_INSTALL_DIR, PYTHON_VENV_DIR, TASK_COMMAND } from './const';
 import Logger from '../loaders/logger';
 import { writeFileWithLock } from '../shared/utils';
 import { DependenceTypes } from '../data/dependence';
 import { FormData } from 'undici';
 import os from 'os';
 import { maybeSudo, isInContainer } from './container';
+import { shellQuote } from '../shared/shellQuote';
 import { resolveFileAccess } from '../shared/fileAccess';
-import { mirrorWriteCommand, normalizeMirror, rewriteMirrorSources } from '../shared/linuxMirror';
+import {
+  mirrorWriteCommand,
+  normalizeMirror,
+  rewriteMirrorSources,
+} from '../shared/linuxMirror';

 export * from './share';

@@ -671,7 +676,9 @@ export async function setSystemTimezone(timezone: string): Promise<boolean> {
      throw new Error('Invalid timezone');
    }

-    await promiseExec(maybeSudo(`ln -sf /usr/share/zoneinfo/${timezone} /etc/localtime`));
+    await promiseExec(
+      maybeSudo(`ln -sf /usr/share/zoneinfo/${timezone} /etc/localtime`),
+    );
    await promiseExec(`echo "${timezone}" | ${maybeSudo('tee /etc/timezone')}`);

    return true;
@@ -685,7 +692,13 @@ export function getGetCommand(type: DependenceTypes, name: string): string {
  const baseCommands = {
    [DependenceTypes.nodejs]: `pnpm ls -g  | grep "${name}" | head -1`,
    [DependenceTypes.python3]: `
-    python3 -c "exec('''
+    ${
+      PYTHON_VENV_DIR
+        ? shellQuote(
+            path.resolve(config.rootPath, PYTHON_VENV_DIR, 'bin/python3'),
+          )
+        : 'python3'
+    } -c "exec('''
 name='${name}'
 try:
    from importlib.metadata import version
@@ -696,9 +709,10 @@ except:
    spec=u.find_spec(name)
    print(name if spec else '')
 ''')"`,
-    [DependenceTypes.linux]: getOsTypeSync() === 'Alpine'
-      ? `apk info -es ${name}`
-      : maybeSudo(`dpkg-query -s ${name}`),
+    [DependenceTypes.linux]:
+      getOsTypeSync() === 'Alpine'
+        ? `apk info -es ${name}`
+        : maybeSudo(`dpkg-query -s ${name}`),
  };

  return baseCommands[type];
@@ -707,16 +721,28 @@ except:
 export function getInstallCommand(type: DependenceTypes, name: string): string {
  const baseCommands = {
    [DependenceTypes.nodejs]: 'pnpm add -g',
-    [DependenceTypes.python3]:
-      'pip3 install --disable-pip-version-check --root-user-action=ignore',
-    [DependenceTypes.linux]: getOsTypeSync() === 'Alpine'
-      ? 'apk add --no-check-certificate'
-      : maybeSudo('apt-get install -y'),
+    [DependenceTypes.python3]: `${
+      PYTHON_VENV_DIR
+        ? shellQuote(
+            path.resolve(config.rootPath, PYTHON_VENV_DIR, 'bin/python3'),
+          ) + ' -m pip'
+        : 'pip3'
+    } install --disable-pip-version-check${
+      PYTHON_VENV_DIR ? '' : ' --root-user-action=ignore'
+    }`,
+    [DependenceTypes.linux]:
+      getOsTypeSync() === 'Alpine'
+        ? 'apk add --no-check-certificate'
+        : maybeSudo('apt-get install -y'),
  };

  let command = baseCommands[type];

-  if (type === DependenceTypes.python3 && PYTHON_INSTALL_DIR) {
+  if (
+    type === DependenceTypes.python3 &&
+    PYTHON_INSTALL_DIR &&
+    !PYTHON_VENV_DIR
+  ) {
    command = `${command} --prefix=${PYTHON_INSTALL_DIR}`;
  }

@@ -729,11 +755,17 @@ export function getUninstallCommand(
 ): string {
  const baseCommands = {
    [DependenceTypes.nodejs]: 'pnpm remove -g',
-    [DependenceTypes.python3]:
-      'pip3 uninstall --disable-pip-version-check --root-user-action=ignore -y',
-    [DependenceTypes.linux]: getOsTypeSync() === 'Alpine'
-      ? 'apk del'
-      : maybeSudo('apt-get remove -y'),
+    [DependenceTypes.python3]: `${
+      PYTHON_VENV_DIR
+        ? shellQuote(
+            path.resolve(config.rootPath, PYTHON_VENV_DIR, 'bin/python3'),
+          ) + ' -m pip'
+        : 'pip3'
+    } uninstall --disable-pip-version-check${
+      PYTHON_VENV_DIR ? '' : ' --root-user-action=ignore'
+    } -y`,
+    [DependenceTypes.linux]:
+      getOsTypeSync() === 'Alpine' ? 'apk del' : maybeSudo('apt-get remove -y'),
  };

  return `${baseCommands[type]} ${name.trim()}`;
@@ -28,7 +28,8 @@
    "panel": "npm run build:back && node static/build/app.js",
    "gen:proto": "protoc --experimental_allow_proto3_optional --plugin=./node_modules/.bin/protoc-gen-ts_proto ./back/protos/*.proto --ts_proto_out=./ --ts_proto_opt=outputServices=grpc-js,env=node,esModuleInterop=true,snakeToCamel=false",
    "prettier": "prettier --write '**/*.{js,jsx,tsx,ts,less,md,json}'",
-    "postinstall": "max setup 2>/dev/null || true"
+    "postinstall": "max setup 2>/dev/null || true",
+    "start:simple": "bash shell/start-simplify.sh"
  },
  "gitHooks": {
    "pre-commit": "lint-staged"
@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+set -e
+
+: "${QL_DIR:?Please set QL_DIR}"
+: "${QL_DATA_DIR:?Please set QL_DATA_DIR}"
+[[ "$QL_DIR" = /* && "$QL_DATA_DIR" = /*/data ]] || { echo "QL_DIR and QL_DATA_DIR must be absolute paths (data directory ends in /data)" >&2; exit 1; }
+command="$1"
+
+export PNPM_HOME="${PNPM_HOME:-${QL_DATA_DIR}/dep_cache/node}"
+if [[ -n "${PYTHON_VENV_DIR:-}" ]]; then
+  [[ "$PYTHON_VENV_DIR" = /* ]] || PYTHON_VENV_DIR="${QL_DIR}/${PYTHON_VENV_DIR}"
+  export PYTHON_VENV_DIR
+  if [[ ! -x "$PYTHON_VENV_DIR/bin/python3" ]]; then
+    python3 -m venv "$PYTHON_VENV_DIR"
+  fi
+  [[ -x "$PYTHON_VENV_DIR/bin/python3" ]] || { echo "Invalid Python venv" >&2; exit 1; }
+  unset PYTHON_HOME PYTHONUSERBASE PYTHONPATH
+  export PATH="${PYTHON_VENV_DIR}/bin:${PATH}:${PNPM_HOME}"
+  export PIP_CACHE_DIR="${PYTHON_VENV_DIR}/pip"
+else
+  export PYTHON_SHORT_VERSION=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
+  export PYTHON_HOME="${PYTHON_HOME:-${QL_DATA_DIR}/dep_cache/python3}"
+  export PYTHONUSERBASE="$PYTHON_HOME"
+  export PATH="${PATH}:${PNPM_HOME}:${PYTHON_HOME}/bin"
+  export PIP_CACHE_DIR="${PYTHON_HOME}/pip"
+  export PYTHONPATH="${PYTHON_HOME}:${PYTHON_HOME}/lib/python${PYTHON_SHORT_VERSION}:${PYTHON_HOME}/lib/python${PYTHON_SHORT_VERSION}/site-packages"
+fi
+export NODE_PATH="${NODE_PATH:-/usr/local/bin:/usr/local/lib/node_modules:${PNPM_HOME}/global/5/node_modules}"
+mkdir -p "$PIP_CACHE_DIR"
+
+cd "${QL_DIR}"
+if [[ ! -e .env ]]; then
+  # Preserve user configuration, including a file created concurrently.
+  (umask 077; set -C; cat .env.example > .env)
+fi
+chmod u+x "${QL_DIR}"/shell/*.sh
+
+. "${QL_DIR}/shell/share.sh"
+. "${QL_DIR}/shell/env.sh"
+
+log_with_style() {
+  local level="$1"
+  local message="$2"
+  local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
+
+  printf "\n[%s] [%7s]  %s\n" "${timestamp}" "${level}" "${message}"
+}
+
+log_with_style "INFO" "🚀  1. 检测配置文件..."
+import_config "$@"
+make_dir /etc/nginx/conf.d
+make_dir /run/nginx
+fix_config
+
+pm2 l &>/dev/null
+
+log_with_style "INFO" "🔄  2. 启动 nginx..."
+nginx -s reload 2>/dev/null || nginx -c /etc/nginx/nginx.conf
+
+log_with_style "INFO" "⚙️  3. 启动 pm2 服务..."
+reload_pm2
+
+if [[ $command != "reload" ]]; then
+  if [[ $AutoStartBot == true ]]; then
+    log_with_style "INFO" "🤖 4. 启动 bot..."
+    nohup ql bot >$dir_log/bot.log 2>&1 &
+  fi
+
+  if [[ $EnableExtraShell == true ]]; then
+    log_with_style "INFO" "🛠️ 5. 执行自定义脚本..."
+    nohup ql extra >$dir_log/extra.log 2>&1 &
+  fi
+
+  pm2 startup
+  pm2 save
+fi
+
+log_with_style "SUCCESS" "🎉 启动成功!"
@@ -77,65 +77,13 @@ if [[ $command != "reload" ]]; then
  npm install -g pnpm@8.3.1 pm2 ts-node typescript@5
 fi

-export PYTHON_SHORT_VERSION=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
-export PNPM_HOME=${QL_DIR}/data/dep_cache/node
-export PYTHON_HOME=${QL_DIR}/data/dep_cache/python3
-export PYTHONUSERBASE=${QL_DIR}/data/dep_cache/python3
-
-export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PNPM_HOME}:${PYTHON_HOME}/bin
-export NODE_PATH=/usr/local/bin:/usr/local/lib/node_modules:${PNPM_HOME}/global/5/node_modules
-export PIP_CACHE_DIR=${PYTHON_HOME}/pip
-export PYTHONPATH=${PYTHON_HOME}:${PYTHON_HOME}/lib/python${PYTHON_SHORT_VERSION}:${PYTHON_HOME}/lib/python${PYTHON_SHORT_VERSION}/site-packages
-
-if [[ $command != "reload" ]]; then
-  pip3 install --prefix ${PYTHON_HOME} requests
-fi
-
-cd "${QL_DIR}"
-if [[ ! -e .env ]]; then
-  # Preserve user configuration, including a file created concurrently.
-  (umask 077; set -C; cat .env.example > .env)
-fi
-chmod 777 ${QL_DIR}/shell/*.sh
-
-. ${QL_DIR}/shell/share.sh
-. ${QL_DIR}/shell/env.sh
-
-log_with_style() {
-  local level="$1"
-  local message="$2"
-  local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
-
-  printf "\n[%s] [%7s]  %s\n" "${timestamp}" "${level}" "${message}"
-}
-
-log_with_style "INFO" "🚀  1. 检测配置文件..."
-import_config "$@"
-make_dir /etc/nginx/conf.d
-make_dir /run/nginx
-fix_config
-
-pm2 l &>/dev/null
-
-log_with_style "INFO" "🔄  2. 启动 nginx..."
-nginx -s reload 2>/dev/null || nginx -c /etc/nginx/nginx.conf
-
-log_with_style "INFO" "⚙️  3. 启动 pm2 服务..."
-reload_pm2
-
-if [[ $command != "reload" ]]; then
-  if [[ $AutoStartBot == true ]]; then
-    log_with_style "INFO" "🤖 4. 启动 bot..."
-    nohup ql bot >$dir_log/bot.log 2>&1 &
+if [[ "$command" != "reload" ]]; then
+  if [[ -n "${PYTHON_VENV_DIR:-}" ]]; then
+    [[ "$PYTHON_VENV_DIR" = /* ]] || export PYTHON_VENV_DIR="${QL_DIR}/${PYTHON_VENV_DIR}"
+    [[ -x "$PYTHON_VENV_DIR/bin/python3" ]] || python3 -m venv "$PYTHON_VENV_DIR"
+    "$PYTHON_VENV_DIR/bin/python3" -m pip install requests
+  else
+    pip3 install --prefix "${PYTHON_HOME:-${QL_DATA_DIR}/dep_cache/python3}" requests
  fi
-
-  if [[ $EnableExtraShell == true ]]; then
-    log_with_style "INFO" "🛠️ 5. 执行自定义脚本..."
-    nohup ql extra >$dir_log/extra.log 2>&1 &
-  fi
-
-  pm2 startup
-  pm2 save
 fi
-
-log_with_style "SUCCESS" "🎉 启动成功!"
+exec /bin/bash "${QL_DIR}/shell/start-simplify.sh" "$@"
@@ -0,0 +1,69 @@
+const test = require('node:test'),
+  assert = require('node:assert/strict'),
+  fs = require('node:fs'),
+  os = require('node:os'),
+  path = require('node:path'),
+  { execFileSync } = require('node:child_process');
+const load = require('../helpers/load-security-module.cjs');
+test('venv installs, uninstalls and probes use the selected Python, including spaces', () => {
+  const methods = load('back/config/util.ts', {
+    './index': { rootPath: '/application' },
+    './const': {
+      PYTHON_VENV_DIR: 'virtual env',
+      PYTHON_INSTALL_DIR: '/legacy',
+    },
+    '../loaders/logger': {},
+    '../shared/utils': {},
+    '../data/dependence': {
+      DependenceTypes: { nodejs: 0, python3: 1, linux: 2 },
+    },
+  });
+  for (const name of [
+    'getInstallCommand',
+    'getUninstallCommand',
+    'getGetCommand',
+  ]) {
+    const cmd = methods[name](1, 'requests');
+    assert.match(cmd, /'\/application\/virtual env\/bin\/python3'/);
+    assert.doesNotMatch(cmd, /--prefix=/);
+  }
+});
+test('venv setup keeps Python packages isolated and retains existing startup configuration', (t) => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-venv-'));
+  t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
+  const ql = path.join(dir, 'application with spaces');
+  fs.mkdirSync(ql);
+  const venv = path.join(dir, 'venv with spaces');
+  execFileSync('python3', ['-m', 'venv', venv]);
+  const source = fs.readFileSync('shell/start-simplify.sh', 'utf8');
+  const snippet = source.slice(
+    source.indexOf('export PNPM_HOME='),
+    source.indexOf('\ncd "${QL_DIR}"'),
+  );
+  const result = execFileSync(
+    '/bin/bash',
+    [
+      '-c',
+      snippet +
+        '\npython3 -c "import sys; print(sys.prefix); print(sys.base_prefix)"',
+    ],
+    {
+      env: {
+        ...process.env,
+        QL_DIR: ql,
+        QL_DATA_DIR: path.join(dir, 'data'),
+        PYTHON_VENV_DIR: venv,
+        PYTHONPATH: '/legacy/modules',
+        PYTHONUSERBASE: '/legacy',
+        PATH: process.env.PATH,
+      },
+      encoding: 'utf8',
+    },
+  )
+    .trim()
+    .split('\n');
+  assert.equal(fs.realpathSync(result[0]), fs.realpathSync(venv));
+  assert.notEqual(result[1], venv);
+  assert.match(source, /set -C; cat .env.example > .env/);
+  assert.doesNotMatch(source, /npm install|pip3 install|chmod 777/);
+});
@@ -7,8 +7,8 @@ const { spawnSync } = require('node:child_process');

 // Run the actual startup prefix, stopping before service initialization.
 const source = fs
-  .readFileSync('shell/start.sh', 'utf8')
-  .split('. ${QL_DIR}/shell/share.sh')[0];
+  .readFileSync('shell/start-simplify.sh', 'utf8')
+  .split('. "${QL_DIR}/shell/share.sh"')[0];
 for (const command of ['start', 'reload']) {
  for (const existing of [false, true]) {
    test(`${command} ${