mirror of
https://github.com/whyour/qinglong.git
synced 2026-10-16 22:35:32 +08:00
feat(python): support isolated venv environments for native deployment
This commit is contained in:
whyour
@@ -21,3 +21,6 @@ JWT_EXPIRES_IN=
|
||||
QINIU_AK=
|
||||
QINIU_SK=
|
||||
QINIU_SCOPE=
|
||||
|
||||
# Python 虚拟环境路径，留空则使用系统 Python
|
||||
# PYTHON_VENV_DIR=./.venv
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
# Node.js 原生部署与 Python venv
|
||||
|
||||
设置 `QL_DIR` 为安装目录，`QL_DATA_DIR` 为以 `/data` 结尾的绝对持久化数据路径。
|
||||
如需隔离 Python，请在启动进程的环境中显式设置 `PYTHON_VENV_DIR`（绝对路径或相对 `QL_DIR` 的路径），例如 `export PYTHON_VENV_DIR=/srv/qinglong/.venv`。未设置时保留现有 Python prefix 行为。
|
||||
|
||||
- `qinglong`（`shell/start.sh`）安装系统依赖和 Node 工具，并安装 Python requests，然后调用公共启动脚本。
|
||||
- `npm run start:simple`（`shell/start-simplify.sh`）使用已准备的 Node、pm2、nginx、jq 等依赖，不安装系统包、全局 npm 包或 requests；显式指定的 venv 不存在时仍会创建。请先在 venv 中安装 requests 和任务所需依赖。
|
||||
- 两种启动方式均保留已有 `.env`；使用 venv 时清除继承的 `PYTHONPATH`/`PYTHONUSERBASE`，依赖查询、安装和卸载使用该 venv 的 Python。
|
||||
|
||||
该模式需具备现有启动流程所要求的 nginx 配置和 pm2 服务权限。生产环境设置变量应通过 systemd 的 `Environment` 或相应进程配置完成。已有系统 Python 依赖不自动迁移至 venv，需在 venv 中重新安装。
|
||||
@@ -27,6 +27,7 @@ export const SAMPLE_FILES = [
|
||||
];
|
||||
|
||||
export const PYTHON_INSTALL_DIR = process.env.PYTHON_HOME;
|
||||
export const PYTHON_VENV_DIR = process.env.PYTHON_VENV_DIR;
|
||||
|
||||
export const NotificationModeStringMap = {
|
||||
0: 'gotify',
|
||||
|
||||
+46
-14
@@ -5,15 +5,20 @@ import psTreeFun from 'ps-tree';
|
||||
import { promisify } from 'util';
|
||||
import { load } from 'js-yaml';
|
||||
import config from './index';
|
||||
import { PYTHON_INSTALL_DIR, TASK_COMMAND } from './const';
|
||||
import { PYTHON_INSTALL_DIR, PYTHON_VENV_DIR, TASK_COMMAND } from './const';
|
||||
import Logger from '../loaders/logger';
|
||||
import { writeFileWithLock } from '../shared/utils';
|
||||
import { DependenceTypes } from '../data/dependence';
|
||||
import { FormData } from 'undici';
|
||||
import os from 'os';
|
||||
import { maybeSudo, isInContainer } from './container';
|
||||
import { shellQuote } from '../shared/shellQuote';
|
||||
import { resolveFileAccess } from '../shared/fileAccess';
|
||||
import { mirrorWriteCommand, normalizeMirror, rewriteMirrorSources } from '../shared/linuxMirror';
|
||||
import {
|
||||
mirrorWriteCommand,
|
||||
normalizeMirror,
|
||||
rewriteMirrorSources,
|
||||
} from '../shared/linuxMirror';
|
||||
|
||||
export * from './share';
|
||||
|
||||
@@ -671,7 +676,9 @@ export async function setSystemTimezone(timezone: string): Promise<boolean> {
|
||||
throw new Error('Invalid timezone');
|
||||
}
|
||||
|
||||
await promiseExec(maybeSudo(`ln -sf /usr/share/zoneinfo/${timezone} /etc/localtime`));
|
||||
await promiseExec(
|
||||
maybeSudo(`ln -sf /usr/share/zoneinfo/${timezone} /etc/localtime`),
|
||||
);
|
||||
await promiseExec(`echo "${timezone}" | ${maybeSudo('tee /etc/timezone')}`);
|
||||
|
||||
return true;
|
||||
@@ -685,7 +692,13 @@ export function getGetCommand(type: DependenceTypes, name: string): string {
|
||||
const baseCommands = {
|
||||
[DependenceTypes.nodejs]: `pnpm ls -g | grep "${name}" | head -1`,
|
||||
[DependenceTypes.python3]: `
|
||||
python3 -c "exec('''
|
||||
${
|
||||
PYTHON_VENV_DIR
|
||||
? shellQuote(
|
||||
path.resolve(config.rootPath, PYTHON_VENV_DIR, 'bin/python3'),
|
||||
)
|
||||
: 'python3'
|
||||
} -c "exec('''
|
||||
name='${name}'
|
||||
try:
|
||||
from importlib.metadata import version
|
||||
@@ -696,7 +709,8 @@ except:
|
||||
spec=u.find_spec(name)
|
||||
print(name if spec else '')
|
||||
''')"`,
|
||||
[DependenceTypes.linux]: getOsTypeSync() === 'Alpine'
|
||||
[DependenceTypes.linux]:
|
||||
getOsTypeSync() === 'Alpine'
|
||||
? `apk info -es ${name}`
|
||||
: maybeSudo(`dpkg-query -s ${name}`),
|
||||
};
|
||||
@@ -707,16 +721,28 @@ except:
|
||||
export function getInstallCommand(type: DependenceTypes, name: string): string {
|
||||
const baseCommands = {
|
||||
[DependenceTypes.nodejs]: 'pnpm add -g',
|
||||
[DependenceTypes.python3]:
|
||||
'pip3 install --disable-pip-version-check --root-user-action=ignore',
|
||||
[DependenceTypes.linux]: getOsTypeSync() === 'Alpine'
|
||||
[DependenceTypes.python3]: `${
|
||||
PYTHON_VENV_DIR
|
||||
? shellQuote(
|
||||
path.resolve(config.rootPath, PYTHON_VENV_DIR, 'bin/python3'),
|
||||
) + ' -m pip'
|
||||
: 'pip3'
|
||||
} install --disable-pip-version-check${
|
||||
PYTHON_VENV_DIR ? '' : ' --root-user-action=ignore'
|
||||
}`,
|
||||
[DependenceTypes.linux]:
|
||||
getOsTypeSync() === 'Alpine'
|
||||
? 'apk add --no-check-certificate'
|
||||
: maybeSudo('apt-get install -y'),
|
||||
};
|
||||
|
||||
let command = baseCommands[type];
|
||||
|
||||
if (type === DependenceTypes.python3 && PYTHON_INSTALL_DIR) {
|
||||
if (
|
||||
type === DependenceTypes.python3 &&
|
||||
PYTHON_INSTALL_DIR &&
|
||||
!PYTHON_VENV_DIR
|
||||
) {
|
||||
command = `${command} --prefix=${PYTHON_INSTALL_DIR}`;
|
||||
}
|
||||
|
||||
@@ -729,11 +755,17 @@ export function getUninstallCommand(
|
||||
): string {
|
||||
const baseCommands = {
|
||||
[DependenceTypes.nodejs]: 'pnpm remove -g',
|
||||
[DependenceTypes.python3]:
|
||||
'pip3 uninstall --disable-pip-version-check --root-user-action=ignore -y',
|
||||
[DependenceTypes.linux]: getOsTypeSync() === 'Alpine'
|
||||
? 'apk del'
|
||||
: maybeSudo('apt-get remove -y'),
|
||||
[DependenceTypes.python3]: `${
|
||||
PYTHON_VENV_DIR
|
||||
? shellQuote(
|
||||
path.resolve(config.rootPath, PYTHON_VENV_DIR, 'bin/python3'),
|
||||
) + ' -m pip'
|
||||
: 'pip3'
|
||||
} uninstall --disable-pip-version-check${
|
||||
PYTHON_VENV_DIR ? '' : ' --root-user-action=ignore'
|
||||
} -y`,
|
||||
[DependenceTypes.linux]:
|
||||
getOsTypeSync() === 'Alpine' ? 'apk del' : maybeSudo('apt-get remove -y'),
|
||||
};
|
||||
|
||||
return `${baseCommands[type]} ${name.trim()}`;
|
||||
|
||||
+2
-1
@@ -28,7 +28,8 @@
|
||||
"panel": "npm run build:back && node static/build/app.js",
|
||||
"gen:proto": "protoc --experimental_allow_proto3_optional --plugin=./node_modules/.bin/protoc-gen-ts_proto ./back/protos/*.proto --ts_proto_out=./ --ts_proto_opt=outputServices=grpc-js,env=node,esModuleInterop=true,snakeToCamel=false",
|
||||
"prettier": "prettier --write '**/*.{js,jsx,tsx,ts,less,md,json}'",
|
||||
"postinstall": "max setup 2>/dev/null || true"
|
||||
"postinstall": "max setup 2>/dev/null || true",
|
||||
"start:simple": "bash shell/start-simplify.sh"
|
||||
},
|
||||
"gitHooks": {
|
||||
"pre-commit": "lint-staged"
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
: "${QL_DIR:?Please set QL_DIR}"
|
||||
: "${QL_DATA_DIR:?Please set QL_DATA_DIR}"
|
||||
[[ "$QL_DIR" = /* && "$QL_DATA_DIR" = /*/data ]] || { echo "QL_DIR and QL_DATA_DIR must be absolute paths (data directory ends in /data)" >&2; exit 1; }
|
||||
command="$1"
|
||||
|
||||
export PNPM_HOME="${PNPM_HOME:-${QL_DATA_DIR}/dep_cache/node}"
|
||||
if [[ -n "${PYTHON_VENV_DIR:-}" ]]; then
|
||||
[[ "$PYTHON_VENV_DIR" = /* ]] || PYTHON_VENV_DIR="${QL_DIR}/${PYTHON_VENV_DIR}"
|
||||
export PYTHON_VENV_DIR
|
||||
if [[ ! -x "$PYTHON_VENV_DIR/bin/python3" ]]; then
|
||||
python3 -m venv "$PYTHON_VENV_DIR"
|
||||
fi
|
||||
[[ -x "$PYTHON_VENV_DIR/bin/python3" ]] || { echo "Invalid Python venv" >&2; exit 1; }
|
||||
unset PYTHON_HOME PYTHONUSERBASE PYTHONPATH
|
||||
export PATH="${PYTHON_VENV_DIR}/bin:${PATH}:${PNPM_HOME}"
|
||||
export PIP_CACHE_DIR="${PYTHON_VENV_DIR}/pip"
|
||||
else
|
||||
export PYTHON_SHORT_VERSION=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
|
||||
export PYTHON_HOME="${PYTHON_HOME:-${QL_DATA_DIR}/dep_cache/python3}"
|
||||
export PYTHONUSERBASE="$PYTHON_HOME"
|
||||
export PATH="${PATH}:${PNPM_HOME}:${PYTHON_HOME}/bin"
|
||||
export PIP_CACHE_DIR="${PYTHON_HOME}/pip"
|
||||
export PYTHONPATH="${PYTHON_HOME}:${PYTHON_HOME}/lib/python${PYTHON_SHORT_VERSION}:${PYTHON_HOME}/lib/python${PYTHON_SHORT_VERSION}/site-packages"
|
||||
fi
|
||||
export NODE_PATH="${NODE_PATH:-/usr/local/bin:/usr/local/lib/node_modules:${PNPM_HOME}/global/5/node_modules}"
|
||||
mkdir -p "$PIP_CACHE_DIR"
|
||||
|
||||
cd "${QL_DIR}"
|
||||
if [[ ! -e .env ]]; then
|
||||
# Preserve user configuration, including a file created concurrently.
|
||||
(umask 077; set -C; cat .env.example > .env)
|
||||
fi
|
||||
chmod u+x "${QL_DIR}"/shell/*.sh
|
||||
|
||||
. "${QL_DIR}/shell/share.sh"
|
||||
. "${QL_DIR}/shell/env.sh"
|
||||
|
||||
log_with_style() {
|
||||
local level="$1"
|
||||
local message="$2"
|
||||
local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
printf "\n[%s] [%7s] %s\n" "${timestamp}" "${level}" "${message}"
|
||||
}
|
||||
|
||||
log_with_style "INFO" "🚀 1. 检测配置文件..."
|
||||
import_config "$@"
|
||||
make_dir /etc/nginx/conf.d
|
||||
make_dir /run/nginx
|
||||
fix_config
|
||||
|
||||
pm2 l &>/dev/null
|
||||
|
||||
log_with_style "INFO" "🔄 2. 启动 nginx..."
|
||||
nginx -s reload 2>/dev/null || nginx -c /etc/nginx/nginx.conf
|
||||
|
||||
log_with_style "INFO" "⚙️ 3. 启动 pm2 服务..."
|
||||
reload_pm2
|
||||
|
||||
if [[ $command != "reload" ]]; then
|
||||
if [[ $AutoStartBot == true ]]; then
|
||||
log_with_style "INFO" "🤖 4. 启动 bot..."
|
||||
nohup ql bot >$dir_log/bot.log 2>&1 &
|
||||
fi
|
||||
|
||||
if [[ $EnableExtraShell == true ]]; then
|
||||
log_with_style "INFO" "🛠️ 5. 执行自定义脚本..."
|
||||
nohup ql extra >$dir_log/extra.log 2>&1 &
|
||||
fi
|
||||
|
||||
pm2 startup
|
||||
pm2 save
|
||||
fi
|
||||
|
||||
log_with_style "SUCCESS" "🎉 启动成功!"
|
||||
+8
-60
@@ -77,65 +77,13 @@ if [[ $command != "reload" ]]; then
|
||||
npm install -g pnpm@8.3.1 pm2 ts-node typescript@5
|
||||
fi
|
||||
|
||||
export PYTHON_SHORT_VERSION=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
|
||||
export PNPM_HOME=${QL_DIR}/data/dep_cache/node
|
||||
export PYTHON_HOME=${QL_DIR}/data/dep_cache/python3
|
||||
export PYTHONUSERBASE=${QL_DIR}/data/dep_cache/python3
|
||||
|
||||
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PNPM_HOME}:${PYTHON_HOME}/bin
|
||||
export NODE_PATH=/usr/local/bin:/usr/local/lib/node_modules:${PNPM_HOME}/global/5/node_modules
|
||||
export PIP_CACHE_DIR=${PYTHON_HOME}/pip
|
||||
export PYTHONPATH=${PYTHON_HOME}:${PYTHON_HOME}/lib/python${PYTHON_SHORT_VERSION}:${PYTHON_HOME}/lib/python${PYTHON_SHORT_VERSION}/site-packages
|
||||
|
||||
if [[ $command != "reload" ]]; then
|
||||
pip3 install --prefix ${PYTHON_HOME} requests
|
||||
if [[ "$command" != "reload" ]]; then
|
||||
if [[ -n "${PYTHON_VENV_DIR:-}" ]]; then
|
||||
[[ "$PYTHON_VENV_DIR" = /* ]] || export PYTHON_VENV_DIR="${QL_DIR}/${PYTHON_VENV_DIR}"
|
||||
[[ -x "$PYTHON_VENV_DIR/bin/python3" ]] || python3 -m venv "$PYTHON_VENV_DIR"
|
||||
"$PYTHON_VENV_DIR/bin/python3" -m pip install requests
|
||||
else
|
||||
pip3 install --prefix "${PYTHON_HOME:-${QL_DATA_DIR}/dep_cache/python3}" requests
|
||||
fi
|
||||
|
||||
cd "${QL_DIR}"
|
||||
if [[ ! -e .env ]]; then
|
||||
# Preserve user configuration, including a file created concurrently.
|
||||
(umask 077; set -C; cat .env.example > .env)
|
||||
fi
|
||||
chmod 777 ${QL_DIR}/shell/*.sh
|
||||
|
||||
. ${QL_DIR}/shell/share.sh
|
||||
. ${QL_DIR}/shell/env.sh
|
||||
|
||||
log_with_style() {
|
||||
local level="$1"
|
||||
local message="$2"
|
||||
local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
printf "\n[%s] [%7s] %s\n" "${timestamp}" "${level}" "${message}"
|
||||
}
|
||||
|
||||
log_with_style "INFO" "🚀 1. 检测配置文件..."
|
||||
import_config "$@"
|
||||
make_dir /etc/nginx/conf.d
|
||||
make_dir /run/nginx
|
||||
fix_config
|
||||
|
||||
pm2 l &>/dev/null
|
||||
|
||||
log_with_style "INFO" "🔄 2. 启动 nginx..."
|
||||
nginx -s reload 2>/dev/null || nginx -c /etc/nginx/nginx.conf
|
||||
|
||||
log_with_style "INFO" "⚙️ 3. 启动 pm2 服务..."
|
||||
reload_pm2
|
||||
|
||||
if [[ $command != "reload" ]]; then
|
||||
if [[ $AutoStartBot == true ]]; then
|
||||
log_with_style "INFO" "🤖 4. 启动 bot..."
|
||||
nohup ql bot >$dir_log/bot.log 2>&1 &
|
||||
fi
|
||||
|
||||
if [[ $EnableExtraShell == true ]]; then
|
||||
log_with_style "INFO" "🛠️ 5. 执行自定义脚本..."
|
||||
nohup ql extra >$dir_log/extra.log 2>&1 &
|
||||
fi
|
||||
|
||||
pm2 startup
|
||||
pm2 save
|
||||
fi
|
||||
|
||||
log_with_style "SUCCESS" "🎉 启动成功!"
|
||||
exec /bin/bash "${QL_DIR}/shell/start-simplify.sh" "$@"
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
const test = require('node:test'),
|
||||
assert = require('node:assert/strict'),
|
||||
fs = require('node:fs'),
|
||||
os = require('node:os'),
|
||||
path = require('node:path'),
|
||||
{ execFileSync } = require('node:child_process');
|
||||
const load = require('../helpers/load-security-module.cjs');
|
||||
test('venv installs, uninstalls and probes use the selected Python, including spaces', () => {
|
||||
const methods = load('back/config/util.ts', {
|
||||
'./index': { rootPath: '/application' },
|
||||
'./const': {
|
||||
PYTHON_VENV_DIR: 'virtual env',
|
||||
PYTHON_INSTALL_DIR: '/legacy',
|
||||
},
|
||||
'../loaders/logger': {},
|
||||
'../shared/utils': {},
|
||||
'../data/dependence': {
|
||||
DependenceTypes: { nodejs: 0, python3: 1, linux: 2 },
|
||||
},
|
||||
});
|
||||
for (const name of [
|
||||
'getInstallCommand',
|
||||
'getUninstallCommand',
|
||||
'getGetCommand',
|
||||
]) {
|
||||
const cmd = methods[name](1, 'requests');
|
||||
assert.match(cmd, /'\/application\/virtual env\/bin\/python3'/);
|
||||
assert.doesNotMatch(cmd, /--prefix=/);
|
||||
}
|
||||
});
|
||||
test('venv setup keeps Python packages isolated and retains existing startup configuration', (t) => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-venv-'));
|
||||
t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
|
||||
const ql = path.join(dir, 'application with spaces');
|
||||
fs.mkdirSync(ql);
|
||||
const venv = path.join(dir, 'venv with spaces');
|
||||
execFileSync('python3', ['-m', 'venv', venv]);
|
||||
const source = fs.readFileSync('shell/start-simplify.sh', 'utf8');
|
||||
const snippet = source.slice(
|
||||
source.indexOf('export PNPM_HOME='),
|
||||
source.indexOf('\ncd "${QL_DIR}"'),
|
||||
);
|
||||
const result = execFileSync(
|
||||
'/bin/bash',
|
||||
[
|
||||
'-c',
|
||||
snippet +
|
||||
'\npython3 -c "import sys; print(sys.prefix); print(sys.base_prefix)"',
|
||||
],
|
||||
{
|
||||
env: {
|
||||
...process.env,
|
||||
QL_DIR: ql,
|
||||
QL_DATA_DIR: path.join(dir, 'data'),
|
||||
PYTHON_VENV_DIR: venv,
|
||||
PYTHONPATH: '/legacy/modules',
|
||||
PYTHONUSERBASE: '/legacy',
|
||||
PATH: process.env.PATH,
|
||||
},
|
||||
encoding: 'utf8',
|
||||
},
|
||||
)
|
||||
.trim()
|
||||
.split('\n');
|
||||
assert.equal(fs.realpathSync(result[0]), fs.realpathSync(venv));
|
||||
assert.notEqual(result[1], venv);
|
||||
assert.match(source, /set -C; cat .env.example > .env/);
|
||||
assert.doesNotMatch(source, /npm install|pip3 install|chmod 777/);
|
||||
});
|
||||
@@ -7,8 +7,8 @@ const { spawnSync } = require('node:child_process');
|
||||
|
||||
// Run the actual startup prefix, stopping before service initialization.
|
||||
const source = fs
|
||||
.readFileSync('shell/start.sh', 'utf8')
|
||||
.split('. ${QL_DIR}/shell/share.sh')[0];
|
||||
.readFileSync('shell/start-simplify.sh', 'utf8')
|
||||
.split('. "${QL_DIR}/shell/share.sh"')[0];
|
||||
for (const command of ['start', 'reload']) {
|
||||
for (const existing of [false, true]) {
|
||||
test(`${command} ${
|
||||
|
||||
Reference in New Issue
Block a user