feat(deps): support Python Git URLs and requirements files

back/config/util.ts

@@ -1,4 +1,5 @@
 import * as fs from 'fs/promises';
+import { statSync } from 'fs';
 import * as path from 'path';
 import { exec, execSync } from 'child_process';
 import psTreeFun from 'ps-tree';
@@ -688,86 +689,150 @@ export async function setSystemTimezone(timezone: string): Promise<boolean> {
   }
 }
 
+// Helper function to check if a name is a GitHub URL
+function getPythonExecutable(): string {
+  return PYTHON_VENV_DIR
+    ? shellQuote(path.resolve(config.rootPath, PYTHON_VENV_DIR, 'bin/python3'))
+    : 'python3';
+}
+
+function getPythonPip(): string {
+  return PYTHON_VENV_DIR ? `${getPythonExecutable()} -m pip` : 'pip3';
+}
+
+function isGitHubUrl(name: string): boolean {
+  // Support git+https://, git+http://, https://, and http:// URLs
+  // This covers GitHub URLs and other git-compatible repositories
+  return !!name.match(/^(git\+https?:\/\/|https?:\/\/)/i);
+}
+
+// Helper function to check if a name is a requirements file
+function isRequirementsFile(name: string): boolean {
+  return !!name.match(/requirements.*\.(txt|in)$/i);
+}
+
+// Helper function to check if a name is a pyproject.toml file
+function isPyprojectToml(name: string): boolean {
+  return name.endsWith('pyproject.toml');
+}
+
 export function getGetCommand(type: DependenceTypes, name: string): string {
+  const trimmedName = name.trim();
+
+  // For Python dependencies installed from GitHub or requirements files,
+  // we can't reliably check if they're installed, so skip the check
+  if (type === DependenceTypes.python3) {
+    if (
+      isGitHubUrl(trimmedName) ||
+      isRequirementsFile(trimmedName) ||
+      isPyprojectToml(trimmedName)
+    ) {
+      // Return a command that will always indicate not installed
+      // This ensures GitHub URLs and requirements files are always installed
+      return 'echo ""';
+    }
+  }
+
   const baseCommands = {
-    [DependenceTypes.nodejs]: `pnpm ls -g  | grep "${name}" | head -1`,
+    [DependenceTypes.nodejs]: `pnpm ls -g  | grep "${trimmedName}" | head -1`,
-    [DependenceTypes.python3]: `
+    [DependenceTypes.python3]: `${getPythonExecutable()} -c ${shellQuote(
-    ${
+      `from importlib.metadata import version\nimport importlib.util as u\nname=${JSON.stringify(
-      PYTHON_VENV_DIR
+        trimmedName,
-        ? shellQuote(
+      )}\ntry:\n print(version(name))\nexcept Exception:\n print(name if u.find_spec(name) else '')`,
-            path.resolve(config.rootPath, PYTHON_VENV_DIR, 'bin/python3'),
+    )}`,
-          )
-        : 'python3'
-    } -c "exec('''
-name='${name}'
-try:
-    from importlib.metadata import version
-    print(version(name))
-except:
-    import importlib.util as u
-    import importlib.metadata as m
-    spec=u.find_spec(name)
-    print(name if spec else '')
-''')"`,
     [DependenceTypes.linux]:
       getOsTypeSync() === 'Alpine'
-        ? `apk info -es ${name}`
+        ? `apk info -es ${trimmedName}`
-        : maybeSudo(`dpkg-query -s ${name}`),
+        : maybeSudo(`dpkg-query -s ${trimmedName}`),
   };
 
   return baseCommands[type];
 }
 
 export function getInstallCommand(type: DependenceTypes, name: string): string {
+  if (type === DependenceTypes.python3) {
+    const prefix =
+      PYTHON_INSTALL_DIR && !PYTHON_VENV_DIR
+        ? ` --prefix=${shellQuote(PYTHON_INSTALL_DIR)}`
+        : '';
+    const pip = `${getPythonPip()} install --disable-pip-version-check${
+      PYTHON_VENV_DIR ? '' : ' --root-user-action=ignore'
+    }${prefix}`;
+    name = name.trim();
+    if (isRequirementsFile(name) || isPyprojectToml(name)) {
+      const file = [config.scriptPath, config.repoPath]
+        .filter(Boolean)
+        .map((root) => resolveFileAccess(root, [name]))
+        .find((file) => {
+          if (!file) return false;
+          try {
+            return statSync(file).isFile();
+          } catch {
+            return false;
+          }
+        });
+      if (!file)
+        throw new Error(
+          'Python dependency file must be inside scripts or repo',
+        );
+      return isRequirementsFile(name)
+        ? `${pip} -r ${shellQuote(file)}`
+        : `${pip} ${shellQuote(path.dirname(file))}`;
+    }
+    return `${pip} -- ${shellQuote(name)}`;
+  }
   const baseCommands = {
     [DependenceTypes.nodejs]: 'pnpm add -g',
-    [DependenceTypes.python3]: `${
-      PYTHON_VENV_DIR
-        ? shellQuote(
-            path.resolve(config.rootPath, PYTHON_VENV_DIR, 'bin/python3'),
-          ) + ' -m pip'
-        : 'pip3'
-    } install --disable-pip-version-check${
-      PYTHON_VENV_DIR ? '' : ' --root-user-action=ignore'
-    }`,
     [DependenceTypes.linux]:
       getOsTypeSync() === 'Alpine'
         ? 'apk add --no-check-certificate'
         : maybeSudo('apt-get install -y'),
   };
-
+  return `${baseCommands[type]} ${name.trim()}`;
-  let command = baseCommands[type];
-
-  if (
-    type === DependenceTypes.python3 &&
-    PYTHON_INSTALL_DIR &&
-    !PYTHON_VENV_DIR
-  ) {
-    command = `${command} --prefix=${PYTHON_INSTALL_DIR}`;
-  }
-
-  return `${command} ${name.trim()}`;
 }
 
 export function getUninstallCommand(
   type: DependenceTypes,
   name: string,
 ): string {
+  if (type === DependenceTypes.python3) {
+    name = name.trim();
+    if (isRequirementsFile(name)) {
+      const file = [config.scriptPath, config.repoPath]
+        .filter(Boolean)
+        .map((root) => resolveFileAccess(root, [name]))
+        .find((file) => {
+          if (!file) return false;
+          try {
+            return statSync(file).isFile();
+          } catch {
+            return false;
+          }
+        });
+      if (!file)
+        throw new Error(
+          'Python dependency file must be inside scripts or repo',
+        );
+      return `${getPythonPip()} uninstall --disable-pip-version-check${
+        PYTHON_VENV_DIR ? '' : ' --root-user-action=ignore'
+      } -y -r ${shellQuote(file)}`;
+    }
+    if (isPyprojectToml(name) || isGitHubUrl(name)) {
+      // A source URL/project can install several distributions and has no safe
+      // inverse. Removing its panel entry leaves installed packages intact.
+      return `printf '%s\\n' ${shellQuote(
+        'Remove installed Python distributions separately by package name.',
+      )}`;
+    }
+    return `${getPythonPip()} uninstall --disable-pip-version-check${
+      PYTHON_VENV_DIR ? '' : ' --root-user-action=ignore'
+    } -y -- ${shellQuote(name)}`;
+  }
   const baseCommands = {
     [DependenceTypes.nodejs]: 'pnpm remove -g',
-    [DependenceTypes.python3]: `${
-      PYTHON_VENV_DIR
-        ? shellQuote(
-            path.resolve(config.rootPath, PYTHON_VENV_DIR, 'bin/python3'),
-          ) + ' -m pip'
-        : 'pip3'
-    } uninstall --disable-pip-version-check${
-      PYTHON_VENV_DIR ? '' : ' --root-user-action=ignore'
-    } -y`,
     [DependenceTypes.linux]:
       getOsTypeSync() === 'Alpine' ? 'apk del' : maybeSudo('apt-get remove -y'),
   };
-
   return `${baseCommands[type]} ${name.trim()}`;
 }
 

back/services/dependence.ts

@@ -36,6 +36,7 @@ export default class DependenceService {
 
   public async create(payloads: Dependence[]): Promise<Dependence[]> {
     const tabs = payloads.map((x) => {
+      getInstallCommand(x.type, x.name);
       const tab = new Dependence({ ...x, status: DependenceStatus.queued });
       return tab;
     });
@@ -59,6 +60,7 @@ export default class DependenceService {
       ...other,
       status: DependenceStatus.queued,
     });
+    getInstallCommand(tab.type, tab.name);
     const newDoc = await this.updateDb(tab);
     this.installDependenceOneByOne([newDoc]);
     return newDoc;
@@ -296,6 +298,23 @@ export default class DependenceService {
     force: boolean = false,
   ) {
     return taskLimit.runDependeny(dependency, () => {
+      let validatedCommand: string;
+      try {
+        validatedCommand = isInstall
+          ? getInstallCommand(dependency.type, dependency.name.trim())
+          : getUninstallCommand(dependency.type, dependency.name.trim());
+      } catch (error) {
+        taskLimit.removeQueuedDependency(dependency);
+        return DependenceModel.update(
+          {
+            status: isInstall
+              ? DependenceStatus.installFailed
+              : DependenceStatus.removeFailed,
+            log: [String(error)],
+          },
+          { where: { id: dependency.id } },
+        ).then(() => null);
+      }
       return new Promise(async (resolve) => {
         if (taskLimit.firstDependencyId !== dependency.id) {
           return resolve(null);
@@ -345,9 +364,7 @@ export default class DependenceService {
           : DependenceStatus.removing;
         await DependenceModel.update({ status }, { where: { id: depIds } });
 
-        let command = isInstall
+        let command = validatedCommand;
-          ? getInstallCommand(dependency.type, depName)
-          : getUninstallCommand(dependency.type, depName);
         if (isLinuxDependence) {
           command = isInstall
             ? `${linuxCommand.install} ${depName.trim()}`
@@ -470,15 +487,20 @@ export default class DependenceService {
 
           const message =
             '\n' +
-            tf('依赖%s%s，结束时间 %s，耗时 %s 秒',
+            tf(
+              '依赖%s%s，结束时间 %s，耗时 %s 秒',
               actionText,
               resultText,
               endTime.format('YYYY-MM-DD HH:mm:ss'),
               String(endTime.diff(startTime, 'second')),
             );
           const exitStatus = isSucceed
-            ? (isInstall ? DependenceStatus.installed : DependenceStatus.removed)
+            ? isInstall
-            : (isInstall ? DependenceStatus.installFailed : DependenceStatus.removeFailed);
+              ? DependenceStatus.installed
+              : DependenceStatus.removed
+            : isInstall
+            ? DependenceStatus.installFailed
+            : DependenceStatus.removeFailed;
           this.sockService.sendMessage({
             type: socketMessageType,
             message,

docs/development/python-dependency-sources.md

@@ -0,0 +1,7 @@
+# Python 依赖来源
+
+依赖管理中可手动添加 Python 包名、`git+https://` 等 pip 支持的源 URL，或脚本/订阅仓库目录内的 requirements 文件、pyproject.toml 路径。文件路径支持目录空格，必须位于持久化 scripts 或 repo 根目录内；越界与指向根目录外的符号链接会被拒绝。
+
+requirements 使用 `pip install -r`，pyproject 使用完整项目目录安装，保留相对依赖文件和项目源码。拉取订阅不会自动安装仓库声明的依赖。
+
+删除 requirements 条目会调用 `pip uninstall -r`。源 URL 或 pyproject 条目无法可靠逆向推断其安装的所有分发包，因此删除该条目只移除面板记录；需另外按包名卸载已安装的包，日志会明确提示。

src/locales/en-US.json

@@ -395,7 +395,7 @@
   "父目录": "Parent Directory",
   "版本": "Version",
   "状态": "Status",
-  "环境变量": "Environment Variables",
+  "环境变量": "Environment variable",
   "生成数据中...": "Generating data...",
   "用于访问所有私有仓库的全局SSH私钥": "Global SSH private key for accessing all private repositories",
   "用户令牌，扫描登录后 我的—>设置->令牌 中获取，参考 https://www.weplusbot.com/": "Token, which can be obtained after scanning and logging in, is available under 'My Account' -> 'Settings' -> 'Tokens'. Please refer to the instructions for detailed steps: https://www.weplusbot.com/",
@@ -634,7 +634,6 @@
   "保存配置": "Save",
   "当前生效值": "Effective value",
   "默认配置": "Default",
-  "环境变量": "Environment variable",
   "客户端 IP 诊断": "Client IP diagnostics",
   "重新诊断": "Run again",
   "Socket 地址": "Socket address",
@@ -699,5 +698,7 @@
   "仍然保存": "Save anyway",
   "已保存，本次未记录历史版本": "Saved without a history snapshot",
   "已保存，但旧历史清理失败，请检查存储权限和空间": "Saved, but old history could not be removed. Check storage permissions and free space.",
-  "保存失败": "Unable to save"
+  "保存失败": "Unable to save",
+  "Python支持多种安装方式：\n1. 包名（如：requests）\n2. GitHub链接（如：git+https://github.com/user/repo.git）\n3. requirements文件路径（如：path/to/requirements.txt）\n4. pyproject.toml文件路径": "Python supports package names, Git URLs, and existing requirements.txt or pyproject.toml files under the scripts or repository directory.",
+  "支持包名、GitHub链接、requirements.txt或pyproject.toml路径": "Package name, Git URL, requirements.txt or pyproject.toml path"
 }

src/locales/zh-CN.json

@@ -634,7 +634,6 @@
   "保存配置": "保存配置",
   "当前生效值": "当前生效值",
   "默认配置": "默认配置",
-  "环境变量": "环境变量",
   "客户端 IP 诊断": "客户端 IP 诊断",
   "重新诊断": "重新诊断",
   "Socket 地址": "Socket 地址",
@@ -699,5 +698,7 @@
   "仍然保存": "仍然保存",
   "已保存，本次未记录历史版本": "已保存，本次未记录历史版本",
   "已保存，但旧历史清理失败，请检查存储权限和空间": "已保存，但旧历史清理失败，请检查存储权限和空间",
-  "保存失败": "保存失败"
+  "保存失败": "保存失败",
+  "Python支持多种安装方式：\n1. 包名（如：requests）\n2. GitHub链接（如：git+https://github.com/user/repo.git）\n3. requirements文件路径（如：path/to/requirements.txt）\n4. pyproject.toml文件路径": "Python支持多种安装方式：\n1. 包名（如：requests）\n2. GitHub链接（如：git+https://github.com/user/repo.git）\n3. requirements文件路径（如：path/to/requirements.txt）\n4. pyproject.toml文件路径",
+  "支持包名、GitHub链接、requirements.txt或pyproject.toml路径": "支持包名、GitHub链接、requirements.txt或pyproject.toml路径"
 }

src/pages/dependence/modal.tsx

@@ -22,6 +22,9 @@ const DependenceModal = ({
 }) => {
   const [form] = Form.useForm();
   const [loading, setLoading] = useState(false);
+  const [selectedType, setSelectedType] = useState(
+    DependenceTypes[defaultType as any],
+  );
 
   const handleOk = async (values: any) => {
     setLoading(true);
@@ -90,7 +93,7 @@ const DependenceModal = ({
           label={intl.get('依赖类型')}
           initialValue={DependenceTypes[defaultType as any]}
         >
-          <Select>
+          <Select onChange={(value) => setSelectedType(value)}>
             {config.dependenceTypes.map((x, i) => (
               <Option key={i} value={i}>
                 {x}
@@ -121,11 +124,24 @@ const DependenceModal = ({
               whitespace: true,
             },
           ]}
+          tooltip={
+            selectedType === DependenceTypes.python3
+              ? intl.get(
+                  'Python支持多种安装方式：\n1. 包名（如：requests）\n2. GitHub链接（如：git+https://github.com/user/repo.git）\n3. requirements文件路径（如：path/to/requirements.txt）\n4. pyproject.toml文件路径',
+                )
+              : undefined
+          }
         >
           <Input.TextArea
             rows={4}
             autoSize={{ minRows: 1, maxRows: 5 }}
-            placeholder={intl.get('请输入依赖名称')}
+            placeholder={
+              selectedType === DependenceTypes.python3
+                ? intl.get(
+                    '支持包名、GitHub链接、requirements.txt或pyproject.toml路径',
+                  )
+                : intl.get('请输入依赖名称')
+            }
           />
         </Form.Item>
         <Form.Item name="remark" label={intl.get('备注')}>

test/back/python-source-dependencies.test.cjs

@@ -0,0 +1,63 @@
+const test = require('node:test'),
+  assert = require('node:assert/strict'),
+  fs = require('node:fs'),
+  os = require('node:os'),
+  path = require('node:path'),
+  { execFileSync } = require('node:child_process');
+const load = require('../helpers/load-security-module.cjs');
+test('Python source arguments round-trip literally and local files stay inside maintained roots', (t) => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-pip-source-'));
+  t.after(() => fs.rmSync(root, { recursive: true, force: true }));
+  const scripts = path.join(root, 'scripts'),
+    repo = path.join(root, 'repo');
+  fs.mkdirSync(scripts);
+  fs.mkdirSync(repo);
+  fs.mkdirSync(path.join(scripts, 'project space'));
+  const requirement = path.join(scripts, 'project space', 'requirements.txt');
+  fs.writeFileSync(requirement, 'requests');
+  fs.writeFileSync(
+    path.join(scripts, 'project space', 'pyproject.toml'),
+    '[project]\nname="sample"',
+  );
+  const outside = path.join(root, 'requirements.txt');
+  fs.writeFileSync(outside, 'outside');
+  fs.symlinkSync(outside, path.join(scripts, 'requirements-link.txt'));
+  const api = load('back/config/util.ts', {
+    './index': { scriptPath: scripts, repoPath: repo },
+    './const': { PYTHON_INSTALL_DIR: '/prefix with spaces' },
+    '../loaders/logger': {},
+    '../shared/utils': {},
+    '../data/dependence': {
+      DependenceTypes: { nodejs: 0, python3: 1, linux: 2 },
+    },
+  });
+  function args(cmd) {
+    return execFileSync('/bin/bash', [
+      '-c',
+      'pip3(){ printf "%s\\0" "$@"; }; ' + cmd,
+    ])
+      .toString()
+      .split('\0')
+      .slice(0, -1);
+  }
+  const req = args(api.getInstallCommand(1, 'project space/requirements.txt'));
+  assert.equal(req.at(-1), requirement);
+  assert.equal(req.at(-2), '-r');
+  assert.ok(req.includes('--prefix=/prefix with spaces'));
+  assert.equal(
+    args(api.getInstallCommand(1, 'project space/pyproject.toml')).at(-1),
+    path.dirname(requirement),
+  );
+  const url = 'git+https://example.invalid/repo.git#egg=pkg;$(echo INJECTED)';
+  assert.equal(args(api.getInstallCommand(1, url)).at(-1), url);
+  assert.throws(() => api.getInstallCommand(1, '../requirements.txt'));
+  assert.throws(() => api.getInstallCommand(1, 'requirements-link.txt'));
+  assert.equal(
+    args(api.getUninstallCommand(1, 'project space/requirements.txt')).at(-1),
+    requirement,
+  );
+  assert.match(
+    api.getUninstallCommand(1, url),
+    /Remove installed Python distributions separately/,
+  );
+});

test/back/python-source-venv.test.cjs

@@ -0,0 +1,70 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { execFileSync } = require('node:child_process');
+const load = require('../helpers/load-security-module.cjs');
+
+test('requirements install, probe and removal stay in the selected venv', (t) => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-source-venv-'));
+  t.after(() => fs.rmSync(root, { recursive: true, force: true }));
+  const venv = path.join(root, 'virtual env');
+  const scripts = path.join(root, 'scripts');
+  fs.mkdirSync(scripts);
+  execFileSync('python3', ['-m', 'venv', venv]);
+  const wheel = path.join(scripts, 'ql_venv_fixture-1.0-py3-none-any.whl');
+  execFileSync('python3', [
+    '-c',
+    `import zipfile,sys
+with zipfile.ZipFile(sys.argv[1],'w') as z:
+ z.writestr('ql_venv_fixture.py','value = 42')
+ z.writestr('ql_venv_fixture-1.0.dist-info/METADATA','Metadata-Version: 2.1\\nName: ql-venv-fixture\\nVersion: 1.0\\n')
+ z.writestr('ql_venv_fixture-1.0.dist-info/WHEEL','Wheel-Version: 1.0\\nGenerator: fixture\\nRoot-Is-Purelib: true\\nTag: py3-none-any\\n')
+ z.writestr('ql_venv_fixture-1.0.dist-info/RECORD','')`,
+    wheel,
+  ]);
+  fs.writeFileSync(path.join(scripts, 'requirements.txt'), wheel + '\n');
+  const api = load('back/config/util.ts', {
+    './index': {
+      rootPath: root,
+      scriptPath: scripts,
+      repoPath: path.join(root, 'repo'),
+    },
+    './const': {
+      PYTHON_VENV_DIR: 'virtual env',
+      PYTHON_INSTALL_DIR: '/unused-global-prefix',
+    },
+    '../loaders/logger': {},
+    '../shared/utils': {},
+    '../data/dependence': {
+      DependenceTypes: { nodejs: 0, python3: 1, linux: 2 },
+    },
+  });
+  const run = (command) =>
+    execFileSync('/bin/bash', ['-c', command], {
+      encoding: 'utf8',
+      env: {
+        ...process.env,
+        PIP_NO_INDEX: '1',
+        PIP_DISABLE_PIP_VERSION_CHECK: '1',
+      },
+    });
+  const install = api.getInstallCommand(1, 'requirements.txt');
+  assert.doesNotMatch(install, /--prefix=/);
+  run(install);
+  assert.match(run(api.getGetCommand(1, 'ql-venv-fixture')), /1\.0/);
+  assert.equal(
+    run(
+      `'${path.join(
+        venv,
+        'bin/python3',
+      )}' -c 'import ql_venv_fixture; print(ql_venv_fixture.value)'`,
+    ).trim(),
+    '42',
+  );
+  run(api.getUninstallCommand(1, 'requirements.txt'));
+  assert.throws(() =>
+    run(`'${path.join(venv, 'bin/python3')}' -c 'import ql_venv_fixture'`),
+  );
+});