From e555fcbb0b57d9bc3fbe946f99e4ac044f28043a Mon Sep 17 00:00:00 2001 From: whyour Date: Thu, 15 Oct 2026 23:36:21 +0800 Subject: [PATCH] fix: honor Python requirement specifiers and portable script discovery --- back/config/util.ts | 34 ++++++- back/services/dependence.ts | 18 +++- shell/otask.sh | 2 +- test/back/dev-e2e-regressions.test.cjs | 85 +++++++++++++++++ test/back/python-specifiers.test.cjs | 127 +++++++++++++++++++++++++ test/back/shell-compatibility.test.cjs | 4 +- 6 files changed, 260 insertions(+), 10 deletions(-) create mode 100644 test/back/python-specifiers.test.cjs diff --git a/back/config/util.ts b/back/config/util.ts index 122319eb..bf5d6132 100644 --- a/back/config/util.ts +++ b/back/config/util.ts @@ -739,10 +739,30 @@ export function getGetCommand(type: DependenceTypes, name: string): string { const baseCommands = { [DependenceTypes.nodejs]: `pnpm ls -g | grep "${trimmedName}" | head -1`, [DependenceTypes.python3]: `${getPythonExecutable()} -c ${shellQuote( - `from importlib.metadata import version\nimport importlib.util as u\nname=${JSON.stringify( - trimmedName, - )}\ntry:\n print(version(name))\nexcept Exception:\n print(name if u.find_spec(name) else '')`, - )}`, + `from importlib.metadata import version, PackageNotFoundError +import importlib.util +import sys +try: + try: + from packaging.requirements import Requirement + except ImportError: + from pip._vendor.packaging.requirements import Requirement + requirement = Requirement(sys.argv[1]) + if requirement.marker and not requirement.marker.evaluate(): + print('not applicable') + else: + try: + installed = version(requirement.name) + except PackageNotFoundError: + if not requirement.specifier and not requirement.extras and not requirement.url: + if importlib.util.find_spec(requirement.name): + print(requirement.name) + else: + if requirement.specifier.contains(installed, prereleases=True): + print(installed) +except Exception: + pass`, + )} ${shellQuote(trimmedName)}`, [DependenceTypes.linux]: getOsTypeSync() === 'Alpine' ? `apk info -es ${trimmedName}` @@ -827,9 +847,13 @@ export function getUninstallCommand( 'Remove installed Python distributions separately by package name.', )}`; } + const packageName = + name.match( + /^([A-Za-z0-9][A-Za-z0-9._-]*)(?:\[[^\]]+\])?(?=\s*(?:[<>=!~;@]|$))/, + )?.[1] || name; return `${getPythonPip()} uninstall --disable-pip-version-check${ PYTHON_VENV_DIR ? '' : ' --root-user-action=ignore' - } -y -- ${shellQuote(name)}`; + } -y -- ${shellQuote(packageName)}`; } const baseCommands = { [DependenceTypes.nodejs]: 'pnpm remove -g', diff --git a/back/services/dependence.ts b/back/services/dependence.ts index 83f360e6..6f9eed31 100644 --- a/back/services/dependence.ts +++ b/back/services/dependence.ts @@ -188,7 +188,10 @@ export default class DependenceService { let depName = dependency.name.trim(); const depVersionStr = versionDependenceCommandTypes[dependency.type]; let depVersion = ''; - if (depName.includes(depVersionStr)) { + if ( + dependency.type !== DependenceTypes.python3 && + depName.includes(depVersionStr) + ) { const symbolRegx = new RegExp( `(.*)${depVersionStr}([0-9\\.\\-\\+a-zA-Z]*)`, ); @@ -316,6 +319,9 @@ export default class DependenceService { : getUninstallCommand(dependency.type, dependency.name.trim()); } catch (error) { taskLimit.removeQueuedDependency(dependency); + if (!isInstall && force) { + return this.removeDb([dependency.id!]).then(() => null); + } return DependenceModel.update( { status: isInstall @@ -398,7 +404,13 @@ export default class DependenceService { this.updateLog(depIds, message); // 判断是否已经安装过依赖 - if (isInstall && !force) { + // pip must resolve optional dependencies even if the base distribution + // is already present; its version alone cannot prove extras are ready. + if ( + isInstall && + !force && + !(isPythonDependence && depName.includes('[')) + ) { let getCommand = getGetCommand(dependency.type, depName); const depVersionStr = versionDependenceCommandTypes[dependency.type]; if (isLinuxDependence) { @@ -406,7 +418,7 @@ export default class DependenceService { } let depVersion = ''; - if (depName.includes(depVersionStr)) { + if (!isPythonDependence && depName.includes(depVersionStr)) { const symbolRegx = new RegExp( `(.*)${depVersionStr}([0-9\\.\\-\\+a-zA-Z]*)`, ); diff --git a/shell/otask.sh b/shell/otask.sh index 20e1b70f..b729fb21 100755 --- a/shell/otask.sh +++ b/shell/otask.sh @@ -44,7 +44,7 @@ gen_array_scripts() { if [[ -f $file ]] && [[ $file == *.js && $file != sendNotify.js ]]; then let i++ array_scripts[i]=$file - array_scripts_name[i]=$(grep -- "new Env" "$file" | awk -F "\(" '{print $2}' | awk -F "\)" '{print $1}' | sed 's:.*\('\''\|"\)\([^"'\'']*\)\('\''\|"\).*:\2:' | sed 's:"::g' | sed "s:'::g" | head -1) + array_scripts_name[i]=$(grep -- "new Env" "$file" | awk -F '[(]' '{print $2}' | awk -F '[)]' '{print $1}' | sed 's:.*\('\''\|"\)\([^"'\'']*\)\('\''\|"\).*:\2:' | sed 's:"::g' | sed "s:'::g" | head -1) [[ -z ${array_scripts_name[i]} ]] && array_scripts_name[i]="<未识别出活动名称>" fi done diff --git a/test/back/dev-e2e-regressions.test.cjs b/test/back/dev-e2e-regressions.test.cjs index a5c26c0d..4a5e56d4 100644 --- a/test/back/dev-e2e-regressions.test.cjs +++ b/test/back/dev-e2e-regressions.test.cjs @@ -151,3 +151,88 @@ test('Alpine container mirror command runs without sudo and writes valid reposit ), ); }); + +test('force deletion removes a source record even when its requirements file is gone', async () => { + const removed = [], + updates = []; + const Service = load('back/services/dependence.ts', { + '../config': {}, + '../config/util': { + getUninstallCommand() { + throw Error('Python dependency file must be inside scripts or repo'); + }, + }, + '../data/dependence': { + Dependence: class {}, + DependenceTypes: types, + DependenceStatus: { ...statuses, removeFailed: 5 }, + DependenceModel: { + destroy: async ({ where }) => removed.push(...where.id), + update: async (values) => updates.push(values), + }, + }, + '../shared/pLimit': { + runDependeny: (_dep, fn) => fn(), + removeQueuedDependency() {}, + }, + './sock': class {}, + '../shared/i18n': {}, + }).default; + const service = new Service({}, {}); + const dep = { id: 1, name: 'gone/requirements.txt', type: 1, status: 2 }; + await service.installOrUninstallDependency(dep, false, false); + assert.deepEqual(removed, []); + assert.equal(updates[0].status, 5); + await service.installOrUninstallDependency(dep, false, true); + assert.deepEqual(removed, [1]); +}); + +test('Python extras run pip even when the base package version is already installed', async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-extra-install-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const marker = path.join(root, 'pip-ran'); + const dep = { + id: 1, + name: 'fixture[optional]>=1', + type: 1, + status: 6, + log: [], + }; + let probes = 0; + const Service = load('back/services/dependence.ts', { + '../config': {}, + '../config/util': { + getInstallCommand: () => `printf installed > '${marker}'`, + getGetCommand: () => { + probes++; + return 'probe'; + }, + promiseExecSuccess: async () => '1.0', + detectOS: async () => 'Alpine', + fileExist: async () => false, + }, + '../data/dependence': { + Dependence: class {}, + DependenceTypes: types, + DependenceStatus: { ...statuses, installing: 0, cancelled: 7 }, + versionDependenceCommandTypes: { 0: '@', 1: '==', 2: '=' }, + DependenceModel: { + findAll: async () => [dep], + update: async (values) => Object.assign(dep, values), + }, + }, + '../shared/pLimit': { + firstDependencyId: 1, + runDependeny: (_dep, fn) => fn(), + removeQueuedDependency() {}, + updateDepLog: (fn) => fn(), + }, + './sock': class {}, + '../shared/i18n': { t: (text) => text, tf: (text) => text }, + }).default; + const service = new Service({}, { sendMessage() {} }); + await service.installOrUninstallDependency(dep, true, false); + assert.equal(probes, 0); + assert.equal(fs.readFileSync(marker, 'utf8'), 'installed'); + assert.equal(dep.status, 1); +}); diff --git a/test/back/python-specifiers.test.cjs b/test/back/python-specifiers.test.cjs new file mode 100644 index 00000000..6552d343 --- /dev/null +++ b/test/back/python-specifiers.test.cjs @@ -0,0 +1,127 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { execFileSync } = require('node:child_process'); +const load = require('../helpers/load-security-module.cjs'); + +test('#3097 Python requirement probes preserve version constraints, extras and installed distributions', async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-specifiers-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const venv = path.join(root, 'virtual env'); + execFileSync('python3', ['-m', 'venv', venv]); + execFileSync('python3', [ + '-c', + `import zipfile,sys,os +for version in ['1.2.3','2.1.0']: + name='ql_specifier_fixture' + with zipfile.ZipFile(os.path.join(sys.argv[1],name+'-'+version+'-py3-none-any.whl'),'w') as z: + z.writestr(name+'.py','value=42\\n') + info=name+'-'+version+'.dist-info/' + z.writestr(info+'METADATA','Metadata-Version: 2.1\\nName: ql-specifier-fixture\\nVersion: '+version+'\\nProvides-Extra: optional\\n') + z.writestr(info+'WHEEL','Wheel-Version: 1.0\\nGenerator: fixture\\nRoot-Is-Purelib: true\\nTag: py3-none-any\\n') + z.writestr(info+'RECORD','')`, + root, + ]); + const api = load('back/config/util.ts', { + './index': { rootPath: root }, + './const': { PYTHON_VENV_DIR: venv, PYTHON_INSTALL_DIR: '/unused-prefix' }, + '../loaders/logger': {}, + '../shared/utils': {}, + '../data/dependence': { + DependenceTypes: { nodejs: 0, python3: 1, linux: 2 }, + }, + }); + const env = { ...process.env, PIP_NO_INDEX: '1', PIP_FIND_LINKS: root }; + for (const key of ['PYTHONPATH', 'PYTHONUSERBASE', 'PYTHON_HOME']) + delete env[key]; + const run = (command) => + execFileSync('/bin/bash', ['-c', command], { + encoding: 'utf8', + env, + stdio: ['ignore', 'pipe', 'pipe'], + }).trim(); + run(api.getInstallCommand(1, 'ql-specifier-fixture==1.2.3')); + const Service = load('back/services/dependence.ts', { + '../config': {}, + '../config/util': { + ...api, + promiseExecSuccess: async (command) => run(command), + }, + '../data/dependence': { + Dependence: class {}, + DependenceTypes: { nodejs: 0, python3: 1, linux: 2 }, + versionDependenceCommandTypes: { 0: '@', 1: '==', 2: '=' }, + }, + '../shared/pLimit': {}, + './sock': class {}, + '../shared/i18n': {}, + }).default; + const service = new Service({}, {}); + assert.equal( + await service.isDependenceInstalled({ + type: 1, + name: 'ql-specifier-fixture==1.2.3.0', + }), + true, + ); + assert.equal( + await service.isDependenceInstalled({ + type: 1, + name: 'ql-specifier-fixture>=2', + }), + false, + ); + for (const suffix of [ + '', + '>=1.0', + '<2', + '>=1,<2', + '~=1.2', + '==1.*', + '!=2.0', + '[optional]>=1', + '==1.2.3.0', + ]) { + assert.equal( + run(api.getGetCommand(1, 'ql-specifier-fixture' + suffix)), + '1.2.3', + suffix, + ); + } + for (const suffix of ['>=2', '<1', '!=1.2.3', '==1.2.30']) { + assert.equal( + run(api.getGetCommand(1, 'ql-specifier-fixture' + suffix)), + '', + suffix, + ); + } + assert.equal(run(api.getGetCommand(1, 'ql_specifier_fixture')), '1.2.3'); + assert.equal(run(api.getGetCommand(1, 'json')), 'json'); + assert.equal( + run(api.getGetCommand(1, 'ql-missing-specifier-package>=1')), + '', + ); + assert.equal( + run(api.getGetCommand(1, 'ql-specifier-fixture; python_version < "0"')), + 'not applicable', + ); + run(api.getInstallCommand(1, 'ql-specifier-fixture>=2')); + assert.equal( + run(api.getGetCommand(1, 'ql-specifier-fixture>=2,<3')), + '2.1.0', + ); + run(api.getUninstallCommand(1, 'ql-specifier-fixture[optional]>=2,<3')); + assert.equal(run(api.getGetCommand(1, 'ql-specifier-fixture')), ''); + assert.equal(fs.existsSync(path.join(root, 'INJECTED')), false); + assert.equal( + run( + api.getGetCommand( + 1, + `ql-specifier-fixture; __import__('os').system('touch ${root}/INJECTED')`, + ), + ), + '', + ); +}); diff --git a/test/back/shell-compatibility.test.cjs b/test/back/shell-compatibility.test.cjs index 0af5b9fa..ed59a51a 100644 --- a/test/back/shell-compatibility.test.cjs +++ b/test/back/shell-compatibility.test.cjs @@ -162,7 +162,7 @@ test('script discovery keeps filenames with spaces, glob characters and leading f.write(`scripts/${name}`, "new Env('fixture');\n"); const result = f.run( extract('shell/otask.sh', 'gen_array_scripts') + - '\ngen_array_scripts; printf "%s\\n" "${array_scripts[@]}"', + '\ngen_array_scripts; printf "%s\\n" "${array_scripts[@]}"; printf "NAME=%s\\n" "${array_scripts_name[@]}"', { dir_scripts: path.join(f.root, 'scripts'), }, @@ -171,6 +171,8 @@ test('script discovery keeps filenames with spaces, glob characters and leading assert.equal(result.stderr, ''); assert.deepEqual(result.stdout.trim().split('\n').sort(), [ '-dash name.js', + 'NAME=fixture', + 'NAME=fixture', 'probe [1].js', ]); });