fix: honor Python requirement specifiers and portable script discovery

back/config/util.ts

@@ -739,10 +739,30 @@ export function getGetCommand(type: DependenceTypes, name: string): string {
   const baseCommands = {
     [DependenceTypes.nodejs]: `pnpm ls -g  | grep "${trimmedName}" | head -1`,
     [DependenceTypes.python3]: `${getPythonExecutable()} -c ${shellQuote(
-      `from importlib.metadata import version\nimport importlib.util as u\nname=${JSON.stringify(
+      `from importlib.metadata import version, PackageNotFoundError
-        trimmedName,
+import importlib.util
-      )}\ntry:\n print(version(name))\nexcept Exception:\n print(name if u.find_spec(name) else '')`,
+import sys
-    )}`,
+try:
+ try:
+  from packaging.requirements import Requirement
+ except ImportError:
+  from pip._vendor.packaging.requirements import Requirement
+ requirement = Requirement(sys.argv[1])
+ if requirement.marker and not requirement.marker.evaluate():
+  print('not applicable')
+ else:
+  try:
+   installed = version(requirement.name)
+  except PackageNotFoundError:
+   if not requirement.specifier and not requirement.extras and not requirement.url:
+    if importlib.util.find_spec(requirement.name):
+     print(requirement.name)
+  else:
+   if requirement.specifier.contains(installed, prereleases=True):
+    print(installed)
+except Exception:
+ pass`,
+    )} ${shellQuote(trimmedName)}`,
     [DependenceTypes.linux]:
       getOsTypeSync() === 'Alpine'
         ? `apk info -es ${trimmedName}`
@@ -827,9 +847,13 @@ export function getUninstallCommand(
         'Remove installed Python distributions separately by package name.',
       )}`;
     }
+    const packageName =
+      name.match(
+        /^([A-Za-z0-9][A-Za-z0-9._-]*)(?:\[[^\]]+\])?(?=\s*(?:[<>=!~;@]|$))/,
+      )?.[1] || name;
     return `${getPythonPip()} uninstall --disable-pip-version-check${
       PYTHON_VENV_DIR ? '' : ' --root-user-action=ignore'
-    } -y -- ${shellQuote(name)}`;
+    } -y -- ${shellQuote(packageName)}`;
   }
   const baseCommands = {
     [DependenceTypes.nodejs]: 'pnpm remove -g',

back/services/dependence.ts

@@ -188,7 +188,10 @@ export default class DependenceService {
     let depName = dependency.name.trim();
     const depVersionStr = versionDependenceCommandTypes[dependency.type];
     let depVersion = '';
-    if (depName.includes(depVersionStr)) {
+    if (
+      dependency.type !== DependenceTypes.python3 &&
+      depName.includes(depVersionStr)
+    ) {
       const symbolRegx = new RegExp(
         `(.*)${depVersionStr}([0-9\\.\\-\\+a-zA-Z]*)`,
       );
@@ -316,6 +319,9 @@ export default class DependenceService {
           : getUninstallCommand(dependency.type, dependency.name.trim());
       } catch (error) {
         taskLimit.removeQueuedDependency(dependency);
+        if (!isInstall && force) {
+          return this.removeDb([dependency.id!]).then(() => null);
+        }
         return DependenceModel.update(
           {
             status: isInstall
@@ -398,7 +404,13 @@ export default class DependenceService {
         this.updateLog(depIds, message);
 
         // 判断是否已经安装过依赖
-        if (isInstall && !force) {
+        // pip must resolve optional dependencies even if the base distribution
+        // is already present; its version alone cannot prove extras are ready.
+        if (
+          isInstall &&
+          !force &&
+          !(isPythonDependence && depName.includes('['))
+        ) {
           let getCommand = getGetCommand(dependency.type, depName);
           const depVersionStr = versionDependenceCommandTypes[dependency.type];
           if (isLinuxDependence) {
@@ -406,7 +418,7 @@ export default class DependenceService {
           }
 
           let depVersion = '';
-          if (depName.includes(depVersionStr)) {
+          if (!isPythonDependence && depName.includes(depVersionStr)) {
             const symbolRegx = new RegExp(
               `(.*)${depVersionStr}([0-9\\.\\-\\+a-zA-Z]*)`,
             );

shell/otask.sh

@@ -44,7 +44,7 @@ gen_array_scripts() {
     if [[ -f $file ]] && [[ $file == *.js && $file != sendNotify.js ]]; then
       let i++
       array_scripts[i]=$file
-      array_scripts_name[i]=$(grep -- "new Env" "$file" | awk -F "\(" '{print $2}' | awk -F "\)" '{print $1}' | sed 's:.*\('\''\|"\)\([^"'\'']*\)\('\''\|"\).*:\2:' | sed 's:"::g' | sed "s:'::g" | head -1)
+      array_scripts_name[i]=$(grep -- "new Env" "$file" | awk -F '[(]' '{print $2}' | awk -F '[)]' '{print $1}' | sed 's:.*\('\''\|"\)\([^"'\'']*\)\('\''\|"\).*:\2:' | sed 's:"::g' | sed "s:'::g" | head -1)
       [[ -z ${array_scripts_name[i]} ]] && array_scripts_name[i]="<未识别出活动名称>"
     fi
   done

test/back/dev-e2e-regressions.test.cjs

@@ -151,3 +151,88 @@ test('Alpine container mirror command runs without sudo and writes valid reposit
     ),
   );
 });
+
+test('force deletion removes a source record even when its requirements file is gone', async () => {
+  const removed = [],
+    updates = [];
+  const Service = load('back/services/dependence.ts', {
+    '../config': {},
+    '../config/util': {
+      getUninstallCommand() {
+        throw Error('Python dependency file must be inside scripts or repo');
+      },
+    },
+    '../data/dependence': {
+      Dependence: class {},
+      DependenceTypes: types,
+      DependenceStatus: { ...statuses, removeFailed: 5 },
+      DependenceModel: {
+        destroy: async ({ where }) => removed.push(...where.id),
+        update: async (values) => updates.push(values),
+      },
+    },
+    '../shared/pLimit': {
+      runDependeny: (_dep, fn) => fn(),
+      removeQueuedDependency() {},
+    },
+    './sock': class {},
+    '../shared/i18n': {},
+  }).default;
+  const service = new Service({}, {});
+  const dep = { id: 1, name: 'gone/requirements.txt', type: 1, status: 2 };
+  await service.installOrUninstallDependency(dep, false, false);
+  assert.deepEqual(removed, []);
+  assert.equal(updates[0].status, 5);
+  await service.installOrUninstallDependency(dep, false, true);
+  assert.deepEqual(removed, [1]);
+});
+
+test('Python extras run pip even when the base package version is already installed', async (t) => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-extra-install-'));
+  t.after(() => fs.rmSync(root, { recursive: true, force: true }));
+  const marker = path.join(root, 'pip-ran');
+  const dep = {
+    id: 1,
+    name: 'fixture[optional]>=1',
+    type: 1,
+    status: 6,
+    log: [],
+  };
+  let probes = 0;
+  const Service = load('back/services/dependence.ts', {
+    '../config': {},
+    '../config/util': {
+      getInstallCommand: () => `printf installed > '${marker}'`,
+      getGetCommand: () => {
+        probes++;
+        return 'probe';
+      },
+      promiseExecSuccess: async () => '1.0',
+      detectOS: async () => 'Alpine',
+      fileExist: async () => false,
+    },
+    '../data/dependence': {
+      Dependence: class {},
+      DependenceTypes: types,
+      DependenceStatus: { ...statuses, installing: 0, cancelled: 7 },
+      versionDependenceCommandTypes: { 0: '@', 1: '==', 2: '=' },
+      DependenceModel: {
+        findAll: async () => [dep],
+        update: async (values) => Object.assign(dep, values),
+      },
+    },
+    '../shared/pLimit': {
+      firstDependencyId: 1,
+      runDependeny: (_dep, fn) => fn(),
+      removeQueuedDependency() {},
+      updateDepLog: (fn) => fn(),
+    },
+    './sock': class {},
+    '../shared/i18n': { t: (text) => text, tf: (text) => text },
+  }).default;
+  const service = new Service({}, { sendMessage() {} });
+  await service.installOrUninstallDependency(dep, true, false);
+  assert.equal(probes, 0);
+  assert.equal(fs.readFileSync(marker, 'utf8'), 'installed');
+  assert.equal(dep.status, 1);
+});

test/back/python-specifiers.test.cjs

@@ -0,0 +1,127 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { execFileSync } = require('node:child_process');
+const load = require('../helpers/load-security-module.cjs');
+
+test('#3097 Python requirement probes preserve version constraints, extras and installed distributions', async (t) => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-specifiers-'));
+  t.after(() => fs.rmSync(root, { recursive: true, force: true }));
+  const venv = path.join(root, 'virtual env');
+  execFileSync('python3', ['-m', 'venv', venv]);
+  execFileSync('python3', [
+    '-c',
+    `import zipfile,sys,os
+for version in ['1.2.3','2.1.0']:
+ name='ql_specifier_fixture'
+ with zipfile.ZipFile(os.path.join(sys.argv[1],name+'-'+version+'-py3-none-any.whl'),'w') as z:
+  z.writestr(name+'.py','value=42\\n')
+  info=name+'-'+version+'.dist-info/'
+  z.writestr(info+'METADATA','Metadata-Version: 2.1\\nName: ql-specifier-fixture\\nVersion: '+version+'\\nProvides-Extra: optional\\n')
+  z.writestr(info+'WHEEL','Wheel-Version: 1.0\\nGenerator: fixture\\nRoot-Is-Purelib: true\\nTag: py3-none-any\\n')
+  z.writestr(info+'RECORD','')`,
+    root,
+  ]);
+  const api = load('back/config/util.ts', {
+    './index': { rootPath: root },
+    './const': { PYTHON_VENV_DIR: venv, PYTHON_INSTALL_DIR: '/unused-prefix' },
+    '../loaders/logger': {},
+    '../shared/utils': {},
+    '../data/dependence': {
+      DependenceTypes: { nodejs: 0, python3: 1, linux: 2 },
+    },
+  });
+  const env = { ...process.env, PIP_NO_INDEX: '1', PIP_FIND_LINKS: root };
+  for (const key of ['PYTHONPATH', 'PYTHONUSERBASE', 'PYTHON_HOME'])
+    delete env[key];
+  const run = (command) =>
+    execFileSync('/bin/bash', ['-c', command], {
+      encoding: 'utf8',
+      env,
+      stdio: ['ignore', 'pipe', 'pipe'],
+    }).trim();
+  run(api.getInstallCommand(1, 'ql-specifier-fixture==1.2.3'));
+  const Service = load('back/services/dependence.ts', {
+    '../config': {},
+    '../config/util': {
+      ...api,
+      promiseExecSuccess: async (command) => run(command),
+    },
+    '../data/dependence': {
+      Dependence: class {},
+      DependenceTypes: { nodejs: 0, python3: 1, linux: 2 },
+      versionDependenceCommandTypes: { 0: '@', 1: '==', 2: '=' },
+    },
+    '../shared/pLimit': {},
+    './sock': class {},
+    '../shared/i18n': {},
+  }).default;
+  const service = new Service({}, {});
+  assert.equal(
+    await service.isDependenceInstalled({
+      type: 1,
+      name: 'ql-specifier-fixture==1.2.3.0',
+    }),
+    true,
+  );
+  assert.equal(
+    await service.isDependenceInstalled({
+      type: 1,
+      name: 'ql-specifier-fixture>=2',
+    }),
+    false,
+  );
+  for (const suffix of [
+    '',
+    '>=1.0',
+    '<2',
+    '>=1,<2',
+    '~=1.2',
+    '==1.*',
+    '!=2.0',
+    '[optional]>=1',
+    '==1.2.3.0',
+  ]) {
+    assert.equal(
+      run(api.getGetCommand(1, 'ql-specifier-fixture' + suffix)),
+      '1.2.3',
+      suffix,
+    );
+  }
+  for (const suffix of ['>=2', '<1', '!=1.2.3', '==1.2.30']) {
+    assert.equal(
+      run(api.getGetCommand(1, 'ql-specifier-fixture' + suffix)),
+      '',
+      suffix,
+    );
+  }
+  assert.equal(run(api.getGetCommand(1, 'ql_specifier_fixture')), '1.2.3');
+  assert.equal(run(api.getGetCommand(1, 'json')), 'json');
+  assert.equal(
+    run(api.getGetCommand(1, 'ql-missing-specifier-package>=1')),
+    '',
+  );
+  assert.equal(
+    run(api.getGetCommand(1, 'ql-specifier-fixture; python_version < "0"')),
+    'not applicable',
+  );
+  run(api.getInstallCommand(1, 'ql-specifier-fixture>=2'));
+  assert.equal(
+    run(api.getGetCommand(1, 'ql-specifier-fixture>=2,<3')),
+    '2.1.0',
+  );
+  run(api.getUninstallCommand(1, 'ql-specifier-fixture[optional]>=2,<3'));
+  assert.equal(run(api.getGetCommand(1, 'ql-specifier-fixture')), '');
+  assert.equal(fs.existsSync(path.join(root, 'INJECTED')), false);
+  assert.equal(
+    run(
+      api.getGetCommand(
+        1,
+        `ql-specifier-fixture; __import__('os').system('touch ${root}/INJECTED')`,
+      ),
+    ),
+    '',
+  );
+});

test/back/shell-compatibility.test.cjs

@@ -162,7 +162,7 @@ test('script discovery keeps filenames with spaces, glob characters and leading
     f.write(`scripts/${name}`, "new Env('fixture');\n");
   const result = f.run(
     extract('shell/otask.sh', 'gen_array_scripts') +
-      '\ngen_array_scripts; printf "%s\\n" "${array_scripts[@]}"',
+      '\ngen_array_scripts; printf "%s\\n" "${array_scripts[@]}"; printf "NAME=%s\\n" "${array_scripts_name[@]}"',
     {
       dir_scripts: path.join(f.root, 'scripts'),
     },
@@ -171,6 +171,8 @@ test('script discovery keeps filenames with spaces, glob characters and leading
   assert.equal(result.stderr, '');
   assert.deepEqual(result.stdout.trim().split('\n').sort(), [
     '-dash name.js',
+    'NAME=fixture',
+    'NAME=fixture',
     'probe [1].js',
   ]);
 });