mirror of
https://github.com/whyour/qinglong.git
synced 2026-10-16 22:35:32 +08:00
fix(python): keep source dependency installs and removals in selected venv
This commit is contained in:
whyour
@@ -27,6 +27,7 @@ export const SAMPLE_FILES = [
|
||||
];
|
||||
|
||||
export const PYTHON_INSTALL_DIR = process.env.PYTHON_HOME;
|
||||
export const PYTHON_VENV_DIR = process.env.PYTHON_VENV_DIR;
|
||||
|
||||
export const NotificationModeStringMap = {
|
||||
0: 'gotify',
|
||||
|
||||
+91
-95
@@ -6,7 +6,7 @@ import psTreeFun from 'ps-tree';
|
||||
import { promisify } from 'util';
|
||||
import { load } from 'js-yaml';
|
||||
import config from './index';
|
||||
import { PYTHON_INSTALL_DIR, TASK_COMMAND } from './const';
|
||||
import { PYTHON_INSTALL_DIR, PYTHON_VENV_DIR, TASK_COMMAND } from './const';
|
||||
import Logger from '../loaders/logger';
|
||||
import { writeFileWithLock } from '../shared/utils';
|
||||
import { DependenceTypes } from '../data/dependence';
|
||||
@@ -15,6 +15,11 @@ import os from 'os';
|
||||
import { maybeSudo, isInContainer } from './container';
|
||||
import { shellQuote } from '../shared/shellQuote';
|
||||
import { resolveFileAccess } from '../shared/fileAccess';
|
||||
import {
|
||||
mirrorWriteCommand,
|
||||
normalizeMirror,
|
||||
rewriteMirrorSources,
|
||||
} from '../shared/linuxMirror';
|
||||
|
||||
export * from './share';
|
||||
|
||||
@@ -685,6 +690,16 @@ export async function setSystemTimezone(timezone: string): Promise<boolean> {
|
||||
}
|
||||
|
||||
// Helper function to check if a name is a GitHub URL
|
||||
function getPythonExecutable(): string {
|
||||
return PYTHON_VENV_DIR
|
||||
? shellQuote(path.resolve(config.rootPath, PYTHON_VENV_DIR, 'bin/python3'))
|
||||
: 'python3';
|
||||
}
|
||||
|
||||
function getPythonPip(): string {
|
||||
return PYTHON_VENV_DIR ? `${getPythonExecutable()} -m pip` : 'pip3';
|
||||
}
|
||||
|
||||
function isGitHubUrl(name: string): boolean {
|
||||
// Support git+https://, git+http://, https://, and http:// URLs
|
||||
// This covers GitHub URLs and other git-compatible repositories
|
||||
@@ -720,7 +735,7 @@ export function getGetCommand(type: DependenceTypes, name: string): string {
|
||||
|
||||
const baseCommands = {
|
||||
[DependenceTypes.nodejs]: `pnpm ls -g | grep "${trimmedName}" | head -1`,
|
||||
[DependenceTypes.python3]: `python3 -c ${shellQuote(
|
||||
[DependenceTypes.python3]: `${getPythonExecutable()} -c ${shellQuote(
|
||||
`from importlib.metadata import version\nimport importlib.util as u\nname=${JSON.stringify(
|
||||
trimmedName,
|
||||
)}\ntry:\n print(version(name))\nexcept Exception:\n print(name if u.find_spec(name) else '')`,
|
||||
@@ -736,10 +751,13 @@ export function getGetCommand(type: DependenceTypes, name: string): string {
|
||||
|
||||
export function getInstallCommand(type: DependenceTypes, name: string): string {
|
||||
if (type === DependenceTypes.python3) {
|
||||
const prefix = PYTHON_INSTALL_DIR
|
||||
? ` --prefix=${shellQuote(PYTHON_INSTALL_DIR)}`
|
||||
: '';
|
||||
const pip = `pip3 install --disable-pip-version-check --root-user-action=ignore${prefix}`;
|
||||
const prefix =
|
||||
PYTHON_INSTALL_DIR && !PYTHON_VENV_DIR
|
||||
? ` --prefix=${shellQuote(PYTHON_INSTALL_DIR)}`
|
||||
: '';
|
||||
const pip = `${getPythonPip()} install --disable-pip-version-check${
|
||||
PYTHON_VENV_DIR ? '' : ' --root-user-action=ignore'
|
||||
}${prefix}`;
|
||||
name = name.trim();
|
||||
if (isRequirementsFile(name) || isPyprojectToml(name)) {
|
||||
const file = [config.scriptPath, config.repoPath]
|
||||
@@ -795,9 +813,9 @@ export function getUninstallCommand(
|
||||
throw new Error(
|
||||
'Python dependency file must be inside scripts or repo',
|
||||
);
|
||||
return `pip3 uninstall --disable-pip-version-check --root-user-action=ignore -y -r ${shellQuote(
|
||||
file,
|
||||
)}`;
|
||||
return `${getPythonPip()} uninstall --disable-pip-version-check${
|
||||
PYTHON_VENV_DIR ? '' : ' --root-user-action=ignore'
|
||||
} -y -r ${shellQuote(file)}`;
|
||||
}
|
||||
if (isPyprojectToml(name) || isGitHubUrl(name)) {
|
||||
// A source URL/project can install several distributions and has no safe
|
||||
@@ -806,9 +824,9 @@ export function getUninstallCommand(
|
||||
'Remove installed Python distributions separately by package name.',
|
||||
)}`;
|
||||
}
|
||||
return `pip3 uninstall --disable-pip-version-check --root-user-action=ignore -y -- ${shellQuote(
|
||||
name,
|
||||
)}`;
|
||||
return `${getPythonPip()} uninstall --disable-pip-version-check${
|
||||
PYTHON_VENV_DIR ? '' : ' --root-user-action=ignore'
|
||||
} -y -- ${shellQuote(name)}`;
|
||||
}
|
||||
const baseCommands = {
|
||||
[DependenceTypes.nodejs]: 'pnpm remove -g',
|
||||
@@ -886,95 +904,73 @@ export async function detectOS(): Promise<
|
||||
return osType;
|
||||
}
|
||||
|
||||
async function getCurrentMirrorDomain(
|
||||
filePath: string,
|
||||
): Promise<string | null> {
|
||||
const fileContent = await fs.readFile(filePath, 'utf8');
|
||||
const lines = fileContent.split('\n');
|
||||
for (const line of lines) {
|
||||
if (line.trim().startsWith('#')) {
|
||||
continue;
|
||||
}
|
||||
const match = line.match(/https?:\/\/[^\/]+/);
|
||||
if (match) {
|
||||
return match[0];
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function replaceDomainInFile(
|
||||
filePath: string,
|
||||
oldDomainWithScheme: string,
|
||||
newDomainWithScheme: string,
|
||||
): Promise<void> {
|
||||
let fileContent = await fs.readFile(filePath, 'utf8');
|
||||
let updatedContent = fileContent.replace(
|
||||
new RegExp(oldDomainWithScheme, 'g'),
|
||||
newDomainWithScheme,
|
||||
);
|
||||
|
||||
if (!newDomainWithScheme.endsWith('/')) {
|
||||
newDomainWithScheme += '/';
|
||||
}
|
||||
|
||||
await writeFileWithLock(filePath, updatedContent);
|
||||
}
|
||||
|
||||
async function _updateLinuxMirror(
|
||||
osType: string,
|
||||
mirrorDomainWithScheme: string,
|
||||
mirror: string,
|
||||
): Promise<string> {
|
||||
const defaults: Record<string, string> = {
|
||||
Debian: 'http://deb.debian.org/debian',
|
||||
Ubuntu: 'http://archive.ubuntu.com/ubuntu',
|
||||
Alpine: 'http://dl-cdn.alpinelinux.org/alpine',
|
||||
};
|
||||
if (!defaults[osType])
|
||||
throw new Error('Unsupported OS type for updating mirrors.');
|
||||
const target = mirror || defaults[osType];
|
||||
normalizeMirror(target);
|
||||
const S = isInContainer() ? 'sudo ' : '';
|
||||
// APT otherwise returns zero when index downloads fail with transient errors.
|
||||
const aptUpdate = `${S}apt-get update --error-on=any`;
|
||||
let filePath: string, currentDomainWithScheme: string | null;
|
||||
switch (osType) {
|
||||
case 'Debian':
|
||||
filePath = '/etc/apt/sources.list.d/debian.sources';
|
||||
currentDomainWithScheme = await getCurrentMirrorDomain(filePath);
|
||||
if (currentDomainWithScheme) {
|
||||
return `${S}sed -i 's|${currentDomainWithScheme}|${
|
||||
mirrorDomainWithScheme || 'http://deb.debian.org'
|
||||
}|g' ${filePath} || (${S}mkdir -p /etc/apt/sources.list.d && echo -e "Types: deb\\nURIs: ${
|
||||
mirrorDomainWithScheme || 'http://deb.debian.org'
|
||||
}\\nSuites: \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-updates\\nComponents: main\\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg" | ${S}tee ${filePath}) && ${aptUpdate}`;
|
||||
} else {
|
||||
return `${S}mkdir -p /etc/apt/sources.list.d && echo -e "Types: deb\\nURIs: ${
|
||||
mirrorDomainWithScheme || 'http://deb.debian.org'
|
||||
}\\nSuites: \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-updates\\nComponents: main\\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg" | ${S}tee ${filePath} && ${aptUpdate}`;
|
||||
const update =
|
||||
osType === 'Alpine' ? 'apk update' : `${S}apt-get update --error-on=any`;
|
||||
let file =
|
||||
osType === 'Alpine'
|
||||
? '/etc/apk/repositories'
|
||||
: `/etc/apt/sources.list.d/${osType.toLowerCase()}.sources`;
|
||||
let content = '';
|
||||
try {
|
||||
content = await fs.readFile(file, 'utf8');
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') throw error;
|
||||
if (osType !== 'Alpine') {
|
||||
try {
|
||||
file = '/etc/apt/sources.list';
|
||||
content = await fs.readFile(file, 'utf8');
|
||||
} catch (legacyError: any) {
|
||||
if (legacyError.code !== 'ENOENT') throw legacyError;
|
||||
file = `/etc/apt/sources.list.d/${osType.toLowerCase()}.sources`;
|
||||
}
|
||||
case 'Ubuntu':
|
||||
filePath = '/etc/apt/sources.list.d/ubuntu.sources';
|
||||
currentDomainWithScheme = await getCurrentMirrorDomain(filePath);
|
||||
if (currentDomainWithScheme) {
|
||||
return `${S}sed -i 's|${currentDomainWithScheme}|${
|
||||
mirrorDomainWithScheme || 'http://archive.ubuntu.com'
|
||||
}|g' ${filePath} || (${S}mkdir -p /etc/apt/sources.list.d && echo -e "Types: deb\\nURIs: ${
|
||||
mirrorDomainWithScheme || 'http://archive.ubuntu.com'
|
||||
}\\nSuites: \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-updates \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-backports\\nComponents: main restricted universe multiverse\\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg" | ${S}tee ${filePath}) && ${aptUpdate}`;
|
||||
} else {
|
||||
return `${S}mkdir -p /etc/apt/sources.list.d && echo -e "Types: deb\\nURIs: ${
|
||||
mirrorDomainWithScheme || 'http://archive.ubuntu.com'
|
||||
}\\nSuites: \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-updates \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-backports\\nComponents: main restricted universe multiverse\\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg" | ${S}tee ${filePath} && ${aptUpdate}`;
|
||||
}
|
||||
case 'Alpine':
|
||||
filePath = '/etc/apk/repositories';
|
||||
currentDomainWithScheme = await getCurrentMirrorDomain(filePath);
|
||||
if (currentDomainWithScheme) {
|
||||
return `sed -i 's|${currentDomainWithScheme}|${
|
||||
mirrorDomainWithScheme || 'http://dl-cdn.alpinelinux.org'
|
||||
}|g' ${filePath} || (mkdir -p /etc/apk && echo -e "\\$(grep VERSION_ID /etc/os-release | cut -d= -f2 | cut -d. -f1,2)/main\\n\\$(grep VERSION_ID /etc/os-release | cut -d= -f2 | cut -d. -f1,2)/community" | sed "s|^|${
|
||||
mirrorDomainWithScheme || 'http://dl-cdn.alpinelinux.org'
|
||||
}/alpine/v|" | tee ${filePath}) && apk update`;
|
||||
} else {
|
||||
return `mkdir -p /etc/apk && echo -e "\\$(grep VERSION_ID /etc/os-release | cut -d= -f2 | cut -d. -f1,2)/main\\n\\$(grep VERSION_ID /etc/os-release | cut -d= -f2 | cut -d. -f1,2)/community" | sed "s|^|${
|
||||
mirrorDomainWithScheme || 'http://dl-cdn.alpinelinux.org'
|
||||
}/alpine/v|" | tee ${filePath} && apk update`;
|
||||
}
|
||||
default:
|
||||
throw Error('Unsupported OS type for updating mirrors.');
|
||||
}
|
||||
}
|
||||
if (!content.trim()) {
|
||||
const release = await fs.readFile('/etc/os-release', 'utf8');
|
||||
if (osType === 'Alpine') {
|
||||
const version = release.match(/^VERSION_ID=["']?(\d+\.\d+)/m)?.[1];
|
||||
if (!version)
|
||||
throw new Error('Cannot determine Alpine repository version');
|
||||
content = `${defaults.Alpine}/v${version}/main\n${defaults.Alpine}/v${version}/community\n`;
|
||||
} else {
|
||||
// The fallback may have read an empty legacy .list. Deb822 stanzas must
|
||||
// be written to a .sources file, leaving the legacy file untouched.
|
||||
file = `/etc/apt/sources.list.d/${osType.toLowerCase()}.sources`;
|
||||
const codename = release.match(/^VERSION_CODENAME=["']?([\w-]+)/m)?.[1];
|
||||
if (!codename)
|
||||
throw new Error('Cannot determine APT repository codename');
|
||||
const ubuntu = osType === 'Ubuntu';
|
||||
content = `Types: deb\nURIs: ${
|
||||
defaults[osType]
|
||||
}\nSuites: ${codename} ${codename}-updates${
|
||||
ubuntu ? ` ${codename}-backports` : ''
|
||||
}\nComponents: ${
|
||||
ubuntu ? 'main restricted universe multiverse' : 'main'
|
||||
}\nSigned-By: /usr/share/keyrings/${osType.toLowerCase()}-archive-keyring.gpg\n`;
|
||||
if (!ubuntu)
|
||||
content += `\nTypes: deb\nURIs: http://security.debian.org/debian-security\nSuites: ${codename}-security\nComponents: main\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg\n`;
|
||||
}
|
||||
}
|
||||
return mirrorWriteCommand(
|
||||
file,
|
||||
rewriteMirrorSources(content, target),
|
||||
S,
|
||||
update,
|
||||
);
|
||||
}
|
||||
|
||||
export async function updateLinuxMirrorFile(mirror: string): Promise<string> {
|
||||
|
||||
@@ -179,7 +179,14 @@ export default ({ app }: { app: Application }) => {
|
||||
app.use(rewrite('/open/*', '/api/$1'));
|
||||
app.use(config.api.prefix, routes());
|
||||
|
||||
app.get('*', (_, res, next) => {
|
||||
app.get('*', (req, res, next) => {
|
||||
// Don't serve index.html for API routes
|
||||
if (
|
||||
req.path === config.api.prefix ||
|
||||
req.path.startsWith(`${config.api.prefix}/`)
|
||||
) {
|
||||
return res.status(404).json({ code: 404, message: 'Not Found' });
|
||||
}
|
||||
const indexPath = path.join(frontendPath, 'index.html');
|
||||
res.sendFile(indexPath, (err) => {
|
||||
if (err) {
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
import { shellQuote } from './shellQuote';
|
||||
|
||||
const repositoryNames =
|
||||
/^(debian(?:-security)?|ubuntu(?:-ports)?|alpine|armbian)$/;
|
||||
|
||||
export function normalizeMirror(mirror: string): URL {
|
||||
if (/\s/.test(mirror)) throw new Error('Invalid Linux mirror URL');
|
||||
const url = new URL(mirror);
|
||||
if (
|
||||
!['http:', 'https:'].includes(url.protocol) ||
|
||||
url.username ||
|
||||
url.password ||
|
||||
url.search ||
|
||||
url.hash
|
||||
) {
|
||||
throw new Error(
|
||||
'Linux mirror must be an HTTP(S) URL without credentials, query or fragment',
|
||||
);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
export function replaceMirrorUrl(current: string, mirror: string): string {
|
||||
const target = normalizeMirror(mirror);
|
||||
const previous = new URL(current);
|
||||
const oldSegments = previous.pathname.split('/').filter(Boolean);
|
||||
const newSegments = target.pathname.split('/').filter(Boolean);
|
||||
const repositoryIndex = oldSegments.findIndex((part) =>
|
||||
repositoryNames.test(part),
|
||||
);
|
||||
const targetIndex = newSegments.findIndex((part) =>
|
||||
repositoryNames.test(part),
|
||||
);
|
||||
const prefix =
|
||||
targetIndex >= 0 ? newSegments.slice(0, targetIndex) : newSegments;
|
||||
if (repositoryIndex >= 0) {
|
||||
const repository = oldSegments[repositoryIndex];
|
||||
let suffix = oldSegments.slice(repositoryIndex + 1);
|
||||
// Repair paths already corrupted by repeated authority-only replacement.
|
||||
while (suffix[0] === repository) suffix.shift();
|
||||
return `${target.origin}/${[...prefix, repository, ...suffix].join('/')}`;
|
||||
}
|
||||
// Unknown repository paths: a full URL replaces the old root, a hostname
|
||||
// preserves it. Never append the newly entered root to the existing root.
|
||||
return `${target.origin}/${(newSegments.length
|
||||
? newSegments
|
||||
: oldSegments
|
||||
).join('/')}`;
|
||||
}
|
||||
|
||||
export function rewriteMirrorSources(content: string, mirror: string): string {
|
||||
normalizeMirror(mirror);
|
||||
return content
|
||||
.split(/(\r?\n[ \t]*\r?\n)/)
|
||||
.map((stanza) => {
|
||||
const signedBy = stanza.match(/^Signed-By:\s*([^\r\n]*)/im)?.[1];
|
||||
return stanza
|
||||
.split('\n')
|
||||
.map((line) => {
|
||||
if (!/^\s*(?:URIs:|deb(?:-src)?\s|https?:\/\/)/.test(line))
|
||||
return line;
|
||||
const legacy = /^\s*deb(?:-src)?\s/.test(line);
|
||||
const key = legacy
|
||||
? line.match(/\bsigned-by=([^\]\s]+)/i)?.[1]
|
||||
: signedBy;
|
||||
// In .list files a # starts an inline comment; its URLs are metadata.
|
||||
const comment = legacy ? line.indexOf('#') : -1;
|
||||
const source = comment < 0 ? line : line.slice(0, comment);
|
||||
const rewritten = source.replace(/https?:\/\/[^\s#]+/g, (url) =>
|
||||
isSystemRepository(url, key) ? replaceMirrorUrl(url, mirror) : url,
|
||||
);
|
||||
return rewritten + (comment < 0 ? '' : line.slice(comment));
|
||||
})
|
||||
.join('\n');
|
||||
})
|
||||
.join('');
|
||||
}
|
||||
|
||||
function isSystemRepository(url: string, signedBy?: string): boolean {
|
||||
const keys =
|
||||
signedBy
|
||||
?.trim()
|
||||
.split(/[\s,]+/)
|
||||
.filter(Boolean) || [];
|
||||
const archiveKeyring =
|
||||
keys.length > 0 &&
|
||||
keys.every((key) =>
|
||||
/\/(?:debian|ubuntu|armbian)-archive-keyring\.(?:gpg|asc)$/.test(key),
|
||||
);
|
||||
// A separate signing key identifies a third-party archive, even if its
|
||||
// repository happens to be called /debian or /ubuntu.
|
||||
if (keys.length > 0 && !archiveKeyring) return false;
|
||||
const segments = new URL(url).pathname.split('/').filter(Boolean);
|
||||
const repositoryIndex = segments.findIndex((part) =>
|
||||
repositoryNames.test(part),
|
||||
);
|
||||
if (repositoryIndex < 0) return false;
|
||||
// Without archive metadata, accept only conventional archive roots. A
|
||||
// vendor path such as Docker's /linux/debian is not the Debian archive.
|
||||
return (
|
||||
archiveKeyring ||
|
||||
repositoryIndex === 0 ||
|
||||
segments
|
||||
.slice(0, repositoryIndex)
|
||||
.every((part) => /^mirrors?$/.test(part)) ||
|
||||
segments[repositoryIndex] === 'alpine'
|
||||
);
|
||||
}
|
||||
|
||||
export function mirrorWriteCommand(
|
||||
file: string,
|
||||
content: string,
|
||||
prefix: string,
|
||||
update: string,
|
||||
): string {
|
||||
return `${prefix}mkdir -p ${shellQuote(
|
||||
file.slice(0, file.lastIndexOf('/')),
|
||||
)} && printf '%s' ${shellQuote(content)} | ${prefix}tee ${shellQuote(
|
||||
file,
|
||||
)} >/dev/null && ${update}`;
|
||||
}
|
||||
Reference in New Issue
Block a user