From e694570f1ae81fc0860eedf43a13e219bc3397c3 Mon Sep 17 00:00:00 2001 From: whyour Date: Mon, 22 Jul 2024 01:07:00 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=20task=5Fbefore.js=20?= =?UTF-8?q?=E5=92=8C=20task=5Fbefore.py=20=E6=96=87=E4=BB=B6=EF=BC=8C?= =?UTF-8?q?=E5=9C=A8=E6=89=A7=E8=A1=8C=E4=BB=BB=E5=8A=A1=E5=89=8D=E6=89=A7?= =?UTF-8?q?=E8=A1=8C=EF=BC=8C=E9=81=BF=E5=85=8D=E7=8E=AF=E5=A2=83=E5=8F=98?= =?UTF-8?q?=E9=87=8F=E8=BF=87=E5=A4=A7=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- back/loaders/initFile.ts | 18 +++++++ docker/docker-entrypoint.sh | 2 +- shell/otask.sh | 10 ++-- shell/preload/sitecustomize.js | 32 +++++++++--- shell/preload/sitecustomize.py | 26 +++++++--- shell/share.sh | 94 ++++++++++++++++------------------ shell/update.sh | 2 +- 7 files changed, 114 insertions(+), 70 deletions(-) diff --git a/back/loaders/initFile.ts b/back/loaders/initFile.ts index 3750da22..5983ccfa 100644 --- a/back/loaders/initFile.ts +++ b/back/loaders/initFile.ts @@ -31,6 +31,8 @@ const scriptNotifyPyFile = path.join(scriptPath, 'notify.py'); const jsNotifyFile = path.join(preloadPath, 'notify.js'); const pyNotifyFile = path.join(preloadPath, 'notify.py'); const TaskBeforeFile = path.join(configPath, 'task_before.sh'); +const TaskBeforeJsFile = path.join(configPath, 'task_before.js'); +const TaskBeforePyFile = path.join(configPath, 'task_before.py'); const TaskAfterFile = path.join(configPath, 'task_after.sh'); const homedir = os.homedir(); const sshPath = path.resolve(homedir, '.ssh'); @@ -53,6 +55,8 @@ export default async () => { const scriptNotifyJsFileExist = await fileExist(scriptNotifyJsFile); const scriptNotifyPyFileExist = await fileExist(scriptNotifyPyFile); const TaskBeforeFileExist = await fileExist(TaskBeforeFile); + const TaskBeforeJsFileExist = await fileExist(TaskBeforeJsFile); + const TaskBeforePyFileExist = await fileExist(TaskBeforePyFile); const TaskAfterFileExist = await fileExist(TaskAfterFile); if (!configDirExist) { @@ -125,6 +129,20 @@ export default async () => { await fs.writeFile(TaskBeforeFile, await fs.readFile(sampleTaskShellFile)); } + if (!TaskBeforeJsFileExist) { + await fs.writeFile( + TaskBeforeJsFile, + '// The JavaScript code that executes before the JavaScript task execution will execute.', + ); + } + + if (!TaskBeforePyFileExist) { + await fs.writeFile( + TaskBeforePyFile, + '# The Python code that executes before the Python task execution will execute.', + ); + } + if (!TaskAfterFileExist) { await fs.writeFile(TaskAfterFile, await fs.readFile(sampleTaskShellFile)); } diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 78def676..deae25a0 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,8 +1,8 @@ #!/bin/bash dir_shell=/ql/shell -. $dir_shell/env.sh . $dir_shell/share.sh +. $dir_shell/env.sh echo -e "======================1. 检测配置文件========================\n" make_dir /etc/nginx/conf.d diff --git a/shell/otask.sh b/shell/otask.sh index 9084e4fd..b1397c67 100755 --- a/shell/otask.sh +++ b/shell/otask.sh @@ -100,7 +100,7 @@ run_normal() { if [[ $isJsOrPythonFile == 'false' ]]; then clear_non_sh_env fi - fileName="${file_param}" taskBefore="${file_task_before}" configDir="${dir_config}" $timeoutCmd $which_program $file_param "${script_params[@]}" + $timeoutCmd $which_program $file_param "${script_params[@]}" } handle_env_split() { @@ -143,7 +143,7 @@ run_concurrent() { export "${env_param}=${array[$i - 1]}" clear_non_sh_env fi - eval fileName="${file_param}" taskBefore="${file_task_before}" configDir="${dir_config}" envParam="${env_param}" numParam="${i}" $timeoutCmd $which_program $file_param "${script_params[@]}" &>$single_log_path & + eval envParam="${env_param}" numParam="${i}" $timeoutCmd $which_program $file_param "${script_params[@]}" &>$single_log_path & done wait @@ -188,7 +188,7 @@ run_designated() { file_param=${file_param/$relative_path\//} fi - fileName="${file_param}" taskBefore="${file_task_before}" configDir="${dir_config}" envParam="${env_param}" numParam="${num_param}" $timeoutCmd $which_program $file_param "${script_params[@]}" + envParam="${env_param}" numParam="${num_param}" $timeoutCmd $which_program $file_param "${script_params[@]}" } ## 运行其他命令 @@ -216,8 +216,8 @@ check_file() { if [[ -f $file_env ]]; then get_env_array if [[ $isJsOrPythonFile == 'true' ]]; then - export NODE_OPTIONS="${NODE_OPTIONS} -r ${preload_js_file}" - export PYTHONPATH="${PYTHONPATH}:${dir_preload}" + export NODE_OPTIONS="${NODE_OPTIONS} -r ${file_preload_js}" + export PYTHONPATH="${PYTHONPATH}:${dir_preload}:${dir_config}" else . $file_env fi diff --git a/shell/preload/sitecustomize.js b/shell/preload/sitecustomize.js index 74ccf729..5b670e44 100644 --- a/shell/preload/sitecustomize.js +++ b/shell/preload/sitecustomize.js @@ -25,12 +25,23 @@ function expandRange(rangeStr, max) { } function run() { + const { + envParam, + numParam, + file_task_before, + file_task_before_js, + dir_scripts, + task_before, + } = process.env; + + require(file_task_before_js); + try { - // TODO: big size const splitStr = '__sitecustomize__'; - let command = `bash -c "source ${process.env.taskBefore} ${process.env.fileName}`; - if (process.env.task_before) { - command = `${command} && echo -e '执行前置命令\n' && eval "${process.env.task_before}" && echo -e '\n执行前置命令结束\n'`; + const fileName = process.argv[1].replace(`${dir_scripts}/`, ''); + let command = `bash -c "source ${file_task_before} ${fileName}`; + if (task_before) { + command = `${command} && echo -e '执行前置命令\n' && eval "${task_before}" && echo -e '\n执行前置命令结束\n'`; } const res = execSync( `${command} && echo "${splitStr}" && NODE_OPTIONS= node -p 'JSON.stringify(process.env)'"`, @@ -45,10 +56,11 @@ function run() { } console.log(output); } catch (error) { - console.log(`run task before error: `, error.message); + if (!error.message.includes('spawnSync /bin/sh E2BIG')) { + console.log(`run task before error: `, error); + } } - const { envParam, numParam } = process.env; if (envParam && numParam) { const array = (process.env[envParam] || '').split('&'); const runArr = expandRange(numParam, array.length); @@ -58,5 +70,9 @@ function run() { } } -initGlobal(); -run(); +try { + initGlobal(); + run(); +} catch (error) { + console.log(`run builtin code error: `, error, '\n'); +} diff --git a/shell/preload/sitecustomize.py b/shell/preload/sitecustomize.py index 38434ead..9e8b631d 100644 --- a/shell/preload/sitecustomize.py +++ b/shell/preload/sitecustomize.py @@ -1,9 +1,10 @@ import os import re -import env import subprocess import json import builtins +import sys +import env from notify import send @@ -44,12 +45,16 @@ def expand_range(range_str, max_value): def run(): + import task_before + try: split_str = "__sitecustomize__" - command = f'bash -c "source {os.getenv("taskBefore")} {os.getenv("fileName")}' + file_name = sys.argv[0].replace(f"{os.getenv('dir_scripts')}/", "") + command = f'bash -c "source {os.getenv("file_task_before")} {file_name}' + task_before = os.getenv("task_before") - if os.getenv("task_before"): - command += f" && echo -e '执行前置命令\n' && eval \"{os.getenv('task_before')}\" && echo -e '\n执行前置命令结束\n'" + if task_before: + command += f" && echo -e '执行前置命令\n' && eval \"{task_before}\" && echo -e '\n执行前置命令结束\n'" python_command = "PYTHONPATH= python3 -c 'import os, json; print(json.dumps(dict(os.environ)))'" command += f' && echo "{split_str}" && {python_command}"' @@ -66,6 +71,12 @@ def run(): except subprocess.CalledProcessError as error: print(f"run task before error: {error}") + except OSError as error: + error_message = str(error) + if "Argument list too long" not in error_message: + print(f"run task before error: {error}") + except Exception as error: + print(f"run task before error: {error}") env_param = os.getenv("envParam") num_param = os.getenv("numParam") @@ -78,5 +89,8 @@ def run(): os.environ[env_param] = env_str -init_global() -run() +try: + init_global() + run() +except Exception as error: + print(f"run builtin code error: {error}\n") diff --git a/shell/share.sh b/shell/share.sh index bd982390..ffbbc436 100755 --- a/shell/share.sh +++ b/shell/share.sh @@ -1,55 +1,57 @@ #!/usr/bin/env bash ## 目录 -dir_root=$QL_DIR -dir_tmp=$dir_root/.tmp -dir_data=$dir_root/data +export dir_root=$QL_DIR +export dir_tmp=$dir_root/.tmp +export dir_data=$dir_root/data if [[ $QL_DATA_DIR ]]; then - dir_data="${QL_DATA_DIR%/}" + export dir_data="${QL_DATA_DIR%/}" fi -dir_shell=$dir_root/shell -dir_preload=$dir_shell/preload -dir_sample=$dir_root/sample -dir_static=$dir_root/static -dir_config=$dir_data/config -dir_scripts=$dir_data/scripts -dir_repo=$dir_data/repo -dir_raw=$dir_data/raw -dir_log=$dir_data/log -dir_db=$dir_data/db -dir_dep=$dir_data/deps -dir_list_tmp=$dir_log/.tmp -dir_update_log=$dir_log/update -ql_static_repo=$dir_repo/static +export dir_shell=$dir_root/shell +export dir_preload=$dir_shell/preload +export dir_sample=$dir_root/sample +export dir_static=$dir_root/static +export dir_config=$dir_data/config +export dir_scripts=$dir_data/scripts +export dir_repo=$dir_data/repo +export dir_raw=$dir_data/raw +export dir_log=$dir_data/log +export dir_db=$dir_data/db +export dir_dep=$dir_data/deps +export dir_list_tmp=$dir_log/.tmp +export dir_update_log=$dir_log/update +export ql_static_repo=$dir_repo/static ## 文件 -file_config_sample=$dir_sample/config.sample.sh -file_env=$dir_preload/env.sh -preload_js_file=$dir_preload/sitecustomize.js -file_sharecode=$dir_config/sharecode.sh -file_config_user=$dir_config/config.sh -file_auth_sample=$dir_sample/auth.sample.json -file_auth_user=$dir_config/auth.json -file_auth_token=$dir_config/token.json -file_extra_shell=$dir_config/extra.sh -file_task_before=$dir_config/task_before.sh -file_task_after=$dir_config/task_after.sh -file_task_sample=$dir_sample/task.sample.sh -file_extra_sample=$dir_sample/extra.sample.sh -file_notify_js_sample=$dir_sample/notify.js -file_notify_py_sample=$dir_sample/notify.py -file_test_js_sample=$dir_sample/ql_sample.js -file_test_py_sample=$dir_sample/ql_sample.py -file_notify_py=$dir_scripts/notify.py -file_notify_js=$dir_scripts/sendNotify.js -file_test_js=$dir_scripts/ql_sample.js -file_test_py=$dir_scripts/ql_sample.py -nginx_app_conf=$dir_root/docker/front.conf -nginx_conf=$dir_root/docker/nginx.conf -dep_notify_py=$dir_dep/notify.py -dep_notify_js=$dir_dep/sendNotify.js +export file_config_sample=$dir_sample/config.sample.sh +export file_env=$dir_preload/env.sh +export file_preload_js=$dir_preload/sitecustomize.js +export file_sharecode=$dir_config/sharecode.sh +export file_config_user=$dir_config/config.sh +export file_auth_sample=$dir_sample/auth.sample.json +export file_auth_user=$dir_config/auth.json +export file_auth_token=$dir_config/token.json +export file_extra_shell=$dir_config/extra.sh +export file_task_before=$dir_config/task_before.sh +export file_task_before_js=$dir_config/task_before.js +export file_task_before_py=$dir_config/task_before.py +export file_task_after=$dir_config/task_after.sh +export file_task_sample=$dir_sample/task.sample.sh +export file_extra_sample=$dir_sample/extra.sample.sh +export file_notify_js_sample=$dir_sample/notify.js +export file_notify_py_sample=$dir_sample/notify.py +export file_test_js_sample=$dir_sample/ql_sample.js +export file_test_py_sample=$dir_sample/ql_sample.py +export file_notify_py=$dir_scripts/notify.py +export file_notify_js=$dir_scripts/sendNotify.js +export file_test_js=$dir_scripts/ql_sample.js +export file_test_py=$dir_scripts/ql_sample.py +export nginx_app_conf=$dir_root/docker/front.conf +export nginx_conf=$dir_root/docker/nginx.conf +export dep_notify_py=$dir_dep/notify.py +export dep_notify_js=$dir_dep/sendNotify.js ## 清单文件 list_crontab_user=$dir_config/crontab.list @@ -177,12 +179,6 @@ fix_config() { echo fi - if [[ ! -f $file_env ]]; then - echo -e "检测到config配置目录下不存在env.sh，创建一个空文件用于初始化...\n" - touch $file_env - echo - fi - if [[ ! -f $file_task_before ]]; then echo -e "复制一份 $file_task_sample 为 $file_task_before\n" cp -fv $file_task_sample $file_task_before diff --git a/shell/update.sh b/shell/update.sh index 3ffd7152..f34fa690 100755 --- a/shell/update.sh +++ b/shell/update.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash dir_shell=$QL_DIR/shell -. $dir_shell/env.sh . $dir_shell/share.sh . $dir_shell/api.sh +. $dir_shell/env.sh send_mark=$dir_shell/send_mark