增加 task_before.js 和 task_before.py 文件，在执行任务前执行，避免环境变量过大报错

back/loaders/initFile.ts

@@ -31,6 +31,8 @@ const scriptNotifyPyFile = path.join(scriptPath, 'notify.py');
 const jsNotifyFile = path.join(preloadPath, 'notify.js');
 const pyNotifyFile = path.join(preloadPath, 'notify.py');
 const TaskBeforeFile = path.join(configPath, 'task_before.sh');
+const TaskBeforeJsFile = path.join(configPath, 'task_before.js');
+const TaskBeforePyFile = path.join(configPath, 'task_before.py');
 const TaskAfterFile = path.join(configPath, 'task_after.sh');
 const homedir = os.homedir();
 const sshPath = path.resolve(homedir, '.ssh');
@@ -53,6 +55,8 @@ export default async () => {
   const scriptNotifyJsFileExist = await fileExist(scriptNotifyJsFile);
   const scriptNotifyPyFileExist = await fileExist(scriptNotifyPyFile);
   const TaskBeforeFileExist = await fileExist(TaskBeforeFile);
+  const TaskBeforeJsFileExist = await fileExist(TaskBeforeJsFile);
+  const TaskBeforePyFileExist = await fileExist(TaskBeforePyFile);
   const TaskAfterFileExist = await fileExist(TaskAfterFile);
 
   if (!configDirExist) {
@@ -125,6 +129,20 @@ export default async () => {
     await fs.writeFile(TaskBeforeFile, await fs.readFile(sampleTaskShellFile));
   }
 
+  if (!TaskBeforeJsFileExist) {
+    await fs.writeFile(
+      TaskBeforeJsFile,
+      '// The JavaScript code that executes before the JavaScript task execution will execute.',
+    );
+  }
+
+  if (!TaskBeforePyFileExist) {
+    await fs.writeFile(
+      TaskBeforePyFile,
+      '# The Python code that executes before the Python task execution will execute.',
+    );
+  }
+
   if (!TaskAfterFileExist) {
     await fs.writeFile(TaskAfterFile, await fs.readFile(sampleTaskShellFile));
   }

docker/docker-entrypoint.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 
 dir_shell=/ql/shell
-. $dir_shell/env.sh
 . $dir_shell/share.sh
+. $dir_shell/env.sh
 
 echo -e "======================1. 检测配置文件========================\n"
 make_dir /etc/nginx/conf.d

shell/otask.sh

@@ -100,7 +100,7 @@ run_normal() {
   if [[ $isJsOrPythonFile == 'false' ]]; then
     clear_non_sh_env
   fi
-  fileName="${file_param}" taskBefore="${file_task_before}" configDir="${dir_config}" $timeoutCmd $which_program $file_param "${script_params[@]}"
+  $timeoutCmd $which_program $file_param "${script_params[@]}"
 }
 
 handle_env_split() {
@@ -143,7 +143,7 @@ run_concurrent() {
       export "${env_param}=${array[$i - 1]}"
       clear_non_sh_env
     fi
-    eval fileName="${file_param}" taskBefore="${file_task_before}" configDir="${dir_config}" envParam="${env_param}" numParam="${i}" $timeoutCmd $which_program $file_param "${script_params[@]}" &>$single_log_path &
+    eval envParam="${env_param}" numParam="${i}" $timeoutCmd $which_program $file_param "${script_params[@]}" &>$single_log_path &
   done
 
   wait
@@ -188,7 +188,7 @@ run_designated() {
     file_param=${file_param/$relative_path\//}
   fi
 
-  fileName="${file_param}" taskBefore="${file_task_before}" configDir="${dir_config}" envParam="${env_param}" numParam="${num_param}" $timeoutCmd $which_program $file_param "${script_params[@]}"
+  envParam="${env_param}" numParam="${num_param}" $timeoutCmd $which_program $file_param "${script_params[@]}"
 }
 
 ## 运行其他命令
@@ -216,8 +216,8 @@ check_file() {
   if [[ -f $file_env ]]; then
     get_env_array
     if [[ $isJsOrPythonFile == 'true' ]]; then
-      export NODE_OPTIONS="${NODE_OPTIONS} -r ${preload_js_file}"
-      export PYTHONPATH="${PYTHONPATH}:${dir_preload}"
+      export NODE_OPTIONS="${NODE_OPTIONS} -r ${file_preload_js}"
+      export PYTHONPATH="${PYTHONPATH}:${dir_preload}:${dir_config}"
     else
       . $file_env
     fi

shell/preload/sitecustomize.js

@@ -25,12 +25,23 @@ function expandRange(rangeStr, max) {
 }
 
 function run() {
+  const {
+    envParam,
+    numParam,
+    file_task_before,
+    file_task_before_js,
+    dir_scripts,
+    task_before,
+  } = process.env;
+
+  require(file_task_before_js);
+
   try {
-    // TODO: big size
     const splitStr = '__sitecustomize__';
-    let command = `bash -c "source ${process.env.taskBefore} ${process.env.fileName}`;
-    if (process.env.task_before) {
-      command = `${command} && echo -e '执行前置命令\n' && eval "${process.env.task_before}" && echo -e '\n执行前置命令结束\n'`;
+    const fileName = process.argv[1].replace(`${dir_scripts}/`, '');
+    let command = `bash -c "source ${file_task_before} ${fileName}`;
+    if (task_before) {
+      command = `${command} && echo -e '执行前置命令\n' && eval "${task_before}" && echo -e '\n执行前置命令结束\n'`;
     }
     const res = execSync(
       `${command} && echo "${splitStr}" && NODE_OPTIONS= node -p 'JSON.stringify(process.env)'"`,
@@ -45,10 +56,11 @@ function run() {
     }
     console.log(output);
   } catch (error) {
-    console.log(`run task before error: `, error.message);
+    if (!error.message.includes('spawnSync /bin/sh E2BIG')) {
+      console.log(`run task before error: `, error);
+    }
   }
 
-  const { envParam, numParam } = process.env;
   if (envParam && numParam) {
     const array = (process.env[envParam] || '').split('&');
     const runArr = expandRange(numParam, array.length);
@@ -58,5 +70,9 @@ function run() {
   }
 }
 
+try {
   initGlobal();
   run();
+} catch (error) {
+  console.log(`run builtin code error: `, error, '\n');
+}

shell/preload/sitecustomize.py

@@ -1,9 +1,10 @@
 import os
 import re
-import env
 import subprocess
 import json
 import builtins
+import sys
+import env
 from notify import send
 
 
@@ -44,12 +45,16 @@ def expand_range(range_str, max_value):
 
 
 def run():
+    import task_before
+
     try:
         split_str = "__sitecustomize__"
-        command = f'bash -c "source {os.getenv("taskBefore")} {os.getenv("fileName")}'
+        file_name = sys.argv[0].replace(f"{os.getenv('dir_scripts')}/", "")
+        command = f'bash -c "source {os.getenv("file_task_before")} {file_name}'
+        task_before = os.getenv("task_before")
 
-        if os.getenv("task_before"):
-            command += f" && echo -e '执行前置命令\n' && eval \"{os.getenv('task_before')}\" && echo -e '\n执行前置命令结束\n'"
+        if task_before:
+            command += f" && echo -e '执行前置命令\n' && eval \"{task_before}\" && echo -e '\n执行前置命令结束\n'"
 
         python_command = "PYTHONPATH= python3 -c 'import os, json; print(json.dumps(dict(os.environ)))'"
         command += f' && echo "{split_str}" && {python_command}"'
@@ -66,6 +71,12 @@ def run():
 
     except subprocess.CalledProcessError as error:
         print(f"run task before error: {error}")
+    except OSError as error:
+        error_message = str(error)
+        if "Argument list too long" not in error_message:
+            print(f"run task before error: {error}")
+    except Exception as error:
+        print(f"run task before error: {error}")
 
     env_param = os.getenv("envParam")
     num_param = os.getenv("numParam")
@@ -78,5 +89,8 @@ def run():
         os.environ[env_param] = env_str
 
 
+try:
     init_global()
     run()
+except Exception as error:
+    print(f"run builtin code error: {error}\n")

shell/share.sh

@@ -1,55 +1,57 @@
 #!/usr/bin/env bash
 
 ## 目录
-dir_root=$QL_DIR
-dir_tmp=$dir_root/.tmp
-dir_data=$dir_root/data
+export dir_root=$QL_DIR
+export dir_tmp=$dir_root/.tmp
+export dir_data=$dir_root/data
 
 if [[ $QL_DATA_DIR ]]; then
-  dir_data="${QL_DATA_DIR%/}"
+  export dir_data="${QL_DATA_DIR%/}"
 fi
 
-dir_shell=$dir_root/shell
-dir_preload=$dir_shell/preload
-dir_sample=$dir_root/sample
-dir_static=$dir_root/static
-dir_config=$dir_data/config
-dir_scripts=$dir_data/scripts
-dir_repo=$dir_data/repo
-dir_raw=$dir_data/raw
-dir_log=$dir_data/log
-dir_db=$dir_data/db
-dir_dep=$dir_data/deps
-dir_list_tmp=$dir_log/.tmp
-dir_update_log=$dir_log/update
-ql_static_repo=$dir_repo/static
+export dir_shell=$dir_root/shell
+export dir_preload=$dir_shell/preload
+export dir_sample=$dir_root/sample
+export dir_static=$dir_root/static
+export dir_config=$dir_data/config
+export dir_scripts=$dir_data/scripts
+export dir_repo=$dir_data/repo
+export dir_raw=$dir_data/raw
+export dir_log=$dir_data/log
+export dir_db=$dir_data/db
+export dir_dep=$dir_data/deps
+export dir_list_tmp=$dir_log/.tmp
+export dir_update_log=$dir_log/update
+export ql_static_repo=$dir_repo/static
 
 ## 文件
-file_config_sample=$dir_sample/config.sample.sh
-file_env=$dir_preload/env.sh
-preload_js_file=$dir_preload/sitecustomize.js
-file_sharecode=$dir_config/sharecode.sh
-file_config_user=$dir_config/config.sh
-file_auth_sample=$dir_sample/auth.sample.json
-file_auth_user=$dir_config/auth.json
-file_auth_token=$dir_config/token.json
-file_extra_shell=$dir_config/extra.sh
-file_task_before=$dir_config/task_before.sh
-file_task_after=$dir_config/task_after.sh
-file_task_sample=$dir_sample/task.sample.sh
-file_extra_sample=$dir_sample/extra.sample.sh
-file_notify_js_sample=$dir_sample/notify.js
-file_notify_py_sample=$dir_sample/notify.py
-file_test_js_sample=$dir_sample/ql_sample.js
-file_test_py_sample=$dir_sample/ql_sample.py
-file_notify_py=$dir_scripts/notify.py
-file_notify_js=$dir_scripts/sendNotify.js
-file_test_js=$dir_scripts/ql_sample.js
-file_test_py=$dir_scripts/ql_sample.py
-nginx_app_conf=$dir_root/docker/front.conf
-nginx_conf=$dir_root/docker/nginx.conf
-dep_notify_py=$dir_dep/notify.py
-dep_notify_js=$dir_dep/sendNotify.js
+export file_config_sample=$dir_sample/config.sample.sh
+export file_env=$dir_preload/env.sh
+export file_preload_js=$dir_preload/sitecustomize.js
+export file_sharecode=$dir_config/sharecode.sh
+export file_config_user=$dir_config/config.sh
+export file_auth_sample=$dir_sample/auth.sample.json
+export file_auth_user=$dir_config/auth.json
+export file_auth_token=$dir_config/token.json
+export file_extra_shell=$dir_config/extra.sh
+export file_task_before=$dir_config/task_before.sh
+export file_task_before_js=$dir_config/task_before.js
+export file_task_before_py=$dir_config/task_before.py
+export file_task_after=$dir_config/task_after.sh
+export file_task_sample=$dir_sample/task.sample.sh
+export file_extra_sample=$dir_sample/extra.sample.sh
+export file_notify_js_sample=$dir_sample/notify.js
+export file_notify_py_sample=$dir_sample/notify.py
+export file_test_js_sample=$dir_sample/ql_sample.js
+export file_test_py_sample=$dir_sample/ql_sample.py
+export file_notify_py=$dir_scripts/notify.py
+export file_notify_js=$dir_scripts/sendNotify.js
+export file_test_js=$dir_scripts/ql_sample.js
+export file_test_py=$dir_scripts/ql_sample.py
+export nginx_app_conf=$dir_root/docker/front.conf
+export nginx_conf=$dir_root/docker/nginx.conf
+export dep_notify_py=$dir_dep/notify.py
+export dep_notify_js=$dir_dep/sendNotify.js
 
 ## 清单文件
 list_crontab_user=$dir_config/crontab.list
@@ -177,12 +179,6 @@ fix_config() {
     echo
   fi
 
-  if [[ ! -f $file_env ]]; then
-    echo -e "检测到config配置目录下不存在env.sh，创建一个空文件用于初始化...\n"
-    touch $file_env
-    echo
-  fi
-
   if [[ ! -f $file_task_before ]]; then
     echo -e "复制一份 $file_task_sample 为 $file_task_before\n"
     cp -fv $file_task_sample $file_task_before

shell/update.sh

@@ -1,9 +1,9 @@
 #!/usr/bin/env bash
 
 dir_shell=$QL_DIR/shell
-. $dir_shell/env.sh
 . $dir_shell/share.sh
 . $dir_shell/api.sh
+. $dir_shell/env.sh
 
 send_mark=$dir_shell/send_mark