Compare commits

...

10 Commits

Author SHA1 Message Date
whyour a743aa8115 更新版本 v2.17.9 2024-07-28 18:40:10 +08:00
whyour fe4516df00 修复执行 task_before 环境变量加载顺序 2024-07-28 18:40:02 +08:00
whyour be2da98674 修复任务中 task_before 参数双引号解析 2024-07-26 11:30:03 +08:00
whyour 0da8155bac 修复 desi 和 conc 命令参数解析 2024-07-25 20:53:11 +08:00
whyour eddd258614 修复 node 前置逻辑 2024-07-24 23:30:18 +08:00
whyour 2ad6437672 更新 config.sample 2024-07-22 23:37:51 +08:00
whyour 162bd9d8cc 修复 config 引入顺序 2024-07-22 23:29:08 +08:00
whyour e694570f1a 增加 task_before.js 和 task_before.py 文件,在执行任务前执行,避免环境变量过大报错 2024-07-22 01:07:00 +08:00
whyour eb5cc3943d Javascript 和 Python 增加内置函数 QLAPI.notify 2024-07-21 01:15:16 +08:00
whyour 1b39d3ab48 JavaScript 和 Python 执行脚本前执行 task_before 2024-07-20 17:28:48 +08:00
16 changed files with 221 additions and 110 deletions
+1
View File
@@ -25,3 +25,4 @@
/.tmp
__pycache__
/shell/preload/env.*
/shell/preload/notify.*
+4
View File
@@ -41,6 +41,8 @@ const systemLogPath = path.join(dataPath, 'syslog/');
const envFile = path.join(preloadPath, 'env.sh');
const jsEnvFile = path.join(preloadPath, 'env.js');
const pyEnvFile = path.join(preloadPath, 'env.py');
const jsNotifyFile = path.join(preloadPath, 'notify.js');
const pyNotifyFile = path.join(preloadPath, 'notify.py');
const confFile = path.join(configPath, 'config.sh');
const crontabFile = path.join(configPath, 'crontab.list');
const authConfigFile = path.join(configPath, 'auth.json');
@@ -92,6 +94,8 @@ export default {
envFile,
jsEnvFile,
pyEnvFile,
jsNotifyFile,
pyNotifyFile,
dbPath,
uploadPath,
configPath,
+23
View File
@@ -28,7 +28,11 @@ const sampleNotifyJsFile = path.join(samplePath, 'notify.js');
const sampleNotifyPyFile = path.join(samplePath, 'notify.py');
const scriptNotifyJsFile = path.join(scriptPath, 'sendNotify.js');
const scriptNotifyPyFile = path.join(scriptPath, 'notify.py');
const jsNotifyFile = path.join(preloadPath, 'notify.js');
const pyNotifyFile = path.join(preloadPath, 'notify.py');
const TaskBeforeFile = path.join(configPath, 'task_before.sh');
const TaskBeforeJsFile = path.join(configPath, 'task_before.js');
const TaskBeforePyFile = path.join(configPath, 'task_before.py');
const TaskAfterFile = path.join(configPath, 'task_after.sh');
const homedir = os.homedir();
const sshPath = path.resolve(homedir, '.ssh');
@@ -51,6 +55,8 @@ export default async () => {
const scriptNotifyJsFileExist = await fileExist(scriptNotifyJsFile);
const scriptNotifyPyFileExist = await fileExist(scriptNotifyPyFile);
const TaskBeforeFileExist = await fileExist(TaskBeforeFile);
const TaskBeforeJsFileExist = await fileExist(TaskBeforeJsFile);
const TaskBeforePyFileExist = await fileExist(TaskBeforePyFile);
const TaskAfterFileExist = await fileExist(TaskAfterFile);
if (!configDirExist) {
@@ -102,6 +108,9 @@ export default async () => {
await fs.writeFile(confFile, await fs.readFile(sampleConfigFile));
}
await fs.writeFile(jsNotifyFile, await fs.readFile(sampleNotifyJsFile));
await fs.writeFile(pyNotifyFile, await fs.readFile(sampleNotifyPyFile));
if (!scriptNotifyJsFileExist) {
await fs.writeFile(
scriptNotifyJsFile,
@@ -120,6 +129,20 @@ export default async () => {
await fs.writeFile(TaskBeforeFile, await fs.readFile(sampleTaskShellFile));
}
if (!TaskBeforeJsFileExist) {
await fs.writeFile(
TaskBeforeJsFile,
'// The JavaScript code that executes before the JavaScript task execution will execute.',
);
}
if (!TaskBeforePyFileExist) {
await fs.writeFile(
TaskBeforePyFile,
'# The Python code that executes before the Python task execution will execute.',
);
}
if (!TaskAfterFileExist) {
await fs.writeFile(TaskAfterFile, await fs.readFile(sampleTaskShellFile));
}
+2 -1
View File
@@ -1,10 +1,11 @@
#!/bin/bash
dir_shell=/ql/shell
. $dir_shell/env.sh
. $dir_shell/share.sh
. $dir_shell/env.sh
echo -e "======================1. 检测配置文件========================\n"
import_config "$@"
make_dir /etc/nginx/conf.d
make_dir /run/nginx
init_nginx
+1 -1
View File
@@ -43,7 +43,7 @@ BotRepoUrl=""
## 通知环境变量
## 1. Server酱
## https://sct.ftqq.com
## https://sct.ftqq.com/r/13363
## 下方填写 SCHKEY 值或 SendKey 值
export PUSH_KEY=""
+2 -4
View File
@@ -4,8 +4,6 @@
* 定时规则
* cron: 1 9 * * *
*/
const { sendNotify } = require('./sendNotify.js'); // commonjs
// import { sendNotify } from './sendNotify'; // es6
console.log('test scripts');
sendNotify('test scripts', 'test desc');
QLAPI.notify('test scripts', 'test desc');
console.log('test desc');
+2 -3
View File
@@ -4,7 +4,6 @@ name: script name
定时规则
cron: 1 9 * * *
"""
import notify
print("test script")
notify.send('test script', 'test desc')
QLAPI.notify('test script', 'test desc')
print("test desc")
+21 -25
View File
@@ -69,23 +69,6 @@ run_nohup() {
nohup node $file_name &>$log_path &
}
check_server() {
if [[ $cpu_warn ]] && [[ $mem_warn ]] && [[ $disk_warn ]]; then
local top_result=$(top -b -n 1)
cpu_use=$(echo "$top_result" | grep CPU | grep -v -E 'grep|PID' | awk '{print $2}' | cut -f 1 -d "%" | head -n 1)
mem_free=$(free -m | grep "Mem" | awk '{print $3}' | head -n 1)
mem_total=$(free -m | grep "Mem" | awk '{print $2}' | head -n 1)
mem_use=$(printf "%d%%" $((mem_free * 100 / mem_total)) | cut -f 1 -d "%" | head -n 1)
disk_use=$(df -P | grep /dev | grep -v -E '(tmp|boot|shm)' | awk '{print $5}' | cut -f 1 -d "%" | head -n 1)
if [[ $cpu_use -gt $cpu_warn ]] && [[ $cpu_warn ]] || [[ $mem_free -lt $mem_warn ]] || [[ $disk_use -gt $disk_warn ]]; then
local resource=$(echo "$top_result" | grep -v -E 'grep|Mem|idle|Load|tr' | awk '{$2="";$3="";$4="";$5="";$7="";print $0}' | head -n 10 | tr '\n' '|' | sed s/\|/\\\\n/g)
notify_api "服务器资源异常警告" "当前CPU占用 $cpu_use% 内存占用 $mem_use% 磁盘占用 $disk_use% \n资源占用详情 \n\n $resource"
fi
fi
}
env_str_to_array() {
. $file_env
local IFS="&"
@@ -117,7 +100,7 @@ run_normal() {
if [[ $isJsOrPythonFile == 'false' ]]; then
clear_non_sh_env
fi
configDir="${dir_config}" $timeoutCmd $which_program $file_param "${script_params[@]}"
$timeoutCmd $which_program $file_param "${script_params[@]}"
}
handle_env_split() {
@@ -160,7 +143,7 @@ run_concurrent() {
export "${env_param}=${array[$i - 1]}"
clear_non_sh_env
fi
eval configDir="${dir_config}" envParam="${env_param}" numParam="${i}" $timeoutCmd $which_program $file_param "${script_params[@]}" &>$single_log_path &
eval envParam="${env_param}" numParam="${i}" $timeoutCmd $which_program $file_param "${script_params[@]}" &>$single_log_path &
done
wait
@@ -205,7 +188,7 @@ run_designated() {
file_param=${file_param/$relative_path\//}
fi
configDir="${dir_config}" envParam="${env_param}" numParam="${num_param}" $timeoutCmd $which_program $file_param "${script_params[@]}"
envParam="${env_param}" numParam="${num_param}" $timeoutCmd $which_program $file_param "${script_params[@]}"
}
## 运行其他命令
@@ -225,8 +208,7 @@ run_else() {
$timeoutCmd $which_program $file_param "$@"
}
## 命令检测
main() {
check_file() {
isJsOrPythonFile="false"
if [[ $1 == *.js ]] || [[ $1 == *.py ]] || [[ $1 == *.pyc ]] || [[ $1 == *.ts ]]; then
isJsOrPythonFile="true"
@@ -234,13 +216,20 @@ main() {
if [[ -f $file_env ]]; then
get_env_array
if [[ $isJsOrPythonFile == 'true' ]]; then
export NODE_OPTIONS="${NODE_OPTIONS} -r ${preload_js_file}"
export PYTHONPATH="${PYTHONPATH}:${dir_preload}"
PREV_NODE_OPTIONS="${NODE_OPTIONS}"
PREV_PYTHONPATH="${PYTHONPATH}"
if [[ $1 == *.js ]] || [[ $1 == *.ts ]]; then
export NODE_OPTIONS="${NODE_OPTIONS} -r ${file_preload_js}"
else
export PYTHONPATH="${PYTHONPATH}:${dir_preload}:${dir_config}"
fi
else
. $file_env
fi
fi
}
main() {
if [[ $1 == *.js ]] || [[ $1 == *.py ]] || [[ $1 == *.pyc ]] || [[ $1 == *.sh ]] || [[ $1 == *.ts ]]; then
if [[ $1 == *.sh ]]; then
timeoutCmd=""
@@ -276,8 +265,15 @@ main() {
}
handle_task_start "${task_shell_params[@]}"
run_task_before "${task_shell_params[@]}"
check_file "${task_shell_params[@]}"
if [[ $isJsOrPythonFile == 'false' ]]; then
run_task_before "${task_shell_params[@]}"
fi
main "${task_shell_params[@]}"
if [[ $isJsOrPythonFile == 'true' ]]; then
export NODE_OPTIONS="${PREV_NODE_OPTIONS}"
export PYTHONPATH="${PREV_PYTHONPATH}"
fi
run_task_after "${task_shell_params[@]}"
clear_env
handle_task_end "${task_shell_params[@]}"
+54 -5
View File
@@ -1,5 +1,5 @@
const { execSync } = require('child_process');
require(`./env.js`);
function expandRange(rangeStr, max) {
const tempRangeStr = rangeStr
.trim()
@@ -10,15 +10,55 @@ function expandRange(rangeStr, max) {
const rangeMatch = part.match(/^(\d+)([-~_])(\d+)$/);
if (rangeMatch) {
const [, start, , end] = rangeMatch.map(Number);
return Array.from({ length: end - start + 1 }, (_, i) => start + i);
const step = start < end ? 1 : -1;
return Array.from(
{ length: Math.abs(end - start) + 1 },
(_, i) => start + i * step,
);
}
return Number(part);
});
}
function run() {
if (process.env.envParam && process.env.numParam) {
const { envParam, numParam } = process.env;
const {
envParam,
numParam,
file_task_before,
file_task_before_js,
dir_scripts,
task_before,
} = process.env;
require(file_task_before_js);
try {
const splitStr = '__sitecustomize__';
const fileName = process.argv[1].replace(`${dir_scripts}/`, '');
let command = `bash -c "source ${file_task_before} ${fileName}`;
if (task_before) {
const escapeTaskBefore = task_before.replace(/"/g, '\\"');
command = `${command} && echo -e '执行前置命令\n' && eval '${escapeTaskBefore}' && echo -e '\n执行前置命令结束\n'`;
}
const res = execSync(
`${command} && echo -e '${splitStr}' && NODE_OPTIONS= node -p 'JSON.stringify(process.env)'"`,
{
encoding: 'utf-8',
},
);
const [output, envStr] = res.split(splitStr);
const newEnvObject = JSON.parse(envStr.trim());
for (const key in newEnvObject) {
process.env[key] = newEnvObject[key];
}
console.log(output);
} catch (error) {
if (!error.message.includes('spawnSync /bin/sh E2BIG')) {
console.log(`run task before error: `, error);
}
}
if (envParam && numParam) {
const array = (process.env[envParam] || '').split('&');
const runArr = expandRange(numParam, array.length);
const arrayRun = runArr.map((i) => array[i - 1]);
@@ -27,4 +67,13 @@ function run() {
}
}
run();
try {
run();
const { sendNotify } = require('./notify.js');
global.QLAPI = {
notify: sendNotify,
};
} catch (error) {
console.log(`run builtin code error: `, error, '\n');
}
+53 -2
View File
@@ -1,5 +1,9 @@
import os
import re
import subprocess
import json
import builtins
import sys
import env
@@ -22,7 +26,8 @@ def expand_range(range_str, max_value):
range_match = re.match(r"^(\d+)([-~_])(\d+)$", part)
if range_match:
start, _, end = map(try_parse_int, range_match.groups())
result.extend(range(start, end + 1))
step = 1 if start < end else -1
result.extend(range(start, end + step, step))
else:
result.append(int(part))
@@ -30,6 +35,40 @@ def expand_range(range_str, max_value):
def run():
import task_before
try:
split_str = "__sitecustomize__"
file_name = sys.argv[0].replace(f"{os.getenv('dir_scripts')}/", "")
command = f'bash -c "source {os.getenv("file_task_before")} {file_name}'
task_before = os.getenv("task_before")
if task_before:
escape_task_before = task_before.replace('"', '\\"')
command += f" && echo -e '执行前置命令\n' && eval '{escape_task_before}' && echo -e '\n执行前置命令结束\n'"
python_command = "PYTHONPATH= python3 -c 'import os, json; print(json.dumps(dict(os.environ)))'"
command += f" && echo -e '{split_str}' && {python_command}\""
res = subprocess.check_output(command, shell=True, encoding="utf-8")
output, env_str = res.split(split_str)
env_json = json.loads(env_str.strip())
for key, value in env_json.items():
os.environ[key] = value
print(output)
except subprocess.CalledProcessError as error:
print(f"run task before error: {error}")
except OSError as error:
error_message = str(error)
if "Argument list too long" not in error_message:
print(f"run task before error: {error}")
except Exception as error:
print(f"run task before error: {error}")
env_param = os.getenv("envParam")
num_param = os.getenv("numParam")
@@ -41,4 +80,16 @@ def run():
os.environ[env_param] = env_str
run()
try:
run()
from notify import send
class BaseApi:
def notify(self, *args, **kwargs):
return send(*args, **kwargs)
QLAPI = BaseApi()
builtins.QLAPI = QLAPI
except Exception as error:
print(f"run builtin code error: {error}\n")
+47 -61
View File
@@ -1,57 +1,57 @@
#!/usr/bin/env bash
## 目录
dir_root=$QL_DIR
dir_tmp=$dir_root/.tmp
dir_data=$dir_root/data
export dir_root=$QL_DIR
export dir_tmp=$dir_root/.tmp
export dir_data=$dir_root/data
if [[ $QL_DATA_DIR ]]; then
dir_data="${QL_DATA_DIR%/}"
export dir_data="${QL_DATA_DIR%/}"
fi
dir_shell=$dir_root/shell
dir_preload=$dir_shell/preload
dir_sample=$dir_root/sample
dir_static=$dir_root/static
dir_config=$dir_data/config
dir_scripts=$dir_data/scripts
dir_repo=$dir_data/repo
dir_raw=$dir_data/raw
dir_log=$dir_data/log
dir_db=$dir_data/db
dir_dep=$dir_data/deps
dir_list_tmp=$dir_log/.tmp
dir_update_log=$dir_log/update
ql_static_repo=$dir_repo/static
export dir_shell=$dir_root/shell
export dir_preload=$dir_shell/preload
export dir_sample=$dir_root/sample
export dir_static=$dir_root/static
export dir_config=$dir_data/config
export dir_scripts=$dir_data/scripts
export dir_repo=$dir_data/repo
export dir_raw=$dir_data/raw
export dir_log=$dir_data/log
export dir_db=$dir_data/db
export dir_dep=$dir_data/deps
export dir_list_tmp=$dir_log/.tmp
export dir_update_log=$dir_log/update
export ql_static_repo=$dir_repo/static
## 文件
file_config_sample=$dir_sample/config.sample.sh
file_env=$dir_preload/env.sh
js_file_env=$dir_preload/env.js
py_file_env=$dir_preload/env.py
preload_js_file=$dir_preload/sitecustomize.js
file_sharecode=$dir_config/sharecode.sh
file_config_user=$dir_config/config.sh
file_auth_sample=$dir_sample/auth.sample.json
file_auth_user=$dir_config/auth.json
file_auth_token=$dir_config/token.json
file_extra_shell=$dir_config/extra.sh
file_task_before=$dir_config/task_before.sh
file_task_after=$dir_config/task_after.sh
file_task_sample=$dir_sample/task.sample.sh
file_extra_sample=$dir_sample/extra.sample.sh
file_notify_js_sample=$dir_sample/notify.js
file_notify_py_sample=$dir_sample/notify.py
file_test_js_sample=$dir_sample/ql_sample.js
file_test_py_sample=$dir_sample/ql_sample.py
file_notify_py=$dir_scripts/notify.py
file_notify_js=$dir_scripts/sendNotify.js
file_test_js=$dir_scripts/ql_sample.js
file_test_py=$dir_scripts/ql_sample.py
nginx_app_conf=$dir_root/docker/front.conf
nginx_conf=$dir_root/docker/nginx.conf
dep_notify_py=$dir_dep/notify.py
dep_notify_js=$dir_dep/sendNotify.js
export file_config_sample=$dir_sample/config.sample.sh
export file_env=$dir_preload/env.sh
export file_preload_js=$dir_preload/sitecustomize.js
export file_sharecode=$dir_config/sharecode.sh
export file_config_user=$dir_config/config.sh
export file_auth_sample=$dir_sample/auth.sample.json
export file_auth_user=$dir_config/auth.json
export file_auth_token=$dir_config/token.json
export file_extra_shell=$dir_config/extra.sh
export file_task_before=$dir_config/task_before.sh
export file_task_before_js=$dir_config/task_before.js
export file_task_before_py=$dir_config/task_before.py
export file_task_after=$dir_config/task_after.sh
export file_task_sample=$dir_sample/task.sample.sh
export file_extra_sample=$dir_sample/extra.sample.sh
export file_notify_js_sample=$dir_sample/notify.js
export file_notify_py_sample=$dir_sample/notify.py
export file_test_js_sample=$dir_sample/ql_sample.js
export file_test_py_sample=$dir_sample/ql_sample.py
export file_notify_py=$dir_scripts/notify.py
export file_notify_js=$dir_scripts/sendNotify.js
export file_test_js=$dir_scripts/ql_sample.js
export file_test_py=$dir_scripts/ql_sample.py
export nginx_app_conf=$dir_root/docker/front.conf
export nginx_conf=$dir_root/docker/nginx.conf
export dep_notify_py=$dir_dep/notify.py
export dep_notify_js=$dir_dep/sendNotify.js
## 清单文件
list_crontab_user=$dir_config/crontab.list
@@ -91,10 +91,6 @@ import_config() {
else
default_cron="$(random_range 0 59) $(random_range 0 23) * * *"
fi
cpu_warn=${CpuWarn}
mem_warn=${MemoryWarn}
disk_warn=${DiskWarn}
}
set_proxy() {
@@ -179,12 +175,6 @@ fix_config() {
echo
fi
if [[ ! -f $file_env ]]; then
echo -e "检测到config配置目录下不存在env.sh,创建一个空文件用于初始化...\n"
touch $file_env
echo
fi
if [[ ! -f $file_task_before ]]; then
echo -e "复制一份 $file_task_sample$file_task_before\n"
cp -fv $file_task_sample $file_task_before
@@ -459,13 +449,11 @@ handle_task_start() {
}
run_task_before() {
[[ $is_macos -eq 0 ]] && check_server
. $file_task_before "$@"
if [[ $task_before ]]; then
echo -e "执行前置命令\n"
eval "$task_before"
eval "$task_before" "$@"
echo -e "\n执行前置命令结束\n"
fi
}
@@ -475,7 +463,7 @@ run_task_after() {
if [[ $task_after ]]; then
echo -e "\n执行后置命令\n"
eval "$task_after"
eval "$task_after" "$@"
echo -e "\n执行后置命令结束"
fi
}
@@ -496,5 +484,3 @@ init_env
detect_termux
detect_macos
define_cmd
import_config $1
+1
View File
@@ -113,6 +113,7 @@ init_begin_time() {
begin_timestamp=$(format_timestamp "$time_format" "$time")
}
import_config "$@"
while getopts ":lm:" opt; do
case $opt in
l)
+2 -1
View File
@@ -1,9 +1,9 @@
#!/usr/bin/env bash
dir_shell=$QL_DIR/shell
. $dir_shell/env.sh
. $dir_shell/share.sh
. $dir_shell/api.sh
. $dir_shell/env.sh
send_mark=$dir_shell/send_mark
@@ -563,6 +563,7 @@ main() {
fi
}
import_config "$@"
main "$@"
exit 0
+1 -1
View File
@@ -349,7 +349,7 @@
"推送到个人QQ: http://127.0.0.1/send_private_msg,群:http://127.0.0.1/send_group_msg": "Push to personal QQ: http://127.0.0.1/send_private_msg, group: http://127.0.0.1/send_group_msg",
"访问密钥": "Access Key",
"如果GOBOT_URL设置 /send_private_msg 则需要填入 user_id=个人QQ 相反如果是 /send_group_msg 则需要填入 group_id=QQ群": "If GOBOT_URL is set to /send_private_msg, enter user_id=personal QQ; if set to /send_group_msg, enter group_id=QQ group",
"Server酱SENDKEY": "ServerChan SENDKEY",
"Server酱SENDKEY": "ServerChan SENDKEY, https://sct.ftqq.com/r/13363",
"PushDeer的Keyhttps://github.com/easychen/pushdeer": "PushDeer Key, https://github.com/easychen/pushdeer",
"PushDeer的自架API endpoint,默认是 https://api2.pushdeer.com/message/push": "PushDeer's self-hosted API endpoint, default is https://api2.pushdeer.com/message/push",
"Bark的信息IP/设备码,例如:https://api.day.app/XXXXXXXX": "Bark information IP/device code, e.g., https://api.day.app/XXXXXXXX",
+1 -1
View File
@@ -349,7 +349,7 @@
"推送到个人QQ: http://127.0.0.1/send_private_msg,群:http://127.0.0.1/send_group_msg": "推送到个人QQ: http://127.0.0.1/send_private_msg,群:http://127.0.0.1/send_group_msg",
"访问密钥": "访问密钥",
"如果GOBOT_URL设置 /send_private_msg 则需要填入 user_id=个人QQ 相反如果是 /send_group_msg 则需要填入 group_id=QQ群": "如果GOBOT_URL设置 /send_private_msg 则需要填入 user_id=个人QQ 相反如果是 /send_group_msg 则需要填入 group_id=QQ群",
"Server酱SENDKEY": "ServerSENDKEY",
"Server酱SENDKEY": "ServerSENDKEYhttps://sct.ftqq.com/r/13363",
"PushDeer的Keyhttps://github.com/easychen/pushdeer": "PushDeer的Keyhttps://github.com/easychen/pushdeer",
"PushDeer的自架API endpoint,默认是 https://api2.pushdeer.com/message/push": "PushDeer的自架API endpoint,默认是 https://api2.pushdeer.com/message/push",
"Bark的信息IP/设备码,例如:https://api.day.app/XXXXXXXX": "Bark的信息IP/设备码,例如:https://api.day.app/XXXXXXXX",
+6 -5
View File
@@ -1,7 +1,8 @@
version: 2.17.8
changeLogLink: https://t.me/jiao_long/419
publishTime: 2024-07-13 08:00
version: 2.17.9
changeLogLink: https://t.me/jiao_long/420
publishTime: 2024-07-23 23:00
changeLog: |
1. 修改环境变量加载逻辑
2. 修复执行任务报错 Argument list too long
1. Javascript 和 Python 增加内置API QLAPI.notify
2. 配置管理增加 task_before.js 和 task_before.py 文件,在执行任务前执行,避免环境变量过大报错
3. 修复执行任务 JavaScript 和 Python 任务前未执行 task_before.sh