优化task.sh

This commit is contained in:
whyour 2022-10-22 00:25:54 +08:00
parent 04613bf9a9
commit eb58774ee9
3 changed files with 296 additions and 297 deletions

266
shell/otask.sh Normal file
View File

@ -0,0 +1,266 @@
#!/usr/bin/env bash
## 导入通用变量与函数
dir_shell=$QL_DIR/shell
. $dir_shell/share.sh
. $dir_shell/api.sh
random_delay() {
local random_delay_max=$RandomDelay
if [[ $random_delay_max ]] && [[ $random_delay_max -gt 0 ]]; then
local file_param=$1
local file_extensions=${RandomDelayFileExtensions-"js"}
local ignored_minutes=${RandomDelayIgnoredMinutes-"0 30"}
if [[ -n $file_extensions ]]; then
if ! echo "$file_param" | grep -qE "\.${file_extensions// /$|\\.}$"; then
# echo -e "\n当前文件需要准点运行, 放弃随机延迟\n"
return
fi
fi
local current_min
current_min=$(date "+%-M")
for minute in $ignored_minutes; do
if [[ $current_min -eq $minute ]]; then
# echo -e "\n当前时间需要准点运行, 放弃随机延迟\n"
return
fi
done
local delay_second=$(($(gen_random_num "$random_delay_max") + 1))
echo -e "\n命令未添加 \"now\",随机延迟 $delay_second 秒后再执行任务,如需立即终止,请按 CTRL+C...\n"
sleep $delay_second
fi
}
## scripts目录下所有可运行脚本数组
gen_array_scripts() {
local dir_current=$(pwd)
local i="-1"
cd $dir_scripts
for file in $(ls); do
if [[ -f $file ]] && [[ $file == *.js && $file != sendNotify.js ]]; then
let i++
array_scripts[i]=$(echo "$file" | perl -pe "s|$dir_scripts/||g")
array_scripts_name[i]=$(grep "new Env" $file | awk -F "'|\"" '{print $2}' | head -1)
[[ -z ${array_scripts_name[i]} ]] && array_scripts_name[i]="<未识别出活动名称>"
fi
done
cd $dir_current
}
## 使用说明
usage() {
gen_array_scripts
echo -e "task命令运行本程序自动添加进crontab的脚本需要输入脚本的绝对路径或去掉 “$dir_scripts/” 目录后的相对路径(定时任务中请写作相对路径),用法为:"
echo -e "1.$cmd_task <file_name> # 依次执行,如果设置了随机延迟,将随机延迟一定秒数"
echo -e "2.$cmd_task <file_name> now # 依次执行,无论是否设置了随机延迟,均立即运行,前台会输出日志,同时记录在日志文件中"
echo -e "3.$cmd_task <file_name> conc <环境变量名称> <账号编号,空格分隔>(可选的) # 并发执行,无论是否设置了随机延迟,均立即运行,前台不产生日志,直接记录在日志文件中,且可指定账号执行"
echo -e "4.$cmd_task <file_name> desi <环境变量名称> <账号编号,空格分隔> # 指定账号执行,无论是否设置了随机延迟,均立即运行"
if [[ ${#array_scripts[*]} -gt 0 ]]; then
echo -e "\n当前有以下脚本可以运行:"
for ((i = 0; i < ${#array_scripts[*]}; i++)); do
echo -e "$(($i + 1)). ${array_scripts_name[i]}${array_scripts[i]}"
done
else
echo -e "\n暂无脚本可以执行"
fi
}
## run nohup$1文件名不含路径带后缀
run_nohup() {
local file_name=$1
nohup node $file_name &>$log_path &
}
check_server() {
cpu_use=$(top -b -n 1 | grep CPU | grep -v -E 'grep|PID' | awk '{print $2}' | cut -f 1 -d "%")
mem_free=$(free -m | grep "Mem" | awk '{print $3}')
mem_total=$(free -m | grep "Mem" | awk '{print $2}')
mem_use=$(printf "%d%%" $((mem_free * 100 / mem_total)) | cut -f 1 -d "%")
disk_use=$(df -P | grep /dev | grep -v -E '(tmp|boot|shm)' | awk '{print $5}' | cut -f 1 -d "%")
if [[ $cpu_use -gt $cpu_warn ]] || [[ $mem_free -lt $mem_warn ]] || [[ $disk_use -gt $disk_warn ]]; then
local resource=$(top -b -n 1 | grep -v -E 'grep|Mem|idle|Load' | awk '{$2="";$3="";$4="";$5="";$7="";print $0}' | head -n 10)
notify_api "服务器资源异常警告" "当前CPU占用 $cpu_use% 内存占用 $mem_use% 磁盘占用 $disk_use% \n资源占用详情 \n\n $resource"
fi
}
handle_task_before() {
begin_time=$(format_time "$time_format" "$time")
begin_timestamp=$(format_timestamp "$time_format" "$time")
echo -e "## 开始执行... $begin_time\n"
[[ $is_macos -eq 0 ]] && check_server
[[ -f $task_error_log_path ]] && cat $task_error_log_path
[[ $ID ]] && update_cron "\"$ID\"" "0" "$$" "$log_path" "$begin_timestamp"
. $file_task_before "$@"
}
handle_task_after() {
. $file_task_after "$@"
local end_time=$(date '+%Y-%m-%d %H:%M:%S')
local end_timestamp=$(date "+%s")
local diff_time=$(($end_timestamp - $begin_timestamp))
[[ $ID ]] && update_cron "\"$ID\"" "1" "" "$log_path" "$begin_timestamp" "$diff_time"
echo -e "\n\n## 执行结束... $end_time 耗时 $diff_time"
echo -e "\n     "
}
## 正常运行单个脚本,$1传入参数
run_normal() {
local file_param=$1
if [[ $# -eq 1 ]]; then
random_delay "$file_param"
fi
cd $dir_scripts
local relative_path="${file_param%/*}"
if [[ ${file_param} != /* ]] && [[ ! -z ${relative_path} ]] && [[ ${file_param} =~ "/" ]]; then
cd ${relative_path}
file_param=${file_param/$relative_path\//}
fi
$timeoutCmd $which_program $file_param
}
## 并发执行时,设定的 RandomDelay 不会生效,即所有任务立即执行
run_concurrent() {
local file_param="$1"
local env_param="$2"
local num_param=$(echo "$3" | perl -pe "s|.*$2(.*)|\1|")
if [[ ! $env_param ]]; then
echo -e "\n 缺少并发运行的环境变量参数"
exit 1
fi
local envs=$(eval echo "\$${env_param}")
local array=($(echo $envs | sed 's/&/ /g'))
local tempArr=$(echo $num_param | perl -pe "s|(\d+)(-\|~\|_)(\d+)|{\1..\3}|g")
local runArr=($(eval echo $tempArr))
runArr=($(awk -v RS=' ' '!a[$1]++' <<<${runArr[@]}))
local n=0
for i in ${runArr[@]}; do
array_run[n]=${array[$i - 1]}
let n++
done
local cookieStr=$(echo ${array_run[*]} | sed 's/\ /\&/g')
[[ ! -z $cookieStr ]] && export ${env_param}=${cookieStr}
local envs=$(eval echo "\$${env_param}")
local array=($(echo $envs | sed 's/&/ /g'))
single_log_time=$(date "+%Y-%m-%d-%H-%M-%S.%N")
cd $dir_scripts
local relative_path="${file_param%/*}"
if [[ ! -z ${relative_path} ]] && [[ ${file_param} =~ "/" ]]; then
cd ${relative_path}
file_param=${file_param/$relative_path\//}
fi
for i in "${!array[@]}"; do
export ${env_param}=${array[i]}
single_log_path="$dir_log/$log_dir/${single_log_time}_$((i + 1)).log"
eval $timeoutCmd $which_program $file_param &>$single_log_path &
done
wait
for i in "${!array[@]}"; do
single_log_path="$dir_log/$log_dir/${single_log_time}_$((i + 1)).log"
cat $single_log_path
[[ -f $single_log_path ]] && rm -f $single_log_path
done
}
run_designated() {
local file_param="$1"
local env_param="$2"
local num_param=$(echo "$3" | perl -pe "s|.*$2(.*)|\1|")
if [[ ! $env_param ]] || [[ ! $num_param ]]; then
echo -e "\n 缺少单独运行的参数 task xxx.js desi Test 1 3"
exit 1
fi
local envs=$(eval echo "\$${env_param}")
local array=($(echo $envs | sed 's/&/ /g'))
local tempArr=$(echo $num_param | perl -pe "s|(\d+)(-\|~\|_)(\d+)|{\1..\3}|g")
local runArr=($(eval echo $tempArr))
runArr=($(awk -v RS=' ' '!a[$1]++' <<<${runArr[@]}))
local n=0
for i in ${runArr[@]}; do
array_run[n]=${array[$i - 1]}
let n++
done
local cookieStr=$(echo ${array_run[*]} | sed 's/\ /\&/g')
[[ ! -z $cookieStr ]] && export ${env_param}=${cookieStr}
cd $dir_scripts
local relative_path="${file_param%/*}"
if [[ ! -z ${relative_path} ]] && [[ ${file_param} =~ "/" ]]; then
cd ${relative_path}
file_param=${file_param/$relative_path\//}
fi
$timeoutCmd $which_program $file_param
}
## 运行其他命令
run_else() {
local file_param="$1"
cd $dir_scripts
local relative_path="${file_param%/*}"
if [[ ! -z ${relative_path} ]] && [[ ${file_param} =~ "/" ]]; then
cd ${relative_path}
file_param=${file_param/$relative_path\//}
fi
shift
local params=$(echo "$@" | sed 's/ /\" \"/g')
$timeoutCmd $which_program $file_param \"$params\"
}
## 命令检测
main() {
if [[ $1 == *.js ]] || [[ $1 == *.py ]] || [[ $1 == *.pyc ]] || [[ $1 == *.sh ]] || [[ $1 == *.ts ]]; then
case $# in
1)
run_normal "$1"
;;
*)
case $2 in
now)
run_normal "$1" "$2"
;;
conc)
run_concurrent "$1" "$3" "$*"
;;
desi)
run_designated "$1" "$3" "$*"
;;
*)
run_else "$@"
;;
esac
;;
esac
elif [[ $# -eq 0 ]]; then
echo
usage
else
run_else "$@"
fi
}
handle_task_before "$@"
main "$@"
handle_task_after "$@"

View File

@ -21,78 +21,12 @@ define_program() {
fi
}
random_delay() {
local random_delay_max=$RandomDelay
if [[ $random_delay_max ]] && [[ $random_delay_max -gt 0 ]]; then
local file_param=$1
local file_extensions=${RandomDelayFileExtensions-"js"}
local ignored_minutes=${RandomDelayIgnoredMinutes-"0 30"}
if [[ -n $file_extensions ]]; then
if ! echo "$file_param" | grep -qE "\.${file_extensions// /$|\\.}$"; then
# echo -e "\n当前文件需要准点运行, 放弃随机延迟\n"
return
fi
fi
local current_min
current_min=$(date "+%-M")
for minute in $ignored_minutes; do
if [[ $current_min -eq $minute ]]; then
# echo -e "\n当前时间需要准点运行, 放弃随机延迟\n"
return
fi
done
local delay_second=$(($(gen_random_num "$random_delay_max") + 1))
echo -e "\n命令未添加 \"now\",随机延迟 $delay_second 秒后再执行任务,如需立即终止,请按 CTRL+C...\n"
sleep $delay_second
fi
}
## scripts目录下所有可运行脚本数组
gen_array_scripts() {
local dir_current=$(pwd)
local i="-1"
cd $dir_scripts
for file in $(ls); do
if [[ -f $file ]] && [[ $file == *.js && $file != sendNotify.js ]]; then
let i++
array_scripts[i]=$(echo "$file" | perl -pe "s|$dir_scripts/||g")
array_scripts_name[i]=$(grep "new Env" $file | awk -F "'|\"" '{print $2}' | head -1)
[[ -z ${array_scripts_name[i]} ]] && array_scripts_name[i]="<未识别出活动名称>"
fi
done
cd $dir_current
}
## 使用说明
usage() {
define_cmd
gen_array_scripts
echo -e "task命令运行本程序自动添加进crontab的脚本需要输入脚本的绝对路径或去掉 “$dir_scripts/” 目录后的相对路径(定时任务中请写作相对路径),用法为:"
echo -e "1.$cmd_task <file_name> # 依次执行,如果设置了随机延迟,将随机延迟一定秒数"
echo -e "2.$cmd_task <file_name> now # 依次执行,无论是否设置了随机延迟,均立即运行,前台会输出日志,同时记录在日志文件中"
echo -e "3.$cmd_task <file_name> conc <环境变量名称> <账号编号,空格分隔>(可选的) # 并发执行,无论是否设置了随机延迟,均立即运行,前台不产生日志,直接记录在日志文件中,且可指定账号执行"
echo -e "4.$cmd_task <file_name> desi <环境变量名称> <账号编号,空格分隔> # 指定账号执行,无论是否设置了随机延迟,均立即运行"
if [[ ${#array_scripts[*]} -gt 0 ]]; then
echo -e "\n当前有以下脚本可以运行:"
for ((i = 0; i < ${#array_scripts[*]}; i++)); do
echo -e "$(($i + 1)). ${array_scripts_name[i]}${array_scripts[i]}"
done
else
echo -e "\n暂无脚本可以执行"
fi
}
## run nohup$1文件名不含路径带后缀
run_nohup() {
local file_name=$1
nohup node $file_name &>$log_path &
}
handle_log_path() {
define_program "$file_param"
local file_param=$1
if [[ -z $file_param ]];then
file_param="task"
fi
local suffix=""
if [[ ! -z $ID ]]; then
@ -118,182 +52,14 @@ handle_log_path() {
make_dir "$dir_log/$log_dir"
}
check_server() {
cpu_use=$(top -b -n 1 | grep CPU | grep -v -E 'grep|PID' | awk '{print $2}' | cut -f 1 -d "%")
mem_free=$(free -m | grep "Mem" | awk '{print $3}')
mem_total=$(free -m | grep "Mem" | awk '{print $2}')
mem_use=$(printf "%d%%" $((mem_free * 100 / mem_total)) | cut -f 1 -d "%")
disk_use=$(df -P | grep /dev | grep -v -E '(tmp|boot|shm)' | awk '{print $5}' | cut -f 1 -d "%")
# eval echo -e "\#\# 当前CPU占用 $cpu_use% 内存占用 $mem_use% 磁盘占用 $disk_use% \\\n" $cmd
if [[ $cpu_use -gt $cpu_warn ]] || [[ $mem_free -lt $mem_warn ]] || [[ $disk_use -gt $disk_warn ]]; then
local resource=$(top -b -n 1 | grep -v -E 'grep|Mem|idle|Load' | awk '{$2="";$3="";$4="";$5="";$7="";print $0}' | head -n 10)
notify_api "服务器资源异常警告" "当前CPU占用 $cpu_use% 内存占用 $mem_use% 磁盘占用 $disk_use% \n资源占用详情 \n\n $resource"
format_params() {
time_format="%Y-%m-%d %H:%M:%S"
timeoutCmd=""
if type timeout &>/dev/null; then
timeoutCmd="timeout -k 10s $command_timeout_time "
fi
}
handle_task_before() {
handle_log_path
begin_time=$(format_time "$time_format" "$time")
begin_timestamp=$(format_timestamp "$time_format" "$time")
eval echo -e "\#\# 开始执行... $begin_time\\\n" $cmd
[[ $is_macos -eq 0 ]] && check_server
[[ -f $task_error_log_path ]] && eval cat $task_error_log_path $cmd
[[ $ID ]] && update_cron "\"$ID\"" "0" "$$" "$log_path" "$begin_timestamp"
eval . $file_task_before "$@" $cmd
}
handle_task_after() {
eval . $file_task_after "$@" $cmd
local end_time=$(date '+%Y-%m-%d %H:%M:%S')
local end_timestamp=$(date "+%s")
local diff_time=$(($end_timestamp - $begin_timestamp))
[[ $ID ]] && update_cron "\"$ID\"" "1" "" "$log_path" "$begin_timestamp" "$diff_time"
eval echo -e "\\\n\\\n\#\# 执行结束... $end_time 耗时 $diff_time" $cmd
eval echo -e "\\\n     " $cmd
}
## 正常运行单个脚本,$1传入参数
run_normal() {
local file_param=$1
if [[ $# -eq 1 ]]; then
random_delay "$file_param"
fi
handle_task_before "$@"
cd $dir_scripts
local relative_path="${file_param%/*}"
if [[ ${file_param} != /* ]] && [[ ! -z ${relative_path} ]] && [[ ${file_param} =~ "/" ]]; then
cd ${relative_path}
file_param=${file_param/$relative_path\//}
fi
eval $timeoutCmd $which_program $file_param $cmd
handle_task_after "$@"
}
## 并发执行时,设定的 RandomDelay 不会生效,即所有任务立即执行
run_concurrent() {
local file_param="$1"
local env_param="$2"
local num_param=$(echo "$3" | perl -pe "s|.*$2(.*)|\1|")
if [[ ! $env_param ]]; then
echo -e "\n 缺少并发运行的环境变量参数"
exit 1
fi
handle_task_before "$@"
local envs=$(eval echo "\$${env_param}")
local array=($(echo $envs | sed 's/&/ /g'))
local tempArr=$(echo $num_param | perl -pe "s|(\d+)(-\|~\|_)(\d+)|{\1..\3}|g")
local runArr=($(eval echo $tempArr))
runArr=($(awk -v RS=' ' '!a[$1]++' <<<${runArr[@]}))
local n=0
for i in ${runArr[@]}; do
array_run[n]=${array[$i - 1]}
let n++
done
local cookieStr=$(echo ${array_run[*]} | sed 's/\ /\&/g')
[[ ! -z $cookieStr ]] && export ${env_param}=${cookieStr}
local envs=$(eval echo "\$${env_param}")
local array=($(echo $envs | sed 's/&/ /g'))
single_log_time=$(date "+%Y-%m-%d-%H-%M-%S.%N")
cd $dir_scripts
local relative_path="${file_param%/*}"
if [[ ! -z ${relative_path} ]] && [[ ${file_param} =~ "/" ]]; then
cd ${relative_path}
file_param=${file_param/$relative_path\//}
fi
for i in "${!array[@]}"; do
export ${env_param}=${array[i]}
single_log_path="$dir_log/$log_dir/${single_log_time}_$((i + 1)).log"
eval $timeoutCmd $which_program $file_param &>$single_log_path &
done
wait
for i in "${!array[@]}"; do
single_log_path="$dir_log/$log_dir/${single_log_time}_$((i + 1)).log"
eval cat $single_log_path $cmd
[[ -f $single_log_path ]] && rm -f $single_log_path
done
handle_task_after "$@"
}
run_designated() {
local file_param="$1"
local env_param="$2"
local num_param=$(echo "$3" | perl -pe "s|.*$2(.*)|\1|")
if [[ ! $env_param ]] || [[ ! $num_param ]]; then
echo -e "\n 缺少单独运行的参数 task xxx.js desi Test 1 3"
exit 1
fi
handle_task_before "$@"
local envs=$(eval echo "\$${env_param}")
local array=($(echo $envs | sed 's/&/ /g'))
local tempArr=$(echo $num_param | perl -pe "s|(\d+)(-\|~\|_)(\d+)|{\1..\3}|g")
local runArr=($(eval echo $tempArr))
runArr=($(awk -v RS=' ' '!a[$1]++' <<<${runArr[@]}))
local n=0
for i in ${runArr[@]}; do
array_run[n]=${array[$i - 1]}
let n++
done
local cookieStr=$(echo ${array_run[*]} | sed 's/\ /\&/g')
[[ ! -z $cookieStr ]] && export ${env_param}=${cookieStr}
cd $dir_scripts
local relative_path="${file_param%/*}"
if [[ ! -z ${relative_path} ]] && [[ ${file_param} =~ "/" ]]; then
cd ${relative_path}
file_param=${file_param/$relative_path\//}
fi
eval $timeoutCmd $which_program $file_param $cmd
handle_task_after "$@"
}
## 运行其他命令
run_else() {
local file_param="$1"
handle_task_before "$@"
cd $dir_scripts
local relative_path="${file_param%/*}"
if [[ ! -z ${relative_path} ]] && [[ ${file_param} =~ "/" ]]; then
cd ${relative_path}
file_param=${file_param/$relative_path\//}
fi
shift
local params=$(echo "$@" | sed 's/ /\" \"/g')
eval $timeoutCmd $which_program $file_param \"$params\" $cmd
handle_task_after "$@"
}
## 命令检测
main() {
show_log="false"
while getopts ":l" opt; do
case $opt in
@ -304,43 +70,10 @@ main() {
done
[[ "$show_log" == "true" ]] && shift $(($OPTIND - 1))
timeoutCmd=""
if type timeout &>/dev/null; then
timeoutCmd="timeout -k 10s $command_timeout_time "
fi
time_format="%Y-%m-%d %H:%M:%S"
if [[ $1 == *.js ]] || [[ $1 == *.py ]] || [[ $1 == *.pyc ]] || [[ $1 == *.sh ]] || [[ $1 == *.ts ]]; then
case $# in
1)
run_normal "$1"
;;
*)
case $2 in
now)
run_normal "$1" "$2"
;;
conc)
run_concurrent "$1" "$3" "$*"
;;
desi)
run_designated "$1" "$3" "$*"
;;
*)
run_else "$@"
;;
esac
;;
esac
format_params
define_program "$@"
handle_log_path "$@"
eval . $dir_shell/otask.sh "$@" "$cmd"
[[ -f "$dir_log/$log_path" ]] && cat "$dir_log/$log_path"
elif [[ $# -eq 0 ]]; then
echo
usage
else
run_else "$@"
fi
}
main "$@"
exit 0

View File

@ -1,4 +1,4 @@
import { LOG_END_SYMBOL } from "./const";
import { LOG_END_SYMBOL } from './const';
export default function browserType() {
// 权重:系统 + 系统版本 > 平台 > 内核 + 载体 + 内核版本 + 载体版本 > 外壳 + 外壳版本
@ -278,5 +278,5 @@ export function depthFirstSearch<
export function logEnded(log: string): boolean {
const endTips = [LOG_END_SYMBOL, '执行结束'];
return endTips.some(x => !log.includes(x));
return endTips.some((x) => log.includes(x));
}