text Shell使用cron自动调起挂掉的程序

Posted 2021-05-02

#!/bin/bash
# 如果检测到没有指定脚本执行, 那么唤起程序
# cronjob配置
# * * * * * /home/ads_app/low_latency_pipeline/online/minute_level_pipeline/cron_restart_spark_script.sh spark_batch_process_main_script.py realtime
source ~/.bashrc
source ~/.bash_profile

script_name=$1
run_mode=$2

cd "$(dirname "$0")"

ps -aux|grep $script_name|grep -v grep|grep -v $0|grep -v "tail"|grep -v "vim" > /dev/null
if [[ 0 != $? ]]; then
    if [[ "realtime" == $run_mode ]]; then
        curl "http://jenkins.jd.com/job/send_ump_alarm/buildWithParameters?token=123456" -d "UMP_KEY=jd.ad.data.lowDelay.rerun&ALARM_INFO="$script_name,restart""
    fi
    export PYSPARK_PYTHON=/usr/local/bin/python3
    spark-submit \
        --jars spark-2-clickhouse.jar \
        --driver-memory 16G \
        --executor-memory 20G \
        --executor-cores 10 \
        --num-executors 100 \
        --conf spark.yarn.maxAppAttempts=4 \
        --conf spark.yarn.am.attemptFailuresValidityInterval=1h \
        --conf spark.yarn.executor.failuresValidityInterval=1h \
        --conf spark.task.maxFailures=8 \
        --py-files sparkpipeline.zip \
        $script_name $run_mode >> $script_name.log \
        2>> $script_name.err.log &
fi

需要注意的有以下几点:

- cronjob配置要用脚本或者命令的绝对路径
- 脚本要source使环境变量生效
- 通过cd手动切换pwd