贴一段Shell脚本，以便日后参考。

Posted 2020-09-30 代号菜鸟

 1 #! /bin/bash
 2 
 3 ICMS_PATH=$(cd $(dirname $0); pwd)
 4 DEPLOY_PATH=$(cd $(dirname $ICMS_PATH); pwd)
 5 JAR_FILE=$DEPLOY_PATH‘/dailydata-1.0-SNAPSHOT.jar‘
 6 INI_FILE=$DEPLOY_PATH‘/cnedmp_etl.ini‘
 7 LOG_FILE=$DEPLOY_PATH‘/shellrun.log‘
 8 CODING=‘GB2312‘
 9 
10 if [ $# -ne 1 ]; then
11     logging INFO "Ussage: $0 <source file path>"
12     exit 1
13 fi
14 
15 #Funtion for Log
16 function logging(){
17     logTime=$(date +"%Y-%m-%d %T")
18     echo "$logTime $1 $2"
19 }
20 source $INI_FILE
21 PASSWORD=`yarn jar $JAR_FILE com.hypers.etl.file.specialFileProcess.BlowFish`
22 SRC_DIR=$1
23 logging INFO "Going to process all the source file under path: $SRC_DIR"
24 cd $SRC_DIR
25 partition=$(echo $SRC_DIR | awk -F/  ‘{print $(NF-1)}‘)
26 logging INFO "The partition is $partition"
27 filelist=$(ls $dir)
28 logging INFO "Will handle $filelist under path $SRC_DIR"
29 logging INFO "-----------------------------"
30 
31 for filename in $filelist
32 do
33 
34 logging INFO "---------------------------"
35 logging INFO "handling file $filename ..."
36 
37 PRE_LOCATION=`mysql -h$HOSTNAME -u$USERNAME -p$PASSWORD $DBNAME  <<EOF 2> /dev/null | tail -n +2
38 SELECT Pre_Location from $CONFNAME where File_Name=‘$filename‘;
39 EOF`
40 year=$(echo $partition | cut -b 1-4)
41 PRE_LOCATION=$PRE_LOCATION/$year
42 logging INFO "get pre file location $PRE_LOCATION for $filename"
43 
44 TARG_LOCATION=`mysql -h$HOSTNAME -u$USERNAME -p$PASSWORD $DBNAME  <<EOF 2> /dev/null | tail -n +2 
45 SELECT Target_Location from $CONFNAME where File_Name=‘$filename‘;
46 EOF`
47 TARG_LOCATION=$TARG_LOCATION/$partition
48 logging INFO "get target file location $TARG_LOCATION for $filename"
49 
50 TARG_TABLE_NAME=`mysql -h$HOSTNAME -u$USERNAME -p$PASSWORD $DBNAME  <<EOF 2> /dev/null | tail -n +2 
51 SELECT Trg_Table_Name from $CONFNAME where File_Name=‘$filename‘;
52 EOF`
53 logging INFO "get target table name $TARG_TABLE_NAME for $filename"
54 
55 hdfs dfs -mkdir -p $PRE_LOCATION && logging INFO "Created $PRE_LOCATION succesfully"
56 hdfs dfs -mkdir -p $TARG_LOCATION && logging INFO "Created $TARG_LOCATION succesfully"
57 
58 hdfs dfs -put  $filename $PRE_LOCATION && logging INFO "Put $filename onto hdfs prelocation $PRE_LOCATION"
59 
60 yarn jar $JAR_FILE com.hypers.etl.file.mr.CLIDriver -i ${PRE_LOCATION}‘/‘${filename} -o ${TARG_LOCATION} -e ${CODING} -d ${partition} -s ‘,‘
61 
62 logging INFO "alter table $TARG_TABLE_NAME to add partition"
63 #sudo su hdfs -c"beeline -u jdbc:hive2://${HOSTNAME}:10000/cnedmp -n hdfs -p $HIVE_SERVER2_PASSWORD -e "alter table $TARG_TABLE_NAME add partition  dt = ‘${partition}‘  location ‘${TARG_LOCATION}‘;" "
64 
65 done