1 #!bin/bash 2 3 DEPLOY_PATH=$(cd "$(dirname "$0")"; pwd) 4 INI_FILE=$DEPLOY_PATH'/cnedmp_etl.ini' 5 . $INI_FILE 6 BASE_PATH=$(cd "$(dirname "$DEPLOY_PATH")"; pwd) 7 MAIN_PATH=$(cd "$(dirname "$BASE_PATH")"; pwd) 8 CURRRENT_DATE=$(date "+%Y-%m-%d") 9 CURRENT_TIME=`date -u +\%Y\%m\%d` 10 JAR_FILE=$DEPLOY_PATH'/dailydata-1.0-SNAPSHOT.jar' 11 LOG_PATH=$BASE_PATH'/log/' 12 LOAD_CHECK_LOG=$BASE_PATH'/log/Load_Check.'$CURRENT_TIME'.log' 13 PASSWORD=`yarn jar $JAR_FILE com.hypers.etl.file.specialFileProcess.BlowFish` 14 15 logging(){ 16 logTime=`date "+%y/%m/%d %T"` 17 echo "$logTime $1 $2" >> $LOAD_CHECK_LOG 18 } 19 20 #Check whether this is the first time running 21 test -e $LOAD_CHECK_LOG || touch $LOAD_CHECK_LOG 22 23 #mysql parameters 24 mysql_user=$USERNAME 25 mysql_password=$PASSWORD 26 mysql_host=$HOSTNAME 27 mysql_port=$PORT 28 mysql_database=$DBNAME 29 30 #Get WARNING LIST 31 sql="select File_Name,Transfer_Status,Load_Status,Job_Status 32 from $JOBNAME 33 where File_Date='$CURRRENT_DATE' 34 and Hive_Partition != 'none' 35 and (Transfer_Status!=2 or Load_Status!=2 or Job_Status!=2) 36 and Load_Status!=3 37 and Job_Status!=3;" 38 39 WARN_list=$(printf "%s " 40 "[client]" 41 "user=${mysql_user}" 42 "password=${mysql_password}" 43 "host=${mysql_host}" 44 "port=${mysql_port}" 45 "database=${mysql_database}" 46 | HOME="/sys" mysql --defaults-file=/dev/stdin -s -e "${sql}") 47 48 #Use to store email cotent 49 #email_message="File_Name, Transfer_Status, Load_Status, Job_Status" 50 51 while read line 52 do 53 test -z "${line}" 54 if [ $? -eq 0 ] 55 then 56 continue 57 fi 58 exist=`cat $LOAD_CHECK_LOG | grep "Job doesn't finish." | grep "$line"` 59 test -z "${exist}" 60 if [ $? -eq 0 ] 61 then 62 # echo $line >> $LOAD_CHECK_LOG 63 # email_message=$email_message"\n"$line 64 logging WARN "Job doesn't finish. | Status: File_Name, Transfer_Status, Load_Status, Job_Status | $line" 65 fi 66 done <<EOF 67 $WARN_list 68 EOF 69 70 #Get ERROR LIST 71 sql="select File_Name,Transfer_Status,Load_Status,Job_Status 72 from $JOBNAME 73 where File_Date='$CURRRENT_DATE' 74 and (Load_Status=3 or Job_Status=3);" 75 76 ERROR_list=$(printf "%s " 77 "[client]" 78 "user=${mysql_user}" 79 "password=${mysql_password}" 80 "host=${mysql_host}" 81 "port=${mysql_port}" 82 "database=${mysql_database}" 83 | HOME="/sys" mysql --defaults-file=/dev/stdin -s -e "${sql}") 84 85 while read line 86 do 87 test -z "${line}" 88 if [ $? -eq 0 ] 89 then 90 continue 91 fi 92 exist=`cat $LOAD_CHECK_LOG | grep "Job run ERROR." | grep "$line"` 93 test -z "${exist}" 94 if [ $? -eq 0 ] 95 then 96 # echo $line >> $LOAD_CHECK_LOG 97 # email_message=$email_message"\n"$line 98 logging FATAL "Job run ERROR. | Status: File_Name, Transfer_Status, Load_Status, Job_Status | $line" 99 fi 100 done <<EOF 101 $ERROR_list 102 EOF 103 104 #Get Row Fail list 105 sql="select File_Name,Transfer_Status,Load_Status,Job_Status, Rows_In_Total, Rows_Succeed, Rows_Fail 106 from $JOBNAME 107 where File_Date='$CURRRENT_DATE' 108 and Rows_Fail!=0 109 and Transfer_Status=2 and Load_Status=2 and Job_Status=2;" 110 111 RF_list=$(printf "%s " 112 "[client]" 113 "user=${mysql_user}" 114 "password=${mysql_password}" 115 "host=${mysql_host}" 116 "port=${mysql_port}" 117 "database=${mysql_database}" 118 | HOME="/sys" mysql --defaults-file=/dev/stdin -s -e "${sql}") 119 120 while read line 121 do 122 test -z "${line}" 123 if [ $? -eq 0 ] 124 then 125 continue 126 fi 127 exist=`cat $LOAD_CHECK_LOG | grep "Job finished with Row Fail." | grep "$line"` 128 test -z "${exist}" 129 if [ $? -eq 0 ] 130 then 131 # echo $line >> $LOAD_CHECK_LOG 132 # email_message=$email_message"\n"$line 133 logging WARN "Job finished with Row Fail. | Status: File_Name, Transfer_Status, Load_Status, Job_Status, Total Row, Succeed Row, Fail Row | $line" 134 fi 135 done <<EOF 136 $RF_list 137 EOF 138 #echo -e $email_message >> testoutput.txt; 139 #$echo -e "hello jason \n jason" | wc -l