zoukankan      html  css  js  c++  java
  • 这两天光写shell了,再贴一段代码,以供日后参考。

      1 #!bin/bash
      2 
      3 DEPLOY_PATH=$(cd "$(dirname "$0")"; pwd)
      4 INI_FILE=$DEPLOY_PATH'/cnedmp_etl.ini'
      5 . $INI_FILE
      6 BASE_PATH=$(cd "$(dirname "$DEPLOY_PATH")"; pwd)
      7 MAIN_PATH=$(cd "$(dirname "$BASE_PATH")"; pwd)
      8 CURRRENT_DATE=$(date "+%Y-%m-%d")
      9 CURRENT_TIME=`date -u +\%Y\%m\%d`
     10 JAR_FILE=$DEPLOY_PATH'/dailydata-1.0-SNAPSHOT.jar'
     11 LOG_PATH=$BASE_PATH'/log/'
     12 LOAD_CHECK_LOG=$BASE_PATH'/log/Load_Check.'$CURRENT_TIME'.log'
     13 PASSWORD=`yarn jar $JAR_FILE com.hypers.etl.file.specialFileProcess.BlowFish`
     14 
     15 logging(){
     16     logTime=`date "+%y/%m/%d %T"`
     17     echo "$logTime $1 $2" >> $LOAD_CHECK_LOG
     18 }
     19 
     20 #Check whether this is the first time running
     21 test -e $LOAD_CHECK_LOG || touch $LOAD_CHECK_LOG
     22 
     23 #mysql parameters
     24 mysql_user=$USERNAME
     25 mysql_password=$PASSWORD
     26 mysql_host=$HOSTNAME
     27 mysql_port=$PORT
     28 mysql_database=$DBNAME
     29 
     30 #Get WARNING LIST
     31 sql="select File_Name,Transfer_Status,Load_Status,Job_Status 
     32 from $JOBNAME 
     33 where File_Date='$CURRRENT_DATE' 
     34 and Hive_Partition != 'none' 
     35 and (Transfer_Status!=2 or Load_Status!=2 or Job_Status!=2)
     36 and Load_Status!=3
     37 and Job_Status!=3;"
     38 
     39 WARN_list=$(printf "%s
    " 
     40     "[client]" 
     41     "user=${mysql_user}" 
     42     "password=${mysql_password}" 
     43     "host=${mysql_host}" 
     44     "port=${mysql_port}" 
     45     "database=${mysql_database}" 
     46 | HOME="/sys" mysql --defaults-file=/dev/stdin -s -e "${sql}")
     47 
     48 #Use to store email cotent
     49 #email_message="File_Name, Transfer_Status, Load_Status, Job_Status"
     50 
     51 while read line
     52 do
     53         test -z "${line}"
     54         if [ $? -eq 0 ]
     55         then
     56             continue
     57         fi
     58         exist=`cat $LOAD_CHECK_LOG | grep "Job doesn't finish." | grep "$line"`
     59         test -z "${exist}"
     60         if [ $? -eq 0 ]
     61         then
     62 #            echo $line >> $LOAD_CHECK_LOG
     63 #            email_message=$email_message"\n"$line
     64             logging WARN "Job doesn't finish. | Status: File_Name, Transfer_Status, Load_Status, Job_Status | $line"
     65         fi
     66 done <<EOF
     67 $WARN_list
     68 EOF
     69 
     70 #Get ERROR LIST
     71 sql="select File_Name,Transfer_Status,Load_Status,Job_Status 
     72 from $JOBNAME 
     73 where File_Date='$CURRRENT_DATE' 
     74 and (Load_Status=3 or Job_Status=3);"
     75 
     76 ERROR_list=$(printf "%s
    " 
     77     "[client]" 
     78     "user=${mysql_user}" 
     79     "password=${mysql_password}" 
     80     "host=${mysql_host}" 
     81     "port=${mysql_port}" 
     82     "database=${mysql_database}" 
     83 | HOME="/sys" mysql --defaults-file=/dev/stdin -s -e "${sql}")
     84 
     85 while read line
     86 do
     87         test -z "${line}"
     88         if [ $? -eq 0 ]
     89         then
     90             continue
     91         fi
     92         exist=`cat $LOAD_CHECK_LOG | grep "Job run ERROR." | grep "$line"`
     93         test -z "${exist}"
     94         if [ $? -eq 0 ]
     95         then
     96 #            echo $line >> $LOAD_CHECK_LOG
     97 #            email_message=$email_message"\n"$line
     98             logging FATAL "Job run ERROR. | Status: File_Name, Transfer_Status, Load_Status, Job_Status | $line"
     99         fi
    100 done <<EOF
    101 $ERROR_list
    102 EOF
    103 
    104 #Get Row Fail list
    105 sql="select File_Name,Transfer_Status,Load_Status,Job_Status, Rows_In_Total, Rows_Succeed, Rows_Fail 
    106 from $JOBNAME 
    107 where File_Date='$CURRRENT_DATE' 
    108 and Rows_Fail!=0 
    109 and Transfer_Status=2 and Load_Status=2 and Job_Status=2;"
    110 
    111 RF_list=$(printf "%s
    " 
    112     "[client]" 
    113     "user=${mysql_user}" 
    114     "password=${mysql_password}" 
    115     "host=${mysql_host}" 
    116     "port=${mysql_port}" 
    117     "database=${mysql_database}" 
    118 | HOME="/sys" mysql --defaults-file=/dev/stdin -s -e "${sql}")
    119 
    120 while read line
    121 do
    122         test -z "${line}"
    123         if [ $? -eq 0 ]
    124         then
    125             continue
    126         fi
    127         exist=`cat $LOAD_CHECK_LOG | grep "Job finished with Row Fail." | grep "$line"`
    128         test -z "${exist}"
    129         if [ $? -eq 0 ]
    130         then
    131 #            echo $line >> $LOAD_CHECK_LOG
    132 #            email_message=$email_message"\n"$line
    133             logging WARN "Job finished with Row Fail. | Status: File_Name, Transfer_Status, Load_Status, Job_Status, Total Row, Succeed Row, Fail Row | $line"
    134         fi
    135 done <<EOF
    136 $RF_list
    137 EOF
    138 #echo -e $email_message >> testoutput.txt;
    139 #$echo -e "hello jason \n jason" | wc -l
  • 相关阅读:
    Hbase基础(十七):亿级数据从 MySQL 到 Hbase 的三种同步方案与实践(一)环境搭建
    Hbase基础(十六):扩展
    离线电商数仓(十九)之电商数据仓库系统(二)数仓分层
    离线电商数仓(十八)之电商数据仓库系统(一)数仓分层
    离线电商数仓(十七)之系统业务数据仓库数据采集(四)数据环境准备
    离线电商数仓(十六)之系统业务数据仓库数据采集(三)业务数据采集模块(二)
    离线电商数仓(十五)之系统业务数据仓库数据采集(二)业务数据采集模块(一)
    Linux ioctl调用
    dpdk Vhost 库
    vhost 控制平面 + handle_kick + VhostOps
  • 原文地址:https://www.cnblogs.com/xdlaoliu/p/7287146.html
Copyright © 2011-2022 走看看