1 #! /bin/bash 2 3 ICMS_PATH=$(cd $(dirname $0); pwd) 4 DEPLOY_PATH=$(cd $(dirname $ICMS_PATH); pwd) 5 JAR_FILE=$DEPLOY_PATH'/dailydata-1.0-SNAPSHOT.jar' 6 INI_FILE=$DEPLOY_PATH'/cnedmp_etl.ini' 7 LOG_FILE=$DEPLOY_PATH'/shellrun.log' 8 CODING='GB2312' 9 10 if [ $# -ne 1 ]; then 11 logging INFO "Ussage: $0 <source file path>" 12 exit 1 13 fi 14 15 #Funtion for Log 16 function logging(){ 17 logTime=$(date +"%Y-%m-%d %T") 18 echo "$logTime $1 $2" 19 } 20 source $INI_FILE 21 PASSWORD=`yarn jar $JAR_FILE com.hypers.etl.file.specialFileProcess.BlowFish` 22 SRC_DIR=$1 23 logging INFO "Going to process all the source file under path: $SRC_DIR" 24 cd $SRC_DIR 25 partition=$(echo $SRC_DIR | awk -F/ '{print $(NF-1)}') 26 logging INFO "The partition is $partition" 27 filelist=$(ls $dir) 28 logging INFO "Will handle $filelist under path $SRC_DIR" 29 logging INFO "-----------------------------" 30 31 for filename in $filelist 32 do 33 34 logging INFO "---------------------------" 35 logging INFO "handling file $filename ..." 36 37 PRE_LOCATION=`mysql -h$HOSTNAME -u$USERNAME -p$PASSWORD $DBNAME <<EOF 2> /dev/null | tail -n +2 38 SELECT Pre_Location from $CONFNAME where File_Name='$filename'; 39 EOF` 40 year=$(echo $partition | cut -b 1-4) 41 PRE_LOCATION=$PRE_LOCATION/$year 42 logging INFO "get pre file location $PRE_LOCATION for $filename" 43 44 TARG_LOCATION=`mysql -h$HOSTNAME -u$USERNAME -p$PASSWORD $DBNAME <<EOF 2> /dev/null | tail -n +2 45 SELECT Target_Location from $CONFNAME where File_Name='$filename'; 46 EOF` 47 TARG_LOCATION=$TARG_LOCATION/$partition 48 logging INFO "get target file location $TARG_LOCATION for $filename" 49 50 TARG_TABLE_NAME=`mysql -h$HOSTNAME -u$USERNAME -p$PASSWORD $DBNAME <<EOF 2> /dev/null | tail -n +2 51 SELECT Trg_Table_Name from $CONFNAME where File_Name='$filename'; 52 EOF` 53 logging INFO "get target table name $TARG_TABLE_NAME for $filename" 54 55 hdfs dfs -mkdir -p $PRE_LOCATION && logging INFO "Created $PRE_LOCATION succesfully" 56 hdfs dfs -mkdir -p $TARG_LOCATION && logging INFO "Created $TARG_LOCATION succesfully" 57 58 hdfs dfs -put $filename $PRE_LOCATION && logging INFO "Put $filename onto hdfs prelocation $PRE_LOCATION" 59 60 yarn jar $JAR_FILE com.hypers.etl.file.mr.CLIDriver -i ${PRE_LOCATION}'/'${filename} -o ${TARG_LOCATION} -e ${CODING} -d ${partition} -s ',' 61 62 logging INFO "alter table $TARG_TABLE_NAME to add partition" 63 #sudo su hdfs -c"beeline -u jdbc:hive2://${HOSTNAME}:10000/cnedmp -n hdfs -p $HIVE_SERVER2_PASSWORD -e "alter table $TARG_TABLE_NAME add partition dt = '${partition}' location '${TARG_LOCATION}';" " 64 65 done