1 #! /bin/bash 2 3 #/python2292/CNEDMp/ETLscript/CNEDMp_etl_v1.0/testhis 4 LOAD_PATH=$(cd $(dirname $0); pwd) 5 #/python2292/CNEDMp/ETLscript/CNEDMp_etl_v1.0 6 DEPLOY_PATH=$(cd $(dirname $LOAD_PATH); pwd) 7 #/python2292/CNEDMp/ETLscript 8 BASE_PATH=$(cd "$(dirname "$DEPLOY_PATH")"; pwd) 9 #/python2292/CNEDMp 10 MAIN_PATH=$(cd "$(dirname "$BASE_PATH")"; pwd) 11 12 FILE_LANDING=$DEPLOY_PATH'/File_Landing.sh' 13 LOAD_HDFS=$DEPLOY_PATH'/Load_hdfs.sh' 14 LOAD_HIVE=$DEPLOY_PATH'/Load_hive.sh' 15 LANDING_PATH=$MAIN_PATH'/landing/' 16 JAR_FILE=$DEPLOY_PATH'/dailydata-1.0-SNAPSHOT.jar' 17 INI_FILE=$DEPLOY_PATH'/cnedmp_etl.ini' 18 LOG_FILE=$DEPLOY_PATH'/shellrun.log' 19 20 CODING='GB2312' 21 22 #Funtion for Log 23 function logging(){ 24 logTime=$(date +"%Y-%m-%d %T") 25 echo "$logTime $1 $2" 26 } 27 28 29 source $INI_FILE 30 31 cd $LOAD_PATH 32 rm -r dataspace 33 mkdir dataspace 34 cd dataspace 35 ziplist=$(cat $LOAD_PATH/ziplist.txt) 36 logging INFO "Get zip file list $ziplist" 37 for zipname in $ziplist 38 do 39 logging INFO "-------------------Handling zipfile $zipname----------------------------" 40 cp /home/infra1/jason/DataFiles/$zipname ./ 41 foldername=$(echo $zipname|awk -F . '{print $1}') 42 echo $foldername 43 mkdir $foldername 44 unzip -d ./$foldername $zipname 45 cd $foldername 46 partition=$foldername 47 filelist=$(cat $LOAD_PATH/filelist.txt) 48 logging INFO "Get file list $filelist" 49 for filename in $filelist 50 do 51 cp $filename $LANDING_PATH && logging INFO "$filename copied to $LANDING_PATH" 52 done 53 cd .. 54 sh $LOAD_PATH/Loading_Meta_Init_history.sh $partition 55 sh $LOAD_PATH/schedule_history.sh $partition 56 cd $LOAD_PATH 57 rm -r $LOAD_PATH/dataspace 58 logging INFO "-------------------zipfile $zipname finished----------------------------" 59 done