#!/bin/bash #/usr/hdp/current/flume-server/bin/flume-ng agent -c conf/ -f /usr/hdp/current/flume-server/Cold_TestConf/spooldircsv1.conf -n a1 -Dflume.root.logger=WARN,console 2>&1 & #sleep 1 #/usr/hdp/current/flume-server/bin/flume-ng agent -c conf/ -f /usr/hdp/current/flume-server/Cold_TestConf/spooldircsv3.conf -n a1 -Dflume.root.logger=WARN,console 2>&1 & #sleep 1 #/usr/hdp/current/spark2-client/bin/spark-submit --class cold_test.realTimeEtl.application.RealtimeEtl --master yarn --deploy-mode client --queue streaming --driver-memory 4g --num-executors 10 --executor-memory 2g --executor-cores 1 --driver-java-options="-Djava.security.auth.login.config=/home/a133_a_daaas005/guo_test/jaas.conf" --conf "spark.executor.extraJavaOptions=-Djava.security.auth.login.config=/home/a133_a_daaas005/guo_test/jaas.conf" --jars /usr/hdp/current/spark2-client/jars/ojdbc6.jar,/usr/hdp/current/spark2-client/jars/config-1.3.1.jar /home/a133_a_daaas005/guo_test/Cold_Test_RealTimeETLCsv1-1.0-SNAPSHOT.jar & #/usr/hdp/current/spark2-client/bin/spark-submit --class app.Cold_test_Csv3 --master local[*] --driver-memory 8g --num-executors 8 --executor-memory 2g --executor-cores 1 --driver-java-options="-Djava.security.auth.login.config=/home/a133_a_daaas005/guo_test/jaas.conf" --conf "spark.executor.extraJavaOptions=-Djava.security.auth.login.config=/home/a133_a_daaas005/guo_test/jaas.conf" --jars /usr/hdp/current/spark2-client/jars/ojdbc6.jar,/usr/hdp/current/spark2-client/jars/config-1.3.1.jar /home/a133_a_daaas005/guo_test/WORKTEST-1.0-OFFICIA.jar #FlumeStartShell /usr/hdp/current/flume-server bin/flume-ng agent -c conf/ -f /usr/hdp/current/flume-server/Cold_TestConf/spooldircsv1.conf -n a1 -Dflume.root.logger=WARN,console >>./ColdTestFlume/run.log 2>&1 & #SparkStreamingStartShell /home/a133_a_daaas005/guo_test /usr/hdp/current/spark2-client/bin/spark-submit --class cold_test.realTimeEtl.application.RealtimeEtl --master yarn --deploy-mode client --queue ETL --driver-memory 10g --num-executors 30 --executor-memory 2g --executor-cores 1 --driver-java-options="-Djava.security.auth.login.config=jaas.conf" --conf "spark.executor.extraJavaOptions=-Djava.security.auth.login.config=jaas.conf" --files "./jaas.conf#jaas.conf,./kafka.service.keytab#kafka.service.keytab" --jars /usr/hdp/current/spark2-client/jars/ojdbc6.jar,/usr/hdp/current/spark2-client/jars/config-1.3.1.jar /home/a133_a_daaas005/guo_test/Cold_Test_RealTimeETLCsv1-1.0-SNAPSHOT.jar #MointorStartShell source activate rmfile /data/landing_zone/BBAC/Cold_Test/script/ java -classpath Cold_Test.MonitorFileAndPutOracle-1.0-SNAPSHOT.jar cold_test.application.MonitorFile >>monitorrun.log 2>&1 &