一次流式处理的submit

考虑很多：

压背、限流、JVM优化，出错的重试等

#!/bin/bash
num_executors=1
executor_memory=1g
driver_memory=1g

executor_cores=1
realtime_queue=root
# backpressure
receiver_max_rate=100
receiver_initial_rate=30
my_job_name="streamingSYN"
main_class="com.df.QZ.HeartOrderChart"

spark-submit --master yarn --deploy-mode cluster 
--name ${my_job_name} 
--class ${main_class} 
--driver-memory ${driver_memory} 
--num-executors ${num_executors} --executor-cores ${executor_cores} --executor-memory ${executor_memory} 
--queue ${realtime_queue} 
--conf spark.driver.extraJavaOptions=-Dlog4j.configuration=log4j-yarn.properties 
--conf spark.executor.extraJavaOptions=-Dlog4j.configuration=log4j-yarn.properties 
--conf spark.serializer=org.apache.spark.serializer.KryoSerializer  
--conf spark.locality.wait=10  
--conf spark.task.maxFailures=8  
--conf spark.ui.killEnabled=false  
--conf spark.logConf=true  
--conf spark.streaming.blockInterval=200  
--conf spark.streaming.receiver.writeAheadLog.enable=true  
--conf spark.streaming.backpressure.enabled=true 
--conf spark.streaming.backpressure.pid.minRate=10  
--conf spark.streaming.receiver.maxRate=${receiver_max_rate} 
--conf spark.streaming.kafka.maxRatePerPartition=${receiver_max_rate}  
--conf spark.streaming.backpressure.initialRate=${receiver_initial_rate}  
--conf spark.yarn.driver.memoryOverhead=512  
--conf spark.yarn.executor.memoryOverhead=1024 
--conf spark.yarn.maxAppAttempts=4 
--conf spark.yarn.am.attemptFailuresValidityInterval=1h 
--conf spark.yarn.max.executor.failures=$((8 * ${num_executors})) 
--conf spark.yarn.executor.failuresValidityInterval=1h 
--driver-java-options "-XX:+UseConcMarkSweepGC -XX:+CMSClassUnloadingEnabled -XX:ParallelCMSThreads=4 -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=70  -XX:CMSFullGCsBeforeCompaction=2 -XX:-UseCompressedOops -XX:+PrintHeapAtGC" 
hdfs://df1:9000/Thermodynamic-1.0-SNAPSHOT.jar 
1 df1:9092,df2:9092,df3:9092 driverinfo cm1 df1:2181,df2:2181,df3:2181