kafka性能测试代码

 bin/kafka-producer-perf-test.sh --num-records 5000000 --record-size 5000 
 --topic kafkatopic2   
  --broker-list 


bin/kafka-run-class.sh org.apache.kafka.clients.tools.ProducerPerformance kafkatopic2 50000000 100 -1 acks=1 
 bootstrap.servers=bdnodedn03.smpv.com:6667,bdnodedn08.smpv.com:6667,bdnodedn09.smpv.com:6667 
  buffer.memory=67108864 batch.size=8196


Producer
Setup
bin/kafka-topics.sh --zookeeper bdnodedn01.smpv.com:2181 --create --topic test-rep-one --partitions 6 --replication-factor 1
bin/kafka-topics.sh --zookeeper bdnodedn01.smpv.com:2181 --create --topic test-rep-two --partitions 6 --replication-factor 3

Single thread, no replication
bin/kafka-run-class.sh org.apache.kafka.tools.ProducerPerformance --print-metrics --topic test-rep-one 
--num-records 6000000 --throughput 100000 
--record-size 100 --producer-props bootstrap.servers=bdnodedn03.smpv.com:6667 
buffer.memory=67108864 batch.size=8196

Single-thread, async 3x replication
bin/kafka-run-class.sh org.apache.kafka.tools.ProducerPerformance --print-metrics --topic test-rep-two 
--num-records 6000000 --throughput 100000 --record-size 100 --producer-props bootstrap.servers=bdnodedn03.smpv.com:6667 
acks=1 buffer.memory=67108864 batch.size=8196

Single-thread, sync 3x replication
bin/kafka-run-class.sh org.apache.kafka.tools.ProducerPerformance --print-metrics --topic test-rep-two 
--num-records 6000000 --throughput 100000 --record-size 100 
--producer-props bootstrap.servers=bdnodedn03.smpv.com:6667 acks=-1 buffer.memory=67108864 batch.size=8196

Effect of message size

for i in 10 100 1000 10000 100000;
do
echo ""
echo $i
bin/kafka-run-class.sh org.apache.kafka.tools.ProducerPerformance --print-metrics 
--topic test-rep-two --num-records  $((1000*1024*1024/$i)) --throughput 100000 
--record-size $i --producer-props bootstrap.servers=bdnodedn03.smpv.com:6667 acks=1 buffer.memory=67108864 batch.size=8196
done;


Consumer

Consumer throughput
bin/kafka-run-class.sh kafka.tools.ConsumerPerformance  --broker-list bdnodedn03.smpv.com:6667 
--messages 6000000 --threads 1 
--topic test-rep-two --print-metrics

3 Consumers

On three servers, run:
bin/kafka-run-class.sh kafka.tools.ConsumerPerformance  --broker-list bdnodedn03.smpv.com:6667 
--messages 6000000 --threads 1 
--topic test-rep-two --print-metrics