zookeeper & kafka 集群

http://cloudurable.com/blog/kafka-architecture/index.html

静态解析

cat >> /etc/hosts << EOF

10.105.70.126 kafka-01
10.154.153.99 kafka-02
10.105.71.175 kafka-03
EOF

配置zookeeper

cat > /root/zookeeper/conf/zoo.cfg << EOF 
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/root/zookeeper/data
clientPort=2181
server.1=kafka-01:2888:3888  
server.2=kafka-02:2888:3888
server.3=kafka-03:2888:3888
EOF

mkdir /root/zookeeper/data

# server.x，x需要在对应的服务器数据目录生成myid文件，内容为x
echo '1' > /root/zookeeper/data/myid

zookeeper启动

启动
/root/zookeeper/bin/zkServer.sh restart

查看状态
/root/zookeeper/bin/zkServer.sh status
状态分别为 Mode: follower or leader

配置kafka

vim /root/kafka/config/server.properties 
broker.id=101   # 每个borker分别配置
listeners=PLAINTEXT://:9092
advertised.listeners=PLAINTEXT://kafka-01:9092  # 每个borker分别配置
log.dirs=/root/kafka-logs
zookeeper.connect=kafka-01:2181,kafka-02:2181,kafka-03:2181

delete.topic.enable=true

log.flush.interval.messages=10000
log.flush.interval.ms=1000

log.retention.minutes=10
log.retention.hours=168
log.retention.bytes=1073741824
log.retention.check.interval.ms=300000

log.segment.delete.delay.ms=1000

log.cleaner.enable=true
log.cleanup.policy=delete

启动

/root/kafka/bin/kafka-server-start.sh -daemon /root/kafka/config/server.properties

测试

#参考 
https://kafka.apache.org/quickstart

#创建主题（1个分区，3个副本）
/root/kafka/bin/kafka-topics.sh --create --zookeeper kafka-01:2181,kafka-02:2181,kafka-03:2181 --replication-factor 3 --partitions 1 --topic test

#查询集群
/root/kafka/bin/kafka-topics.sh --describe --zookeeper kafka-01:2181,kafka-02:2181,kafka-03:2181

#生产者
/root/kafka/bin/kafka-console-producer.sh --broker-list kafka-01:9092,kafka-02:9092,kafka-03:9092 --topic test

#消费者
/root/kafka/bin/kafka-console-consumer.sh --bootstrap-server kafka-01:9092,kafka-02:9092,kafka-03:9092 --from-beginning --topic test

#删除主题
/root/kafka/bin/kafka-topics.sh --delete --zookeeper kafka-01:2181,kafka-02:2181,kafka-03:2181 --topic test

#配置__consumer_offsets自动删除
/root/kafka/bin/kafka-configs.sh --zookeeper kafka-01:2181,kafka-02:2181,kafka-03:2181 --entity-type topics --entity-name __consumer_offsets --describe

/root/kafka/bin/kafka-configs.sh --zookeeper kafka-01:2181,kafka-02:2181,kafka-03:2181 --entity-type topics --entity-name __consumer_offsets --alter --add-config cleanup.policy=delete