centos7部署kafka+zookeeper集群
实验环境：CentOS Linux release 7.6.1810 (Core)
提前安装好java环境变量
一、安装zookeeper
yum install java-1.8.0
1、配置本地hosts
cat /etc/hosts
192.168.121.30 es-node01
192.168.121.31 es-node02
192.168.121.32 es-node03
2、以下所有操作都在3个节点中执行，下载编译好的zookeeper程序包，并执行以下操作
cd /data
wget http://mirror.bit.edu.cn/apache/zookeeper/stable/apache-zookeeper-3.5.7-bin.tar.gz
tar -xf apache-zookeeper-3.5.7-bin.tar.gz
mv apache-zookeeper-3.5.7-bin zookeeper
mkdir /data/zookeeper/data
3、更改配置文件,每个节点配置一样
cd /data/zookeeper/conf
more zoo.cfg|grep -Ev "^#|^$"
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/data/zookeeper/data
clientPort=2181
server.1=192.168.121.30:2888:3888
server.2=192.168.121.31:2888:3888
server.3=192.168.121.32:2888:3888
4、设置zookeeper集群id，每个节点id不能相同
echo "1" > /data/zookeeper/data/myid
echo "2" > /data/zookeeper/data/myid
echo "3" > /data/zookeeper/data/myid
5、配置开机自启
[root@es-node01 conf]# cat /usr/lib/systemd/system/zookeeper.service
[Unit]
Description=zookeeper.service
After=network.target
[Service]
Type=forking
ExecStart=/data/zookeeper/bin/zkServer.sh start
ExecStop=/data/zookeeper/bin/zkServer.sh stop
ExecReload=/data/zookeeper/bin/zkServer.sh restart
[Install]
WantedBy=multi-user.target
6、启动命令如下
systemctl daemon-reload
systemctl start zookeeper.service
systemctl enbale zookeeper.service
6、查看zookeeper集群是否正常
查看zk的状态命令（可以看到当前节点是follower还是leader）：
bin/zkServer.sh status
一、安装kafka
1、以下所有操作都在3个节点中执行，下载编译好的kafka程序包，并执行以下操作
cd /data
wget http://mirror.bit.edu.cn/apache/kafka/2.4.0/kafka_2.13-2.4.0.tgz
scp -r kafka_2.13-2.4.0.tgz root@192.168.121.31:/data
scp -r kafka_2.13-2.4.0.tgz root@192.168.121.32:/data
tar -xf kafka_2.13-2.4.0.tgz
mv kafka_2.13-2.4.0 kafka
mkdir /data/kafka/kafkalogs
2、编辑kafka配置文件
cd /data/kafka/config
[root@es-node01 config]# cat server.properties |grep -Ev "^#|^$"
broker.id=1
listeners=PLAINTEXT://es-node01:9092
host.name=es-node01
advertised.listeners=PLAINTEXT://es-node01:9092
advertised.host.name=es-node01
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/data/kafka/log
num.partitions=3
delete.topic.enable=true
replica.fetch.max.bytes=5242880
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=3
transaction.state.log.replication.factor=3
transaction.state.log.min.isr=3
message.max.byte=5242880
log.cleaner.enable=true
log.retention.hours=72
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=192.168.121.30:2181,192.168.121.31:2181,192.168.121.32:2181
zookeeper.connection.timeout.ms=6000
group.initial.rebalance.delay.ms=0
三个节点除了broker.id=1(broker.id=2/broker.id=3)和主机名es-node01需要修改，避免重复，其他配置信息保持一致即可
3、配置开机自启
cat /usr/lib/systemd/system/kafka.service
[Unit]
Description=kafka server daemon
After=kafka.target
[Service]
Type=forking
ExecStart=/data/kafka/bin/kafka-server-start.sh -daemon /data/kafka/config/server.properties
ExecReload=/data/kafka/bin/kafka-server-stop.sh && sleep 2 && /data/kafka/bin/kafka-server-start.sh -daemon /data/kafka/config/server.properties
ExecStop=/data/kafka/bin/kafka-server-stop.sh
Restart=always
[Install]
WantedBy=multi-user.target
4、启动命令如下
systemctl daemon-reload
systemctl start kafka
systemctl status kafka
systemctl enbale kafka
5、测试集群是否可用
创建一个名称为SyslogTopic的 3副本3分区的topic
./bin/kafka-topics.sh --create --zookeeper 192.168.121.30:2181,192.168.121.31:2181,192.168.121.32:2181 --replication-factor 3 --partitions 3 --topic SyslogTopic
展示topic
./bin/kafka-topics.sh --list --zookeeper 192.168.121.30:2181,192.168.121.31:2181,192.168.121.32:2181
查看topic详细信息
./bin/kafka-topics.sh --describe --zookeeper 192.168.121.30:2181,192.168.121.31:2181,192.168.121.32:2181 --topic SyslogTopic
模拟生产者发送消息(该操作在node1节点上执行)
./bin/kafka-console-producer.sh --broker-list 192.168.121.30:9092 --topic SyslogTopic
模拟消费者消费消息(该操作在node2节点上执行)
./bin/kafka-console-consumer.sh --bootstrap-server 192.168.121.30:9092,192.168.121.31:9092,192.168.121.32:9092 --topic SyslogTopic
删除名称为SyslogTopic的topic
./bin/kafka-topics.sh --delete --topic SyslogTopic --zookeeper 192.168.121.30:2181,192.168.121.31:2181,192.168.121.32:2181
查看SyslogTopic是否还存在
./bin/kafka-topics.sh --describe --zookeeper 192.168.121.30:2181,192.168.121.31:2181,192.168.121.32:2181 --topic SyslogTopic