redis 集群搭建、扩容和收缩。（师从张亚老师）

安装redis
wget http://download.redis.io/releases/redis_3.2.9.tar.gz

tar zxf redis-3.2.9.tar.gz -C /opt/redis_cluster/

ln -s /opt/redis_cluster/redis-3.2.9 /opt/redis_cluster/redis

cd /opt/redis_cluster/redis

make && make install

集群结构规划，每台上面两个实例
db01 10.0.0.51 6380,6381
db02 10.0.0.52 6380,6381
db03 10.0.0.53 6380,6381

主从关系
db01:6380 >> db02:6381
db02:6380 >> db03:6381
db03:6380 >> db01:6381

#################################
db01配置 （10.0.0.51）

mkdir -p /opt/redis_cluster/redis_{6380,6381}/{conf,logs,pid}
mkdir –p /data/redis_cluster/redis_{6380,6381}

cat >/opt/redis_cluster/redis_6380/conf/redis_6380.conf<<EOF
bind 10.0.0.51
port 6380
daemonize yes
pidfile "/opt/redis_cluster/redis_6380/pid/redis_6380.pid"
logfile "/opt/redis_cluster/redis_6380/logs/redis_6380.log"
dbfilename "redis_6380.rdb"
dir "/data/redis_cluster/redis_6380/"
cluster-enabled yes
cluster-config-file nodes_6380.conf
cluster-node-timeout 15000
EOF

cat >/opt/redis_cluster/redis_6381/conf/redis_6381.conf<<EOF
bind 10.0.0.51
port 6381
daemonize yes
pidfile "/opt/redis_cluster/redis_6381/pid/redis_6381.pid"
logfile "/opt/redis_cluster/redis_6381/logs/redis_6381.log"
dbfilename "redis_6381.rdb"
dir "/data/redis_cluster/redis_6381/"
cluster-enabled yes
cluster-config-file nodes_6381.conf
cluster-node-timeout 15000
EOF

redis-server /opt/redis_cluster/redis_6380/conf/redis_6380.conf
redis-server /opt/redis_cluster/redis_6381/conf/redis_6381.conf
############################
db02配置 （10.0.0.52）
mkdir -p /opt/redis_cluster/redis_{6380,6381}/{conf,logs,pid}
mkdir –p /data/redis_cluster/redis_{6380,6381}

cat >/opt/redis_cluster/redis_6380/conf/redis_6380.conf<<EOF
bind 10.0.0.52
port 6380
daemonize yes
pidfile "/opt/redis_cluster/redis_6380/pid/redis_6380.pid"
logfile "/opt/redis_cluster/redis_6380/logs/redis_6380.log"
dbfilename "redis_6380.rdb"
dir "/data/redis_cluster/redis_6380/"
cluster-enabled yes
cluster-config-file nodes_6380.conf
cluster-node-timeout 15000
EOF

cat >/opt/redis_cluster/redis_6381/conf/redis_6381.conf<<EOF
bind 10.0.0.52
port 6381
daemonize yes
pidfile "/opt/redis_cluster/redis_6381/pid/redis_6381.pid"
logfile "/opt/redis_cluster/redis_6381/logs/redis_6381.log"
dbfilename "redis_6381.rdb"
dir "/data/redis_cluster/redis_6381/"
cluster-enabled yes
cluster-config-file nodes_6381.conf
cluster-node-timeout 15000
EOF

redis-server /opt/redis_cluster/redis_6380/conf/redis_6380.conf
redis-server /opt/redis_cluster/redis_6381/conf/redis_6381.conf
############################
db03配置
mkdir -p /opt/redis_cluster/redis_{6380,6381}/{conf,logs,pid}
mkdir –p /data/redis_cluster/redis_{6380,6381}

cat >/opt/redis_cluster/redis_6380/conf/redis_6380.conf<<EOF
bind 10.0.0.53
port 6380
daemonize yes
pidfile "/opt/redis_cluster/redis_6380/pid/redis_6380.pid"
logfile "/opt/redis_cluster/redis_6380/logs/redis_6380.log"
dbfilename "redis_6380.rdb"
dir "/data/redis_cluster/redis_6380/"
cluster-enabled yes
cluster-config-file nodes_6380.conf
cluster-node-timeout 15000
EOF

cat >/opt/redis_cluster/redis_6381/conf/redis_6381.conf<<EOF
bind 10.0.0.53
port 6381
daemonize yes
pidfile "/opt/redis_cluster/redis_6381/pid/redis_6381.pid"
logfile "/opt/redis_cluster/redis_6381/logs/redis_6381.log"
dbfilename "redis_6381.rdb"
dir "/data/redis_cluster/redis_6381/"
cluster-enabled yes
cluster-config-file nodes_6381.conf
cluster-node-timeout 15000
EOF

redis-server /opt/redis_cluster/redis_6380/conf/redis_6380.conf
redis-server /opt/redis_cluster/redis_6381/conf/redis_6381.conf
##############一条命令部署集群##################

redis-cli --cluster create 192.168.2.101:6380 192.168.2.101:6381 192.168.2.102:6380 192.168.2.102:6381 192.168.2.103:6380 192.168.2.103:6381 --cluster-replicas 1

################################

部署完毕之后
netstat -tunple|grep redis
检查是否启动两个实例6380 6381，以及实例的内部通讯端口16380 16381
################################
手动配置节点
db01 上可以完成加入所有节点的操作
10.0.0.51:6380> CLUSTER MEET 10.0.0.51 6381
10.0.0.51:6380> CLUSTER MEET 10.0.0.52 6381
10.0.0.51:6380> CLUSTER MEET 10.0.0.53 6381
10.0.0.51:6380> CLUSTER MEET 10.0.0.52 6380
10.0.0.51:6380> CLUSTER MEET 10.0.0.53 6380

加完之后使用cluster node 查询当前集群节点状况
或者/data/redis_cluster/redis_6380/nodes_6380.conf 内也可以查看。

10.0.0.51:6380> cluster nodes
68af205aad42909db61013ae2d0f9d2ec49cb5b9 10.0.0.52:6380 master - 0 1562243748164 2 connected 5462-10922
215158ede75cadd1c9a8fccb99278d0da3c5de48 10.0.0.51:6380 myself,master - 0 0 1 connected 0-5461
f5248261ef32638fc11966cdeedff96ab197b812 10.0.0.53:6380 master - 0 1562243745141 0 connected 10923-16383
c847d86eb040a5cbeaeddef225beecba22f401b2 10.0.0.51:6381 master - 0 1562243742121 3 connected
0815edc37378ce8c1bed1b46a460b410f231937d 10.0.0.53:6381 master - 0 1562243749173 5 connected
eb67970ebb0eb3512f09b7be79128bf736eaccb5 10.0.0.52:6381 master - 0 1562243746147 4 connected
##################################
槽位分配

redis-cli -h 10.0.0.51 -p 6380 cluster addslots {0..5461}
redis-cli -h 10.0.0.52 -p 6380 cluster addslots {5462..10922}
redis-cli -h 10.0.0.53 -p 6380 cluster addslots {10923..16383}

分配完之后，可以通过cluster nodes 查看三个主机点的操作分配情况
通过cluster info 查看集群状态

##################################
主从关系设定
redis-cli -c -h db01 -p 6381 cluster nodes|grep -v "6381"|awk '{print $1,$2}'
215158ede75cadd1c9a8fccb99278d0da3c5de48 10.0.0.51:6380
68af205aad42909db61013ae2d0f9d2ec49cb5b9 10.0.0.52:6380
f5248261ef32638fc11966cdeedff96ab197b812 10.0.0.53:6380
查询出实例id号

执行完成主从关系
redis-cli -c -h db01 -p 6381 cluster replicate 68af205aad42909db61013ae2d0f9d2ec49cb5b9
redis-cli -c -h db02 -p 6381 cluster replicate f5248261ef32638fc11966cdeedff96ab197b812
redis-cli -c -h db03 -p 6381 cluster replicate 215158ede75cadd1c9a8fccb99278d0da3c5de48

###########################
集群内插入值测试(-c 参数)
redis-cli -c -h db01 -p 6380 set k1 v1

大批量插入值
for i in {0..1000};do redis-cli -c -h db01 -p 6380 set 58NB_${i} 58V5_${i};done

测试数据量分布情况，
redis-cli -h db01 -c -p 6380 DBSIZE
###############################
模拟故障转移
kill 掉db02上的6380，通过 redis-cli -h db01 -c -p 6380 cluster nodes
可以观察到 db01的6381从节点 升级为主节点，并接管了db02上原来6380的槽。
[root@db01 ~]# redis-cli -h db01 -c -p 6380 cluster nodes
10.0.0.52:6380 master,fail - 1562251836868 1562251833945 2 disconnected
10.0.0.51:6380 myself,master - 0 0 1 connected 0-5461
10.0.0.53:6380 master - 0 1562251999666 0 connected 10923-16383
10.0.0.51:6381 master - 0 1562251998652 6 connected 5462-10922
10.0.0.53:6381 slave 215158ede75cadd1c9a8fccb99278d0da3c5de48 0 1562251999159 5 connected
10.0.0.52:6381 slave f5248261ef32638fc11966cdeedff96ab197b812 0 1562252000675 4 connected
###############################
故障恢复
启动db02的6380
redis-server /opt/redis_cluster/redis_6380/conf/redis_6380.conf
观察db02上的6380的日志情况
[root@db02 ~]# cat /opt/redis_cluster/redis_6380/logs/redis_6380.log
8619:S 04 Jul 22:59:54.217 * Connecting to MASTER 10.0.0.51:6381
8619:S 04 Jul 22:59:54.217 * MASTER <-> SLAVE sync started
8619:S 04 Jul 22:59:54.217 * Non blocking connect for SYNC fired the event.
8619:S 04 Jul 22:59:54.218 * Master replied to PING, replication can continue...
8619:S 04 Jul 22:59:54.218 * Partial resynchronization not possible (no cached master)
8619:S 04 Jul 22:59:54.223 * Full resync from master: c75849d817eac34104812970ea0d80ef06448e91:1
8619:S 04 Jul 22:59:54.303 * MASTER <-> SLAVE sync: receiving 10524 bytes from master
8619:S 04 Jul 22:59:54.303 * MASTER <-> SLAVE sync: Flushing old data
8619:S 04 Jul 22:59:54.303 * MASTER <-> SLAVE sync: Loading DB in memory
8619:S 04 Jul 22:59:54.304 * MASTER <-> SLAVE sync: Finished with success

观察db01上的6381的日志
[root@db01 ~]# cat /opt/redis_cluster/redis_6381/logs/redis_6381.log
32564:M 04 Jul 22:50:53.868 # Cluster state changed: ok
32564:M 04 Jul 22:59:53.294 * Clear FAIL state for node 68af205aad42909db61013ae2d0f9d2ec49cb5b9: master without slots is reachable again.
32564:M 04 Jul 22:59:54.217 * Slave 10.0.0.52:6380 asks for synchronization
32564:M 04 Jul 22:59:54.217 * Full resync requested by slave 10.0.0.52:6380
32564:M 04 Jul 22:59:54.217 * Starting BGSAVE for SYNC with target: disk
32564:M 04 Jul 22:59:54.219 * Background saving started by pid 37006
37006:C 04 Jul 22:59:54.230 * DB saved on disk
37006:C 04 Jul 22:59:54.230 * RDB: 6 MB of memory used by copy-on-write
32564:M 04 Jul 22:59:54.300 * Background saving terminated with success
32564:M 04 Jul 22:59:54.301 * Synchronization with slave 10.0.0.52:6380 succeeded
可以看到db02:6380 成为了db01:6381的从库

若需要db02:6380 重新恢复主身份，执行cluster failover
10.0.0.52:6380> CLUSTER FAILOVER
执行后 db01:6381 再次成为了db02:6380的从库

###########################################
工具搭建redis cluster
yum makecache fast
yum install rubygems
gem sources --remove https://rubygems.org/
gem sources -a http://mirrors.aliyun.com/rubygems/
gem update –system
gem install redis -v 3.3.5

清理掉之前手工创建的集群
pkill redis
rm -rf /data/redis_cluster/redis_6380/*
rm -rf /data/redis_cluster/redis_6381/*

在db01上创建集群信息
cd /opt/redis_cluster/redis/src/
./redis-trib.rb create --replicas 1 10.0.0.51:6380 10.0.0.52:6380 10.0.0.53:6380 10.0.0.51:6381 10.0.0.52:6381 10.0.0.53:6381

检查集群状态
./redis-trib.rb check 10.0.0.51:6380

检查槽位状态
./redis-trib.rb rebalance 10.0.0.51:6380

当前复制关系现状
10.0.0.51:6381-->10.0.0.52:6380 876e7ced4441cda59aa19d51051af6459a5c90d4
10.0.0.52:6381-->10.0.0.51:6380 ac14a416ef65d4d03fb4ad528ecbd7271296ba3a
10.0.0.53:6381-->10.0.0.53:6380 c2349ca206f3747c140a83cfef10e78845bed2b3

修正错误(db03:6380>>db03:6381) 的主从关系
redis-cli -c -h db02 -p 6381 cluster replicate c2349ca206f3747c140a83cfef10e78845bed2b3
redis-cli -c -h db03 -p 6381 cluster replicate ac14a416ef65d4d03fb4ad528ecbd7271296ba3a

修正之后的状态
10.0.0.51:6381-->10.0.0.52:6380 876e7ced4441cda59aa19d51051af6459a5c90d4
10.0.0.52:6381-->10.0.0.53:6380 c2349ca206f3747c140a83cfef10e78845bed2b3
10.0.0.53:6381-->10.0.0.51:6380 ac14a416ef65d4d03fb4ad528ecbd7271296ba3a

####################################################
集群扩展，新加入一个db04
db03配置
mkdir -p /opt/redis_cluster/redis_{6380,6381}/{conf,logs,pid}
mkdir –p /data/redis_cluster/redis_{6380,6381}

cat >/opt/redis_cluster/redis_6380/conf/redis_6380.conf<<EOF
bind 10.0.0.54
port 6380
daemonize yes
pidfile "/opt/redis_cluster/redis_6380/pid/redis_6380.pid"
logfile "/opt/redis_cluster/redis_6380/logs/redis_6380.log"
dbfilename "redis_6380.rdb"
dir "/data/redis_cluster/redis_6380/"
cluster-enabled yes
cluster-config-file nodes_6380.conf
cluster-node-timeout 15000
EOF

cat >/opt/redis_cluster/redis_6381/conf/redis_6381.conf<<EOF
bind 10.0.0.54
port 6381
daemonize yes
pidfile "/opt/redis_cluster/redis_6381/pid/redis_6381.pid"
logfile "/opt/redis_cluster/redis_6381/logs/redis_6381.log"
dbfilename "redis_6381.rdb"
dir "/data/redis_cluster/redis_6381/"
cluster-enabled yes
cluster-config-file nodes_6381.conf
cluster-node-timeout 15000
EOF

redis-server /opt/redis_cluster/redis_6380/conf/redis_6380.conf
redis-server /opt/redis_cluster/redis_6381/conf/redis_6381.conf

集群管理工具发现节点
cd /opt/redis_cluster/redis/src/
./redis-trib.rb add-node 10.0.0.54:6380 10.0.0.51:6380
./redis-trib.rb add-node 10.0.0.54:6381 10.0.0.51:6380

分配槽位
./redis-trib.rb reshard 10.0.0.51:6380
设置要迁移多少个槽位进来
4096 >> yes >> all >> yes

db04:6380 的节点id：3f58930807729c245ad9729478ad16e1b1f580fc
db01:6380 的节点id：ac14a416ef65d4d03fb4ad528ecbd7271296ba3a
修正新加入的节点 主从关系
redis-cli -c -h db03 -p 6381 cluster replicate 3f58930807729c245ad9729478ad16e1b1f580fc
redis-cli -c -h db04 -p 6381 cluster replicate ac14a416ef65d4d03fb4ad528ecbd7271296ba3a

最后主从关系
db01:6381-->db02:6380
db02:6381-->db03:6380
db03:6381-->db04:6380
db04:6381-->db01:6380
################################################
集群收缩