使用ruby工具部署cluster05

六、使用工具搭建redis集群

1.环境准备

节点	IP	端口
节点1	172.16.1.51	6379,6380
节点2	172.16.1.52	6379,6380
节点3	172.16.1.53	6379,6380

2.搭建redis

3.启动所有redis (不要做别的 )

4.安装集群插件 （随便找一台集群中的实例）

#EPEL源安装ruby支持
[root@db01 ~]# yum install ruby rubygems -y

#查看gem源
[root@db01 ~]# gem sources -l
*** CURRENT SOURCES ***

http://rubygems.org/

#添加阿里云的gem源
[root@db01 ~]# gem sources -a http://mirrors.aliyun.com/rubygems/
http://mirrors.aliyun.com/rubygems/ added to sources 

#删除国外gem源
[root@db01 ~]# gem sources  --remove https://rubygems.org/
http://rubygems.org/ removed from sources

#再次查看gem源
[root@db01 ~]# gem sources -l

#使用gem安装redis的ruby插件
[root@db01 ~]# gem install redis -v 3.3.3
Successfully installed redis-3.3.3
1 gem installed
Installing ri documentation for redis-3.3.3...
Installing RDoc documentation for redis-3.3.3...

#安装了ruby插件，该文件才能使用（配置环境变量：1.export 2.软链接 3.cp 4.make && make install）
[root@db02 ~]# ll /usr/local/redis/src/redis-trib.rb 
-rwxrwxr-x 1 root root 60852 Jun 13  2018 /usr/local/redis/src/redis-trib.rb

5. redis-trib.rb命令

[root@db01 ~]# redis-trib.rb 
create			#创建一个集群(ip:port ip:port)
	--replicas 1	自动做主从关系，每台机器有一个副本
check			#检查集群,选择集群中任意一台实例即可（ip:port）
info			#集群状态,选择集群中任意一台实例即可（ip:port）
fix				#修复集群
reshard			#重新分配槽位
rebalance		#重新平衡槽位数量
add-node		#添加节点
del-node		#删除节点
set-timeout		#设置超时时间
call			#向集群所有机器输入命令
import			#导入数据
help            #帮助

6.关联所有节点

[root@db01 ~]# redis-trib.rb create --replicas 1 172.16.1.51:6379 172.16.1.52:6379 172.16.1.53:6379 172.16.1.52:6380 172.16.1.53:6380 172.16.1.51:6380 
>>> Creating cluster
>>> Performing hash slots allocation on 6 nodes...
Using 3 masters:
172.16.1.51:6379
172.16.1.52:6379
172.16.1.53:6379
Adding replica 172.16.1.52:6380 to 172.16.1.51:6379
Adding replica 172.16.1.51:6380 to 172.16.1.52:6379
Adding replica 172.16.1.53:6380 to 172.16.1.53:6379
...
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.

#使用该命令前集群内不能有数据，因为要分槽（有数据的话需要删除node.conf后重启）
#node节点不要有密码
#ruby工具的好处是Redis的关联和主从一起做了
#使用工具做主从的话，始终有一台实例自己做自己的主从，所以需要自己手动重新配置主从

7.查看集群状态

[root@db01 ~]# redis-cli -h 172.16.1.51 -p 6379 CLUSTER NODES
e4794215d9d3548e9c514c10626ce618be19ebfb 172.16.1.53:6380 slave d27553035a3e91c78d375208c72b756e9b2523d4 0 1596767315453 5 connected
d27553035a3e91c78d375208c72b756e9b2523d4 172.16.1.53:6379 master - 0 1596767315453 3 connected 10923-16383
5ad7bd957133eac9c3a692b35f8ae72258cf0ece 172.16.1.51:6379 myself,master - 0 0 1 connected 0-5460
fee551a90c8646839f66fa0cd1f6e5859e9dd8e0 172.16.1.52:6380 slave 5ad7bd957133eac9c3a692b35f8ae72258cf0ece 0 1596767313429 4 connected
1d10edbc5ed08f85d2afc21cd338b023b9dd61b4 172.16.1.51:6380 slave 7c79559b280db9d9c182f3a25c718efe9e934fc7 0 1596767313935 6 connected
7c79559b280db9d9c182f3a25c718efe9e934fc7 172.16.1.52:6379 master - 0 1596767314949 2 connected 5461-10922

#可以看出所有节点都已经关联，主从关系是交叉，但是总有一台是自己指向自己

#显示key 槽位 主从
[root@db04 ~]# redis-trib.rb info 172.16.1.52:6379
172.16.1.52:6379 (af5e32c9...) -> 335 keys | 5461 slots | 1 slaves.
172.16.1.53:6379 (3b234ef4...) -> 326 keys | 5462 slots | 1 slaves.
172.16.1.54:6379 (e514df72...) -> 339 keys | 5461 slots | 1 slaves.
[OK] 1000 keys in 3 masters.
0.06 keys per slot on average.

8.重新做主从

#由于使用工具，始终有一台机器从库本机的从库，所以要重新分配主从
172.16.1.52:6380> CLUSTER REPLICATE d27553035a3e91c78d375208c72b756e9b2523d4
OK
172.16.1.53:6380> CLUSTER REPLICATE 5ad7bd957133eac9c3a692b35f8ae72258cf0ece
OK

9.插入数据测试

[root@db04 ~]# vim kv.sh 
for i in range{1..1000};do
        redis-cli -c -p 6379 -h 172.16.1.52 set k${i} v${i}
done

[root@db01 ~]# redis-trib.rb info 172.16.1.52:6379
172.16.1.52:6379 (7c79559b...) -> 332 keys | 5462 slots | 1 slaves.
172.16.1.51:6379 (5ad7bd95...) -> 341 keys | 5461 slots | 1 slaves.
172.16.1.53:6379 (d2755303...) -> 327 keys | 5461 slots | 1 slaves.
[OK] 1000 keys in 3 masters.
0.06 keys per slot on average.

[root@db01 ~]# redis-trib.rb info 172.16.1.52:6379
172.16.1.52:6379 (7c79559b...) -> 661 keys | 5462 slots | 1 slaves.
172.16.1.51:6379 (5ad7bd95...) -> 674 keys | 5461 slots | 1 slaves.
172.16.1.53:6379 (d2755303...) -> 665 keys | 5461 slots | 1 slaves.
[OK] 2000 keys in 3 masters.
0.12 keys per slot on average.

web连接Redis时的ip配置

七、redis集群节点

#添加和删除节点的流程
1.新节点添加槽位
2.源节点中的数据进行迁移
3.源节点数据迁移完毕
4.迁移下一个槽位的数据，依次循环

1.添加节点

1）准备新机器

[root@db02 ~]# mkdir /service/redis/{6381,6382}
[root@db02 ~]# vim /service/redis/6381/redis.conf
[root@db02 ~]# vim /service/redis/6382/redis.conf

#启动新节点
[root@db02 ~]# redis-server /service/redis/6381/redis.conf 
[root@db02 ~]# redis-server /service/redis/6382/redis.conf 

#集群每台实例1个副本的话，每次添加2个节点
#集群每台实例2个副本的话，每次添加3个节点

2）将新节点添加到集群

#方法一，关联
[root@db01 ~]# redis-trib.rb add-node 172.16.1.52:6381 172.16.1.51:6379
#方法二，关联
[root@db01 ~]# redis-cli -p 6379 -h 172.16.1.51 cluster meet 172.16.1.52:6381

#查看节点信息
[root@db01 ~]# redis-trib.rb info 172.16.1.51:6379
[root@db01 ~]# redis-cli -p 6379 -h 172.16.1.51 cluster nodes

3）重新分配槽位

[root@db01 ~]# redis-trib.rb reshard 172.16.1.51:6379
#你想移动多少个槽位到新节点
How many slots do you want to move (from 1 to 16384)? 4096
#新节点的ID是什么
What is the receiving node ID? a298dbd22c10b8492d9ff4295504c50666f4fb2e
#输入源节点的ID，如果是所有节点直接使用all
Please enter all the source node IDs.
  Type 'all' to use all the nodes as source nodes for the hash slots.
  Type 'done' once you entered all the source nodes IDs.
source node #1：all （从集群所有主节点中拿xx个槽位，凑够4096个再给新节点）（指定从哪个id拿的话，要done）
#你确定要这么分配？
Do you want to proceed with the proposed reshard plan (yes/no)? yes

#分配完毕，查看分配结果
[root@db01 ~]# redis-cli -p 6379 -h 172.16.1.51 cluster nodes
e4794215d9d3548e9c514c10626ce618be19ebfb 172.16.1.53:6380 slave 5ad7bd957133eac9c3a692b35f8ae72258cf0ece 0 1596769469815 5 connected
d27553035a3e91c78d375208c72b756e9b2523d4 172.16.1.53:6379 master - 0 1596769468805 3 connected 12288-16383
5ad7bd957133eac9c3a692b35f8ae72258cf0ece 172.16.1.51:6379 myself,master - 0 0 1 connected 1365-5460
fee551a90c8646839f66fa0cd1f6e5859e9dd8e0 172.16.1.52:6380 slave d27553035a3e91c78d375208c72b756e9b2523d4 0 1596769467797 4 connected
1d10edbc5ed08f85d2afc21cd338b023b9dd61b4 172.16.1.51:6380 slave 7c79559b280db9d9c182f3a25c718efe9e934fc7 0 1596769469513 6 connected
a298dbd22c10b8492d9ff4295504c50666f4fb2e 172.16.1.52:6381 master - 0 1596769468302 7 connected 0-1364 5461-6826 10923-12287
7c79559b280db9d9c182f3a25c718efe9e934fc7 172.16.1.52:6379 master - 0 1596769468302 2 connected 6827-10922

[root@db01 ~]# 
[root@db01 ~]# redis-trib.rb info 172.16.1.51:6379
172.16.1.51:6379 (5ad7bd95...) -> 499 keys | 4096 slots | 1 slaves.
172.16.1.53:6379 (d2755303...) -> 501 keys | 4096 slots | 1 slaves.
172.16.1.52:6381 (a298dbd2...) -> 502 keys | 4096 slots | 0 slaves.
172.16.1.52:6379 (7c79559b...) -> 498 keys | 4096 slots | 1 slaves.
[OK] 2000 keys in 4 masters.
0.12 keys per slot on average.

4）添加新节点的副本

[root@db02 ~]# redis-cli -h 172.16.1.52 -p 6382 cluster replicate a298dbd22c10b8492d9ff4295504c50666f4fb2e  （主库id）
#或者
[root@db01 ~]# redis-trib.rb add-node --slave --master-id a298dbd22c10b8492d9ff4295504c50666f4fb2e 172.16.1.52:6382 172.16.1.51:6379  （该副本主库的id + 新从库ip:port + 集群中任意一台机器）
		
[root@db01 ~]# redis-cli -p 6379 -h 172.16.1.51 cluster nodes
[root@db01 ~]# redis-trib.rb info 172.16.1.51:6379
172.16.1.51:6379 (5ad7bd95...) -> 499 keys | 4096 slots | 1 slaves.
172.16.1.53:6379 (d2755303...) -> 501 keys | 4096 slots | 1 slaves.
172.16.1.52:6381 (a298dbd2...) -> 502 keys | 4096 slots | 1 slaves.
172.16.1.52:6379 (7c79559b...) -> 498 keys | 4096 slots | 1 slaves.
[OK] 2000 keys in 4 masters.
0.12 keys per slot on average.

#调整主从，尽量保证每一台机器的副本都不于主节点在一台机器

5）模拟故障（槽位迁移过程中意外终止）

#分配槽位的过程中
[root@db01 ~]# redis-trib.rb reshard 172.16.1.51:6379

#执行Ctrl+c

#查看集群状态一些命令看不出来有错
[root@db01 ~]# redis-cli -p 6379 -h 172.16.1.51 cluster info
[root@db01 ~]# redis-cli -p 6379 -h 172.16.1.51 cluster nodes
[root@db01 ~]# redis-trib.rb info 172.16.1.51:6379

#必须使用工具检查集群
[root@db01 ~]# redis-trib.rb check 172.16.1.51:6379

#错误一：172.16.1.52:6379节点正在导出槽位，172.16.1.52:6381节点正在导入槽位
>>> Check for open slots...
[WARNING] Node 172.16.1.52:6381 has slots in importing state (6885).
[WARNING] Node 172.16.1.52:6379 has slots in migrating state (6885).
[WARNING] The following slots are open: 6885
>>> Check slots coverage...

#错误二：172.16.1.52:6379节点正在导出槽位
>>> Check for open slots...
[WARNING] Node 172.16.1.52:6379 has slots in migrating state (6975).
[WARNING] The following slots are open: 6975
>>> Check slots coverage...

#错误三：
>>> Check for open slots...
[WARNING] Node 172.16.1.52:6381 has slots in importing state (7093).
[WARNING] The following slots are open: 7093
>>> Check slots coverage...

6）修复故障

#使用fix修复集群
[root@db01 ~]# redis-trib.rb fix 172.16.1.52:6379

#将槽位平均分配
1.平均之前
[root@db01 ~]# redis-trib.rb info 172.16.1.51:6379
172.16.1.51:6379 (5ad7bd95...) -> 499 keys | 4096 slots | 1 slaves.
172.16.1.53:6379 (d2755303...) -> 501 keys | 4096 slots | 1 slaves.
172.16.1.52:6381 (a298dbd2...) -> 648 keys | 5320 slots | 1 slaves.
172.16.1.52:6379 (7c79559b...) -> 352 keys | 2872 slots | 1 slaves.
[OK] 2000 keys in 4 masters.
0.12 keys per slot on average.

2.平均分配（如果节点之间槽位数量相差较小，不会平均分配）
[root@db01 ~]# redis-trib.rb rebalance 172.16.1.51:6379
>>> Performing Cluster Check (using node 172.16.1.51:6379)
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.
>>> Rebalancing across 4 nodes. Total weight = 4
Moving 1224 slots from 172.16.1.52:6381 to 172.16.1.52:6379
#####################################################################################################

3平均分配之后
[root@db01 ~]# redis-trib.rb info 172.16.1.51:6379
172.16.1.51:6379 (5ad7bd95...) -> 499 keys | 4096 slots | 1 slaves.
172.16.1.53:6379 (d2755303...) -> 501 keys | 4096 slots | 1 slaves.
172.16.1.52:6381 (a298dbd2...) -> 492 keys | 4096 slots | 1 slaves.
172.16.1.52:6379 (7c79559b...) -> 508 keys | 4096 slots | 1 slaves.
[OK] 2000 keys in 4 masters.
0.12 keys per slot on average.

2.删除节点

1）重新分配槽位

#重新分配
[root@db01 ~]# redis-trib.rb reshard 172.16.1.51:6379
How many slots do you want to move (from 1 to 16384)? 1365   #要分配的槽位数量
What is the receiving node ID? #接收槽位的节点id
Please enter all the source node IDs.
  Type 'all' to use all the nodes as source nodes for the hash slots.
  Type 'done' once you entered all the source nodes IDs.
Source node #1: 要删除的节点id
Source node #2: done
Do you want to proceed with the proposed reshard plan (yes/no)? yes

#第二次重新分配
[root@db01 ~]# redis-trib.rb reshard 172.16.1.51:6379
How many slots do you want to move (from 1 to 16384)? 1365
What is the receiving node ID? 7c79559b280db9d9c182f3a25c718efe9e934fc7
Please enter all the source node IDs.
  Type 'all' to use all the nodes as source nodes for the hash slots.
  Type 'done' once you entered all the source nodes IDs.
Source node #1:a298dbd22c10b8492d9ff4295504c50666f4fb2e
Source node #2:done
Do you want to proceed with the proposed reshard plan (yes/no)? yes

#第三次重新分配
[root@db01 ~]# redis-trib.rb reshard 172.16.1.51:6379
How many slots do you want to move (from 1 to 16384)? 1366
What is the receiving node ID? d27553035a3e91c78d375208c72b756e9b2523d4
Please enter all the source node IDs.
  Type 'all' to use all the nodes as source nodes for the hash slots.
  Type 'done' once you entered all the source nodes IDs.
Source node #1:a298dbd22c10b8492d9ff4295504c50666f4fb2e
Source node #2:done
Do you want to proceed with the proposed reshard plan (yes/no)? yes

2）查看重新分配以后的节点信息

[root@db01 ~]# redis-trib.rb info 172.16.1.51:6379
172.16.1.51:6379 (5ad7bd95...) -> 664 keys | 5461 slots | 1 slaves.
172.16.1.53:6379 (d2755303...) -> 665 keys | 5462 slots | 2 slaves.
172.16.1.52:6381 (a298dbd2...) -> 0 keys | 0 slots | 0 slaves.
172.16.1.52:6379 (7c79559b...) -> 671 keys | 5461 slots | 1 slaves.
[OK] 2000 keys in 4 masters.
0.12 keys per slot on average.

3）删除节点

#删除没有数据的主节点
[root@db01 ~]# redis-trib.rb del-node 172.16.1.52:6381 a298dbd22c10b8492d9ff4295504c50666f4fb2e
>>> Removing node a298dbd22c10b8492d9ff4295504c50666f4fb2e from cluster 172.16.1.52:6381
>>> Sending CLUSTER FORGET messages to the cluster...
>>> SHUTDOWN the node.

#删除从节点
[root@db01 ~]# redis-trib.rb del-node 172.16.1.52:6382 47e3638a203488218d8c62deb82e768598977ba4
>>> Removing node 47e3638a203488218d8c62deb82e768598977ba4 from cluster 172.16.1.52:6382
>>> Sending CLUSTER FORGET messages to the cluster...
>>> SHUTDOWN the node.

#删除时不能删除有数据的主节点，从节点可以随便删除
[root@db01 ~]# redis-trib.rb del-node 172.16.1.53:6379 d27553035a3e91c78d375208c72b756e9b2523d4
>>> Removing node d27553035a3e91c78d375208c72b756e9b2523d4 from cluster 172.16.1.53:6379
[ERR] Node 172.16.1.53:6379 is not empty! Reshard data away and try again.

八、redis数据迁移

参考文档

1.安装迁移工具

#1.安装依赖
[root@db02 ~]# yum install -y automake libtool autoconf bzip2

#2.拉取工具,#或者上传包
[root@db02 ~]# git clone https://github.com/vipshop/redis-migrate-tool

#3.安装
[root@db02 ~]# cd redis-migrate-tool/
[root@db02 redis-migrate-tool]# autoreconf -fvi
[root@db02 redis-migrate-tool]# ./configure
[root@db02 redis-migrate-tool]# make

2.编写数据迁移脚本

[root@db02 redis-migrate-tool]# vim tocluster.sh

[source]
type: single
servers:
 - 172.16.1.52:6381

[target]
type: redis cluster
servers:
 - 172.16.1.54:6379

[common]
listen: 0.0.0.0:8888

3.单节点生成数据

[root@db03 ~]# vim data.sh 
#!/bin/bash
for i in {1001..2000};do
    redis-cli -c -p 6381 -h 172.16.1.52 set k${i} v${i}
done

[root@db03 ~]# sh data.sh 

4.迁移数据

[root@db02 redis-migrate-tool]# src/redis-migrate-tool -c tocluster.sh &

#把web中的后端地址 单节点ip:port ，修改为集群中ip:port ip:port

九、数据审计

redis自带的审计

[root@db02 redis-migrate-tool]# redis-cli -p 6379 
127.0.0.1:6379> set k1 ddddddddddddddddddddddddddddddddddddeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk
OK
127.0.0.1:6379> get k1
"ddddddddddddddddddddddddddddddddddddeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk"

#该工具只能找出最大的key，只能一个个处理
[root@db02 ~]# redis-cli -p 6379 --bigkeys

# Scanning the entire keyspace to find biggest keys as well as
# average sizes per key type.  You can use -i 0.1 to sleep 0.1 sec
# per 100 SCAN commands (not usually needed).

[00.00%] Biggest string found so far 'k1080' with 5 bytes
[00.00%] Biggest string found so far 'krange145' with 9 bytes
[96.50%] Biggest string found so far 'k1' with 345 bytes

1.安装工具

#1.安装依赖
[root@db02 ~]# yum install -y python-pip python-devel

#2.安装工具
[root@db02 ~]# pip install rdbtools python-lzf

#3.下载或上传
[root@db02 ~]# git clone https://github.com/sripathikrishnan/redis-rdb-tools
#或者上传
[root@db02 ~]# tar xf redis-rdb-tools.tar.gz

#4.安装
[root@db02 ~]# cd redis-rdb-tools
[root@db02 redis-rdb-tools]# python setup.py install

2.确认生成rdb文件

[root@db02 6381]# redis-cli -p 6381
127.0.0.1:6381> bgsave		#保存
Background saving started
127.0.0.1:6381> quit
[root@db02 6381]# ll
total 44
-rw-r--r-- 1 root root 26206 Aug  7 15:18 dump.rdb

3.使用工具分析文件

#使用工具生成CSV表格，下载下来进行分析
[root@db02 6381]# rdb -c memory ./dump.rdb -f memory.csv