redis cluster

1.集群信息

redis密码：******

序号 | ip | 端口 
:---|:---|:---
 1 | 192.168.127.172 | 6379
 2 | 192.168.127.173 | 6379
 3 | 192.168.127.174 | 6379
 4 | 192.168.127.175 | 6379
 5 | 192.168.127.176 | 6379
 6 | 192.168.127.177 | 6379

2.目录信息

cat << EOF >>  /etc/motd
redis bin path: /usr/local/redis/bin
redis conf path: /usr/local/redis/etc
redis data path: /usr/local/redis/data
redis log path: /usr/local/redis/log
redis restart cmd: systemctl redis stop/start/restart/status 
EOF

3.安装

wget http://download.redis.io/releases/redis-4.0.11.tar.gz
tar -zxvf redis-4.0.11.tar.gz
cd redis-4.0.11
make && make install PREFIX=/usr/local/redis-4.0.1
ln -s /usr/local/redis-4.0.1 /usr/local/redis
mkdir -p /usr/local/redis/etc  /usr/local/redis/data /usr/local/redis/log

4.配置文件/usr/local/redis/etc/redis.conf

bind 192.168.127.177 127.0.0.1
protected-mode yes
port 6379
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize yes
supervised no
pidfile /var/run/redis_6379.pid
loglevel notice
logfile "/usr/local/redis/log/redis.log"
masterauth dXzMHN2MaHUX
requirepass dXzMHN2MaHUX
databases 16
save 900 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
dir /usr/local/redis/data
slave-serve-stale-data yes
slave-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
slave-priority 100
appendonly no
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
lua-time-limit 5000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
aof-rewrite-incremental-fsync yes
cluster-enabled yes
cluster-node-timeout 5000
cluster-slave-validity-factor 0
cluster-require-full-coverage no
repl-ping-slave-period 1
repl-timeout 10
slave-serve-stale-data yes
slave-read-only yes
maxmemory 20gb

key值过期策略默认noeviction

5.安装集群依赖软件

yum install rubygems
curl -sSL https://rvm.io/mpapis.asc | gpg2 --import -
curl -L get.rvm.io | bash -s stable 
source /usr/local/rvm/scripts/rvm
rvm list known
rvm install  2.4.0
rvm use  2.4.0
rvm remove 2.0.0
ruby  --version
gem install redis

• 淘宝源-备用

gem sources -a https://ruby.taobao.org/
gem install redis

6.因为设置了redis密码，需要配置client.rb密码

[root@mycat01 work]# 
# 增加密码，或者配置完集群后增加密码
# /usr/local/rvm/gems/ruby-2.4.0/gems/redis-4.0.2/lib/redis/client.rb
      :password => "dXzMHN2MaHUX",

7.创建集群

[root@mall-redism1 ~]# /usr/local/redis/bin/redis-trib.rb create --replicas 1 192.168.127.172:6379 192.168.127.173:6379 192.168.127.174:6379 192.168.127.175:6379 192.168.127.176:6379 192.168.127.177:6379 
>>> Creating cluster
>>> Performing hash slots allocation on 6 nodes...
Using 3 masters:
192.168.127.172:6379
192.168.127.173:6379
192.168.127.174:6379
Adding replica 192.168.127.176:6379 to 192.168.127.172:6379
Adding replica 192.168.127.177:6379 to 192.168.127.173:6379
Adding replica 192.168.127.175:6379 to 192.168.127.174:6379
M: 94f5a34679cd3be24167c43b64fc356d6d60c6ab 192.168.127.172:6379
   slots:0-5460 (5461 slots) master
M: d6947aa9f626fd0b2c087ef1cb1d514cc191fd70 192.168.127.173:6379
   slots:5461-10922 (5462 slots) master
M: 68e2e0927ee21dcd3568381e279a77e956da979d 192.168.127.174:6379
   slots:10923-16383 (5461 slots) master
S: 29eb3393f1f446e933a3786d5eda89dedd23d8fb 192.168.127.175:6379
   replicates 68e2e0927ee21dcd3568381e279a77e956da979d
S: ae956c7d1fa18754c49c937800c692756bab1be6 192.168.127.176:6379
   replicates 94f5a34679cd3be24167c43b64fc356d6d60c6ab
S: 6731c9e20bedf2a1322c056a2b91ee63fd59729f 192.168.127.177:6379
   replicates d6947aa9f626fd0b2c087ef1cb1d514cc191fd70
Can I set the above configuration? (type 'yes' to accept): yes
>>> Nodes configuration updated
>>> Assign a different config epoch to each node
>>> Sending CLUSTER MEET messages to join the cluster
Waiting for the cluster to join......
>>> Performing Cluster Check (using node 192.168.127.172:6379)
M: 94f5a34679cd3be24167c43b64fc356d6d60c6ab 192.168.127.172:6379
   slots:0-5460 (5461 slots) master
   1 additional replica(s)
M: 68e2e0927ee21dcd3568381e279a77e956da979d 192.168.127.174:6379
   slots:10923-16383 (5461 slots) master
   1 additional replica(s)
S: 6731c9e20bedf2a1322c056a2b91ee63fd59729f 192.168.127.177:6379
   slots: (0 slots) slave
   replicates d6947aa9f626fd0b2c087ef1cb1d514cc191fd70
S: 29eb3393f1f446e933a3786d5eda89dedd23d8fb 192.168.127.175:6379
   slots: (0 slots) slave
   replicates 68e2e0927ee21dcd3568381e279a77e956da979d
S: ae956c7d1fa18754c49c937800c692756bab1be6 192.168.127.176:6379
   slots: (0 slots) slave
   replicates 94f5a34679cd3be24167c43b64fc356d6d60c6ab
M: d6947aa9f626fd0b2c087ef1cb1d514cc191fd70 192.168.127.173:6379
   slots:5461-10922 (5462 slots) master
   1 additional replica(s)
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.

8.使用效果

[root@mall-redism1 etc]# redis-cli -c -a dXzMHN2MaHUX
Warning: Using a password with '-a' option on the command line interface may not be safe.
127.0.0.1:6379> set k1 v1
-> Redirected to slot [12706] located at 192.168.127.174:6379
OK
192.168.127.174:6379> set k2 v2
-> Redirected to slot [449] located at 192.168.127.172:6379
OK
192.168.127.172:6379> set k3 v3
OK
192.168.127.172:6379> set k4 v4
-> Redirected to slot [8455] located at 192.168.127.173:6379
OK
192.168.127.173:6379> set k5 v5
-> Redirected to slot [12582] located at 192.168.127.174:6379
OK
192.168.127.174:6379> set k6 v6
-> Redirected to slot [325] located at 192.168.127.172:6379
OK
192.168.127.172:6379> keys *
1) "k6"
2) "k3"
3) "k2"

9.查看状态

[root@mall-rediss1 bmviewer]# redis-trib.rb check 127.0.0.1:6379
>>> Performing Cluster Check (using node 127.0.0.1:6379)
S: 29eb3393f1f446e933a3786d5eda89dedd23d8fb 127.0.0.1:6379
   slots: (0 slots) slave
   replicates 68e2e0927ee21dcd3568381e279a77e956da979d
M: 68e2e0927ee21dcd3568381e279a77e956da979d 192.168.127.174:6379
   slots:10923-16383 (5461 slots) master
   1 additional replica(s)
M: 94f5a34679cd3be24167c43b64fc356d6d60c6ab 192.168.127.172:6379
   slots:0-5460 (5461 slots) master
   1 additional replica(s)
S: ae956c7d1fa18754c49c937800c692756bab1be6 192.168.127.176:6379
   slots: (0 slots) slave
   replicates 94f5a34679cd3be24167c43b64fc356d6d60c6ab
M: 6731c9e20bedf2a1322c056a2b91ee63fd59729f 192.168.127.177:6379
   slots:5461-10922 (5462 slots) master
   1 additional replica(s)
S: d6947aa9f626fd0b2c087ef1cb1d514cc191fd70 192.168.127.173:6379
   slots: (0 slots) slave
   replicates 6731c9e20bedf2a1322c056a2b91ee63fd59729f
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.

10.service方式启动：/etc/init.d/redis

#!/bin/sh
#
# Simple Redis init.d script conceived to work on Linux systems
# as it does use of the /proc filesystem.

REDISPORT=6379
EXEC=/usr/local/bin/redis-server
CLIEXEC=/usr/local/bin/redis-cli
PASSWD=foobared

PIDFILE=/var/run/redis_${REDISPORT}.pid
CONF="/usr/local/redis/etc/redis_6379.conf"

case "$1" in
    start)
        if [ -f $PIDFILE ]
        then
                echo "$PIDFILE exists, process is already running or crashed"
        else
                echo "Starting Redis server..."
                $EXEC $CONF
        fi
        ;;
    stop)
        if [ ! -f $PIDFILE ]
        then
                echo "$PIDFILE does not exist, process is not running"
        else
                PID=$(cat $PIDFILE)
                echo "Stopping ..."
                $CLIEXEC -a $PASSWD -p $REDISPORT shutdown
                while [ -x /proc/${PID} ]
                do
                    echo "Waiting for Redis to shutdown ..."
                    sleep 1
                done
                echo "Redis stopped"
        fi
        ;;
    *)
        echo "Please use start or stop as first argument"
        ;;
esac

11.centos7 systemctl启动

# vi /lib/systemd/system/redis.service
[Unit]
Description=Redis
After=syslog.target network.target remote-fs.target nss-lookup.target
 
[Service]
Type=forking
ExecStart=/usr/local/redis/bin/redis-server /usr/local/redis/etc/redis.conf
#ExecReload=/bin/kill -s HUP $MAINPID
ExecStop=/usr/local/redis/bin/redis-cli -a dXzMHN2MaHUX -p 6379 shutdown
PrivateTmp=true
 
[Install]
WantedBy=multi-user.target
# ln -s /lib/systemd/system/redis.service /etc/systemd/system/multi-user.target.wants/redis.service

12.关闭一个主库，可以自动切换到从库

# 关闭前
[root@mall-redism1 ~]# redis-trib.rb check 192.168.127.172:6379
>>> Performing Cluster Check (using node 192.168.127.172:6379)
M: 94f5a34679cd3be24167c43b64fc356d6d60c6ab 192.168.127.172:6379
   slots:0-5460 (5461 slots) master
   1 additional replica(s)
S: 29eb3393f1f446e933a3786d5eda89dedd23d8fb 192.168.127.175:6379
   slots: (0 slots) slave
   replicates 68e2e0927ee21dcd3568381e279a77e956da979d
M: 68e2e0927ee21dcd3568381e279a77e956da979d 192.168.127.174:6379
   slots:10923-16383 (5461 slots) master
   1 additional replica(s)
M: 6731c9e20bedf2a1322c056a2b91ee63fd59729f 192.168.127.177:6379
   slots:5461-10922 (5462 slots) master
   1 additional replica(s)
S: ae956c7d1fa18754c49c937800c692756bab1be6 192.168.127.176:6379
   slots: (0 slots) slave
   replicates 94f5a34679cd3be24167c43b64fc356d6d60c6ab
S: d6947aa9f626fd0b2c087ef1cb1d514cc191fd70 192.168.127.173:6379
   slots: (0 slots) slave
   replicates 6731c9e20bedf2a1322c056a2b91ee63fd59729f
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.

# 关闭
[root@mall-redism1 ~]# redis-cli -c -a dXzMHN2MaHUX            
Warning: Using a password with '-a' option on the command line interface may not be safe.
127.0.0.1:6379> shutdown 
not connected> exit
[root@mall-redism1 ~]# ps -ef|grep redis
root     22608 21971  0 11:26 pts/0    00:00:00 grep --color=auto redis
[root@mall-redism1 ~]# 

# 关闭后，172被175接管
[root@mall-redism1 ~]# redis-trib.rb check 192.168.127.175:6379       
>>> Performing Cluster Check (using node 192.168.127.175:6379)
S: 29eb3393f1f446e933a3786d5eda89dedd23d8fb 192.168.127.175:6379
   slots: (0 slots) slave
   replicates 68e2e0927ee21dcd3568381e279a77e956da979d
M: 68e2e0927ee21dcd3568381e279a77e956da979d 192.168.127.174:6379
   slots:10923-16383 (5461 slots) master
   1 additional replica(s)
M: ae956c7d1fa18754c49c937800c692756bab1be6 192.168.127.176:6379
   slots:0-5460 (5461 slots) master
   0 additional replica(s)
M: 6731c9e20bedf2a1322c056a2b91ee63fd59729f 192.168.127.177:6379
   slots:5461-10922 (5462 slots) master
   1 additional replica(s)
S: d6947aa9f626fd0b2c087ef1cb1d514cc191fd70 192.168.127.173:6379
   slots: (0 slots) slave
   replicates 6731c9e20bedf2a1322c056a2b91ee63fd59729f
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.

# 启动关闭的172 maser后变成slave
[root@mall-redism1 ~]# redis-trib.rb check 192.168.127.175:6379       
>>> Performing Cluster Check (using node 192.168.127.175:6379)
S: 29eb3393f1f446e933a3786d5eda89dedd23d8fb 192.168.127.175:6379
   slots: (0 slots) slave
   replicates 68e2e0927ee21dcd3568381e279a77e956da979d
M: 68e2e0927ee21dcd3568381e279a77e956da979d 192.168.127.174:6379
   slots:10923-16383 (5461 slots) master
   1 additional replica(s)
S: 94f5a34679cd3be24167c43b64fc356d6d60c6ab 192.168.127.172:6379
   slots: (0 slots) slave
   replicates ae956c7d1fa18754c49c937800c692756bab1be6
M: ae956c7d1fa18754c49c937800c692756bab1be6 192.168.127.176:6379
   slots:0-5460 (5461 slots) master
   1 additional replica(s)
M: 6731c9e20bedf2a1322c056a2b91ee63fd59729f 192.168.127.177:6379
   slots:5461-10922 (5462 slots) master
   1 additional replica(s)
S: d6947aa9f626fd0b2c087ef1cb1d514cc191fd70 192.168.127.173:6379
   slots: (0 slots) slave
   replicates 6731c9e20bedf2a1322c056a2b91ee63fd59729f
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.

13.日志警告处理

WARNING you have Transparent Huge Pages (THP) support enabled in your kernel. This will create latency and mem
ory usage issues with Redis. To fix this issue run the command 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' as root, and add i
t to your /etc/rc.local in order to retain the setting after a reboot. Redis must be restarted after THP is disabled

15.压测

# Intel(R) Xeon(R) CPU E5-4620 0 @ 2.20GHz
[bmviewer@mall-redism2 ~]$ redis-benchmark -c 9000 -h 192.168.127.172 -p 6379 -t set,get -n 1000000 -a dXzMHN2MaHUX -d 10000 -q 
SET: 32953.27 requests per second
GET: 52454.89 requests per second

# Intel(R) Xeon(R) CPU E5-2690 v2 @ 3.00GHz
[root@master ~]# redis-benchmark -c 9000 -h 172.16.10.154 -p 6379 -t set,get -n 1000000 -a foobared -d 10000 -q   
SET: 25188.92 requests per second
GET: 55682.39 requests per second

# Power
KYL-DB-MYSQL01:/usr/local/redis/bin # redis-benchmark -c 9000 -h 172.16.10.154 -p 6379 -t set,get -n 1000000 -a foobared -d 10000 -q
SET: 4390.70 requests per second
GET: 10911.79 requests per second

cache

CREATE TABLE `system_config` (
  `config_key` varchar(255) NOT NULL COMMENT '配置key',
  `config_value` varchar(512) NOT NULL COMMENT '配置value',
  `info` varchar(255) NOT NULL COMMENT '配置说明',
  `status` tinyint(4) NOT NULL COMMENT '1:可用,0:不可用',
  `order_id` int(11) NOT NULL COMMENT '顺序',
  PRIMARY KEY (`config_key`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='系统配置'

192.168.127.176:6379:20480
192.168.127.172:6379:20480
192.168.127.177:6379:20480
192.168.127.173:6379:20480
192.168.127.174:6379:20480
192.168.127.175:6379:20480

shell监控密码问题

[bmviewer@mall-rediss3 etc]$ redis-cli -a dXzMHN2MaHUX info memory|grep maxmemory|awk 'NR==1{print $0}'              
Warning: Using a password with '-a' option on the command line interface may not be safe.
maxmemory:21474836480
[bmviewer@mall-rediss3 etc]$ redis-cli -a dXzMHN2MaHUX info memory 2>/dev/null |grep maxmemory         
maxmemory:21474836480
maxmemory_human:20.00G
maxmemory_policy:noeviction