Keepalived高可用集群应用

一、Keepalived相关介绍

1、Keepalived简介

　Keepalived软件起初是专门为LVS负载均衡软件设计的用来管理并监控LVS集群系统中各个服务节点的状态，后来又加入了可以实现高可用的VRRP功能。因此，Keepalived除了能够管理LVS软件外，还可以作为其他服务的高可用解决方案软件。

　Keepalived软件主要是通过VRRP协议实现高可用功能的，VRRP是Virtual Router Redundancy Protocol（虚拟路由器冗余协议）的缩写.VRRP出现的目的就是为了解决静态路由单点故障问题的，它能够保证当个别节点宕机时，整个网络可以不间断地运行。所以，Keepalived一方面具有配置管理LVS功能，同时还具有对LVS下面节点进行健康检查的功能，另一方面也可实现系统网络服务的高可用功能。

　Keepalived软件的官网为 http://www.keepalived.org 。

2、Keepalived服务功能

　1. 管理LVS负载均衡

　2. 实现对LVS集群节点健康检查功能

　3. 作为系统网络服务的高可能功能

3、Keepalived高可用故障切换转移原理

　Keepalived高可用服务对之间的故障切换转移，是通过VRRP来实现的；在Keepalived正常工作时，主Master节点会不断地向备Backup节点发送(多播的方式)心跳信息，当主节点发生故障时，就无法发送心跳信息，备节点也就因此无法继续检测到来自主机点的心跳了，于是调用自身的接管程序，接管主节点的IP资源及服务，而当主节点恢复时，备节点又会释放接管的IP资源及服务，恢复到原来的备用角色。

4、Keepalived配置文件介绍

　1. 全局定义部分

global_defs {
   notification_email {        #设置警报邮箱
     acassen@firewall.loc    #邮箱信息
     failover@firewall.loc
     sysadmin@firewall.loc
   }
   notification_email_from Alexandre.Cassen@firewall.loc    #设置发件人地址
   smtp_server 192.168.200.1        #设置smtp server地址
   smtp_connect_timeout 30          #设置smtp超时连接时间
   router_id LVS_DEVEL                 #路由或主机标识，注意唯一性
   vrrp_mcast_group4 224.100.100.100   #多播地址，不设置默认为224.0.0.18
}

　2. vrrp实例定义区块

vrrp_instance VI_1 {      #VRRP实例定义区块名字是VI_1
    state MASTER          #表示当前实例VI_1的角色状态，MASTER或BACKUP
    interface eth0        #对外提供服务的网络接口
    virtual_router_id 51  #虚拟路由ID唯一标识,范围0-255,主备两台服务器此处ID要相同
    priority 100          #优先级 范围1-254,越大越优先
    advert_int 1          #为同步通知间隔,主备之间通信检查的时间间隔,默认为1秒
    authentication {      #认证机制，同一实例主备认证密码要相同
        auth_type PASS    #认证类型，有PASS与HA两种
        auth_pass 1111    #密码，最长不超过8位
    }
    virtual_ipaddress {   #虚拟IP地址
        192.168.200.16    #此格式ip a显示 ifconfig不显示
        192.168.200.17/24 dev eth0 label eth0:1    #绑定接口为eth0，别名为eth0:1
    }
}

　3. 虚拟服务器定义部分

virtual_server 10.10.10.2 80 {   #设置虚拟服务器,指定虚拟IP和端口
    delay_loop 6             #健康检查时间为6秒
    lb_algo rr               #设置负载调度算法 rr|wrr|sh|dh|lc|wlc|lblc|lblcr|sed|nq 
    lb_kind NAT              #设置负载均衡机制 有NAT,TUN和DR三种模式
    persistence_timeout 50   #持久连接时长，50秒无响应则重新分配节点
    protocol TCP             #服务协议，仅支持tcp

    sorry_server 127.0.0.1 80    #所有RS故障时，备用服务器的地址

    real_server 192.168.200.2 80 {  #RS1节点 
        weight 1      #权重
        HTTP_GET {    #节点健康检测,应用层检测HTTP_GET|SSL_GET,传输层检测TCP_CHECK
            url {
              path /testurl/test.jsp  #定义要监控的URL
              status_code 200         #判断上述检测机制为健康状态的响应码
              digest 640205b7b0fc66c1ea91c463fac6334d  #判断为健康状态的响应内容校验码
            }
            connect_timeout 3      #请求连接超时时长
            nb_get_retry 3         #重试次数
            delay_before_retry 3   #重试之前的延迟时长
        }
    }

    real_server 192.168.200.3 80 {  #RS2节点
        weight 1
        HTTP_GET {
            url {
              path /testurl/test.jsp
              status_code 200 
              digest 640205b7b0fc66c1ea91c463fac6334c
            }
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}

#传输层检测 TCP_CHECK 
TCP_CHECK {
connect_ip <IP ADDRESS>:向当前RS的哪个IP地址发起健康状态检测请求
connect_port <PORT>:向当前RS的哪个PORT发起健康状态检测请求
bindto <IP ADDRESS>:发出健康状态检测请求时使用的源地址
bind_port <PORT>:发出健康状态检测请求时使用的源端口
connect_timeout <INTEGER>:连接请求的超时时长
}

　4. 脚本的调用方法

#在vrrp_instance VI_1 语句块最后面加下面行
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"

#!/bin/bash
#
contact='root@localhost' 

notify() {
    mailsubject="$(hostname) to be $1, vip floating"
    mailbody="$(date +'%F %T'): vrrp transition, $(hostname) changed to be $1" 
    echo "$mailbody" | mail -s "$mailsubject" $contact
}

case $1 in
master)
    notify master
    ;; 
backup)
    notify backup
    ;;
fault)
    notify fault
    ;; 
*)
    echo "Usage: $(basename $0) {master|backup|fault}" exit 1
    ;;
esac

 

二、相关配置实验

1、Keepalived单主配置实现

　系统：CentOS7.6

　主机：两台，一台主节点(192.168.214.27)，一台备节点(192.168.214.37)，VIP(192.168.214.100)

　软件包：keepalived（光盘yum源）

　(1) 两台主机分别安装keepalived

[root@centos7-27 ~]# yum install -y keepalived
[root@centos7-37 ~]# yum install -y keepalived

　(2) 主Master节点配置

[root@centos7-27 ~]# cp /etc/keepalived/keepalived.conf{,.bak}   #备份
[root@centos7-27 ~]# vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {
   notification_email {
     admin@localhost
   }
   notification_email_from keepalive@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node1
   vrrp_mcast_group4 224.100.100.100
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 66
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.214.100/16 dev eth0 label eth0:1
    }
} 

　(3) 备Backup节点配置

[root@centos7-37 ~]# cp /etc/keepalived/keepalived.conf{,.bak}
[root@centos7-37 ~]# vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {
   notification_email {
     admin@localhost
   }
   notification_email_from keepalive@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node2     #此处修改
   vrrp_mcast_group4 224.100.100.100
}

vrrp_instance VI_1 {
    state BACKUP    #此处修改
    interface eth0
    virtual_router_id 66   
    priority 80     #此处修改
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.214.100/16 dev eth0 label eth0:1
    }
}

　(4) 启动keepalived，然后进行测试

[root@centos7-27 ~]# systemctl start keepalived
[root@centos7-37 ~]# systemctl start keepalived
[root@centos7-27 ~]# ip a|grep 192.168.214.100    #可以看到VIP绑在主节点上
    inet 192.168.214.100/16 scope global secondary eth0:1
[root@centos7-37 ~]# ip a|grep 192.168.214.100
[root@centos7-27 ~]# systemctl stop keepalived   #在主节点上关闭keepalived
[root@centos7-27 ~]# ip a|grep 192.168.214.100    #主节点已无VIP
[root@centos7-37 ~]# ip a|grep 192.168.214.100    #可以看到VIP已漂移至备节点
    inet 192.168.214.100/16 scope global secondary eth0:1

2、Keepalived双主备配置实现

　系统：CentOS7.6

　主机：两台，一台主节点(192.168.214.27)，一台备节点(192.168.214.37)，VIP1(192.168.214.100)，VIP2(192.168.214.200)

　软件包：keepalived（光盘yum源）

　(1) 两台主机分别安装keepalived

[root@centos7-27 ~]# yum install -y keepalived
[root@centos7-37 ~]# yum install -y keepalived

　(2) 主Master节点配置

[root@centos7-27 ~]# vim /etc/keepalived/keepalived.conf
[root@centos7-27 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     admin@localhost
   }
   notification_email_from keepalive@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node1
   vrrp_mcast_group4 224.100.100.100
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 66
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.214.100/16 dev eth0 label eth0:1
    }
}

vrrp_instance VI_2 {    #添加实例VI_2
    state BACKUP    #第二个实例为备节点
    interface eth0
    virtual_router_id 88   #另一个虚拟路由ID
    priority 80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 654321
    }
    virtual_ipaddress {
        192.168.214.200/16 dev eth0 label eth0:2  #VIP2
    }
} 

　(3) 备Backup节点配置

[root@centos7-37 ~]# vim /etc/keepalived/keepalived.conf
[root@centos7-37 ~]# cat /etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {
   notification_email {
     admin@localhost
   }
   notification_email_from keepalive@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node2
   vrrp_mcast_group4 224.100.100.100
}

vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 66
    priority 80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.214.100/16 dev eth0 label eth0:1
    }
}

vrrp_instance VI_2 {   #添加实例VI_2
    state MASTER    #修改此处
    interface eth0
    virtual_router_id 88   #修改此处
    priority 100    #修改此处
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 654321
    }
    virtual_ipaddress {
        192.168.214.200/16 dev eth0 label eth0:2    #VIP2
    }
} 

　(4) 重启keepalived服务，然后进行测试

[root@centos7-27 ~]# systemctl restart keepalived
[root@centos7-37 ~]# systemctl restart keepalived
#正常情况下，VIP1在主节点上，VIP2在备节点上
[root@centos7-27 ~]# ip a|grep -E "192.168.214.100|192.168.214.200"
    inet 192.168.214.100/16 scope global secondary eth0:1
[root@centos7-37 ~]# ip a|grep -E "192.168.214.100|192.168.214.200"
    inet 192.168.214.200/16 scope global secondary eth0:2
#现在关闭主节点的keepalived服务，VIP1与VIP2应该都在备节点上
[root@centos7-27 ~]# systemctl stop keepalived
[root@centos7-27 ~]# ip a|grep -E "192.168.214.100|192.168.214.200"  #主节点无VIP1了
[root@centos7-37 ~]# ip a|grep -E "192.168.214.100|192.168.214.200"  #都在备节点上了
    inet 192.168.214.200/16 scope global secondary eth0:2
    inet 192.168.214.100/16 scope global secondary eth0:1
#现在启动主节点的keepalived服务，看是否回归正常
#以下可以看到是正常的了
[root@centos7-27 ~]# systemctl start keepalived
[root@centos7-27 ~]# ip a|grep -E "192.168.214.100|192.168.214.200"
    inet 192.168.214.100/16 scope global secondary eth0:1
[root@centos7-37 ~]# ip a|grep -E "192.168.214.100|192.168.214.200"
    inet 192.168.214.200/16 scope global secondary eth0:2

3、Keepalived+LVS配置实现

　系统：CentOS7.6

　主机：四台

　　两台keepalived主备：一台主节点(192.168.214.27/16)，一台备节点(192.168.214.37/16)，VIP1(192.168.214.100)

　　两台RS服务器：RS1 (192.168.214.47/16)，RS2 (192.168.214.57/16)

　软件包：keepalived，ipvsadm，httpd（光盘yum源）

　(1) 两台keepalived服务器安装 keepalived，ipvsadm服务

[root@centos7-27 ~]# yum install -y keepalived ipvsadm
[root@centos7-37 ~]# yum install -y keepalived ipvsadm

　(2) 配置keepalived主备及RS服务器

[root@centos7-27 ~]# cp /etc/keepalived/keepalived.conf{,.bak}
[root@centos7-27 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from keepalived@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node1
   vrrp_mcast_group4 224.100.100.100
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 66
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.214.100/32 dev eth0 label eth0:1
    }
}

virtual_server 192.168.214.100 80 {   #虚拟服务器
    delay_loop 6
    lb_algo wrr 
    lb_kind DR
    protocol TCP

    sorry_server 127.0.0.1 80

    real_server 192.168.214.47 80 {  #RS1
        weight 1
        HTTP_GET {
            url { 
              path /
              status_code 200
        }
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }

    real_server 192.168.214.57 80 {  #RS2
        weight 1
        HTTP_GET {
            url { 
              path /
          status_code 200
            }
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}
#从节点配置与以上大致一样，只需修改三项
# router_id node1 ----> router_id node2
# state MASTER ----> state BACKUP
# priority 100  ---->  priority 80

　(3) 配置RS1与RS2服务器，先安装httpd服务，再配置RS服务器的VIP与内核参数(这里使用脚本配置)

[root@centos7-47 ~]# yum install -y httpd
[root@centos7-47 ~]# echo "<h1>`hostname`</h1>" > /var/www/html/index.html     #准备主页
[root@centos7-47 ~]# systemctl start httpd   #启动httpd服务
[root@centos7-57 ~]# yum install -y httpd
[root@centos7-57 ~]# echo "<h1>`hostname`</h1>" > /var/www/html/index.html     #准备主页
[root@centos7-57 ~]# systemctl start httpd   #启动httpd服务
[root@centos7-47 ~]# bash lvs_dr_rs.sh start   #脚本配置VIP及相关内核参数
[root@centos7-57 ~]# bash lvs_dr_rs.sh start   #脚本配置VIP及相关内核参数

[root@centos7-47 ~]# cat lvs_dr_rs.sh 
#!/bin/bash

vip='192.168.214.100'
mask='255.255.255.255'
dev='lo:1'

case $1 in
start)
    echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
    echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
    echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
    echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
    ifconfig $dev $vip netmask $mask #broadcast $vip up
    #route add -host $vip dev $dev
    echo "The RS Server is Ready!"
    ;;
stop)
    ifconfig $dev down
    echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
    echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
    echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
    echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
    echo "The RS Server is Canceled!"
    ;;
*)
    echo "Usage:$(basename $0) start|stop"
    exit 1
    ;;
esac

　(4) 在keepalived主节点与备节点启动keepalived服务，使用ipvsadm查看LVS集群，并查看VIP的绑定情况

[root@centos7-27 ~]# systemctl start keepalived
[root@centos7-37 ~]# systemctl start keepalived
[root@centos7-27 ~]# ipvsadm -Ln  #可以看到lvs集群生成了
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.214.100:80 wrr
  -> 192.168.214.47:80            Route   1      0          0         
  -> 192.168.214.57:80            Route   1      0          0         
[root@centos7-27 ~]# ip a |grep 192.168.214.100  #VIP也绑在主节点上
    inet 192.168.214.100/32 scope global eth0:1

　(5) 在客户端测试LVS的调度情况及故障转移情况

[root@centos7 ~]# while true;do curl 192.168.214.100 ;sleep 1;done  #可以看到调度正常
<h1>centos7-47</h1>
<h1>centos7-57</h1>
<h1>centos7-47</h1>
<h1>centos7-57</h1>
<h1>centos7-47</h1>
<h1>centos7-57</h1>
<h1>centos7-47</h1>
<h1>centos7-57</h1>
...
#下面先测试VS服务器(keepalived)的主备故障转移
[root@centos7-27 ~]# systemctl stop keepalived
[root@centos7-27 ~]# ip a |grep 192.168.214.100  #VIP已不在主节点上了
[root@centos7-37 ~]# ip a |grep 192.168.214.100  #VIP已转移到从节点
    inet 192.168.214.100/32 scope global eth0:1
[root@centos7 ~]# while true;do curl 192.168.214.100 ;sleep 1;done
#访问也未断
<h1>centos7-47</h1>
<h1>centos7-57</h1>
<h1>centos7-47</h1>
<h1>centos7-57</h1>
<h1>centos7-47</h1>
<h1>centos7-57</h1>
...
[root@centos7-27 ~]# systemctl start keepalived  #重启主节点
[root@centos7-27 ~]# ip a |grep 192.168.214.100  #VIP重新回到主节点了
    inet 192.168.214.100/32 scope global eth0:1
[root@centos7-37 ~]# ip a |grep 192.168.214.100  #VIP已不在从节点上了
#下面测试RS服务器故障时，lvs的调度情况
#一开始是轮询的，现在关掉RS1的httpd服务
[root@centos7-47 ~]# systemctl stop httpd
[root@centos7 ~]# while true;do curl 192.168.214.100 ;sleep 1;done
#检查了几次，发现RS1断了，后续访问全调度给RS2了
<h1>centos7-47</h1>
<h1>centos7-57</h1>
<h1>centos7-47</h1>
<h1>centos7-57</h1>
curl: (7) Failed connect to 192.168.214.100:80; Connection refused
<h1>centos7-57</h1>
curl: (7) Failed connect to 192.168.214.100:80; Connection refused
<h1>centos7-57</h1>
curl: (7) Failed connect to 192.168.214.100:80; Connection refused
<h1>centos7-57</h1>
curl: (7) Failed connect to 192.168.214.100:80; Connection refused
<h1>centos7-57</h1>
curl: (7) Failed connect to 192.168.214.100:80; Connection refused
<h1>centos7-57</h1>
curl: (7) Failed connect to 192.168.214.100:80; Connection refused
<h1>centos7-57</h1>
<h1>centos7-57</h1>
<h1>centos7-57</h1>
...
#接下恢复RS1的httpd服务
[root@centos7-47 ~]# systemctl start httpd
[root@centos7 ~]# while true;do curl 192.168.214.100 ;sleep 1;done
#等RS1重新连接正常后，可以看到后续也参与了调度
...
<h1>centos7-57</h1>
<h1>centos7-57</h1>
<h1>centos7-57</h1>
<h1>centos7-57</h1>
<h1>centos7-47</h1>
<h1>centos7-57</h1>
<h1>centos7-47</h1>
<h1>centos7-57</h1>
<h1>centos7-47</h1>
<h1>centos7-57</h1>
<h1>centos7-47</h1>
<h1>centos7-57</h1>
<h1>centos7-47</h1>
...