通过keepalived实现多主集群方案

一、 环境说明:
1、服务器列表：
proxy01:
eth0: 192.168.56.11 eth2: 192.168.156.11

proxy02:
eth0: 192.168.56.12 eth2: 192.168.156.12

proxy03:
eth0: 192.168.56.13 eth2: 192.168.156.13

2、拓扑图:

二、 实验步骤
1、 在所有节点上操作

# yum install –y keepalived haproxy

2、 proxy01节点配置

[root@proxy01 ~]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   router_id LVS_DEVEL
   vrrp_skip_check_adv_addr
#   vrrp_strict
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_sync_group VG1 {
    group {
        VI_1
        VI_11
    }
}

vrrp_sync_group VG2 {
    group {
        VI_2
        VI_12
    }
}

vrrp_sync_group VG3 {
    group {
        VI_3
        VI_13
    }
}

vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 51
    ##nopreempt 启用抢占模式
    priority 10
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
}
    use_vmac #启用vmac模式，会创建虚拟机接口，vip绑定到虚拟接口上
    vmac_xmit_base #发送和接收VRRP包的虚拟MAC地址
    unicast_src_ip 192.168.56.11
    unicast_peer {
        192.168.56.12
        192.168.56.13
    }
    virtual_ipaddress {
        192.168.56.101
    }
}

vrrp_instance VI_2 {
    state BACKUP
    interface eth0
    virtual_router_id 52
    priority 20
    ##nopreempt
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
}
    use_vmac
    vmac_xmit_base
    unicast_src_ip 192.168.56.11
    unicast_peer {
        192.168.56.12
        192.168.56.13
}
    virtual_ipaddress {
        192.168.56.102
    }
}

vrrp_instance VI_3 {
    state BACKUP
    interface eth0
    virtual_router_id 53
    priority 30
    advert_int 1
    ##nopreempt
    authentication {
        auth_type PASS
        auth_pass 1111
}
    use_vmac
    vmac_xmit_base
    unicast_src_ip 192.168.56.11
    unicast_peer {
        192.168.56.12
        192.168.56.13
    }
    virtual_ipaddress {
        192.168.56.103
    }
}

vrrp_instance VI_11 {
    state BACKUP
    interface eth2
    virtual_router_id 51
    ##nopreempt
    priority 10
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
}
    use_vmac
    vmac_xmit_base
    unicast_src_ip 192.168.156.11
    unicast_peer {
        192.168.156.12
        192.168.156.13
    }
    virtual_ipaddress {
        192.168.156.101
    }
}

vrrp_instance VI_12 {
    state BACKUP
    interface eth2
    virtual_router_id 52
    priority 20
    #nopreempt
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
}
    use_vmac
    vmac_xmit_base
    unicast_src_ip 192.168.156.11
    unicast_peer {
        192.168.156.12
        192.168.156.13
    }
    virtual_ipaddress {
        192.168.156.102
    }
}

vrrp_instance VI_13 {
    state BACKUP
    interface eth2
    virtual_router_id 53
    priority 30
    advert_int 1
    #nopreempt
    authentication {
        auth_type PASS
        auth_pass 1111
}
    use_vmac
    vmac_xmit_base
    unicast_src_ip 192.168.156.11
    unicast_peer {
        192.168.156.12
        192.168.156.13
    }
    virtual_ipaddress {
        192.168.156.103
    }
}

　　

3、 proxy02节点配置

[root@proxy02 ~]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   router_id LVS_DEVEL
   vrrp_skip_check_adv_addr
#   vrrp_strict
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_sync_group VG1 {
    group {
        VI_1
        VI_11
    }
}

vrrp_sync_group VG2 {
    group {
        VI_2
        VI_12
    }
}

vrrp_sync_group VG3 {
    group {
        VI_3
        VI_13
    }
}

vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 51
    #nopreempt
    priority 20
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
}
    use_vmac
    vmac_xmit_base
    unicast_src_ip 192.168.56.12
    unicast_peer {
        192.168.56.11
        192.168.56.13
    }
    virtual_ipaddress {
        192.168.56.101
    }
}

vrrp_instance VI_2 {
    state BACKUP
    interface eth0
    virtual_router_id 52
    #nopreempt
    priority 30
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
}
    use_vmac
    vmac_xmit_base
    unicast_src_ip 192.168.56.12
    unicast_peer {
        192.168.56.11
        192.168.56.13
    }
    virtual_ipaddress {
        192.168.56.102
    }
}

vrrp_instance VI_3 {
    state BACKUP
    interface eth0
    virtual_router_id 53
    #nopreempt
    priority 10
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
}
    use_vmac
    vmac_xmit_base
    unicast_src_ip 192.168.56.12
    unicast_peer {
        192.168.56.11
        192.168.56.13
    }
    virtual_ipaddress {
        192.168.56.103
    }
}

vrrp_instance VI_11 {
    state BACKUP
    interface eth2
    virtual_router_id 51
    #nopreempt
    priority 20
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
}
    use_vmac
    vmac_xmit_base
    unicast_src_ip 192.168.156.12
    unicast_peer {
        192.168.156.11
        192.168.156.13
    }
    virtual_ipaddress {
        192.168.156.101
    }
}

vrrp_instance VI_12 {
    state BACKUP
    interface eth2
    virtual_router_id 52
    #nopreempt
    priority 30
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
}
    use_vmac
    vmac_xmit_base
    unicast_src_ip 192.168.156.12
    unicast_peer {
        192.168.156.11
        192.168.156.13
    }
    virtual_ipaddress {
        192.168.156.102
    }
}

vrrp_instance VI_13 {
    state BACKUP
    interface eth2
    virtual_router_id 53
    #nopreempt
    priority 10
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
}
    use_vmac
    vmac_xmit_base
    unicast_src_ip 192.168.156.12
    unicast_peer {
        192.168.156.11
        192.168.156.13
    }
    virtual_ipaddress {
        192.168.156.103
    }
}

　　

4、 proxy03节点配置

[root@proxy03 ~]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
#   notification_email {
#     acassen@firewall.loc
#     failover@firewall.loc
#     sysadmin@firewall.loc
#   }
#   notification_email_from Alexandre.Cassen@firewall.loc
#   smtp_server 192.168.200.1
#   smtp_connect_timeout 30
   router_id LVS_DEVEL
   vrrp_skip_check_adv_addr
#   vrrp_strict
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_sync_group VG1 {
    group {
        VI_1
        VI_11
    }
}

vrrp_sync_group VG2 {
    group {
        VI_2
        VI_12
    }
}

vrrp_sync_group VG3 {
    group {
        VI_3
        VI_13
    }
}

vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 51
    #nopreempt
    priority 30
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
}
    use_vmac
    vmac_xmit_base
    unicast_src_ip 192.168.56.13
    unicast_peer {
        192.168.56.11
        192.168.56.12
    }
    virtual_ipaddress {
        192.168.56.101
    }
}

vrrp_instance VI_2 {
    state BACKUP
    interface eth0
    virtual_router_id 52
    #nopreempt
    priority 10
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
}
    use_vmac
    vmac_xmit_base
    unicast_src_ip 192.168.56.13
    unicast_peer {
        192.168.56.11
        192.168.56.12
    }
    virtual_ipaddress {
        192.168.56.102
    }
}

vrrp_instance VI_3 {
    state BACKUP
    interface eth0
    virtual_router_id 53
    #nopreempt
    priority 20
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
}
    use_vmac
    vmac_xmit_base
    unicast_src_ip 192.168.56.13
    unicast_peer {
        192.168.56.11
        192.168.56.12
    }
    virtual_ipaddress {
        192.168.56.103
    }
}

vrrp_instance VI_11 {
    state BACKUP
    interface eth2
    virtual_router_id 51
    #nopreempt
    priority 30
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
}
    use_vmac
    vmac_xmit_base
    unicast_src_ip 192.168.156.13
    unicast_peer {
        192.168.156.11
        192.168.156.12
    }
    virtual_ipaddress {
        192.168.156.101
    }
}

vrrp_instance VI_12 {
    state BACKUP
    interface eth2
    virtual_router_id 52
    #nopreempt
    priority 10
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
}
    use_vmac
    vmac_xmit_base
    unicast_src_ip 192.168.156.13
    unicast_peer {
        192.168.156.11
        192.168.156.12
    }
    virtual_ipaddress {
        192.168.156.102
    }
}

vrrp_instance VI_13 {
    state BACKUP
    interface eth2
    virtual_router_id 53
    #nopreempt
    priority 20
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
}
    use_vmac
    vmac_xmit_base
    unicast_src_ip 192.168.156.13
    unicast_peer {
        192.168.156.11
        192.168.156.12
    }
    virtual_ipaddress {
        192.168.156.103
    }
}

　　

5、 启动服务

在所有节点操作

# systemctl enable keepalived.service && systemctl start keepalived.service

6、 结果验证

[root@proxy01 ~]# ip add 
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN qlen 1
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 08:00:27:4b:3e:34 brd ff:ff:ff:ff:ff:ff
    inet 192.168.56.11/24 brd 192.168.56.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::a00:27ff:fe4b:3e34/64 scope link 
       valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 08:00:27:17:a2:bc brd ff:ff:ff:ff:ff:ff
    inet 10.0.3.15/24 brd 10.0.3.255 scope global dynamic eth1
       valid_lft 85732sec preferred_lft 85732sec
    inet6 fe80::a00:27ff:fe17:a2bc/64 scope link 
       valid_lft forever preferred_lft forever
4: eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 08:00:27:ff:0e:a8 brd ff:ff:ff:ff:ff:ff
    inet 192.168.156.11/24 brd 192.168.156.255 scope global eth2
       valid_lft forever preferred_lft forever
    inet6 fe80::a00:27ff:feff:ea8/64 scope link 
       valid_lft forever preferred_lft forever
5: vrrp.51@eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN qlen 1000
    link/ether 00:00:5e:00:01:33 brd ff:ff:ff:ff:ff:ff
6: vrrp.52@eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN qlen 1000
    link/ether 00:00:5e:00:01:34 brd ff:ff:ff:ff:ff:ff
7: vrrp.53@eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN qlen 1000
    link/ether 00:00:5e:00:01:35 brd ff:ff:ff:ff:ff:ff
    inet 192.168.56.103/32 scope global vrrp.53
       valid_lft forever preferred_lft forever
8: vrrp1.51@eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN qlen 1000
    link/ether 00:00:5e:00:01:33 brd ff:ff:ff:ff:ff:ff
9: vrrp1.52@eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN qlen 1000
    link/ether 00:00:5e:00:01:34 brd ff:ff:ff:ff:ff:ff
10: vrrp1.53@eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN qlen 1000
    link/ether 00:00:5e:00:01:35 brd ff:ff:ff:ff:ff:ff
    inet 192.168.156.103/32 scope global vrrp1.53
       valid_lft forever preferred_lft forever

[root@proxy02 ~]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN qlen 1
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 08:00:27:ee:7d:7e brd ff:ff:ff:ff:ff:ff
    inet 192.168.56.12/24 brd 192.168.56.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::a00:27ff:feee:7d7e/64 scope link 
       valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 08:00:27:a4:ce:2f brd ff:ff:ff:ff:ff:ff
    inet 10.0.3.15/24 brd 10.0.3.255 scope global dynamic eth1
       valid_lft 85717sec preferred_lft 85717sec
    inet6 fe80::a00:27ff:fea4:ce2f/64 scope link 
       valid_lft forever preferred_lft forever
4: eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 08:00:27:6c:16:4d brd ff:ff:ff:ff:ff:ff
    inet 192.168.156.12/24 brd 192.168.156.255 scope global eth2
       valid_lft forever preferred_lft forever
    inet6 fe80::a00:27ff:fe6c:164d/64 scope link 
       valid_lft forever preferred_lft forever
5: vrrp.51@eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN qlen 1000
    link/ether 00:00:5e:00:01:33 brd ff:ff:ff:ff:ff:ff
6: vrrp.52@eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN qlen 1000
    link/ether 00:00:5e:00:01:34 brd ff:ff:ff:ff:ff:ff
    inet 192.168.56.102/32 scope global vrrp.52
       valid_lft forever preferred_lft forever
7: vrrp.53@eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN qlen 1000
    link/ether 00:00:5e:00:01:35 brd ff:ff:ff:ff:ff:ff
8: vrrp1.51@eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN qlen 1000
    link/ether 00:00:5e:00:01:33 brd ff:ff:ff:ff:ff:ff
9: vrrp1.52@eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN qlen 1000
    link/ether 00:00:5e:00:01:34 brd ff:ff:ff:ff:ff:ff
    inet 192.168.156.102/32 scope global vrrp1.52
       valid_lft forever preferred_lft forever
10: vrrp1.53@eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN qlen 1000
    link/ether 00:00:5e:00:01:35 brd ff:ff:ff:ff:ff:ff

[root@proxy03 ~]# ip add 
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN qlen 1
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 08:00:27:87:1a:f0 brd ff:ff:ff:ff:ff:ff
    inet 192.168.56.13/24 brd 192.168.56.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::a00:27ff:fe87:1af0/64 scope link 
       valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 08:00:27:6e:be:14 brd ff:ff:ff:ff:ff:ff
    inet 10.0.3.15/24 brd 10.0.3.255 scope global dynamic eth1
       valid_lft 85696sec preferred_lft 85696sec
    inet6 fe80::a00:27ff:fe6e:be14/64 scope link 
       valid_lft forever preferred_lft forever
4: eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 08:00:27:b0:64:83 brd ff:ff:ff:ff:ff:ff
    inet 192.168.156.13/24 brd 192.168.156.255 scope global eth2
       valid_lft forever preferred_lft forever
    inet6 fe80::a00:27ff:feb0:6483/64 scope link 
       valid_lft forever preferred_lft forever
5: vrrp.51@eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN qlen 1000
    link/ether 00:00:5e:00:01:33 brd ff:ff:ff:ff:ff:ff
    inet 192.168.56.101/32 scope global vrrp.51
       valid_lft forever preferred_lft forever
6: vrrp.52@eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN qlen 1000
    link/ether 00:00:5e:00:01:34 brd ff:ff:ff:ff:ff:ff
7: vrrp.53@eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN qlen 1000
    link/ether 00:00:5e:00:01:35 brd ff:ff:ff:ff:ff:ff
8: vrrp1.51@eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN qlen 1000
    link/ether 00:00:5e:00:01:33 brd ff:ff:ff:ff:ff:ff
    inet 192.168.156.101/32 scope global vrrp1.51
       valid_lft forever preferred_lft forever
9: vrrp1.52@eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN qlen 1000
    link/ether 00:00:5e:00:01:34 brd ff:ff:ff:ff:ff:ff
10: vrrp1.53@eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN qlen 1000
    link/ether 00:00:5e:00:01:35 brd ff:ff:ff:ff:ff:ff

7、 总结
　　之前我们用keepalived做集群时一般使用它构建服务器主从，也就是只有一个vip，并且这个vip只是在主节点上，当主节点宕机时，vip漂移到从节点上，从而实现高可用。但随着业务的发展，单个节点随之成为业务的性能瓶颈，及时我们使用的负载均衡再强大，服务器配置再高，也不可能单节点抗住所有流量。而通过这种方案，在keepalived的主从基础上扩展一下，通过配置多个vip，每个keepalived节点互为主从，正常情况下保证所有服务器都能拥有一个vip，然后通过dns负载均衡技术，将业务流量转发到每个vip。从而在一定程度上避免了单服务器的性能瓶颈。