LVS之DR模式实战及高可用性

author：JevonWei
版权声明：原创作品

---

LVS-DR实现同网段调度web模式

• 拓扑环境

网络环境

RS1   
    RIP 192.168.198.138/24
    VIP 192.168.198.100/32
    GW  192.168.198.130
RS2 
    RIP 192.168.198.132/24
    VIP 192.168.198.100/32
    GW  192.168.198.130
VS
    DIP 192.168.198.128/24
    VIP 192.168.198.100/32
    GW  192.168.198.130
    
route 
    192.168.198.130/24
    172.16.253.166/16
Client 
    172.16.254.150/16
    GW 172.16.253.166
RS1,RS2的网关指向192.168.198.130

RS1

[root@RS1 html]# route del default gw 192.168.198.128
[root@RS1 html]# route add default gw 192.168.198.130

[root@RS1 ~]# iptables -F
[root@RS1 ~]# yum -y install httpd
[root@RS1 ~]# vim /var/www/html/index.html 
    welcome to RS1
[root@RS1 ~]# service httpd start

RS2

[root@RS2 network-scripts]# route add -net 172.16.0.0/16 gw 192.168.198.130
[root@RS2 network-scripts]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
172.16.0.0      192.168.198.130 255.255.0.0     UG    0      0        0 ens34
192.168.198.0   0.0.0.0         255.255.255.0   U     100    0        0 ens34
[root@RS2 ~]# iptables -F
[root@RS2 ~]# yum -y install httpd
[root@RS2 ~]# vim /var/www/html/index.html 
    welcome to RS2
[root@RS2 ~]# systemctl start httpd 

VS

添加网关路由信息
[root@VS ~]# route add default gw 192.168.198.130
[root@VS ~]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.198.130 0.0.0.0         UG    0      0        0 ens34
192.168.122.0   0.0.0.0         255.255.255.0   U     0      0        0 virbr0
192.168.198.0   0.0.0.0         255.255.255.0   U     100    0        0 ens34

[root@VS ~]# vim lvs_dr.sh
#! /bin/bash
vip=192.168.198.100
server=$vip:80
rip1=192.168.198.138
rip2=192.168.198.132
sch=wlc
dev=ens34:1   \绑定网卡ens34
case $1 in
start)
    ifconfig $dev $vip/32 broadcast $vip  \绑定vip到ens34网卡上
    iptables -F
    ipvsadm -A -t $server -s $sch
    ipvsadm -a -t $server -r $rip1 -g -w 3
    ipvsadm -a -t $server -r $rip2 -g -w 1
    ;;
stop)
    ipvsadm -C
    ipconfig $dev down
    ;;
*)
    echo "Usage:$(basename $0) start|stop"
    exit 1
    ;;
esac

[root@danran ~]# bash lvs_dr.sh start
[root@danran ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
  TCP  192.168.198.100:80 wlc
  -> 192.168.198.132:80           Route   1      0          0         
  -> 192.168.198.138:80           Route   3      0          0   

添加RS1和RS2的VIP

dr_vip_rs.sh 为添加RS服务端VIP地址的脚本
[root@RS1 ~]# vim dr_vip_rs.sh 
#!/bin/bash
#
vip=192.168.198.100
mask='255.255.255.255'
dev=lo:1
case $1 in
start)
   echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
   echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
   echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
   echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
   ifconfig $dev $vip netmask $mask broadcast $vip up
   route add -host $vip dev $dev
   echo "VS server is Ready "
   ;;
stop)
   ifconfig $dev down
   echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
   echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
   echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
   echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
   echo "VS server is Cancel"
   ;;
*)
   echo "Usage $(basename $0) start|stop"
   exit 1
   ;;
esac   
[root@RS1 ~]# bash dr_vip_rs.sh start
VS server is Ready 
[root@RS2 ~]# bash dr_vip_rs.sh start
VS server is Ready 

client

添加路由
[root@danran ~]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
169.254.0.0     0.0.0.0         255.255.0.0     U     1002   0        0 eth0
172.16.0.0      0.0.0.0         255.255.0.0     U     0      0        0 eth0
0.0.0.0         172.16.0.1      0.0.0.0         UG    0      0        0 eth0

[root@danran ~]# route del default gw 172.16.0.1 
[root@danran ~]# route add default gw 172.16.253.166 \添加默认路由
[root@danran ~]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
169.254.0.0     0.0.0.0         255.255.0.0     U     1002   0        0 eth0
172.16.0.0      0.0.0.0         255.255.0.0     U     0      0        0 eth0
0.0.0.0         172.16.253.166  0.0.0.0         UG    0      0        0 eth0

测试  
[root@danran ~]# for i in {1..10};do curl --connect-timeout 1 192.168.198.100 ;sleep 1;done      
welcome to RS2
welcome to RS1
welcome to RS1
welcome to RS1
welcome to RS2
welcome to RS1
welcome to RS1
welcome to RS1
welcome to RS2
welcome to RS1

LVS-DR实现跨网段

网络拓扑

网络环境

RS1   
    RIP 192.168.198.138/24
    VIP 192.168.80.100/32
    GW  192.168.198.130
RS2 
    RIP 192.168.198.132/24
    VIP 192.168.80.100/32
    GW  192.168.198.130
VS
    DIP 192.168.198.128/24
    VIP 192.168.198.100/32
    GW  192.168.198.130
    
route 
    192.168.198.130/24
    192.168.80.130/8
    172.16.253.166/16
    GW  192.168.198.130
Client 
    172.16.254.150/16
    GW 172.16.253.166
RS1,RS2的网关指向192.168.198.130

route

ens38网卡添加第二个IP
[root@route network-scripts]# nmcli connection modify ens38 +ipv4.addresses 192.168.80.130/8
[root@route ~]# nmcli connection up ens38  \启动ens38网卡
[root@route ~]# ip a
[root@route ~]# route add default gw 192.168.198.130

VS

编辑LVS_DR的配置脚本
[root@VS ~]# vim lvs_dr.sh 
#! /bin/bash
vip=192.168.80.100
server=$vip:80
rip1=192.168.198.138
rip2=192.168.198.132
sch=rr
dev=ens34:1
case $1 in
start)
    ifconfig $dev $vip/32 broadcast $vip
    ipvsadm -A -t $server -s $sch
    ipvsadm -a -t $server -r $rip1 -g -w 3
    ipvsadm -a -t $server -r $rip2 -g -w 1
    ;;
stop)
    ipvsadm -C
    ifconfig $dev down
    ;;
*)
    echo "Usage:$(basename $0) start|stop"
    exit 1
    ;;
esac

添加网关及默认路由
[root@VS ~]# route add default gw 192.168.198.130 
[root@VS ~]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.198.130 0.0.0.0         UG    0      0        0 ens34
192.168.122.0   0.0.0.0         255.255.255.0   U     0      0        0 virbr0
192.168.198.0   0.0.0.0         255.255.255.0   U     100    0        0 ens34

RS1和RS2配置vip IP

[root@RS1 ~]# vim dr_vip_rs.sh 
#!/bin/bash
#
vip=192.168.80.100
mask='255.255.255.255'
dev=lo:1
case $1 in
start)
   echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
   echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
   echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
   echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
   ifconfig $dev $vip netmask $mask broadcast $vip up
   # route add -host $vip dev $dev
   echo "VS server is Ready "
   ;;
stop)
   ifconfig $dev down
   echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
   echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
   echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
   echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
   echo "VS server is Cancel"
   ;;
*)
   echo "Usage $(basename $0) start|stop"
   exit 1
   ;;
esac

[root@RS1 ~]# bash dr_vip_rs.sh start
VS server is Ready 
[root@RS2 ~]#  bash dr_vip_rs.sh start
VS server is Ready 

路由信息
[root@RS2 ~]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.198.130 0.0.0.0         UG    100    0        0 ens34
192.168.198.0   0.0.0.0         255.255.255.0   U     100    0        0 ens34
[root@RS1 ~]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
192.168.198.0   0.0.0.0         255.255.255.0   U     0      0        0 eth1
0.0.0.0         192.168.198.130 0.0.0.0         UG    0      0        0 eth1

client

[root@client ~]# for i in {1..10};do curl 192.168.80.100 ;done
welcome to RS2
welcome to RS1
welcome to RS2
welcome to RS1
welcome to RS2
welcome to RS1
welcome to RS2
welcome to RS1
welcome to RS2
welcome to RS1

将http和https两个不同的服务打标签,从而使http和https做成一个集群服务

FireWall Mark技术

VS

[root@VS ~]# iptables -t mangle -A PREROUTING -d 192.168.80.100 -p tcp -m multiport --dports 80,443 -j MARK --set-mark 10
[root@VS ~]# vim lvs_dr_vs_fwm.sh
#! /bin/bash
vip=192.168.80.100
server=10
rip1=192.168.198.138
rip2=192.168.198.132
sch=rr
dev=ens34:1
case $1 in
start)
    ifconfig $dev $vip/32 broadcast $vip
    ipvsadm -A -f $server -s $sch
    ipvsadm -a -f $server -r $rip1 -g -w 3
    ipvsadm -a -f $server -r $rip2 -g -w 1
    ;;
stop)
    ipvsadm -C
    ifconfig $dev down
    ;;
*)
    echo "Usage:$(basename $0) start|stop"
    exit 1
    ;;
esac

[root@VS ~]# bash lvs_dr_vs_fwm.sh start
[root@VS ~]# ipvsadm -Ln                
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
  FWM  10 rr
  -> 192.168.198.132:0            Route   1      0          0         
  -> 192.168.198.138:0            Route   3      0          0 

client

[root@client ~]# curl 192.168.80.100；curl -k https://192.168.80.100

实现DR持久连接

PFWMC基于防火墙的持久连接

VS

    [root@VS ~]# iptables -t mangle -A PREROUTING -d 192.168.80.100 -p tcp -m multiport --dports 80,443 -j MARK --set-mark 10
    [root@VS ~]# vim lvs_dr_vs_fwm.sh 
    #! /bin/bash
    vip=192.168.80.100
    server=10
    rip1=192.168.198.138
   rip2=192.168.198.132
    sch=rr
    dev=ens34:1
    case $1 in
    start)
        ifconfig $dev $vip/32 broadcast $vip
        ipvsadm -A -f $server -s $sch -p 600  \-p 设置持久连接为600s
        ipvsadm -a -f $server -r $rip1 -g -w 3
        ipvsadm -a -f $server -r $rip2 -g -w 1
        ;;
    stop)
        ipvsadm -C
        ifconfig $dev down
        ;;
    *)
        echo "Usage:$(basename $0) start|stop"
        exit 1
    ;    ;
    esac

[root@VS ~]# bash lvs_dr_vs_fwm.sh start
[root@VS ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
  FWM  10 rr persistent 600 \持久连接为600s
  -> 192.168.198.132:0            Route   1      0          0         
  -> 192.168.198.138:0            Route   3      0          0 

client

[root@client ~]# curl 192.168.80.100 
welcome to RS2
[root@client ~]# curl 192.168.80.100 
welcome to RS2
[root@client ~]# curl 192.168.80.100 
welcome to RS2
[root@client ~]# curl https://192.168.80.100 
welcome to RS2
[root@client ~]# curl https://192.168.80.100 
welcome to RS2

PCC基于0端口的持久连接

VS

[root@VS ~]# iptables -t mangle -A PREROUTING -d 192.168.80.100 -p tcp -m multiport --dports 80,443 -j MARK --set-mark 10
[root@VS ~]# vim lvs_dr_vs_per.sh       
#! /bin/bash
vip=192.168.80.100
server=$vip:0
rip1=192.168.198.138
rip2=192.168.198.132
sch=rr
dev=ens34:1
case $1 in
start)
    ifconfig $dev $vip netmask 255.255.255.255 broadcast $vip
    ipvsadm -A -t $server -s $sch -p 600
    ipvsadm -a -t $server -r $rip1 -g -w 3
    ipvsadm -a -t $server -r $rip2 -g -w 1
    ;;
stop)
    ipvsadm -C
    ifconfig $dev down
    ;;
*)
    echo "Usage:$(basename $0) start|stop"
    exit 1
    ;;
esac

[root@VS ~]# bash lvs_dr_vs_per.sh start
[root@VS ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.80.100:0 rr persistent 600
  -> 192.168.198.132:0            Route   1      0          0         
  -> 192.168.198.138:0            Route   3      0          0

LVS高可用性

VS(编写脚本判断RS服务器是否故障)

[root@VS ~]# vim lvs_dr_vs.sh
#! /bin/bash
vip=192.168.80.100
server=$vip:80
rip1=192.168.198.138
rip2=192.168.198.132
sch=rr
dev=ens34:1
case $1 in
start)
    ifconfig $dev $vip/32 broadcast $vip
    ipvsadm -A -t $server -s $sch 
    ipvsadm -a -t $server -r $rip1 -g -w 3
    ipvsadm -a -t $server -r $rip2 -g -w 1
    ;;
stop)
    ipvsadm -C
    ifconfig $dev down
    ;;
*)
    echo "Usage:$(basename $0) start|stop"
    exit 1
    ;;
esac

[root@VS ~]# bash lvs_dr_vs.sh start

ldirectord实现LVS的高可用性

当RS服务端崩溃时，自动从LVS中删除

VS

[root@VS ~]# iptables -t mangle -A PREROUTING -d 192.168.80.100 -p tcp -m multiport --dports 80,443 -j MARK --set-mark 10

下载ldirectord软件包(pub/Source/7.x86/crmsh/)
[root@VS ~]# yum -y install ldirectord-3.9.6-0rc1.1.1.x86_64.rpm       \需有完整yum源
[root@VS ~]# rpm -ql ldirectord
/etc/ha.d
/etc/ha.d/resource.d
/etc/ha.d/resource.d/ldirectord
/etc/logrotate.d/ldirectord
/usr/lib/ocf/resource.d/heartbeat/ldirectord
/usr/lib/systemd/system/ldirectord.service
/usr/sbin/ldirectord
/usr/share/doc/ldirectord-3.9.6
/usr/share/doc/ldirectord-3.9.6/COPYING
/usr/share/doc/ldirectord-3.9.6/ldirectord.cf
/usr/share/man/man8/ldirectord.8.gz
[root@VS ~]# cp /usr/share/doc/ldirectord-3.9.6/ldirectord.cf /etc/ha.d
[root@VS ~]# vim /etc/ha.d/ldirectord.cf
    checktimeout=3 \超时时间
    checkinterval=1 \检查间隔
    fallback=127.0.0.1:80 \Sorry Server,错误的网页
    autoreload=yes \自动加载配置文件
    logfile="/var/log/ldirectord.log" \日志文件
    quiescent=no \当RS宕机时是否将RS记录从ipvsadm记录中删除，no表示宕机即删除  
    virtual=192.168.80.100:80 \VS服务端IP
    real=192.168.198.138:80 gate 2 \RS服务端IP，gate表示dr类型
    real=192.168.198.132:80 gate 1 \RS服务端IP，gate表示dr类型
    fallback=127.0.0.1:80 gate
    service=http
    scheduler=wrr \调度算法
    protocol=tcp \tcp协议
    checktype=negotiate  
    checkport=80 \检查端口
    request="index.html" \检查网页
    receive="danran" \检查网页字符，若包含该字符，则表示RS服务端正常
[root@VS ~]# systemctl start ldirectord
[root@VS ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
  TCP  192.168.80.100:80 rr
  -> 192.168.198.132:80           Route   1      0          0         
  -> 192.168.198.138:80           Route   1      0          0  

client

[root@client ~]# curl 192.168.80.100
welcome to RS2
[root@client ~]# curl 192.168.80.100
welcome to RS1
[root@client ~]# curl 192.168.80.100
welcome to RS2
[root@client ~]# curl 192.168.80.100
welcome to RS1

使用标签实现ldirectord将多个服务定义为一个集群服务

使用打标签时需删除protocol=tcp选项

[root@VS ~]# iptables -t mangle -A PREROUTING -d 192.168.80.100 -p tcp -m multiport --dports 80,443 -j MARK --set-mark 10 \标签定义为10
[root@VS ~]# iptables -t mangle -nvL
Chain PREROUTING (policy ACCEPT 41 packets, 3944 bytes)
pkts bytes target     prot opt in     out     source               destination         
0     0 MARK       tcp  --  *      *       0.0.0.0/0            192.168.80.100       multiport dports 80,443 MARK set 0xa

[root@VS ~]# vim /etc/ha.d/ldirectord.cf
    checktimeout=3 \超时时间
    checkinterval=1 \检查间隔
    fallback=127.0.0.1:80 \Sorry Server,错误的网页
    autoreload=yes \自动加载配置文件
    logfile="/var/log/ldirectord.log" \日志文件
    quiescent=no \当RS宕机时是否将RS记录从ipvsadm记录中删除，no表示宕机即删除  
    virtual=10  \VS标签为10
        real=192.168.198.138:80 gate 2 \RS服务端IP，gate表示dr类型
        real=192.168.198.132:80 gate 1 \RS服务端IP，gate表示dr类型
        fallback=127.0.0.1:80 gate
        service=http
        scheduler=wrr \调度算法
        checktype=negotiate  
        checkport=80 \检查端口
        request="index.html" \检查网页
        receive="danran" \检查网页字符，若包含该字符，则表示RS服务端正常
[root@VS ~]# systemctl start ldirectord
[root@VS ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
  FWM  10 rr
  -> 192.168.198.132:80           Route   1      0          0         
  -> 192.168.198.138:80           Route   1      0          0