一次lvs迁移记录

需求：从117.119.33.99迁移到122.14.206.125，lvs为dr模式，系统版本为debian7

1、安装lvs和keepalived

# aptitude install -y ipvsadm keepalived
# /etc/init.d/ipvsadm start
# update-rc.d ipvsadm enable
# /etc/init.d/keepalived start
# update-rc.d keepalived enable

2、修改配置文件

主配置文件使用了include进行拆分

# vim keepalived.conf

include  /etc/keepalived/vi_bgp_VI1_yizhuang.inc
include  /etc/keepalived/vi_bgp_VI2_yizhuang.inc
#include  /etc/keepalived/vs_pub_yizhuang.inc
include  /etc/keepalived/vs_web1_yizhuang.inc
include  /etc/keepalived/vs_newad_yizhuang.inc
include  /etc/keepalived/vs_public1_yizhuang.inc
include  /etc/keepalived/vs_turbo2sdk_yizhuang.inc
#include  /etc/keepalived/vs_china-max_yizhuang.inc
include  /etc/keepalived/vs_dsproxy_yizhuang.inc
#include  /etc/keepalived/vs_china-max6_yizhuang.inc
include  /etc/keepalived/vs_china-turbo-2_yizhuang.inc
include /etc/keepalived/vs_cms_yizhuang.inc

节点角色配置，master和backup节点仅该文件不同，其他配置完全一样，可以用rsync同步

# vim state_VI1.conf

#uy-s-07
     state MASTER
     priority 150
#uy-s-45
#    state BACKUP
#    priority 100

# vim state_VI2.conf

#uy-s-07
     state BACKUP
     priority 100
#uy-s-45
#    state MASTER
#    priority 150

vrrp实例配置

# vim vi_bgp_VI1_yizhuang.inc

vrrp_instance VII_1 {
    virtual_router_id 102
    interface eth0
    include /etc/keepalived/state_VI1.conf
    preempt_delay 120
    garp_master_delay 0
    garp_master_refresh 5
    lvs_sync_daemon_interface eth0
    authentication {
        auth_type PASS
        auth_pass opsdk
    }

    virtual_ipaddress {
        #web1 in yizhuang
        122.14.206.128

        #pub in yizhuang
        122.14.206.127

        #new ad in yizhuang
        122.14.206.126

        #new wood water in yizhuang
        122.14.206.125

        #turbo2sdk in yizhuang
        122.14.206.122

        #china-max in yizhuang
        122.14.206.121

        #china-max6 in yizhuang
        122.14.206.120

        #china-max6 in yizhuang
        122.14.206.119

        #china-turbo-2 in yizhuang
        122.14.206.118
    }
}

# vim vi_bgp_VI2_yizhuang.inc

vrrp_instance VII_2 {
    virtual_router_id 202
    interface eth0
    include /etc/keepalived/state_VI2.conf
    preempt_delay 120
    garp_master_delay 0
    garp_master_refresh 5
    lvs_sync_daemon_interface eth0
    authentication {
        auth_type PASS
        auth_pass opsdk
    }

    virtual_ipaddress {
        #duanzi in yizhuang
        122.14.206.117
    }
}

virtual_server配置

# vim vs_public1_yizhuang.inc

###122.14.206.125:80 122.14.206.125:80 public1###

virtual_server 122.14.206.125 80 {
    lb_algo sh
    lb_kind DR
    persistence_timeout 0
    delay_loop 20
    protocol TCP

    # uy04-04 has address 122.14.206.140
    real_server 122.14.206.140 80 {
        weight 10
        TCP_CHECK {
            connect_timeout 10
        }
    }

    # uy04-05 has address 122.14.206.141
    real_server 122.14.206.141 80 {
        weight 10
        TCP_CHECK {
            connect_timeout 10
        }
    }

    # uy07-01 has address 122.14.206.143
    real_server 122.14.206.143 80 {
        weight 10
        TCP_CHECK {
            connect_timeout 10
        }
    }

    # uy07-02 has address 122.14.206.144
    real_server 122.14.206.144 80 {
        weight 10
        TCP_CHECK {
            connect_timeout 10
        }
    }

    # uy07-05 has address 122.14.206.146
    real_server 122.14.206.146 80 {
        weight 10
        TCP_CHECK {
            connect_timeout 10
        }
    }


    # uy07-06 has address 122.14.206.147
    real_server 122.14.206.147 80 {
        weight 10
        TCP_CHECK {
            connect_timeout 10
        }
    }
}

###122.14.206.125:443 122.14.206.125:443 public1###

virtual_server 122.14.206.125 443 {
    lb_algo sh
    lb_kind DR
    persistence_timeout 0
    delay_loop 20
    protocol TCP

    # uy01-01 has address 122.14.206.140
    real_server 122.14.206.140 443 {
        weight 10
        TCP_CHECK {
            connect_timeout 10
        }
    }

    # uy01-02 has address 122.14.206.141
    real_server 122.14.206.141 443 {
        weight 10
        TCP_CHECK {
            connect_timeout 10
        }
    }

    # uy01-03 has address 122.14.206.143
    real_server 122.14.206.143 443 {
        weight 10
        TCP_CHECK {
            connect_timeout 10
        }
    }

    # uy01-04 has address 122.14.206.144
    real_server 122.14.206.144 443 {
        weight 10
        TCP_CHECK {
            connect_timeout 10
        }
    }

    # uy01-05 has address 122.14.206.146
    real_server 122.14.206.146 443 {
        weight 10
        TCP_CHECK {
            connect_timeout 10
        }
    }

    # uy01-06 has address 122.14.206.147
    real_server 122.14.206.147 443 {
        weight 10
        TCP_CHECK {
            connect_timeout 10
        }
    }
}

# vim vs_dsproxy_yizhuang.inc

###122.14.206.125:2012 122.14.206.125:2012 dsproxy###

virtual_server 122.14.206.125 2012 {
    lb_algo rr
    lb_kind DR
    persistence_timeout 0
    delay_loop 20
    protocol UDP

    # uy04-10-v09 has address 192.168.10.88
    real_server 122.14.206.141 2012 {
        weight 5
        MISC_CHECK {
            misc_path "/etc/keepalived/UDP_CHECK.sh 192.168.10.88 2012"
            misc_timeout 10
        }
    }

    # uy01-02-v12 has address 192.168.10.89
    real_server 122.14.206.140 2012 {
        weight 5
        MISC_CHECK {
            misc_path "/etc/keepalived/UDP_CHECK.sh 192.168.10.89 2012"
            misc_timeout 10
        }
    }
}

udp检测脚本

# vim /etc/keepalived/UDP_CHECK.sh

#!/bin/bash
/bin/nc -unvz -w 1 $1 $2 2>&1 | grep open &> /dev/null
exit $?

配置文件同步及服务重载脚本，这里使用了git管理配置文件

# vim rsync2backup.sh

#!/bin/bash

git add ./*
git commit -m "commit on: $(date +%F' '%T)" -a
/etc/init.d/keepalived reload
cd /etc/keepalived
rsync -avz --delete --exclude=.git --exclude=state*.conf --exclude=rsync2backup.sh ./ 192.168.1.250:`pwd`/
ssh 192.168.1.250 '/etc/init.d/keepalived reload'

3、配置Real Server，这里有6台Real Server

a、在lo网卡上配置vip

# ifconfig lo:125 122.14.206.125 netmask 255.255.255.255 up

# vim /etc/network/interfaces
auto lo:125
iface lo:125 inet static
address 122.14.206.125
netmask 255.255.255.255

b、修改arp内核参数

# vim /etc/sysctl.conf
net.ipv4.conf.lo.arp_ignore = 1
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2
net.ipv4.ip_forward = 1
net.ipv4.nf_conntrack_max = 2048000
net.netfilter.nf_conntrack_max = 2048000

# sysctl -p

c、挑两台Real Server为udp服务设置iptables转发规则

# iptables -t nat -I PREROUTING -d 122.14.206.125 -p udp --dport 2012 -j DNAT --to 192.168.10.88:2012
# iptables -t nat -I POSTROUTING -p udp --dport 2012 -j SNAT --to-source 122.14.206.125

# iptables -vnL -t nat
# iptables-save

4、重载服务

# /etc/init.d/keepalived reload

这里可以使用同步脚本

5、查看lvs规则

# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=1048576)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  122.14.206.125:80 sh
  -> 122.14.206.140:80            Route   10     37942      5108
  -> 122.14.206.141:80            Route   10     37034      5506
  -> 122.14.206.143:80            Route   10     46782      5907
  -> 122.14.206.144:80            Route   10     37518      5844
  -> 122.14.206.146:80            Route   10     37340      4928
  -> 122.14.206.147:80            Route   10     46501      6221
TCP  122.14.206.125:443 sh
  -> 122.14.206.140:443           Route   10     3999       4061
  -> 122.14.206.141:443           Route   10     3904       3692
  -> 122.14.206.143:443           Route   10     5116       4551
  -> 122.14.206.144:443           Route   10     4529       4298
  -> 122.14.206.146:443           Route   10     3944       3685
  -> 122.14.206.147:443           Route   10     3916       3719
UDP  122.14.206.125:2012 rr
  -> 122.14.206.140:2012          Route   5      0          260
  -> 122.14.206.141:2012          Route   5      0          261

6、修改本地hosts，将域名指向新的vip在本地测试，这里略过

7、确认没问题就可以切换dns了，修改dns然后重载dns服务

# vim oupeng.com.zone.ALL
;outspot6-yizhuang      IN      A       117.119.33.99 ;117.119.33.11,117.119.33.12,117.119.33.13,117.119.33.14,117.119.33.41
outspot6-yizhuang       IN      A       122.14.206.125 ;122.14.206.140,122.14.206.141,122.14.206.143,122.14.206.144,122.14.206.146,122.14.206.147

至此，迁移就全部完成了，可以尝试访问一下lvs代理的服务确认一切正常。