keepalived实现nginx负载均衡机高可用

keepalived实现nginx负载均衡机高可用

环境说明

主机名称	IP地址	系统版本
master	192.168.110.11	redhat 8
backup	192.168.110.12	redhat 8

本次高可用虚拟IP（VIP）地址暂定为 192.168.110.200

keepalived安装

配置主master

//master
#关闭防火墙和selinux
systemctl disable --now firewalld
sed -ri 's/^(SELINUX=).*/1disabled/g' /etc/sysconfig/selinux
setenforce 0

#安装keepalived
[root@master ~]# yum -y install keepalived

配置备backup

//backup
#关闭防火墙和selinux
systemctl disable --now firewalld
sed -ri 's/^(SELINUX=).*/1disabled/g' /etc/sysconfig/selinux
setenforce 0

#安装keepalived
[root@backup ~]# yum -y install keepalived

nginx安装

在主master上安装nginx

//master
#安装nginx
[root@master ~]# yum -y install nginx

#备份网页文件，创建新的测试网页
[root@master ~]# cd /usr/share/nginx/html/
[root@master html]# mv index.html{,.bak}
[root@master html]# echo 'master' > index.html
[root@master html]# ls
404.html  50x.html  index.html  index.html.bak  nginx-logo.png  poweredby.png
[root@master html]# cat index.html
master

#设置nginx开机自启
[root@master html]# systemctl enable --now nginx
Created symlink /etc/systemd/system/multi-user.target.wants/nginx.service → /usr/lib/systemd/system/nginx.service.
[root@master html]# ss -antl
State     Recv-Q     Send-Q         Local Address:Port         Peer Address:Port    
LISTEN    0          128                  0.0.0.0:22                0.0.0.0:*       
LISTEN    0          128                  0.0.0.0:80                0.0.0.0:*       
LISTEN    0          128                     [::]:22                   [::]:*       
LISTEN    0          128                     [::]:80                   [::]:*       

#可以访问
[root@master html]# curl localhost
master

在备backup上安装nginx

//backup
#安装nginx
[root@backup ~]# yum -y install nginx

#备份网页文件，创建新的测试网页
[root@backup ~]# cd /usr/share/nginx/html/
[root@backup html]# mv index.html{,.bak}
[root@backup html]# echo 'backup' > index.html
[root@backup html]# ls
404.html  50x.html  index.html  index.html.bak  nginx-logo.png  poweredby.png
[root@backup html]# cat index.html
backup

#设置nginx开机自启
[root@backup html]# systemctl enable --now nginx
Created symlink /etc/systemd/system/multi-user.target.wants/nginx.service → /usr/lib/systemd/system/nginx.service.
[root@backup html]# ss -antl
State     Recv-Q     Send-Q         Local Address:Port         Peer Address:Port    
LISTEN    0          128                  0.0.0.0:22                0.0.0.0:*       
LISTEN    0          128                  0.0.0.0:80                0.0.0.0:*       
LISTEN    0          128                     [::]:22                   [::]:*       
LISTEN    0          128                     [::]:80                   [::]:*       

#可以访问
[root@backup html]# curl localhost
backup

keepalived配置

配置主keepalived

//master
#备份文件
[root@master ~]# cd /etc/keepalived/
[root@master keepalived]# mv keepalived.conf{,.bak}
[root@master keepalived]# ls
keepalived.conf  keepalived.conf.bak

#配置文件
[root@master keepalived]# vim keepalived.conf
! Configuration File for keepalived

global_defs {
   router_id lb01
}

vrrp_instance VI_1 {
    state MASTER
    interface ens160
    virtual_router_id 51
    priority 100 
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1
    }
    virtual_ipaddress {
        192.168.110.200
    }
}

virtual_server 192.168.110.200 80 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    persistence_timeout 50
    protocol TCP

    real_server 192.168.110.11 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }

    real_server 192.168.110.12 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}

#设置开机自启
[root@master ~]# systemctl enable --now keepalived
Created symlink /etc/systemd/system/multi-user.target.wants/keepalived.service → /usr/lib/systemd/system/keepalived.service.

配置备keepalived

//backup
#备份文件
[root@backup ~]# cd /etc/keepalived/
[root@backup keepalived]# mv keepalived.conf{,.bak}
[root@backup keepalived]# ls
keepalived.conf  keepalived.conf.bak

#配置文件
[root@backup keepalived]# vim keepalived.conf
! Configuration File for keepalived

global_defs {
   router_id lb02
}

vrrp_instance VI_1 {
    state BACKUP
    interface ens160
    virtual_router_id 51
    priority 90
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1
    }
    virtual_ipaddress {
        192.168.110.200
    }
}

virtual_server 192.168.110.200 80 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    persistence_timeout 50
    protocol TCP

    real_server 192.168.110.11 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }

    real_server 192.168.110.12 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}

#设置开机自启
[root@backup ~]# systemctl enable --now keepalived
Created symlink /etc/systemd/system/multi-user.target.wants/keepalived.service → /usr/lib/systemd/system/keepalived.service.

查看VIP在哪里

在master上查看

//master
#查看IP信息
[root@master ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens160: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:0c:29:ea:5b:7a brd ff:ff:ff:ff:ff:ff
    inet 192.168.110.11/24 brd 192.168.110.255 scope global noprefixroute ens160
       valid_lft forever preferred_lft forever
    inet 192.168.110.200/32 scope global ens160     //此处可以看到VIP
       valid_lft forever preferred_lft forever

在backup上查看

//backup
#查看IP信息
[root@backup ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens160: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:0c:29:85:c0:f0 brd ff:ff:ff:ff:ff:ff
    inet 192.168.110.12/24 brd 192.168.110.255 scope global noprefixroute ens160
       valid_lft forever preferred_lft forever

修改内核参数，开启监听VIP功能

此步可做可不做，该功能可用于仅监听VIP的时候

在master上修改内核参数

#master
[root@master ~]# echo 'net.ipv4.ip_nonlocal_bind = 1' >>/etc/sysctl.conf
[root@master ~]# sysctl -p
net.ipv4.ip_nonlocal_bind = 1
[root@master ~]# cat /proc/sys/net/ipv4/ip_nonlocal_bind
1

在backup上修改内核参数

#slave
[root@backup ~]# echo 'net.ipv4.ip_nonlocal_bind = 1' >>/etc/sysctl.conf
[root@backup ~]# sysctl -p
net.ipv4.ip_nonlocal_bind = 1
[root@backup ~]# cat /proc/sys/net/ipv4/ip_nonlocal_bind
1

让keepalived监控nginx负载均衡机

keepalived通过脚本来监控nginx负载均衡机的状态

在master上编写脚本

此处的脚本名称应避免与服务名相同，推荐用服务名的首字母代替，如check_n，不要给脚本起名check_nginx

//master
#创建脚本目录
[root@master ~]# mkdir /scripts
[root@master ~]# cd /scripts/

#check_n 脚本
[root@master scripts]# vim check_n.sh 
#!/bin/bash
nginx_status=$(ps -ef|grep -Ev "grep|$0"|grep 'nginx'|wc -l)
if [ $nginx_status -lt 1 ];then
    systemctl stop keepalived
fi

#notify 脚本
[root@master scripts]# vim notify.sh
#!/bin/bash
VIP=$2
sendmail (){
        subject="${VIP}'s server keepalived state is translate"
        content="`date +'%F %T'`: `hostname`'s state change to master"
        echo $content | mail -s "$subject" leidazhuang123@163.com
}
case "$1" in
  master)
        nginx_status=$(ps -ef|grep -Ev "grep|$0"|grep 'nginx'|wc -l)
        if [ $nginx_status -lt 1 ];then
            systemctl start nginx
        fi
        sendmail
  ;;
  backup)
        nginx_status=$(ps -ef|grep -Ev "grep|$0"|grep 'nginx'|wc -l)
        if [ $nginx_status -gt 0 ];then
            systemctl stop nginx
        fi
  ;;
  *)
        echo "Usage:$0 master|backup VIP"
  ;;
esac

#给脚本设置执行权限
[root@master scripts]# chmod +x check_n.sh
[root@master scripts]# chmod +x notify.sh
[root@master scripts]# ll
total 8
-rwxr-xr-x. 1 root root 147 May 24 11:05 check_n.sh
-rwxr-xr-x. 1 root root 663 May 24 11:06 notify.sh

在backup上编写脚本

//backup
#创建脚本目录
[root@backup ~]# mkdir /scripts
[root@backup ~]# cd /scripts/

#check_n 脚本
[root@backup scripts]# vim check_n.sh
#!/bin/bash
nginx_status=$(ps -ef|grep -Ev "grep|$0"|grep 'nginx'|wc -l)
if [ $nginx_status -lt 1 ];then
    systemctl stop keepalived
fi

# notify 脚本
[root@backup scripts]# vim notify.sh
#!/bin/bash
VIP=$2
sendmail (){
        subject="${VIP}'s server keepalived state is translate"
        content="`date +'%F %T'`: `hostname`'s state change to master"
        echo $content | mail -s "$subject" leidazhuang123@163.com
}
case "$1" in
  master)
        nginx_status=$(ps -ef|grep -Ev "grep|$0"|grep 'nginx'|wc -l)
        if [ $nginx_status -lt 1 ];then
            systemctl start nginx
        fi
        sendmail
  ;;
  backup)
        nginx_status=$(ps -ef|grep -Ev "grep|$0"|grep 'nginx'|wc -l)
        if [ $nginx_status -gt 0 ];then
            systemctl stop nginx
        fi
  ;;
  *)
        echo "Usage:$0 master|backup VIP"
  ;;
esac

#给脚本设置执行权限
[root@backup scripts]# chmod +x /scripts/check_n.sh
[root@backup scripts]# chmod +x notify.sh
[root@backup scripts]# ll
total 8
-rwxr-xr-x. 1 root root 147 May 24 11:08 check_n.sh
-rwxr-xr-x. 1 root root 663 May 24 11:08 notify.sh

配置keepalived，加入监控脚本的配置

配置主keepalived

//master
#添加配置
[root@master ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   router_id lb01
}

vrrp_script nginx_check {
    script "/scripts/check_n.sh"
    interval 10
    weight -20
}

vrrp_instance VI_1 {
    state MASTER
    interface ens160
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1
    }
    virtual_ipaddress {
        192.168.110.200
    }
    track_script {
        nginx_check
    }
    notify_master "/scripts/notify.sh master 192.168.110.200"
    notify_backup "/scripts/notify.sh backup 192.168.110.200"
}

virtual_server 192.168.110.200 80 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    persistence_timeout 50
    protocol TCP

    real_server 192.168.110.11 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }

    real_server 192.168.110.12 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}

#重启服务
systemctl restart keepalived

配置备keepalived

//backup
#添加配置
[root@backup ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   router_id lb02
}

vrrp_instance VI_1 {
    state BACKUP
    interface ens160
    virtual_router_id 51
    priority 90
    nopreempt
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1
    }
    virtual_ipaddress {
        192.168.110.200
    }
    notify_master "/scripts/notify.sh master 192.168.110.200"
    notify_backup "/scripts/notify.sh backup 192.168.110.200"
}

virtual_server 192.168.110.200 80 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    persistence_timeout 50
    protocol TCP

    real_server 192.168.110.11 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }

    real_server 192.168.110.12 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}

#重启服务
systemctl restart keepalived

模拟故障测试

启用keepalived，开启nginx服务

//master
#启用keepalived，开启nginx服务
[root@master ~]# systemctl start keepalived
[root@master ~]# systemctl start nginx
[root@master ~]# ss -antl
State     Recv-Q     Send-Q         Local Address:Port         Peer Address:Port    
LISTEN    0          128                  0.0.0.0:22                0.0.0.0:*       
LISTEN    0          128                  0.0.0.0:80                0.0.0.0:*       
LISTEN    0          128                     [::]:22                   [::]:*       
LISTEN    0          128                     [::]:80                   [::]:* 

//backup
#启用keepalived，不开启nginx服务，因为当master挂掉之后，会自动抢占
[root@backup ~]# systemctl start keepalived
[root@backup ~]# ss -antl
State     Recv-Q     Send-Q         Local Address:Port         Peer Address:Port    
LISTEN    0          128                  0.0.0.0:22                0.0.0.0:*       
LISTEN    0          128                     [::]:22                   [::]:* 

模拟master主挂掉了，因为启用了高可用，backup备会自动继承

//模拟客户机
#nginx正常运行
[root@localhost ~]# curl 192.168.110.200
master

//master
#模拟master挂掉了，此时backup的nginx会自动启动
[root@master ~]# systemctl stop nginx
[root@master ~]# ss -antl
State     Recv-Q     Send-Q         Local Address:Port         Peer Address:Port    
LISTEN    0          128                  0.0.0.0:22                0.0.0.0:*       
LISTEN    0          128                     [::]:22                   [::]:* 

//backup
#此时backup的nginx会自动启动
[root@backup ~]# ss -antl
State     Recv-Q     Send-Q         Local Address:Port         Peer Address:Port    
LISTEN    0          128                  0.0.0.0:22                0.0.0.0:*       
LISTEN    0          128                  0.0.0.0:80                0.0.0.0:*       
LISTEN    0          128                     [::]:22                   [::]:*       
LISTEN    0          128                     [::]:80                   [::]:* 

//此时用客户机访问VIP 192.168.110.200
#backup会自动接管nginx服务
[root@localhost ~]# curl 192.168.110.200
backup

此时我们模拟master被抢救回来，backup会停掉nginx；客户端访问VIP，此时会自动访问master

//master
#先启动nginx，再启动keepalived
[root@master ~]# systemctl start nginx
[root@master ~]# systemctl start keepalived
[root@master ~]# ss -antl
State     Recv-Q     Send-Q         Local Address:Port         Peer Address:Port    
LISTEN    0          128                  0.0.0.0:22                0.0.0.0:*       
LISTEN    0          128                     [::]:22                   [::]:*       
[root@master ~]# ss -antl
State     Recv-Q     Send-Q         Local Address:Port         Peer Address:Port    
LISTEN    0          128                  0.0.0.0:22                0.0.0.0:*       
LISTEN    0          128                  0.0.0.0:80                0.0.0.0:*       
LISTEN    0          128                     [::]:22                   [::]:*       
LISTEN    0          128                     [::]:80                   [::]:* 

//客户端访问
#此时访问VIP会访问到master
[root@localhost ~]# curl 192.168.110.200
master

//backup
#backup会停掉nginx，自动让位
[root@backup ~]# ss -antl
State     Recv-Q     Send-Q         Local Address:Port         Peer Address:Port    
LISTEN    0          128                  0.0.0.0:22                0.0.0.0:*       
LISTEN    0          128                     [::]:22                   [::]:*   

大功告成，以上就是全部步骤！！！