Keepalived+Nginx 高可用集群 (主从模式)
集群架构图
安装keepalived
[root@localhost ~]# yum install -y keepalived
查看状态
[root@localhost ~]# rpm -qa|grep keepalived
keepalived-1.3.5-16.el7.x86_64
查看配置
[root@localhost ~]# cd /etc/keepalived/
[root@localhost keepalived]# ls
keepalived.conf
# 备份配置文件
[root@localhost keepalived]# cp keepalived.conf keepalived.conf.bak
[root@localhost keepalived]# ls
keepalived.conf keepalived.conf.bak
修改配置文件
vrrp_script chk_http_port {
# 检测nginx状态脚本路径
script "/etc/nginx/script/nginx_check.sh"
interval 2 # 检测脚本执行的间隔
weight 2
}
vrrp_instance VI_1 {
state BACKUP # 主机 MASTER，备机BACKUP
interface ens33 # 网卡名称
virtual_router_id 51 # 主，备机的virtual_router_id必须相同
priority 90 # 主，备机取不同的优先级，主机值较大，备份机值较小
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.1.111 # VRRP H虚拟地址
}
}
修改本地hosts配置文件
192.168.1.111 www.123.com
新增检测nginx状态脚本
#!/bin/bash
A=`ps -C nginx –no-header |wc -l`
if [ $A -eq 0 ];then
/usr/local/nginx/sbin/nginx
sleep 2
if [ `ps -C nginx --no-header |wc -l` -eq 0 ];then
killall keepalived
fi
fi
注意：将此脚本放入keepalived配置的路径下，主备Nginx各一份。
修改备机Nginx配置
http {
upstream myserver {
server 192.168.1.11:8080 weight=1;
server 192.168.1.11:8081 weight=10;
}
server {
listen 80;
# listen [::]:80 default_server;
server_name www.123.com;
root /usr/share/nginx/html;
# Load configuration files for the default server block.
include /etc/nginx/default.d/*.conf;
location / {
proxy_pass http://myserver;
}
}
}
启动
启动主机Nginx
[root@localhost ~]# systemctl start nginx
[root@localhost ~]# systemctl status nginx
● nginx.service - The nginx HTTP and reverse proxy server
Loaded: loaded (/usr/lib/systemd/system/nginx.service; disabled; vendor preset: disabled)
Active: active (running) since Sun 2020-04-05 14:32:15 CST; 5s ago
Process: 92510 ExecStart=/usr/sbin/nginx (code=exited, status=0/SUCCESS)
Process: 92506 ExecStartPre=/usr/sbin/nginx -t (code=exited, status=0/SUCCESS)
Process: 92504 ExecStartPre=/usr/bin/rm -f /run/nginx.pid (code=exited, status=0/SUCCESS)
Main PID: 92512 (nginx)
CGroup: /system.slice/nginx.service
├─92512 nginx: master process /usr/sbin/nginx
├─92513 nginx: worker process
├─92514 nginx: worker process
├─92515 nginx: worker process
└─92516 nginx: worker process
Apr 05 14:32:15 localhost systemd[1]: Starting The nginx HTTP and reverse proxy server...
Apr 05 14:32:15 localhost nginx[92506]: nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
Apr 05 14:32:15 localhost nginx[92506]: nginx: configuration file /etc/nginx/nginx.conf test is successful
Apr 05 14:32:15 localhost systemd[1]: Started The nginx HTTP and reverse proxy server.
启动主机keepalived
[root@localhost ~]# systemctl start keepalived
[root@localhost ~]# systemctl status keepalived
● keepalived.service - LVS and VRRP High Availability Monitor
Loaded: loaded (/usr/lib/systemd/system/keepalived.service; disabled; vendor preset: disabled)
Active: active (running) since Sun 2020-04-05 14:33:13 CST; 5s ago
Process: 92572 ExecStart=/usr/sbin/keepalived $KEEPALIVED_OPTIONS (code=exited, status=0/SUCCESS)
Main PID: 92573 (keepalived)
CGroup: /system.slice/keepalived.service
├─92573 /usr/sbin/keepalived -D
├─92574 /usr/sbin/keepalived -D
└─92575 /usr/sbin/keepalived -D
Apr 05 14:33:14 localhost Keepalived_vrrp[92575]: VRRP_Instance(VI_1) Transition to MASTER STATE
Apr 05 14:33:15 localhost Keepalived_vrrp[92575]: VRRP_Instance(VI_1) Entering MASTER STATE
Apr 05 14:33:15 localhost Keepalived_vrrp[92575]: VRRP_Instance(VI_1) setting protocol iptable drop rule
Apr 05 14:33:15 localhost Keepalived_vrrp[92575]: VRRP_Instance(VI_1) setting protocol VIPs.
Apr 05 14:33:15 localhost Keepalived_vrrp[92575]: Sending gratuitous ARP on ens33 for 192.168.1.111
Apr 05 14:33:15 localhost Keepalived_vrrp[92575]: VRRP_Instance(VI_1) Sending/queueing gratuitous ARPs on ens33 for 192.168.1.111
Apr 05 14:33:15 localhost Keepalived_vrrp[92575]: Sending gratuitous ARP on ens33 for 192.168.1.111
Apr 05 14:33:15 localhost Keepalived_vrrp[92575]: Sending gratuitous ARP on ens33 for 192.168.1.111
Apr 05 14:33:15 localhost Keepalived_vrrp[92575]: Sending gratuitous ARP on ens33 for 192.168.1.111
Apr 05 14:33:15 localhost Keepalived_vrrp[92575]: Sending gratuitous ARP on ens33 for 192.168.1.111
启动备机Nginx
[root@localhost nginx]# systemctl start nginx
[root@localhost nginx]# systemctl status nginx
● nginx.service - The nginx HTTP and reverse proxy server
Loaded: loaded (/usr/lib/systemd/system/nginx.service; disabled; vendor preset: disabled)
Active: active (running) since Sun 2020-04-05 22:04:26 CST; 7s ago
Process: 19901 ExecStart=/usr/sbin/nginx (code=exited, status=0/SUCCESS)
Process: 19898 ExecStartPre=/usr/sbin/nginx -t (code=exited, status=0/SUCCESS)
Process: 19896 ExecStartPre=/usr/bin/rm -f /run/nginx.pid (code=exited, status=0/SUCCESS)
Main PID: 19903 (nginx)
CGroup: /system.slice/nginx.service
├─19903 nginx: master process /usr/sbin/nginx
├─19904 nginx: worker process
├─19905 nginx: worker process
├─19906 nginx: worker process
└─19907 nginx: worker process
Apr 05 22:04:26 localhost.localdomain systemd[1]: Starting The nginx HTTP and reverse proxy server...
Apr 05 22:04:26 localhost.localdomain nginx[19898]: nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
Apr 05 22:04:26 localhost.localdomain nginx[19898]: nginx: configuration file /etc/nginx/nginx.conf test is successful
Apr 05 22:04:26 localhost.localdomain systemd[1]: Started The nginx HTTP and reverse proxy server.
启动备机keepalived
[root@localhost nginx]# systemctl start keepalived
[root@localhost nginx]# systemctl status keepalived
● keepalived.service - LVS and VRRP High Availability Monitor
Loaded: loaded (/usr/lib/systemd/system/keepalived.service; disabled; vendor preset: disabled)
Active: active (running) since Sun 2020-04-05 22:05:16 CST; 8s ago
Process: 19915 ExecStart=/usr/sbin/keepalived $KEEPALIVED_OPTIONS (code=exited, status=0/SUCCESS)
Main PID: 19916 (keepalived)
CGroup: /system.slice/keepalived.service
├─19916 /usr/sbin/keepalived -D
├─19917 /usr/sbin/keepalived -D
└─19918 /usr/sbin/keepalived -D
Apr 05 22:05:16 localhost.localdomain Keepalived_healthcheckers[19917]: Activating healthchecker for service [192.168.200.100]:443
Apr 05 22:05:16 localhost.localdomain Keepalived_healthcheckers[19917]: Activating healthchecker for service [10.10.10.2]:1358
Apr 05 22:05:16 localhost.localdomain Keepalived_healthcheckers[19917]: Activating healthchecker for service [10.10.10.2]:1358
Apr 05 22:05:16 localhost.localdomain Keepalived_healthcheckers[19917]: Activating healthchecker for service [10.10.10.3]:1358
Apr 05 22:05:16 localhost.localdomain Keepalived_healthcheckers[19917]: Activating healthchecker for service [10.10.10.3]:1358
Apr 05 22:05:16 localhost.localdomain Keepalived_vrrp[19918]: VRRP_Instance(VI_1) Entering BACKUP STATE
Apr 05 22:05:16 localhost.localdomain Keepalived_vrrp[19918]: VRRP sockpool: [ifindex(2), proto(112), unicast(0), fd(10,11)]
Apr 05 22:05:22 localhost.localdomain Keepalived_healthcheckers[19917]: Timeout connecting server [192.168.200.2]:1358.
Apr 05 22:05:22 localhost.localdomain Keepalived_healthcheckers[19917]: Timeout connecting server [192.168.200.4]:1358.
Apr 05 22:05:23 localhost.localdomain Keepalived_healthcheckers[19917]: Timeout connecting server [192.168.200.5]:1358.
检测
排查
- 是否关联虚拟ip
[root@localhost ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:d6:85:50 brd ff:ff:ff:ff:ff:ff
inet 192.168.1.11/24 brd 192.168.1.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet 192.168.1.111/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::3e71:f6ff:5b69:2539/64 scope link noprefixroute
valid_lft forever preferred_lft forever
- 是否可以ping通虚拟ip
[root@localhost ~]# ping 192.168.1.111
PING 192.168.1.111 (192.168.1.111) 56(84) bytes of data.
ping不通解决方案：原因是keepalived.conf配置中默认vrrp_strict打开了，需要把它注释掉。重启keepalived即可ping通。
优化keepalived配置
global_defs {
notification_email {
acassen@firewall.loc
failover@firewall.loc
sysadmin@firewall.loc
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 192.168.200.1
smtp_connect_timeout 30
router_id LVS_DEVEL
vrrp_skip_check_adv_addr
# vrrp_strict
vrrp_garp_interval 0
vrrp_gna_interval 0
}
注意：备机中的配置文件也要一起修改
重启keepalived
[root@localhost ~]# systemctl restart keepalived
[root@localhost ~]# systemctl status keepalived
● keepalived.service - LVS and VRRP High Availability Monitor
Loaded: loaded (/usr/lib/systemd/system/keepalived.service; disabled; vendor preset: disabled)
Active: active (running) since Sun 2020-04-05 14:46:31 CST; 15s ago
Process: 93230 ExecStart=/usr/sbin/keepalived $KEEPALIVED_OPTIONS (code=exited, status=0/SUCCESS)
Main PID: 93231 (keepalived)
CGroup: /system.slice/keepalived.service
├─93231 /usr/sbin/keepalived -D
├─93232 /usr/sbin/keepalived -D
└─93233 /usr/sbin/keepalived -D
Apr 05 14:46:38 localhost Keepalived_vrrp[93233]: Sending gratuitous ARP on ens33 for 192.168.1.111
Apr 05 14:46:38 localhost Keepalived_vrrp[93233]: Sending gratuitous ARP on ens33 for 192.168.1.111
Apr 05 14:46:38 localhost Keepalived_healthcheckers[93232]: Timeout connecting server [192.168.200.5]:1358.
Apr 05 14:46:40 localhost Keepalived_healthcheckers[93232]: Timeout connecting server [192.168.200.3]:1358.
Apr 05 14:46:40 localhost Keepalived_healthcheckers[93232]: Timeout connecting server [192.168.201.100]:443.
Apr 05 14:46:43 localhost Keepalived_healthcheckers[93232]: Timeout connecting server [192.168.200.2]:1358.
Apr 05 14:46:44 localhost Keepalived_healthcheckers[93232]: Timeout connecting server [192.168.200.4]:1358.
Apr 05 14:46:44 localhost Keepalived_healthcheckers[93232]: Timeout connecting server [192.168.200.5]:1358.
Apr 05 14:46:46 localhost Keepalived_healthcheckers[93232]: Timeout connecting server [192.168.200.3]:1358.
Apr 05 14:46:46 localhost Keepalived_healthcheckers[93232]: Timeout connecting server [192.168.201.100]:443.
备机同样操作。
校验
关闭主机keepalived
[root@localhost ~]# systemctl stop keepalived
[root@localhost ~]# systemctl status keepalived
● keepalived.service - LVS and VRRP High Availability Monitor
Loaded: loaded (/usr/lib/systemd/system/keepalived.service; disabled; vendor preset: disabled)
Active: inactive (dead)
Apr 05 14:47:28 localhost Keepalived_healthcheckers[93232]: Timeout reading data to remote SMTP server [192.168.200.1]:25.
Apr 05 14:47:28 localhost Keepalived_healthcheckers[93232]: Timeout reading data to remote SMTP server [192.168.200.1]:25.
Apr 05 14:50:50 localhost systemd[1]: Stopping LVS and VRRP High Availability Monitor...
Apr 05 14:50:50 localhost Keepalived[93231]: Stopping
Apr 05 14:50:50 localhost Keepalived_healthcheckers[93232]: Stopped
Apr 05 14:50:50 localhost Keepalived_vrrp[93233]: VRRP_Instance(VI_1) sent 0 priority
Apr 05 14:50:50 localhost Keepalived_vrrp[93233]: VRRP_Instance(VI_1) removing protocol VIPs.
Apr 05 14:50:51 localhost Keepalived_vrrp[93233]: Stopped
Apr 05 14:50:51 localhost Keepalived[93231]: Stopped Keepalived v1.3.5 (03/19,2017), git commit v1.3.5-6-g6fa32f2
Apr 05 14:50:51 localhost systemd[1]: Stopped LVS and VRRP High Availability Monitor.
检测
Keepalived+Nginx 高可用集群 (双主模式)
集群架构图
修改配置
[root@localhost ~]# cd /etc/keepalived/
[root@localhost keepalived]# ls
keepalived.conf keepalived.conf.bak
# 建议将主从模式配置备份
[root@localhost keepalived]# cp keepalived.conf keepalived.conf.ms_bk
[root@localhost keepalived]# ls
keepalived.conf keepalived.conf.bak keepalived.conf.ms_bk
修改192.168.1.12配置
vrrp_instance VI_1 {
state BACKUP # 主机 MASTER，备机 BACKUP
interface ens33 # 网卡名称
virtual_router_id 51 # 主，备机的virtual_router_id必须相同
priority 100 # 主，备机取不同的优先级，主机值较大，备份机值较小
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.1.111/24 dev ens33 label ens33:1 # VRRP H虚拟地址
}
}
vrrp_instance VI_2 {
state MASTER # 主机 MASTER，备机BACKUP
interface ens33 # 网卡名称
virtual_router_id 52 # 主，备机的virtual_router_id必须相同
priority 150 # 主，备机取不同的优先级，主机值较大，备份机值较小
advert_int 1
authentication {
auth_type PASS
auth_pass 2222
}
virtual_ipaddress {
192.168.1.112/24 dev ens33 label ens33:2 # VRRP H虚拟地址
}
}
修改192.168.1.11配置
vrrp_instance VI_1 {
state MASTER # 主机 MASTER，备机BACKUP
interface ens33 # 网卡名称
virtual_router_id 51 # 主，备机的virtual_router_id必须相同
priority 150 # 主，备机取不同的优先级，主机值较大，备份机值较小
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.1.111/24 dev ens33 label ens33:1 # VRRP H虚拟地址
}
}
vrrp_instance VI_2 {
state BACKUP # 主机 MASTER，备机BACKUP
interface ens33 # 网卡名称
virtual_router_id 52 # 主，备机的virtual_router_id必须相同
priority 100 # 主，备机取不同的优先级，主机值较大，备份机值较小
advert_int 1
authentication {
auth_type PASS
auth_pass 2222
}
virtual_ipaddress {
192.168.1.112/24 dev ens33 label ens33:2 # VRRP H虚拟地址
}
}
启动keepalived
[root@localhost ~]# systemctl start keepalived
检测
# 192.168.1.11
[root@localhost ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:d6:85:50 brd ff:ff:ff:ff:ff:ff
inet 192.168.1.11/24 brd 192.168.1.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet 192.168.1.111/24 scope global secondary ens33:1
valid_lft forever preferred_lft forever
inet6 fe80::3e71:f6ff:5b69:2539/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@localhost ~]# ping 192.168.1.111
PING 192.168.1.111 (192.168.1.111) 56(84) bytes of data.
64 bytes from 192.168.1.111: icmp_seq=1 ttl=64 time=0.027 ms
64 bytes from 192.168.1.111: icmp_seq=2 ttl=64 time=0.068 ms
64 bytes from 192.168.1.111: icmp_seq=3 ttl=64 time=0.070 ms
^C
--- 192.168.1.111 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2005ms
rtt min/avg/max/mdev = 0.027/0.055/0.070/0.019 ms
[root@localhost ~]# ping 192.168.1.112
PING 192.168.1.112 (192.168.1.112) 56(84) bytes of data.
64 bytes from 192.168.1.112: icmp_seq=1 ttl=64 time=0.477 ms
64 bytes from 192.168.1.112: icmp_seq=2 ttl=64 time=0.510 ms
64 bytes from 192.168.1.112: icmp_seq=3 ttl=64 time=0.529 ms
^C
--- 192.168.1.112 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2004ms
rtt min/avg/max/mdev = 0.477/0.505/0.529/0.028 ms
# 192.168.1.12
[root@localhost ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:93:75:6a brd ff:ff:ff:ff:ff:ff
inet 192.168.1.12/24 brd 192.168.1.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet 192.168.1.112/24 scope global secondary ens33:2
valid_lft forever preferred_lft forever
inet6 fe80::3353:a636:630b:4a4f/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@localhost ~]# ping 192.168.1.111
PING 192.168.1.111 (192.168.1.111) 56(84) bytes of data.
64 bytes from 192.168.1.111: icmp_seq=1 ttl=64 time=0.766 ms
64 bytes from 192.168.1.111: icmp_seq=2 ttl=64 time=0.857 ms
64 bytes from 192.168.1.111: icmp_seq=3 ttl=64 time=0.554 ms
^C
--- 192.168.1.111 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2012ms
rtt min/avg/max/mdev = 0.554/0.725/0.857/0.130 ms
[root@localhost ~]# ping 192.168.1.112
PING 192.168.1.112 (192.168.1.112) 56(84) bytes of data.
64 bytes from 192.168.1.112: icmp_seq=1 ttl=64 time=0.050 ms
64 bytes from 192.168.1.112: icmp_seq=2 ttl=64 time=0.072 ms
64 bytes from 192.168.1.112: icmp_seq=3 ttl=64 time=0.071 ms
^C
--- 192.168.1.112 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2006ms
rtt min/avg/max/mdev = 0.050/0.064/0.072/0.012 ms
关闭一台keepalived
[root@localhost ~]# systemctl stop keepalived
[root@localhost ~]# systemctl status keepalived
● keepalived.service - LVS and VRRP High Availability Monitor
Loaded: loaded (/usr/lib/systemd/system/keepalived.service; disabled; vendor preset: disabled)
Active: inactive (dead)
Apr 06 08:51:30 localhost Keepalived_healthcheckers[9477]: Timeout reading data to remote SMTP server [192.168.200.1]:25.
Apr 06 08:51:30 localhost Keepalived_healthcheckers[9477]: Timeout reading data to remote SMTP server [192.168.200.1]:25.
Apr 06 09:22:20 localhost Keepalived[9476]: Stopping
Apr 06 09:22:20 localhost systemd[1]: Stopping LVS and VRRP High Availability Monitor...
Apr 06 09:22:20 localhost Keepalived_vrrp[9478]: VRRP_Instance(VI_1) sent 0 priority
Apr 06 09:22:20 localhost Keepalived_vrrp[9478]: VRRP_Instance(VI_1) removing protocol VIPs.
Apr 06 09:22:20 localhost Keepalived_healthcheckers[9477]: Stopped
Apr 06 09:22:21 localhost Keepalived_vrrp[9478]: Stopped
Apr 06 09:22:21 localhost Keepalived[9476]: Stopped Keepalived v1.3.5 (03/19,2017), git commit v1.3.5-6-g6fa32f2
Apr 06 09:22:21 localhost systemd[1]: Stopped LVS and VRRP High Availability Monitor.
