haproxy+keepalived+rabbitmq3.8实现集群的高可用

rabbitmq部署在机房，添加镜像集群后，需要做高可用，当单机故障时可以切换到另外一台

1.修改rabbitmq的配置，引入修改相关配置的文件

# vim /usr/local/rabbitmq/sbin/rabbitmq-defaults
# 添加这行
CONFIG_FILE=${SYS_PREFIX}/etc/rabbitmq/rabbitmq.conf

# 添加修改端口的配置
vim /usr/local/rabbitmq/etc/rabbitmq/rabbitmq.conf
#数据管理端口（默认端口为5672）
listeners.tcp.default=5673
#界面管理端口（默认端口为15672）
management.tcp.port=15672
management.tcp.ip=0.0.0.0

chown -R rabbitmq.rabbitmq /etc/rabbitmq/

2.配置负载均衡程序haproxy

添加haproxy的相关配置
useradd haproxy -s /sbin/nologin

# 编写 haproxy 的启动脚本

vim /etc/init.d/haproxy

#!/bin/sh
#
# chkconfig: - 85 15
# description: HA-Proxy is a TCP/HTTP reverse proxy which is particularly suited 
#              for high availability environments.
# processname: haproxy
# config: /etc/haproxy/haproxy.cfg
# pidfile: /var/run/haproxy.pid

# Script Author: Simon Matter <simon.matter@invoca.ch>
# Version: 2004060600

# Source function library.
if [ -f /etc/init.d/functions ]; then
  . /etc/init.d/functions
elif [ -f /etc/rc.d/init.d/functions ] ; then
  . /etc/rc.d/init.d/functions
else
  exit 0
fi

# Source networking configuration.
. /etc/sysconfig/network

# Check that networking is up.
[ ${NETWORKING} = "no" ] && exit 0

# This is our service name
BASENAME=`basename $0`
if [ -L $0 ]; then
  BASENAME=`find $0 -name $BASENAME -printf %l`
  BASENAME=`basename $BASENAME`
fi

BIN=/usr/sbin/$BASENAME

CFG=/etc/$BASENAME/$BASENAME.cfg
[ -f $CFG ] || exit 1

PIDFILE=/var/run/$BASENAME.pid
LOCKFILE=/var/lock/subsys/$BASENAME

RETVAL=0

start() {
  quiet_check
  if [ $? -ne 0 ]; then
    echo "Errors found in configuration file, check it with '$BASENAME check'."
    return 1
  fi

  echo -n "Starting $BASENAME: "
  daemon $BIN -D -f $CFG -p $PIDFILE
  RETVAL=$?
  echo
  [ $RETVAL -eq 0 ] && touch $LOCKFILE
  return $RETVAL
}

stop() {
  echo -n "Shutting down $BASENAME: "
  killproc $BASENAME -USR1
  RETVAL=$?
  echo
  [ $RETVAL -eq 0 ] && rm -f $LOCKFILE
  [ $RETVAL -eq 0 ] && rm -f $PIDFILE
  return $RETVAL
}

restart() {
  quiet_check
  if [ $? -ne 0 ]; then
    echo "Errors found in configuration file, check it with '$BASENAME check'."
    return 1
  fi
  stop
  start
}

reload() {
  if ! [ -s $PIDFILE ]; then
    return 0
  fi

  quiet_check
  if [ $? -ne 0 ]; then
    echo "Errors found in configuration file, check it with '$BASENAME check'."
    return 1
  fi
  $BIN -D -f $CFG -p $PIDFILE -sf $(cat $PIDFILE)
}

check() {
  $BIN -c -q -V -f $CFG
}

quiet_check() {
  $BIN -c -q -f $CFG
}

rhstatus() {
  status $BASENAME
}

condrestart() {
  [ -e $LOCKFILE ] && restart || :
}

# See how we were called.
case "$1" in
  start)
    start
    ;;
  stop)
    stop
    ;;
  restart)
    restart
    ;;
  reload)
    reload
    ;;
  condrestart)
    condrestart
    ;;
  status)
    rhstatus
    ;;
  check)
    check
    ;;
  *)
    echo $"Usage: $BASENAME {start|stop|restart|reload|condrestart|status|check}"
    exit 1
esac
 
exit $?

3.添加haproxy的配置

# cat /etc/haproxy/haproxy.cfg 
#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
    # to have these messages end up in /var/log/haproxy.log you will
    # need to:
    #
    # 1) configure syslog to accept network log events.  This is done
    #    by adding the '-r' option to the SYSLOGD_OPTIONS in
    #    /etc/sysconfig/syslog
    #
    # 2) configure local2 events to go to the /var/log/haproxy.log
    #   file. A line like the following can be added to
    #   /etc/sysconfig/syslog
    #
    #    local2.*                       /var/log/haproxy.log
    #
    log         127.0.0.1 local2
 
    chroot      /usr/local/haproxy
    pidfile     /usr/local/haproxy/haproxy.pid
    maxconn     50000
    user        haproxy
    group       haproxy
    daemon
    nbproc      1
 
    # turn on stats unix socket
    stats socket /usr/local/haproxy/stats
#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------

defaults
    mode                    tcp
    log                     global
#   option                  httplog
    option                  tcplog
    option                  dontlognull
    option             http-server-close
#   option forwardfor  except 127.0.0.0/8
#   option     redispath
    retries                 3
    timeout queue           1m
    timeout connect         10s
    timeout client          2m
    timeout server          2m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000


#---------------------------------------------------------------------
# HAProxy statistics backend
#---------------------------------------------------------------------
listen haproxy-monitoring 
    bind *:4321
    mode  http
    stats enable
    stats refresh           30s
    stats uri               /stats
    stats realm             HAProxy Statistics
    stats auth              admin:admin
    stats hide-version
 
#---------------------------------------------------------------------
# main frontend which proxys to the backends
#---------------------------------------------------------------------
listen rabbitmq_cluster 
    bind 0.0.0.0:5672
    mode tcp
    balance roundrobin
    server rabbitmq_01 192.168.254.187:5673 check inter 2000 rise 2 fall 3
    server rabbitmq_02 192.168.254.196:5673 check inter 2000 rise 2 fall 3

#---------------------------------------------------------------------
# main frontend which proxys to the backends
#---------------------------------------------------------------------
listen rabbitmq_web
    bind 0.0.0.0:27651
    mode tcp
    balance roundrobin
    server rabbitmq_01 192.168.254.187:15672 check inter 2000 rise 2 fall 3
    server rabbitmq_02 192.168.254.196:15672 check inter 2000 rise 2 fall 3
 
#---------------------------------------------------------------------
# round robin balancing between the various backends

# 配置开机自动启动

chkconfig --add haproxy
chkconfig haproxy on
chkconfig --list haproxy


4.配置keepvid
# 自动启动haproxy的监控脚本
mkdir /etc/keepalived/
vim /etc/keepalived/check_haproxy.sh

#!/bin/bash
A=`ps -C haproxy --no-header | wc -l`
if [ $A -eq 0 ];then
#/usr/local/haproxy/sbin/haproxy -f /etc/haproxy/haproxy.cfg &
killall haproxy
/etc/init.d/haproxy start
echo "haproxy start done" >>/etc/haproxy/haproxy_start.log
sleep 3
if [ `ps -C haproxy --no-header | wc -l` -eq 0 ];then
/etc/init.d/keepalived stop
echo "keepalived stop"
fi
fi

chmod +x /etc/keepalived/check_haproxy.sh
# keepalived的启动文件

vim /etc/init.d/keepalived

#!/bin/sh
#
# Startup script for the Keepalived daemon
#
# processname: keepalived
# pidfile: /var/run/keepalived.pid
# config: /etc/keepalived/keepalived.conf
# chkconfig: - 21 79
# description: Start and stop Keepalived

# Source function library
. /etc/rc.d/init.d/functions

# Source configuration file (we set KEEPALIVED_OPTIONS there)
. /etc/sysconfig/keepalived

RETVAL=0

prog="keepalived"

start() {
    echo -n $"Starting $prog: "
    daemon keepalived ${KEEPALIVED_OPTIONS}
    RETVAL=$?
    echo
    [ $RETVAL -eq 0 ] && touch /var/lock/subsys/$prog
}

stop() {
    echo -n $"Stopping $prog: "
    killproc keepalived
    RETVAL=$?
    echo
    [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/$prog
}

reload() {
    echo -n $"Reloading $prog: "
    killproc keepalived -1
    RETVAL=$?
    echo
}

# See how we were called.
case "$1" in
    start)
        start
        ;;
    stop)
        stop
        ;;
    reload)
        reload
        ;;
    restart)
        stop
        start
        ;;
    condrestart)
        if [ -f /var/lock/subsys/$prog ]; then
            stop
            start
        fi
        ;;
    status)
        status keepalived
        RETVAL=$?
        ;;
    *)
        echo "Usage: $0 {start|stop|reload|restart|condrestart|status}"
        RETVAL=1
esac

exit $RETVAL

# 配置 keepvid.conf

# master
vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived
global_defs {
   notification_email {
   pengll@chinasoft.com
        }
   notification_email_from wsadmin@chinasoft.com
   smtp_server 1.1.1.2
   smtp_connect_timeout 30
   router_id LVS_MQ_DEVEL
}

vrrp_script chk_haproxy_status {
                script "/bin/bash /etc/keepalived/check_haproxy.sh"
                interval 3
                weight 2
}

vrrp_instance VI_1 {
    state MASTER  ##备份服务器上将MASTER改为BACKUP   
    interface eth0
    virtual_router_id 71
    garp_master_delay 2 #主从切换时间，单位为秒。
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass rabbitmqpass
    }

track_script {
        chk_haproxy_status
        }

    virtual_ipaddress {
        192.168.254.86
    }
}

vrrp_instance VI_2 {
    state MASTER
    interface eth1
    virtual_router_id 71
    garp_master_delay 2 #主从切换时间，单位为秒。
    priority 88
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass rabbitmqpass
    }

track_script {
        chk_haproxy_status
        }

    virtual_ipaddress {
        1.1.1.1
    }
}

# slave
vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived
global_defs {
   notification_email {
   pengll@chinasoft.com
        }
   notification_email_from wsadmin@chinasoft.com
   smtp_server 1.1.1.2
   smtp_connect_timeout 30
   router_id LVS_MQ_DEVEL
}

vrrp_script chk_haproxy_status {
                script "/bin/sh /etc/keepalived/check_haproxy.sh"
                interval 5    #运行间隔
                weight 2
}

vrrp_instance VI_1 {
    state BACKUP  ##备份服务器上将MASTER改为BACKUP   
    interface eth0
    virtual_router_id 71
    garp_master_delay 2 #主从切换时间，单位为秒。
    priority 99 ##此处需要比master值小
    advert_int 1   ###MASTER与BACKUP节点间同步检查的时间间隔,单位为秒,两个节点设置必须一样
    authentication {
        auth_type PASS
        auth_pass rabbitmqpass
    }

track_script {
        chk_haproxy_status
        }

    virtual_ipaddress {
        192.168.254.86
    }
}

vrrp_instance VI_2 {
    state BACKUP
    interface eth1
    virtual_router_id 71
    garp_master_delay 2 #主从切换时间，单位为秒。
    priority 87 ##此处需要比master值小
    advert_int 1 #MASTER与BACKUP节点间同步检查的时间间隔,单位为秒,两个节点设置必须一样
    authentication {
        auth_type PASS
        auth_pass rabbitmqpass
    }

track_script {
        chk_haproxy_status
        }

    virtual_ipaddress {
        1.1.1.1
    }
}

# 系统配置
vim /etc/sysconfig/keepalived

# Options for keepalived. See `keepalived --help' output and keepalived(8) and
# keepalived.conf(5) man pages for a list of all options. Here are the most
# common ones :
#
# --vrrp               -P    Only run with VRRP subsystem.
# --check              -C    Only run with Health-checker subsystem.
# --dont-release-vrrp  -V    Dont remove VRRP VIPs & VROUTEs on daemon stop.
# --dont-release-ipvs  -I    Dont remove IPVS topology on daemon stop.
# --dump-conf          -d    Dump the configuration data.
# --log-detail         -D    Detailed log messages.
# --log-facility       -S    0-7 Set local syslog facility (default=LOG_DAEMON)
#

KEEPALIVED_OPTIONS="-D -d -S 0"

cp /usr/local/keepalived/sbin/keepalived /usr/sbin/

# 添加开机自动启动keepalived
chkconfig --add keepalived
chkconfig keepalived on
chkconfig --list keepalived

# Options for keepalived. See `keepalived --help' output and keepalived(8) and
# keepalived.conf(5) man pages for a list of all options. Here are the most
# common ones :
#
# --vrrp               -P    Only run with VRRP subsystem.
# --check              -C    Only run with Health-checker subsystem.
# --dont-release-vrrp  -V    Dont remove VRRP VIPs & VROUTEs on daemon stop.
# --dont-release-ipvs  -I    Dont remove IPVS topology on daemon stop.
# --dump-conf          -d    Dump the configuration data.
# --log-detail         -D    Detailed log messages.
# --log-facility       -S    0-7 Set local syslog facility (default=LOG_DAEMON)
#

KEEPALIVED_OPTIONS="-D -d -S 0"