B05-openstack高可用-haproxy集群部署

1. 安装haproxy

在全部控制节点安装haproxy，以controller01节点为例

[root@controller03 ~]# yum -y install haproxy

2：配置haproxy.cfg

[root@controller01 ~]# cp /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.cfg.bak

[root@controller01 ~]# cat /etc/haproxy/haproxy.cfg
global
  log      127.0.0.1     local0
  chroot   /var/lib/haproxy
  daemon
  group    haproxy
  user     haproxy
  maxconn  4000
  pidfile  /var/run/haproxy.pid
  stats    socket /var/lib/haproxy/stats

defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000


# haproxy监控页
listen stats
  bind 0.0.0.0:1080
  mode http
  stats enable
  stats uri /
  stats realm OpenStack Haproxy
  stats auth admin:admin
  stats  refresh 30s
  stats  show-node
  stats  show-legends
  stats  hide-version

# horizon服务
 listen dashboard_cluster
  bind  10.100.214.200:80
  balance  source
  option  tcpka
  option  httpchk
  option  tcplog
  server controller01 10.100.214.201:80 check inter 2000 rise 2 fall 5
  server controller02 10.100.214.202:80 check inter 2000 rise 2 fall 5
  server controller03 10.100.214.203:80 check inter 2000 rise 2 fall 5

# mariadb服务；
# 设置controller01节点为master，controller02/03节点为backup，一主多备的架构可规避数据不一致性；
# 另外官方示例为检测9200（心跳）端口，测试在mariadb服务宕机的情况下，虽然”/usr/bin/clustercheck”脚本已探测不到服务，但受xinetd控制的9200端口依然正常，导致haproxy始终将请求转发到mariadb服务宕机的节点，暂时修改为监听3306端口
listen galera_cluster
  bind 10.100.214.200:3306
  balance  source
  mode    tcp
  server controller01 10.100.214.203:3306 check inter 2000 rise 2 fall 5
  server controller02 10.100.214.202:3306 backup check inter 2000 rise 2 fall 5
  server controller03 10.100.214.203:3306 backup check inter 2000 rise 2 fall 5

# 为rabbirmq提供ha集群访问端口，供openstack各服务访问；
# 如果openstack各服务直接连接rabbitmq集群，这里可不设置rabbitmq的负载均衡
 listen rabbitmq_cluster
   bind 10.100.214.200:5673
   mode tcp
   option tcpka
   balance roundrobin
   timeout client  3h
   timeout server  3h
   option  clitcpka
   server controller01 10.100.214.201:5672 check inter 10s rise 2 fall 5
   server controller02 10.100.214.202:5672 check inter 10s rise 2 fall 5
   server controller03 10.100.214.203:5672 check inter 10s rise 2 fall 5

# glance_api服务
 listen glance_api_cluster
  bind  10.100.214.200:9292
  balance  source
  option  tcpka
  option  httpchk
  option  tcplog
  server controller01 10.100.214.201:9292 check inter 2000 rise 2 fall 5
  server controller02 10.100.214.202:9292 check inter 2000 rise 2 fall 5
  server controller03 10.100.214.203:9292 check inter 2000 rise 2 fall 5

# glance_registry服务
 listen glance_registry_cluster
  bind 10.100.214.200:9191
  balance  source
  option  tcpka
  option  tcplog
  server controller01 10.100.214.201:9191 check inter 2000 rise 2 fall 5
  server controller02 10.100.214.202:9191 check inter 2000 rise 2 fall 5
  server controller03 10.100.214.203:9191 check inter 2000 rise 2 fall 5

# keystone_admin_internal_api服务
 listen keystone_admin_cluster
  bind 10.100.214.200:35357
  balance  source
  option  tcpka
  option  httpchk
  option  tcplog
  server controller01 10.100.214.201:35357 check inter 2000 rise 2 fall 5
  server controller02 10.100.214.202:35357 check inter 2000 rise 2 fall 5
  server controller03 10.100.214.203:35357 check inter 2000 rise 2 fall 5

# keystone_public _api服务
 listen keystone_public_cluster
  bind 10.100.214.200:5000
  balance  source
  option  tcpka
  option  httpchk
  option  tcplog
  server controller01 10.100.214.201:5000 check inter 2000 rise 2 fall 5
  server controller02 10.100.214.202:5000 check inter 2000 rise 2 fall 5
  server controller03 10.100.214.203:5000 check inter 2000 rise 2 fall 5

# 兼容aws ec2-api
 listen nova_ec2_api_cluster
  bind 10.100.214.200:8773
  balance  source
  option  tcpka
  option  tcplog
  server controller01 10.100.214.201:8773 check inter 2000 rise 2 fall 5
  server controller02 10.100.214.202:8773 check inter 2000 rise 2 fall 5
  server controller03 10.100.214.203:8773 check inter 2000 rise 2 fall 5

 listen nova_compute_api_cluster
  bind 10.100.214.200:8774
  balance  source
  option  tcpka
  option  httpchk
  option  tcplog
  server controller01 10.100.214.201:8774 check inter 2000 rise 2 fall 5
  server controller02 10.100.214.202:8774 check inter 2000 rise 2 fall 5
  server controller03 10.100.214.203:8774 check inter 2000 rise 2 fall 5

 listen nova_placement_cluster
  bind 10.100.214.200:8778
  balance  source
  option  tcpka
  option  tcplog
  server controller01 10.100.214.201:8778 check inter 2000 rise 2 fall 5
  server controller02 10.100.214.202:8778 check inter 2000 rise 2 fall 5
  server controller03 10.100.214.203:8778 check inter 2000 rise 2 fall 5

 listen nova_metadata_api_cluster
  bind 10.100.214.200:8775
  balance  source
  option  tcpka
  option  tcplog
  server controller01 10.100.214.201:8775 check inter 2000 rise 2 fall 5
  server controller02 10.100.214.202:8775 check inter 2000 rise 2 fall 5
  server controller03 10.100.214.203:8775 check inter 2000 rise 2 fall 5

 listen nova_vncproxy_cluster
  bind 10.100.214.200:6080
  balance  source
  option  tcpka
  option  tcplog
  server controller01 10.100.214.201:6080 check inter 2000 rise 2 fall 5
  server controller02 10.100.214.202:6080 check inter 2000 rise 2 fall 5
  server controller03 10.100.214.203:6080 check inter 2000 rise 2 fall 5

 listen neutron_api_cluster
  bind 10.100.214.200:9696
  balance  source
  option  tcpka
  option  httpchk
  option  tcplog
  server controller01 10.100.214.201:9696 check inter 2000 rise 2 fall 5
  server controller02 10.100.214.202:9696 check inter 2000 rise 2 fall 5
  server controller03 10.100.214.203:9696 check inter 2000 rise 2 fall 5

 listen cinder_api_cluster
  bind 10.100.214.200:8776
  balance  source
  option  tcpka
  option  httpchk
  option  tcplog
  server controller01 10.100.214.201:8776 check inter 2000 rise 2 fall 5
  server controller02 10.100.214.202:8776 check inter 2000 rise 2 fall 5
  server controller03 10.100.214.203:8776 check inter 2000 rise 2 fall 5

将配置文件拷贝到其他节点中：

[root@controller01 ~]# scp /etc/haproxy/haproxy.cfg 10.100.214.202:/etc/haproxy/haproxy.cfg
[root@controller01 ~]# scp /etc/haproxy/haproxy.cfg 10.100.214.203:/etc/haproxy/haproxy.cfg

3：配置内核参数

# 全部控制节点修改内核参数，以controller01节点为例；
# net.ipv4.ip_nonlocal_bind：是否允许no-local ip绑定，关系到haproxy实例与vip能否绑定并切换；
# net.ipv4.ip_forward：是否允许转发

[root@controller01 ~]# echo "net.ipv4.ip_nonlocal_bind = 1" >>/etc/sysctl.conf
[root@controller01 ~]# echo "net.ipv4.ip_forward = 1" >>/etc/sysctl.conf
[root@controller01 ~]# sysctl -p
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ip_forward = 1

4：启动服务

[root@controller01 ~]# systemctl enable haproxy && systemctl restart  haproxy && systemctl status haproxy

5：设置pcs资源：

# 任意控制节点操作即可，以controller01节点为例；
# 添加资源lb-haproxy-clone

[root@controller01 ~]# pcs resource create lb-haproxy systemd:haproxy --clone 
[root@controller01 ~]# pcs resource
 vip    (ocf::heartbeat:IPaddr2):    Started controller01
 Clone Set: lb-haproxy-clone [lb-haproxy]
     Started: [ controller01 controller02 controller03 ]

# 设置资源启动顺序，先vip再lb-haproxy-clone；
# 通过“cibadmin --query --scope constraints”可查看资源约束配置

[root@controller01 ~]# pcs constraint order start vip then lb-haproxy-clone kind=Optional --force
Adding vip lb-haproxy-clone (kind: Optional) (Options: first-action=start then-action=start)

 # 官方建议设置vip运行在haproxy active的节点，通过绑定lb-haproxy-clone与vip服务，将两种资源约束在1个节点； # 约束后，从资源角度看，其余暂时没有获得vip的节点的haproxy会被pcs关闭

 [root@controller01 ~]# pcs constraint colocation add lb-haproxy-clone with vip --force
[root@controller01 ~]# pcs resource
 vip    (ocf::heartbeat:IPaddr2):    Started controller01
 Clone Set: lb-haproxy-clone [lb-haproxy]
     Started: [ controller01 ]
     Stopped: [ controller02 controller03 ]