zoukankan      html  css  js  c++  java
  • Haproxy压测

    目的:测试Haproxy压测情况

    环境:

    Ha服务器:8核16G虚机,后端6个2核4G,压测客户端3个2核4G

    安装和优化:

    一、Haproxy

    #cd /opt/soft
    #wget http://www.haproxy.org/download/1.8/src/haproxy-1.8.14.tar.gz
    #tar zxvf haproxy-1.8.14.tar.gz
    #cd haproxy-1.8.14
    安装:
    #make TARGET=linux2628  ARCH=X86_64 PREFIX=/usr/local/haproxy
    #make install PREFIX=/usr/local/haproxy
    添加用户:
    #useradd -s /sbin/nologin -M haproxy
    id haproxy
    #mkdir /etc/haproxy/
    
    配置systemctl
    #cp /opt/soft/haproxy-1.8.14/examples/haproxy.init /etc/init.d/haproxy
    #chmod +x /etc/init.d/haproxy
    #vi /etc/init.d/haproxy
    

    /etc/init.d/haproxy

    #!/bin/sh
    #
    # chkconfig: - 85 15
    # description: HA-Proxy is a TCP/HTTP reverse proxy which is particularly suited 
    #              for high availability environments.
    # processname: haproxy
    # config: /etc/haproxy/haproxy.cfg
    # pidfile: /var/run/haproxy.pid
    
    # Source function library.
    if [ -f /etc/init.d/functions ]; then
      . /etc/init.d/functions
    elif [ -f /etc/rc.d/init.d/functions ] ; then
      . /etc/rc.d/init.d/functions
    else
      exit 0
    fi
    
    # Source networking configuration.
    . /etc/sysconfig/network
    
    # Check that networking is up.
    [ ${NETWORKING} = "no" ] && exit 0
    
    # This is our service name
    BASENAME=`basename $0`
    if [ -L $0 ]; then
      BASENAME=`find $0 -name $BASENAME -printf %l`
      BASENAME=`basename $BASENAME`
    fi
    
    BIN=/usr/sbin/$BASENAME
    
    CFG=/etc/$BASENAME/$BASENAME.cfg
    [ -f $CFG ] || exit 1
    
    PIDFILE=/var/run/$BASENAME.pid
    LOCKFILE=/var/lock/subsys/$BASENAME
    
    RETVAL=0
    
    start() {
      quiet_check
      if [ $? -ne 0 ]; then
        echo "Errors found in configuration file, check it with '$BASENAME check'."
        return 1
      fi
    
      echo -n "Starting $BASENAME: "
      daemon $BIN -D -f $CFG -p $PIDFILE
      RETVAL=$?
      echo
      [ $RETVAL -eq 0 ] && touch $LOCKFILE
      return $RETVAL
    }
    
    stop() {
      echo -n "Shutting down $BASENAME: "
      killproc $BASENAME -USR1
      RETVAL=$?
      echo
      [ $RETVAL -eq 0 ] && rm -f $LOCKFILE
      [ $RETVAL -eq 0 ] && rm -f $PIDFILE
      return $RETVAL
    }
    
    restart() {
      quiet_check
      if [ $? -ne 0 ]; then
        echo "Errors found in configuration file, check it with '$BASENAME check'."
        return 1
      fi
      stop
      start
    }
    
    reload() {
      if ! [ -s $PIDFILE ]; then
        return 0
      fi
    
      quiet_check
      if [ $? -ne 0 ]; then
        echo "Errors found in configuration file, check it with '$BASENAME check'."
        return 1
      fi
      $BIN -D -f $CFG -p $PIDFILE -sf $(cat $PIDFILE)
    }
    
    check() {
      $BIN -c -q -V -f $CFG
    }
    
    quiet_check() {
      $BIN -c -q -f $CFG
    }
    
    rhstatus() {
      status $BASENAME
    }
    
    condrestart() {
      [ -e $LOCKFILE ] && restart || :
    }
    
    # See how we were called.
    case "$1" in
      start)
        start
        ;;
      stop)
        stop
        ;;
      restart)
        restart
        ;;
      reload)
        reload
        ;;
      condrestart)
        condrestart
        ;;
      status)
        rhstatus
        ;;
      check)
        check
        ;;
      *)
        echo $"Usage: $BASENAME {start|stop|restart|reload|condrestart|status|check}"
        exit 1
    esac
     
    exit $?
    

      Haproxy配置:

    global
    group root
        user root
        daemon
        nbproc 7
        log 127.0.0.1 local3
        pidfile /var/run/haproxy/pid/haproxy.pid
        ulimit-n 1000000 ##这里增大文件打开数
        max-spread-checks  1000ms
        maxconn     3000000 ##这里放开限制
        maxconnrate 3000000
        maxsessrate 3000000
        #maxsslconn  3000000
        #maxsslrate  3000000
        spread-checks 20
        stats timeout 60s
        stats maxconn 500
        stats socket /var/run/haproxy/sock/haproxy.sock mode 600 level admin process 1
    
    #---------------------------------------------------------------------
    # common defaults that all the 'listen' and 'backend' sections will
    # use if not designated in their block
    #---------------------------------------------------------------------
    defaults
        mode http
        #mode tcp
        maxconn     3000000 ##这里最好与上面一致,否则默认限制2000,找了好久
        option  abortonclose
        option redispatch
        option forwardfor
        balance roundrobin
        log 127.0.0.1 local3 err
        retries 3
        option clitcpka
        option srvtcpka
    #---------------------------------------------------------------------
    # main frontend which proxys to the backends
    #---------------------------------------------------------------------
    listen 172.28.20.102:81
        balance roundrobin
        bind *:81  
        option tcp-check
        option httplog
        #option dontlognull
    	#test
        timeout http-keep-alive 70s
        timeout http-request    70s 
        timeout connect 100000
        timeout queue   100000
        timeout client  100000
        timeout server  100000
        timeout check   100000
        cookie SERVERID insert indirect nocache
        server  105_1 172.28.20.105:8282 check
        server  105_2 172.28.20.105:8283 check
        server  105_3 172.28.20.105:8284 check
        server  105_4 172.28.20.105:8285 check
        server  105_5 172.28.20.105:8286 check
        server  105_6 172.28.20.105:8287 check
        server  108_1 172.28.20.108:8282 check
        server  108_2 172.28.20.108:8283 check
        server  108_3 172.28.20.108:8284 check
        server  108_4 172.28.20.108:8285 check
        server  108_5 172.28.20.108:8286 check
        server  108_6 172.28.20.108:8287 check
        server  167_1 172.28.16.167:8282 check
        server  167_2 172.28.16.167:8283 check
        server  167_3 172.28.16.167:8284 check
        server  168_1 172.28.16.168:8282 check
        server  168_2 172.28.16.168:8283 check
        server  168_3 172.28.16.168:8284 check
        server  169_1 172.28.16.169:8282 check
        server  169_2 172.28.16.169:8283 check
        server  169_3 172.28.16.169:8284 check
        server  170_1 172.28.16.170:8282 check
        server  170_2 172.28.16.170:8283 check
        server  170_3 172.28.16.170:8284 check
    

     #内核参数优化(haproxy,后端服务器,客户端均要)

    #vi /etc/sysctl.conf

    ### 系统中所允许的文件句柄的最大数目
    fs.file-max = 12553500
    ### 单个进程所允许的文件句柄的最大数目
    fs.nr_open = 12453500
    ### 内核允许使用的共享内存大 Controls the maximum number of shared memory segments, in pages
    kernel.shmall = 4294967296
    ###单个共享内存段的最大值 Controls the maximum shared segment size, in bytes
    kernel.shmmax = 68719476736
    ### 内核中消息队列中消息的最大值 Controls the maximum size of a message, in bytes
    kernel.msgmax = 65536
    ### 系统救援工具
    kernel.sysrq = 0
    ### 在每个网络接口接收数据包的速率比内核处理这些包的速率快时,允许送到缓存队列的数据包的最大数目
    net.core.netdev_max_backlog = 2000000
    ### 默认的TCP数据接收窗口大小(字节)
    net.core.rmem_default = 699040
    ### 最大的TCP数据接收窗口(字节)
    net.core.rmem_max = 50331648
    ### 默认的TCP数据发送窗口大小(字节)
    net.core.wmem_default = 131072
    ### 最大的TCP数据发送窗口(字节)
    net.core.wmem_max = 33554432
    ### 定义了系统中每一个端口最大的监听队列的长度,这是个全局的参数
    net.core.somaxconn = 65535
    
    ### TCP/UDP协议允许使用的本地端口号
    net.ipv4.ip_local_port_range = 15000 65000
    net.ipv4.ip_nonlocal_bind = 1
    ### 对于本端断开的socket连接,TCP保持在FIN-WAIT-2状态的时间(秒)
    net.ipv4.tcp_fin_timeout = 7
    ### TCP发送keepalive探测消息的间隔时间(秒),用于确认TCP连接是否有效
    net.ipv4.tcp_keepalive_time = 300
    net.ipv4.tcp_max_orphans = 3276800
    ### 对于还未获得对方确认的连接请求,可保存在队列中的最大数目
    net.ipv4.tcp_max_syn_backlog = 655360
    net.ipv4.tcp_max_tw_buckets = 6000000
    ### 确定TCP栈应该如何反映内存使用,每个值的单位都是内存页(通常是4KB)
    ### 第一个值是内存使用的下限;第二个值是内存压力模式开始对缓冲区使用应用压力的上限;第三个值是内存使用的上限.
    net.ipv4.tcp_mem = 94500000 915000000 927000000
    ### 为自动调优定义socket使用的内存。
    ### 第一个值是为socket接收缓冲区分配的最少字节数;
    ### 第二个值是默认值(该值会被rmem_default覆盖),缓冲区在系统负载不重的情况下可以增长到这个值;
    ### 第三个值是接收缓冲区空间的最大字节数(该值会被rmem_max覆盖)
    net.ipv4.tcp_rmem = 32768 699040 50331648
    ### 为自动调优定义socket使用的内存。
    ### 第一个值是为socket发送缓冲区分配的最少字节数;
    ### 第二个值是默认值(该值会被wmem_default覆盖),缓冲区在系统负载不重的情况下可以增长到这个值;
    ### 第三个值是发送缓冲区空间的最大字节数(该值会被wmem_max覆盖)
    net.ipv4.tcp_wmem = 32768 131072 33554432
    net.ipv4.tcp_slow_start_after_idle = 0
    net.ipv4.tcp_synack_retries = 2
    ### 表示是否打开TCP同步标签(syncookie),同步标签可以防止一个套接字在有过多试图连接到达时引起过载
    ### 内核必须打开了CONFIG_SYN_COOKIES项进行编译,
    net.ipv4.tcp_syncookies = 1
    net.ipv4.tcp_syn_retries = 2
    ### 表示开启TCP连接中TIME-WAIT sockets的快速回收,默认为0,表示关闭
    net.ipv4.tcp_tw_recycle = 1
    ### 允许将TIME-WAIT sockets重新用于新的TCP连接,默认为0,表示关闭
    net.ipv4.tcp_tw_reuse = 1
    ### 启用RFC 1323定义的window scaling,要支持超过64KB的TCP窗口,必须启用该值(1表示启用),
    ### TCP窗口最大至1GB,TCP连接双方都启用时才生效,默认为1
    net.ipv4.tcp_window_scaling = 1
    ### 最大限度使用物理内存
    vm.swappiness = 0

    生效下

    #/sbin/sysctl -p

    vi /etc/security/limits.conf

    * soft nofile 65535
    * hard nofile 65535
    * soft nproc 65535
    * hard nproc 65535

      

    二、后端服务端

    服务器优化,参考haproxy修改sysctl.conf和limits.conf

    简单写了个springboot web端,创建springboot工程,加入以下两个类,编译成jar包。每个服务器运行3个实例

    启动脚本runhellos.sh

    #!/bin/sh
    counter=0
    while [ $counter -le 2 ]
    do
            port=$((8282+$counter))
            java -jar helloworld-0.0.1-SNAPSHOT.jar --server.port=$port 2>&1 >/dev/null &
            ((counter++))
    done
    echo "all servers create"
    

      关闭:pkill java

    主程序HelloController.class

    package com.wwd;
    
    import javax.servlet.http.HttpServletRequest;
    
    import org.springframework.boot.SpringApplication;
    import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
    import org.springframework.boot.autoconfigure.SpringBootApplication;
    import org.springframework.boot.context.embedded.ConfigurableEmbeddedServletContainer;
    import org.springframework.boot.context.embedded.EmbeddedServletContainerCustomizer;
    import org.springframework.stereotype.Controller;
    import org.springframework.web.bind.annotation.RequestMapping;
    import org.springframework.web.bind.annotation.ResponseBody;
    
    @Controller
    @EnableAutoConfiguration
    @SpringBootApplication
    public class HelloController {
    
        @RequestMapping("/hello")
        @ResponseBody
        String home(HttpServletRequest request) {
        	String sleep = request.getHeader("sleep");
        	if(!"".equals(sleep) && null != sleep){
        		System.out.println("sleeptime:"+(long)((Math.random()*Integer.parseInt(sleep))));
    	    	try {
    				Thread.sleep((long)((Math.random()*Integer.parseInt(sleep))));
    			} catch (NumberFormatException e) {
    				e.printStackTrace();
    			} catch (InterruptedException e) {
    				e.printStackTrace();
    			}
        	}
        	
            return "Hello ,spring boot!";
        }
    
        public static void main(String[] args) throws Exception {
            SpringApplication.run(HelloController.class, args);
            //运行之后在浏览器中访问:http://localhost:8080/hello
        }
    
    
        
       /* @Override  
        public void customize(ConfigurableEmbeddedServletContainer configurableEmbeddedServletContainer) {  
      
            
        }*/  
        
    }
    

     tomcat参数调整(因为要保持连接,设置的超时连接时间较长,不然压不上去。

    /*
     * Project: helloworld
     * 
     * File Created at 2018年10月11日
     * 
     * Copyright 2016 CMCC Corporation Limited.
     * All rights reserved.
     *
     * This software is the confidential and proprietary information of
     * ZYHY Company. ("Confidential Information").  You shall not
     * disclose such Confidential Information and shall use it only in
     * accordance with the terms of the license.
     */
    package com.wwd;
    
    import org.apache.catalina.connector.Connector;
    import org.apache.coyote.http11.Http11NioProtocol;
    import org.springframework.boot.context.embedded.EmbeddedServletContainer;
    import org.springframework.boot.context.embedded.ServletContextInitializer;
    import org.springframework.boot.context.embedded.tomcat.TomcatEmbeddedServletContainerFactory;
    import org.springframework.stereotype.Component;
    
    @Component()
    public class MyEmbeddedServletContainerFactory extends TomcatEmbeddedServletContainerFactory {
    	public EmbeddedServletContainer getEmbeddedServletContainer(ServletContextInitializer... initializers) {
    		// 设置端口
    //		this.setPort(8081);
    		return super.getEmbeddedServletContainer(initializers);
    	}
    
    	protected void customizeConnector(Connector connector) {
    		super.customizeConnector(connector);
    		Http11NioProtocol protocol = (Http11NioProtocol) connector.getProtocolHandler();
    		// 设置最大连接数
    		protocol.setMaxConnections(8000);
    		// 设置最大线程数
    		protocol.setMaxThreads(400);
    		protocol.setConnectionTimeout(3000000);
    	}
    }
    
    /**
     * Revision history
     * -------------------------------------------------------------------------
     * 
     * Date Author Note
     * -------------------------------------------------------------------------
     * 2018年10月11日 WWD create
     */

    三、压测工具

    开始用jmeter和ab去压测,只能测试tps信息,连接数压不上去。

    ab -c 300 -n 100000 http://172.28.20.102:81/hello

    引入贝吉塔Vegeta

    wget https://github.com/tsenart/vegeta/releases/download/v8.0.0/vegeta-8.0.0-linux-amd64.tar.gz
    tar zxvf vegeta-8.0.0-linux-amd64.tar.gz
    ln -s /opt/soft/vegeta /usr/local/sbin/vegeta
    

     2核4G,压测命令和参数如下,

    echo "GET http://172.28.20.102:81/hello" | vegeta -cpus=2 attack -duration=10m -header="sleep:60000" -rate=2000 -workers=200 -timeout=1800s | tee reports.bin | vegeta report
    

    -duration 10m为持续时间

    -rate 2000为每秒2000个请求

    -workers 200个线程  

    sleep为设置后端随机响应时间,以保持连接。

    四、测试对比结果

     调整sleep时间和增加压测客户端数量,查看estab数量

    查看命令: ss -s&&free -g

     

    能到24W的连接数

     但内存和CPU使用率不高,有待 进一步测试。

  • 相关阅读:
    ADF中遍历VO中的行数据(Iterator)
    程序中实现两个DataTable的Left Join效果(修改了,网上第二个DataTable为空,所处的异常)
    ArcGIS api for javascript——鼠标悬停时显示信息窗口
    ArcGIS api for javascript——查询,然后单击显示信息窗口
    ArcGIS api for javascript——查询,立刻打开信息窗口
    ArcGIS api for javascript——显示多个查询结果
    ArcGIS api for javascript——用图表显示查询结果
    ArcGIS api for javascript——查询没有地图的数据
    ArcGIS api for javascript——用第二个服务的范围设置地图范围
    ArcGIS api for javascript——显示地图属性
  • 原文地址:https://www.cnblogs.com/wuweidong/p/9773380.html
Copyright © 2011-2022 走看看