百度数仓Palo-Doris并发压测性能

硬件环境

 

 

表结构

CREATE TABLE IF NOT EXISTS   tj9
(
    vin varchar (30) not null,
        inday date not null,
        type1 int not null,
        type2 int not null,
        type3 int not null,
        p1 float not null,
        p2 float not null,
        p3 float not null,
        p4 float not null,
        p5 float not null,
        p6 float not null,
        p7 float not null,
        p8 float not null,
        p9 float not null,
        p0 float not null 
        
)
ENGINE=olap
DUPLICATE KEY(vin,inday)
PARTITION BY RANGE(inday)
(
      partition p1 values less than('2021-12-01'),
      partition p2 values less than('2022-02-01'),
      partition p3 values less than('2022-04-01'),
      partition p4 values less than('2022-06-01'),
      partition p5 values less than('2022-08-01'),
      partition p6 values less than('2022-10-01'),
      partition p7 values less than('2022-12-01'),
      partition p8 values less than('2023-02-01'),
      partition p9 values less than('2023-04-01'),
      partition p10 values less than('2023-06-01'),
      partition p11 values less than('2023-08-01'),
      partition p12 values less than('2023-10-01'),
      partition p13 values less than('2023-12-01'),
      partition p14 values less than('2024-02-01'),
      partition p15 values less than('2024-04-01'),
      partition p16 values less than('2024-06-01'),
      partition p17 values less than('2024-08-01'),
      partition p18 values less than('2024-10-01'),
      partition p19 values less than('2024-12-01'),
        partition p20 values less than('2025-02-01'),
      partition p21 values less than('2025-04-01'),
      partition p22 values less than('2025-06-01'),
      partition p23 values less than('2025-08-01'),
      partition p24 values less than('2025-10-01'),
      partition p25 values less than('2025-12-01')
)
DISTRIBUTED BY HASH(vin) BUCKETS 50
PROPERTIES
(
 "replication_num" = "3"
);

数据内容

数据量

客户端压测程序

package org.doris;

import java.sql.*;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.Random;

public class Yace2 {

    public static void main(String[] args) throws Exception {

        for (int i = 0; i < 50; i++) {

            Thread t1 = new Thread(new Com());

            t1.start();
        }

    }


    public static class Com implements Runnable {

        Connection con = null;
        ResultSet rs = null;
        PreparedStatement ps = null;
        String JDBC_DRIVER = "com.mysql.cj.jdbc.Driver";
        String CONNECTION_URL = "jdbc:mysql://192.168.154.107:9030/test?rewriteBatchedStatements=true";
        String driverClassName = "com.mysql.cj.jdbc.Driver";    //启动驱动
        String url = "jdbc:mysql://192.168.154.107:9030/test";    //设置连接路径
        String username = "root";

        public Com() {
            try {
//            com.cloudera.impala.jdbc41.Driver v = null;
                Class.forName(driverClassName);
                con = DriverManager.getConnection(url, "root", null);

            } catch (Exception e) {
                e.printStackTrace();
            }
        }

        @Override
        public void run() {


            while (true) {
                try {
                    Random rand = new Random();

                    int entry_time_start = rand.nextInt(700) + 1;

                    int entry_time_end = entry_time_start + 30;

//                String sql = "select a.tid,count(*),sum(std_mileage),avg(diff_mileage),avg(gps_mileage),avg(meter_mileage),avg(ecu_mileage),avg(total_fuel_cons),avg(diff_fuel_cons) ,avg(std_fuel_cons) from kudu_via_city_pdi3 as a where entry_time between "+entry_time_start+" and "+entry_time_end+" and exists (select null from car2 as b where a.tid=b.tid) group by a.tid order by sum(std_mileage) asc limit 1 offset 0";


                    /*java.util.Date date = new Date();
                    Calendar calendar = new GregorianCalendar();
                    calendar.setTime(date);
                    calendar.add(calendar.DATE, i); //把日期往后增加一天,整数  往后推,负数往前移动
                    date = calendar.getTime(); //这个时间就是日期往后推一天的结果*/
                    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");


                    Date dstart = sdf.parse("2021-11-05");

                    Calendar calendar = new GregorianCalendar();
                    calendar.setTime(dstart);
                    calendar.add(calendar.DATE, rand.nextInt(1000)); //把日期往后增加一天,整数  往后推,负数往前移动
                    dstart = calendar.getTime(); //这个时间就是日期往后推一天的结果

                    calendar = new GregorianCalendar();
                    calendar.setTime(dstart);
                    calendar.add(calendar.DATE, 30);

                    Date dend = calendar.getTime();

                    String sstart = sdf.format(dstart);
                    String send = sdf.format(dend);

                    String sql = "select count(*),sum(p3),sum(p1),vin from tj9 where inday between '"+sstart+"' and '"+send+"' group by vin order by sum(p1) desc limit 1";


                    java.util.Date d1 = new java.util.Date();
                    ps = con.prepareStatement(sql);
                    rs = ps.executeQuery();
                    while (rs.next()) {
                        System.out.println(Thread.currentThread() + "===>" + d1 + "-----" + new java.util.Date() + "----" + rs.getString(2));

                    }
                } catch (Exception e) {
                    e.printStackTrace();
                } finally {
                    try {
                        rs.close();
                        ps.close();

                    } catch (SQLException e) {
                        e.printStackTrace();
                    }
                }

            }

        }
    }

}

响应时间

Thread[Thread-26,5,main]===>Sat Nov 06 12:08:21 CST 2021-----Sat Nov 06 12:08:26 CST 2021----28.022251464426517
Thread[Thread-38,5,main]===>Sat Nov 06 12:08:20 CST 2021-----Sat Nov 06 12:08:26 CST 2021----33.72479600459337
Thread[Thread-45,5,main]===>Sat Nov 06 12:08:20 CST 2021-----Sat Nov 06 12:08:26 CST 2021----34.93087751418352
Thread[Thread-39,5,main]===>Sat Nov 06 12:08:20 CST 2021-----Sat Nov 06 12:08:26 CST 2021----30.66674555838108
Thread[Thread-6,5,main]===>Sat Nov 06 12:08:22 CST 2021-----Sat Nov 06 12:08:26 CST 2021----34.45231296867132
Thread[Thread-36,5,main]===>Sat Nov 06 12:08:21 CST 2021-----Sat Nov 06 12:08:26 CST 2021----27.272395104169846
Thread[Thread-35,5,main]===>Sat Nov 06 12:08:20 CST 2021-----Sat Nov 06 12:08:27 CST 2021----30.620394926518202
Thread[Thread-31,5,main]===>Sat Nov 06 12:08:23 CST 2021-----Sat Nov 06 12:08:27 CST 2021----35.72697024047375
Thread[Thread-30,5,main]===>Sat Nov 06 12:08:21 CST 2021-----Sat Nov 06 12:08:27 CST 2021----23.390043392777443
Thread[Thread-10,5,main]===>Sat Nov 06 12:08:21 CST 2021-----Sat Nov 06 12:08:27 CST 2021----24.27678807079792
Thread[Thread-20,5,main]===>Sat Nov 06 12:08:21 CST 2021-----Sat Nov 06 12:08:27 CST 2021----29.040063969790936
Thread[Thread-48,5,main]===>Sat Nov 06 12:08:22 CST 2021-----Sat Nov 06 12:08:27 CST 2021----33.46218280121684
Thread[Thread-37,5,main]===>Sat Nov 06 12:08:21 CST 2021-----Sat Nov 06 12:08:27 CST 2021----31.903710812330246
Thread[Thread-42,5,main]===>Sat Nov 06 12:08:21 CST 2021-----Sat Nov 06 12:08:27 CST 2021----27.873187253251672
Thread[Thread-18,5,main]===>Sat Nov 06 12:08:23 CST 2021-----Sat Nov 06 12:08:27 CST 2021----31.810679212212563
Thread[Thread-21,5,main]===>Sat Nov 06 12:08:23 CST 2021-----Sat Nov 06 12:08:27 CST 2021----33.88313761353493
Thread[Thread-2,5,main]===>Sat Nov 06 12:08:21 CST 2021-----Sat Nov 06 12:08:27 CST 2021----26.80970578826964
Thread[Thread-3,5,main]===>Sat Nov 06 12:08:23 CST 2021-----Sat Nov 06 12:08:27 CST 2021----31.7162510342896
Thread[Thread-7,5,main]===>Sat Nov 06 12:08:21 CST 2021-----Sat Nov 06 12:08:27 CST 2021----30.19490994521766
Thread[Thread-9,5,main]===>Sat Nov 06 12:08:22 CST 2021-----Sat Nov 06 12:08:27 CST 2021----27.030669568106532
Thread[Thread-43,5,main]===>Sat Nov 06 12:08:23 CST 2021-----Sat Nov 06 12:08:27 CST 2021----34.46626507490873
Thread[Thread-40,5,main]===>Sat Nov 06 12:08:22 CST 2021-----Sat Nov 06 12:08:27 CST 2021----27.155178541317582
Thread[Thread-47,5,main]===>Sat Nov 06 12:08:23 CST 2021-----Sat Nov 06 12:08:27 CST 2021----33.44230469316244
Thread[Thread-11,5,main]===>Sat Nov 06 12:08:23 CST 2021-----Sat Nov 06 12:08:28 CST 2021----30.59896296542138
Thread[Thread-32,5,main]===>Sat Nov 06 12:08:22 CST 2021-----Sat Nov 06 12:08:28 CST 2021----30.65741043537855
Thread[Thread-41,5,main]===>Sat Nov 06 12:08:22 CST 2021-----Sat Nov 06 12:08:28 CST 2021----32.03111154586077
Thread[Thread-23,5,main]===>Sat Nov 06 12:08:22 CST 2021-----Sat Nov 06 12:08:28 CST 2021----28.389698467217386
Thread[Thread-25,5,main]===>Sat Nov 06 12:08:22 CST 2021-----Sat Nov 06 12:08:28 CST 2021----28.751262992620468
Thread[Thread-12,5,main]===>Sat Nov 06 12:08:23 CST 2021-----Sat Nov 06 12:08:28 CST 2021----30.460573315620422
Thread[Thread-15,5,main]===>Sat Nov 06 12:08:25 CST 2021-----Sat Nov 06 12:08:28 CST 2021----27.92404123954475
Thread[Thread-34,5,main]===>Sat Nov 06 12:08:23 CST 2021-----Sat Nov 06 12:08:28 CST 2021----33.9197644777596
Thread[Thread-0,5,main]===>Sat Nov 06 12:08:24 CST 2021-----Sat Nov 06 12:08:28 CST 2021----29.629240840673447
Thread[Thread-46,5,main]===>Sat Nov 06 12:08:23 CST 2021-----Sat Nov 06 12:08:28 CST 2021----36.7463488727808
Thread[Thread-5,5,main]===>Sat Nov 06 12:08:24 CST 2021-----Sat Nov 06 12:08:29 CST 2021----31.927827052772045
Thread[Thread-49,5,main]===>Sat Nov 06 12:08:23 CST 2021-----Sat Nov 06 12:08:29 CST 2021----29.674843708984554
Thread[Thread-22,5,main]===>Sat Nov 06 12:08:23 CST 2021-----Sat Nov 06 12:08:29 CST 2021----30.460573315620422

后台负载