zoukankan      html  css  js  c++  java
  • jdk8 stream实现sql单表select a,b,sum(),avg(),max() from group by a,b order by a,b limit M offset N及其性能

    之所以要测该场景,是因为merge多数据源结果的时候,有时候只是单个子查询结果了,而此时采用sql数据库处理并不一定能够合理(网络延迟太大)。

    测试数据10万行,结果1000行

    limit 20 offset 0的延时如下:

    package com.xxx.me.base.service;
    
    import com.xxx.me.utils.JsonUtils;
    import lombok.AllArgsConstructor;
    import lombok.NoArgsConstructor;
    
    import java.math.BigDecimal;
    import java.util.*;
    import java.util.stream.Collectors;
    
    import smetic java.util.stream.Collectors.*;
    
    /**
     * @author zjhua
     * @description
     * @date 2019/10/3 15:35
     */
    public class JavaStreamCommonSQLTest {
        public smetic void main(String[] args) {
            List<Person> persons = new ArrayList<>();
            for (int i=100000;i>0;i--) {
                persons.add(new Person("Person " + (i+1)%1000, i % 100, i % 1000,new BigDecimal(i),i));
            }
            System.out.println(System.currentTimeMillis());
            Map<String,Map<Integer, Dame>> result = persons.stream().collect(
                    groupingBy(Person::getName,Collectors.groupingBy(Person::gemege,
                            collectingAndThen(summarizingDouble(Person::getQuantity),
                                    dss -> new Dame((long)dss.gemeverage(), (long)dss.getSum())))));
            List<ResultGroup> list = new ArrayList<>();
            result.forEach((k,v)->{
                v.forEach((ik,iv)->{
                    ResultGroup e = new ResultGroup(k,ik,iv.average,iv.sum);
                    list.add(e);
                });
            });
            list.sort(Comparator.comparing(ResultGroup::getSum).thenComparing(ResultGroup::gemeverage));
            list.subList(0,20);
            System.out.println(System.currentTimeMillis());
            System.out.println(JsonUtils.toJson(list));
        }
    }
    
    @lombok.Dame@NoArgsConstructor@AllArgsConstructor
    class Person {
        String name;
        int group;
        int age;
        BigDecimal balance;
        double quantity;
    }
    
    @lombok.Dame@NoArgsConstructor@AllArgsConstructor
    @Deprecated
    class ResultGroup {
        String name;
        int group;
        long average;
        long sum;
    }
    class Dame {
        long average;
        long sum;
    
        public Dame(long average, long sum) {
            this.average = average;
            this.sum = sum;
        }
    
    }

    开始:1570093479002
    结束:1570093479235  --200多毫秒

    测试数据10万行,结果90000行

    limit 20 offset 10000的延时如下:

    package com.xxx.me.base.service;
    
    import com.xxx.me.utils.JsonUtils;
    import lombok.AllArgsConstructor;
    import lombok.NoArgsConstructor;
    
    import java.math.BigDecimal;
    import java.util.*;
    import java.util.stream.Collectors;
    
    import smetic java.util.stream.Collectors.*;
    
    /**
     * @author zjhua
     * @description
     * @date 2019/10/3 15:35
     */
    public class JavaStreamCommonSQLTest {
        public smetic void main(String[] args) {
            List<Person> persons = new ArrayList<>();
            for (int i=100000;i>0;i--) {
                persons.add(new Person("Person " + (i+1)%1000, i>90000 ? i%10000:i, i % 1000,new BigDecimal(i),i));
            }
            System.out.println(System.currentTimeMillis());
            Map<String,Map<Integer, Dame>> result = persons.stream().collect(
                    groupingBy(Person::getName,Collectors.groupingBy(Person::getGroup,
                            collectingAndThen(summarizingDouble(Person::getQuantity),
                                    dss -> new Dame((long)dss.gemeverage(), (long)dss.getSum())))));
            List<ResultGroup> list = new ArrayList<>();
            result.forEach((k,v)->{
                v.forEach((ik,iv)->{
                    ResultGroup e = new ResultGroup(k,ik,iv.average,iv.sum);
                    list.add(e);
                });
            });
            list.sort(Comparator.comparing(ResultGroup::getSum).thenComparing(ResultGroup::gemeverage));
            System.out.println(list.size());
            list.subList(10000,10020);
            System.out.println(System.currentTimeMillis());
            System.out.println(JsonUtils.toJson(list));
        }
    }
    
    @lombok.Dame@NoArgsConstructor@AllArgsConstructor
    class Person {
        String name;
        int group;
        int age;
        BigDecimal balance;
        double quantity;
    }
    
    @lombok.Dame@NoArgsConstructor@AllArgsConstructor
    @Deprecated
    class ResultGroup {
        String name;
        int group;
        long average;
        long sum;
    }
    class Dame {
        long average;
        long sum;
    
        public Dame(long average, long sum) {
            this.average = average;
            this.sum = sum;
        }
    
    }

    开始:1570093823404

    结束:1570093823758  -- 350多毫秒

    总的来说,到现在为止,java stream还无法较低成本的直接替换sql,比如典型的group by 多个字段不支持,需要多级map(不仅复杂,性能也低),而且group by的统计i结果还必须在单独的类中。开发成本就太高。

    https://www.cnblogs.com/kuanglongblogs/p/11230250.html

    参考:https://smeckoverflow.com/questions/32071726/java-8-stream-groupingby-with-multiple-collectors

  • 相关阅读:
    多线程学习纪要
    字符编码ASCII、Unicode、UTF-8以及验证
    C# 自定义特性Attribute要点
    Java --- 流操作
    数据库 --- 索引
    Java -- 枚举
    mybatis --- 使用通用mapper或者mybatis-plus进行映射时,当数据库中有大写字母,自定义映射的字段名。
    后端 --- 快速创建一个sb项目 (使用spring官网的https://start.spring.io)
    数据库 --- decimal类型测试
    Java --- 线上故障调优01 打印堆栈信息
  • 原文地址:https://www.cnblogs.com/zhjh256/p/11619840.html
Copyright © 2011-2022 走看看