zoukankan      html  css  js  c++  java
  • 给定a、b两个文件,各存放50亿个url,每个url各占64字节,内存限制是4G,让你找出a、b文件共同的url?

    package com.hadoop.hdfs;
    
    import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
    import org.junit.Test;
    
    import java.io.*;
    import java.util.HashMap;
    import java.util.HashSet;
    
    public class Suanfa1 {
        @Test
        public void a1() throws IOException {
            BufferedReader bufferedReader = new BufferedReader(new FileReader("D:/aa.txt"));
    //        BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter("D://"))
            String str1 = "";
            while ((str1 = bufferedReader.readLine())!=null){
                int i = (int) (hashCode(str1)%1000);
                BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter("D://aa"+String.valueOf(i)+".txt"));
                bufferedWriter.write(str1);
                bufferedWriter.close();
                System.out.println(i);
            }
            bufferedReader.close();
        }
    
        public void a2() throws IOException {
            BufferedReader bufferedReader = new BufferedReader(new FileReader("D:/bb.txt"));
    //        BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter("D://"))
            String str1 = "";
            while ((str1 = bufferedReader.readLine())!=null){
                int i = (int) (hashCode(str1)%1000);
                BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter("D://bb"+String.valueOf(i)+".txt"));
                bufferedWriter.write(str1);
                bufferedWriter.close();
            }
            bufferedReader.close();
        }
    
        public long hashCode(String str) {
            long h = 0;
            if (h == 0) {
                int off = 0;
                char val[] = str.toCharArray();
                long len = str.length();
                for (long i = 0; i < len; i++) {
                    h = 31 * h + val[off++];
                }
            }
            return h;
        }
    
        @Test
        public void a3() throws IOException {
            a1();
            a2();
            for (int i = 0; i < 1000; i++) {
                BufferedReader bufferedReader1 = new BufferedReader(new FileReader("D://aa"+String.valueOf(i)+".txt"));
                BufferedReader bufferedReader2 = new BufferedReader(new FileReader("D://bb"+String.valueOf(i)+".txt"));
                HashSet set = new HashSet();
                String input1 = "";
                while ((input1 = bufferedReader1.readLine())!=null){
                    set.add(hashCode(bufferedReader1.readLine()));
                }
    
                String input2 = "";
                while ((input2 = bufferedReader2.readLine())!=null){
                    if (set.contains(hashCode(input2))){
                        System.out.println(input2);
                    }
                }
            }
    
        }
    }
    我凝视这恒星,等待这那场风暴,我已经准备好了
  • 相关阅读:
    tomcat server.xml 配置示例
    Vue学习1:实例及生命周期
    flex布局
    从输入一个URL到页面完全显示发生了什么?
    webstorm配置eslint【标记错误,修复错误】
    JavaScript实现八大内部排序算法
    es6(六):module模块(export,import)
    es6(五):class关键字(extends,super,static)
    es6(四):Symbol,Set,Map
    es6(三):es6中函数的扩展(参数默认值、rest参数、箭头函数)
  • 原文地址:https://www.cnblogs.com/cheng5350/p/11740754.html
Copyright © 2011-2022 走看看