zoukankan      html  css  js  c++  java
  • MapReduce单表关联

    数据:
    找出孩子的爷爷奶奶姥姥老爷

    child parent
    Tom Lucy
    Tom Jack
    Jone Lucy
    Jone Jack
    Lucy Marry
    Lucy Jesse
    Jack Alice
    Jack Jesse
    Terry Alice
    Terry Jesse
    Philip Terry
    Philip Alma
    Mark Terry
    Mark Alma

    结果:

    Jone    Alice
    Tom    Alice
    Jone    Jesse
    Tom    Jesse
    Jone    Marry
    Tom    Marry
    Jone    Jesse
    Tom    Jesse
    Mark    Alice
    Philip    Alice
    Mark    Jesse
    Philip    Jesse

    Mapper:

    一个坑:每次放入context.write()的时候都需要重新new 一个Text出来。不可以用原来的Text.set()方法

    package _SingleTable;
    
    
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Mapper;
    
    import java.io.IOException;
    
    /**
     * @Author:Dapeng
     * @Discription:
     * @Date:Created in 上午 10:11 2018/11/8 0008
     */
    public class SingleTableMap extends Mapper<LongWritable,Text,Text,Text> {
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
    
    
            String line = value.toString();
            String[] wordArr = line.split("\s+");
            if(!"child".equals(wordArr[0])){
                //设置parents
                context.write(new Text(wordArr[0]),new Text("1:" + wordArr[1]));
                //设置son
                context.write(new Text(wordArr[1]),new Text("2:" + wordArr[0]));
            }
    
        }
    }

    Reducer

    package _SingleTable;
    
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Reducer;
    
    import java.io.IOException;
    import java.util.ArrayList;
    import java.util.List;
    
    /**
     * @Author:Dapeng
     * @Discription:
     * @Date:Created in 上午 10:11 2018/11/8 0008
     */
    public class SingleTableReduce extends Reducer<Text,Text,Text,Text> {
        @Override
        protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
    
    
            List<String> parents = new ArrayList<String>();
            List<String> childs = new ArrayList<String>();
            Text t1 = new Text();
            Text t2 = new Text();
    
            for(Text t : values){
                String str = t.toString();
                String[] s = str.split(":");
    
                if ("1".equals(s[0])) {
                    parents.add(s[1]);
                } else if("2".equals(s[0])) {
                    childs.add(s[1]);
                }
    
            }
    
           for(String p :parents){
                for(String c:childs){
                    t1.set(p);
                    t2.set(c);
                    context.write(t2,t1);
                }
           }
        }
    }
    package _SingleTable;
    
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    
    import java.io.IOException;
    
    /**
     * @Author:Dapeng
     * @Discription:
     * @Date:Created in 上午 10:11 2018/11/8 0008
     */
    public class SingleTableMain {
        public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
            //0.创建一个job
            Configuration conf = new Configuration();
            Job job = Job.getInstance(conf,"single_table");
            job.setJarByClass(SingleTableMain.class);
            //1.输入文件
            //默认用TextInputFormat
            FileInputFormat.addInputPath(job,new Path("file:/D:/hadoopFile/singleTable/data.txt"));
            //2.编写mapper
            job.setMapperClass(SingleTableMap.class);
            //设置输出的格式
            job.setMapOutputKeyClass(Text.class);
            job.setMapOutputValueClass(Text.class);
            //3.shuffle
    
            //4.reduce
            job.setReducerClass(SingleTableReduce.class);
            //设置输出的格式
            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(Text.class);
            //5.输出
            FileOutputFormat.setOutputPath(job,new Path("file:/D:/hadoopFile/singleTable/out"));
    
            //6.运行
    
            boolean result = job.waitForCompletion(true);
            System.out.println(result);
        }
    }
  • 相关阅读:
    APK中java代码反编译
    android应用分析之apk文件结构
    2016第8周五
    优秀前端需要具备的经验
    postgreSQL数据类型转换字符串和数值
    架构设计要考虑的几个方面
    2016第8周一
    常见的交互设计方法
    开源项目使用经验原则
    ExtJS与JQuery对照
  • 原文地址:https://www.cnblogs.com/da-peng/p/9930765.html
Copyright © 2011-2022 走看看