zoukankan      html  css  js  c++  java
  • MapReduce ----数据去重

    三个文件

    2017-03-10 a
    2017-03-11 b
    2017-03-12 d
    2017-03-13 d
    2017-03-14
    2017-03-15 a

    2017-03-10 e
    2017-03-11 b
    2017-03-12 c
    2017-03-13
    2017-03-14 h
    2017-03-15 a
    2017-03-17 p

    2017-03-10
    2017-03-11 b
    2017-03-12
    2017-03-13 d
    2017-03-14
    2017-03-15 f
    2017-03-16 o


    import java.io.IOException;
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.hadoop.mapreduce.Reducer;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    
    public class Deup{
        public static class Map extends Mapper<Object, Text, Text, Text>{
            @Override
            protected void map(Object key, Text value, Mapper<Object, Text, Text, Text>.Context context)
                    throws IOException, InterruptedException {
                context.write(value, new Text(""));
            }
        }
        
        public static class Reduce extends Reducer<Text, Text, Text, Text>{
            @Override
            protected void reduce(Text key, Iterable<Text> values, Reducer<Text, Text, Text, Text>.Context context)
                    throws IOException, InterruptedException {
                context.write(key, new Text(""));
            }
        }
        public static void main(String[] args) throws Exception {
            Configuration conf=new Configuration();
            @SuppressWarnings("deprecation")
            Job job=new Job(conf);
            job.setJarByClass(Deup.class);
            job.setMapperClass(Map.class);
            job.setCombinerClass(Reduce.class);
            job.setReducerClass(Reduce.class);
           job.setOutputKeyClass(Text.class);
           job.setOutputValueClass(Text.class);  
            FileInputFormat.addInputPath(job, new Path(args[0]));
            FileOutputFormat.setOutputPath(job, new Path(args[1]));
            System.exit(job.waitForCompletion(true)?0:1);
            
        }
    }

    结果

    2017-03-10    
    2017-03-10 a    
    2017-03-10 e    
    2017-03-11 b    
    2017-03-12    
    2017-03-12 c    
    2017-03-12 d    
    2017-03-13    
    2017-03-13 d    
    2017-03-14    
    2017-03-14 h    
    2017-03-15 a    
    2017-03-15 f    
    2017-03-16 o    
    2017-03-17 p   

  • 相关阅读:
    大学的最后时光---我的毕设日记(二)
    大学的最后时光---我的毕设日记(一)
    记公司同事的一次集体活动
    ssh框架基本原理
    plsql基本操作记录
    eclipse快捷键。
    面向对象的特性:封装继承多态
    设置label中的对齐方式
    pixmap和label设置图片自适应大小
    类的继承
  • 原文地址:https://www.cnblogs.com/tk55/p/6557489.html
Copyright © 2011-2022 走看看