zoukankan      html  css  js  c++  java
  • MapReduce ----数据去重

    三个文件

    2017-03-10 a
    2017-03-11 b
    2017-03-12 d
    2017-03-13 d
    2017-03-14
    2017-03-15 a

    2017-03-10 e
    2017-03-11 b
    2017-03-12 c
    2017-03-13
    2017-03-14 h
    2017-03-15 a
    2017-03-17 p

    2017-03-10
    2017-03-11 b
    2017-03-12
    2017-03-13 d
    2017-03-14
    2017-03-15 f
    2017-03-16 o


    import java.io.IOException;
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.hadoop.mapreduce.Reducer;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    
    public class Deup{
        public static class Map extends Mapper<Object, Text, Text, Text>{
            @Override
            protected void map(Object key, Text value, Mapper<Object, Text, Text, Text>.Context context)
                    throws IOException, InterruptedException {
                context.write(value, new Text(""));
            }
        }
        
        public static class Reduce extends Reducer<Text, Text, Text, Text>{
            @Override
            protected void reduce(Text key, Iterable<Text> values, Reducer<Text, Text, Text, Text>.Context context)
                    throws IOException, InterruptedException {
                context.write(key, new Text(""));
            }
        }
        public static void main(String[] args) throws Exception {
            Configuration conf=new Configuration();
            @SuppressWarnings("deprecation")
            Job job=new Job(conf);
            job.setJarByClass(Deup.class);
            job.setMapperClass(Map.class);
            job.setCombinerClass(Reduce.class);
            job.setReducerClass(Reduce.class);
           job.setOutputKeyClass(Text.class);
           job.setOutputValueClass(Text.class);  
            FileInputFormat.addInputPath(job, new Path(args[0]));
            FileOutputFormat.setOutputPath(job, new Path(args[1]));
            System.exit(job.waitForCompletion(true)?0:1);
            
        }
    }

    结果

    2017-03-10    
    2017-03-10 a    
    2017-03-10 e    
    2017-03-11 b    
    2017-03-12    
    2017-03-12 c    
    2017-03-12 d    
    2017-03-13    
    2017-03-13 d    
    2017-03-14    
    2017-03-14 h    
    2017-03-15 a    
    2017-03-15 f    
    2017-03-16 o    
    2017-03-17 p   

  • 相关阅读:
    socket的半包,粘包与分包的问题
    /dev/tty /dev/ttyS0 /dev/tty0,/dev/null区别
    23种设计模式小结
    嵌入式试题
    怎样给filter加入自己定义接口及调用
    实现Launcher默认壁纸、选择壁纸定制化功能
    javascript中的稀疏数组(sparse array)和密集数组
    【Linux学习】Linux的文件权限(一)
    HDU--5280(dp或枚举)
    UISegmentedControl UISlider
  • 原文地址:https://www.cnblogs.com/tk55/p/6557489.html
Copyright © 2011-2022 走看看