zoukankan      html  css  js  c++  java
  • hadoop 2.x 简单实现wordCount

    简单实现hadoop程序,包括:hadoop2.x的实现写法

    import org.apache.hadoop.conf.Configured;

    import org.apache.hadoop.fs.Path;

    import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.hadoop.mapreduce.Reducer;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    import org.apache.hadoop.util.Tool;
    import org.apache.hadoop.util.ToolRunner;

    import java.io.IOException;
    import java.util.StringTokenizer;

    /**
    * Created by dell on 2016/7/3.
    */
    public class WordCount extends Configured implements Tool {

    public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> {
        private final static IntWritable one = new IntWritable();
        private Text word = new Text();
        public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
            StringTokenizer itr = new StringTokenizer(value.toString());
            while(itr.hasMoreElements()) {
            word.set(itr.nextToken());
            context.write(word,one);
            }
        }
    }
    public static class IntSumReducer extends Reducer<Text,IntWritable,Text,IntWritable>{
        private IntWritable result = new IntWritable();
        public void Reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
            int sum = 0;
            for(IntWritable val:values) {
            sum += val.get();
        }
            result.set(sum);
            context.write(key,result);
        }
    }
    @Override
    public int run(String[] args) throws Exception {
         Job job = Job.getInstance(getConf());
         job.setJarByClass(WordCount.class);
         job.setMapperClass(TokenizerMapper.class);
         job.setCombinerClass(IntSumReducer.class);
         job.setReducerClass(IntSumReducer.class);
         job.setNumReduceTasks(Integer.parseInt(args[2])); //设置reducer个数
         job.setOutputKeyClass(Text.class);
         job.setOutputValueClass(IntWritable.class);
         FileInputFormat.addInputPath(job,new Path(args[0]));
         FileOutputFormat.setOutputPath(job,new Path(args[1]));
         job.waitForCompletion(true);
         return 0;
    }
        public static void main(String[] args) throws Exception {
            int res = ToolRunner.run(new Configuration(),new WordCount(),args);
            System.exit(res);
        }
    }

    我不生产知识 我只是知识的搬运工
  • 相关阅读:
    使用 ASP.NET Core MVC 创建 Web API(五)
    使用 ASP.NET Core MVC 创建 Web API(四)
    使用 ASP.NET Core MVC 创建 Web API(三)
    使用 ASP.NET Core MVC 创建 Web API(二)
    使用 ASP.NET Core MVC 创建 Web API(一)
    学习ASP.NET Core Razor 编程系列十九——分页
    学习ASP.NET Core Razor 编程系列十八——并发解决方案
    一个屌丝程序猿的人生(九十八)
    一个屌丝程序猿的人生(九十七)
    一个屌丝程序猿的人生(九十五)
  • 原文地址:https://www.cnblogs.com/yyy-blog/p/5638958.html
Copyright © 2011-2022 走看看