zoukankan      html  css  js  c++  java
  • Hadoop之MapReduce单词计数经典实例

    MapReduce实现单词计数示意图

    这里写图片描述

    MapReduce实现单词计数实例代码(Java)

    ①Mapper

    import java.io.IOException;
    
    import org.apache.commons.lang.StringUtils;
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Mapper;
    
    public class WM extends Mapper<LongWritable, Text, Text, LongWritable> {
        protected void map(LongWritable key, Text value, Context context)
                throws IOException, InterruptedException {
    
            String line = value.toString();
            String[] words = StringUtils.split(line, " ");
            for (String word : words) {
                context.write(new Text(word), new LongWritable(1));
            }
    
        }
    }

    ②Reducer

    import java.io.IOException;
    
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Reducer;
    
    public class WR extends Reducer<Text, LongWritable, Text, LongWritable> {
        protected void reduce(Text word, Iterable<LongWritable> counts,
                Context context) throws IOException, InterruptedException {     
            int counter = 0;
    
            for (LongWritable count : counts) {
                long i = count.get();
                counter += i;
            }
    
            context.write(word, new LongWritable(counter));
    
        }
    }

    ③主程序

    import java.io.IOException;
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    import org.apache.hadoop.mapreduce.Job;
    
    public class Launcher {
        public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
            Configuration conf = new Configuration();
    
            Job job = Job.getInstance(conf);
    
            job.setMapperClass(WM.class);
    
            job.setMapOutputKeyClass(Text.class);
    
            job.setMapOutputValueClass(LongWritable.class);
    
            job.setReducerClass(WR.class);
    
            job.setOutputKeyClass(Text.class);
    
            job.setOutputValueClass(LongWritable.class);
    
            job.setJarByClass(Launcher.class);
    
            FileInputFormat.setInputPaths(job, "e://hello//words.txt");
    
            FileOutputFormat.setOutputPath(job, new Path("e://hello//output"));
    
            job.waitForCompletion(true);
        }
    }
  • 相关阅读:
    Mysql: 一个死锁场景的解决
    Mysql: 创建新的账号
    安全评测:BashDoor
    BeanFactory和FactoryBean的区别:
    linux 下将war解压到当前目录
    学习BoolanC++笔记_01(C++面向对象高级编程(上)第一周)
    学习BoolanC++笔记_00(预备)
    Linux (Centos) 上安装Nexus OSS 3 并搭建docker私有仓库
    linux(Centos7)搭建rpm仓库
    树莓派4b安装 manjaro linux
  • 原文地址:https://www.cnblogs.com/wanxi/p/6476217.html
Copyright © 2011-2022 走看看