zoukankan      html  css  js  c++  java
  • Hadoop 学习笔记 (十一) MapReduce 求平均成绩

    china:
    张三 78
    李四 89
    王五 96
    赵六 67
    english
    张三 80
    李四 82
    王五    84
    赵六 86
    math
    张三 88
    李四 99
    王五 66
    赵六 77




    import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.FloatWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.reduce.IntSumReducer;
    public class MyAverage { public static class FormatMapper extends Mapper<Object, Text, Text, IntWritable>{ private IntWritable val = new IntWritable(); public void map(Object key, Text value, Context context) throws IOException, InterruptedException{ String line[] = value.toString().split("\s"); val.set(Integer.parseInt(line[1])); context.write(new Text(line[0]), val); } } public static class AverageReducer extends Reducer<Text, IntWritable, Text, FloatWritable>{ public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException{ int count = 0; int sum = 0; for (IntWritable val : values) { sum += val.get(); count ++; } float ave = (float)sum / count; context.write(key, new FloatWritable(ave)); } } public static void main(String[] args) throws Exception { String dir_in = "hdfs://localhost:9000/in_average"; String dir_out = "hdfs://localhost:9000/out_average"; Path in = new Path(dir_in); Path out = new Path(dir_out); Configuration conf = new Configuration(); Job averageJob = new Job(conf, "my_average"); averageJob.setJarByClass(MyAverage.class); averageJob.setInputFormatClass(TextInputFormat.class); averageJob.setMapperClass(FormatMapper.class); averageJob.setCombinerClass(IntSumReducer.class); //countJob.setPartitionerClass(HashPartitioner.class); averageJob.setMapOutputKeyClass(Text.class); averageJob.setMapOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(averageJob, in); averageJob.setReducerClass(AverageReducer.class); //averageJob.setNumReduceTasks(1); averageJob.setOutputKeyClass(Text.class); averageJob.setOutputValueClass(FloatWritable.class); //countJob.setOutputFormatClass(SequenceFileOutputFormat.class); FileOutputFormat.setOutputPath(averageJob, out); averageJob.waitForCompletion(true); } }

    张三    82.0
    李四    90.0
    王五    82.0
    赵六    76.666664
    
    
    
     
  • 相关阅读:
    在MAC OS X系统上面安装mysql
    在项目中使用DSOFramer需要注意的一些地方
    记一次python安装PIL库所遇到的事
    Calendar.compareTo 比较时间的大小
    String 时间类型怎么进行比较大小?
    ElasticSearch--二、基本语法(创建索引,查询数据)
    ElasticSearch--一、使用场景以及对应软件配置安装
    Linux下nginx反向代理负载均衡几种方式以及配置
    node环境使用lowdb轻量数据库以及基本用法
    jQuery 日常笔记
  • 原文地址:https://www.cnblogs.com/i80386/p/3608273.html
Copyright © 2011-2022 走看看