zoukankan      html  css  js  c++  java
  • 简单的计算最值的MapReduce程序


    import java.io.IOException;
    import java.util.StringTokenizer;
    import java.util.*;
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.hadoop.mapreduce.Reducer;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

    public class max_number {

      public static class TokenizerMapper
           extends Mapper<Object, Text, Text, IntWritable>{

        private final static IntWritable one = new IntWritable(1);
     
        public void map(Object key, Text value, Context context
                        ) throws IOException, InterruptedException {
        String line=value.toString();
          StringTokenizer itr = new StringTokenizer(line,"\n");
          while (itr.hasMoreTokens()) {
        StringTokenizer tkl=new StringTokenizer(itr.nextToken());
        String sKey=tkl.nextToken();
        String sNum=tkl.nextToken();
        Text key=new Text(sKey);
        int numInt=Integer.parseInt(sScore);
            context.write(key, new IntWritable(numInt));
          }
        }
      }

      public static class IntSumReducer
           extends Reducer<Text,IntWritable,Text,IntWritable> {
        private IntWritable result = new IntWritable();

        public void reduce(Text key, Iterable<IntWritable> values,
                           Context context
                           ) throws IOException, InterruptedException {
          int max = -219876543;
      Iterator<IntWritable>iterator=values.iterator();
          while(iterator.hasNext())
          {
            int max1= iterator.next().get();
        if(max1>=max){max=max1;}

        
    }  context.write(key, new IntWritable(max));
        }
      }

      public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf, "max number");
        job.setJarByClass(max_number.class);
        job.setMapperClass(TokenizerMapper.class);
        job.setCombinerClass(IntSumReducer.class);
        job.setReducerClass(IntSumReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));
        System.exit(job.waitForCompletion(true) ? 0 : 1);
      }
    }

    运行需要的设置

    导入hadoop的class和java_home路径
    export JAVA_HOME=/usr/java/default
    export PATH=${JAVA_HOME}/bin:${PATH}
    export HADOOP_CLASSPATH=${JAVA_HOME}/lib/tools.jar

    --编译写好的MapReduce java程序
    $ $Hadoop_Home/bin/hadoop com.sun.tools.javac.Main max_number.java
    $ jar cf maxnumber.jar max_number*.class
    --上传文件到hdfs
    $Hadoop_Home/bin/hadoop fs --copyFromLocal /home/hadoop/hadoop/random.txt /test/random
    --执行
     $Hadoop_Home/bin/hadoop jar maxnumber.jar max_number /test/random /output11/minnumber2 //练习执行的脚本


    查询输出Output:

    $ $Hadoop_Home/bin/hadoop fs -cat /output11/minnumber2/part-r-00000`

  • 相关阅读:
    Linkerd 2.10(Step by Step)—将 GitOps 与 Linkerd 和 Argo CD 结合使用
    Linkerd 2.10(Step by Step)—多集群通信
    Linkerd 2.10(Step by Step)—使用 Kustomize 自定义 Linkerd 的配置
    Linkerd 2.10(Step by Step)—控制平面调试端点
    Linkerd 2.10(Step by Step)—配置超时
    Linkerd 2.10(Step by Step)—配置重试
    Linkerd 2.10(Step by Step)—配置代理并发
    本地正常运行,线上环境诡异异常原因集合
    Need to invoke method 'xxx' declared on target class 'yyy', but not found in any interface(s) of the exposed proxy type
    alpine 安装常用命令
  • 原文地址:https://www.cnblogs.com/wangq17/p/5965145.html
Copyright © 2011-2022 走看看