zoukankan      html  css  js  c++  java
  • 大数据笔记(十二)——使用MRUnit进行单元测试

    package demo.wc;
    
    import java.util.ArrayList;
    import java.util.List;
    
    import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mrunit.mapreduce.MapDriver;
    import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
    import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
    import org.junit.Test;
    
    public class MRUnitWordCount {
    
        @Test
        public void testMapper() throws Exception{
            //设置一个环境变量(没有可能会报错)
            System.setProperty("hadoop.home.dir", "D:\temp\hadoop-2.4.1\hadoop-2.4.1");
            
            //创建一个测试对象
            WordCountMapper mapper = new WordCountMapper();
            
            //创建一个MapDriver进行单元测试
            MapDriver<LongWritable, Text, Text, IntWritable> driver = new MapDriver<>(mapper);
            
            //ָ指定Map的输入值: k1  v1
            driver.withInput(new LongWritable(1), new Text("I love Beijing"));
            
            //ָ指定Map的输出值:k2   v2  ----> 期望值
            driver.withOutput(new Text("I"), new IntWritable(1))
                  .withOutput(new Text("love"), new IntWritable(1))
                  .withOutput(new Text("Beijing"), new IntWritable(1));
            
            //ִ执行单元测试,对比 期望的结果和实际的结果
            driver.runTest();
        }
        
        @Test
        public void testReducer() throws Exception{
            //设置一个环境变量
            System.setProperty("hadoop.home.dir", "D:\temp\hadoop-2.4.1\hadoop-2.4.1");
            
            //创建一个测试对象
            WordCountReducer reducer = new WordCountReducer();
            
            //创建一个ReduceDriver进行单元测试
            ReduceDriver<Text, IntWritable, Text, IntWritable> driver = new ReduceDriver<>(reducer);
            
            //构造v3:List
            List<IntWritable> value3 = new ArrayList<>();
            value3.add(new IntWritable(1));
            value3.add(new IntWritable(1));
            value3.add(new IntWritable(1));
            
            
            //指定reducer的输入
            driver.withInput(new Text("Beijing"), value3);
            
            
            //指定reducer的输出
            driver.withOutput(new Text("Beijing"), new IntWritable(3));
            
            
            //ִ执行测试
            driver.runTest();
        }
        
        @Test
        public void testJob() throws Exception{
            //设置一个环境变量
            System.setProperty("hadoop.home.dir", "D:\temp\hadoop-2.4.1\hadoop-2.4.1");
            
            //创建一个测试对象
            WordCountMapper mapper = new WordCountMapper();        
            WordCountReducer reducer = new WordCountReducer();        
            
            //创建一个Driver
            //MapReduceDriver<K1, V1, K2, V2, K4, V4>
            MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable>
                    driver = new MapReduceDriver<>(mapper,reducer);
            
            //指定Map输入的数据
            driver.withInput(new LongWritable(1), new Text("I love Beijing"))
                  .withInput(new LongWritable(4), new Text("I love China"))
                  .withInput(new LongWritable(7), new Text("Beijing is the capital of China"));
            
            //ָ指定Reducer的输出
    //        driver.withOutput(new Text("I"), new IntWritable(2))
    //              .withOutput(new Text("love"), new IntWritable(2))
    //              .withOutput(new Text("Beijing"), new IntWritable(2))
    //              .withOutput(new Text("China"), new IntWritable(2))
    //              .withOutput(new Text("is"), new IntWritable(1))
    //              .withOutput(new Text("the"), new IntWritable(1))
    //              .withOutput(new Text("capital"), new IntWritable(1))
    //              .withOutput(new Text("of"), new IntWritable(1));
            
            //指定Reducer的输出(默认排序规则)
            driver.withOutput(new Text("Beijing"), new IntWritable(2))
                  .withOutput(new Text("China"), new IntWritable(2))
                  .withOutput(new Text("I"), new IntWritable(2))
                  .withOutput(new Text("capital"), new IntWritable(1))
                  .withOutput(new Text("is"), new IntWritable(1))
                  .withOutput(new Text("love"), new IntWritable(2))
                  .withOutput(new Text("of"), new IntWritable(1))
                  .withOutput(new Text("the"), new IntWritable(1));
            
            driver.runTest();
        }
    }
  • 相关阅读:
    SpringBoot项目启动遇到的问题记录
    关于点击按钮提交前进行数据校验
    idea插件的位置
    sqlite
    xamarin.forms 使用依赖注入
    vs2019查找替换,使用正则表达式功能
    EFCore 事务提交
    windows磁盘【文件和文件夹遇到-打不开or不能删除or损坏】的解决方式
    xamarin.forms 中使用Forms9Patch插件来显示图片遇到的问题
    net5 依赖注入的时候,遇到的问题:Cannot consume scoped service from singleton IHostedService
  • 原文地址:https://www.cnblogs.com/lingluo2017/p/8540275.html
Copyright © 2011-2022 走看看