zoukankan      html  css  js  c++  java
  • 大数据笔记(十二)——使用MRUnit进行单元测试

    package demo.wc;
    
    import java.util.ArrayList;
    import java.util.List;
    
    import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mrunit.mapreduce.MapDriver;
    import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
    import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
    import org.junit.Test;
    
    public class MRUnitWordCount {
    
        @Test
        public void testMapper() throws Exception{
            //设置一个环境变量(没有可能会报错)
            System.setProperty("hadoop.home.dir", "D:\temp\hadoop-2.4.1\hadoop-2.4.1");
            
            //创建一个测试对象
            WordCountMapper mapper = new WordCountMapper();
            
            //创建一个MapDriver进行单元测试
            MapDriver<LongWritable, Text, Text, IntWritable> driver = new MapDriver<>(mapper);
            
            //ָ指定Map的输入值: k1  v1
            driver.withInput(new LongWritable(1), new Text("I love Beijing"));
            
            //ָ指定Map的输出值:k2   v2  ----> 期望值
            driver.withOutput(new Text("I"), new IntWritable(1))
                  .withOutput(new Text("love"), new IntWritable(1))
                  .withOutput(new Text("Beijing"), new IntWritable(1));
            
            //ִ执行单元测试,对比 期望的结果和实际的结果
            driver.runTest();
        }
        
        @Test
        public void testReducer() throws Exception{
            //设置一个环境变量
            System.setProperty("hadoop.home.dir", "D:\temp\hadoop-2.4.1\hadoop-2.4.1");
            
            //创建一个测试对象
            WordCountReducer reducer = new WordCountReducer();
            
            //创建一个ReduceDriver进行单元测试
            ReduceDriver<Text, IntWritable, Text, IntWritable> driver = new ReduceDriver<>(reducer);
            
            //构造v3:List
            List<IntWritable> value3 = new ArrayList<>();
            value3.add(new IntWritable(1));
            value3.add(new IntWritable(1));
            value3.add(new IntWritable(1));
            
            
            //指定reducer的输入
            driver.withInput(new Text("Beijing"), value3);
            
            
            //指定reducer的输出
            driver.withOutput(new Text("Beijing"), new IntWritable(3));
            
            
            //ִ执行测试
            driver.runTest();
        }
        
        @Test
        public void testJob() throws Exception{
            //设置一个环境变量
            System.setProperty("hadoop.home.dir", "D:\temp\hadoop-2.4.1\hadoop-2.4.1");
            
            //创建一个测试对象
            WordCountMapper mapper = new WordCountMapper();        
            WordCountReducer reducer = new WordCountReducer();        
            
            //创建一个Driver
            //MapReduceDriver<K1, V1, K2, V2, K4, V4>
            MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable>
                    driver = new MapReduceDriver<>(mapper,reducer);
            
            //指定Map输入的数据
            driver.withInput(new LongWritable(1), new Text("I love Beijing"))
                  .withInput(new LongWritable(4), new Text("I love China"))
                  .withInput(new LongWritable(7), new Text("Beijing is the capital of China"));
            
            //ָ指定Reducer的输出
    //        driver.withOutput(new Text("I"), new IntWritable(2))
    //              .withOutput(new Text("love"), new IntWritable(2))
    //              .withOutput(new Text("Beijing"), new IntWritable(2))
    //              .withOutput(new Text("China"), new IntWritable(2))
    //              .withOutput(new Text("is"), new IntWritable(1))
    //              .withOutput(new Text("the"), new IntWritable(1))
    //              .withOutput(new Text("capital"), new IntWritable(1))
    //              .withOutput(new Text("of"), new IntWritable(1));
            
            //指定Reducer的输出(默认排序规则)
            driver.withOutput(new Text("Beijing"), new IntWritable(2))
                  .withOutput(new Text("China"), new IntWritable(2))
                  .withOutput(new Text("I"), new IntWritable(2))
                  .withOutput(new Text("capital"), new IntWritable(1))
                  .withOutput(new Text("is"), new IntWritable(1))
                  .withOutput(new Text("love"), new IntWritable(2))
                  .withOutput(new Text("of"), new IntWritable(1))
                  .withOutput(new Text("the"), new IntWritable(1));
            
            driver.runTest();
        }
    }
  • 相关阅读:
    225. 用队列实现栈
    415. 字符串相加
    rabbitmq的基本使用
    3. 无重复字符的最长子串
    面试题59
    面试题30. 包含min函数的栈
    面试题09. 用两个栈实现队列
    287. 寻找重复数
    1137. 第 N 个泰波那契数
    70. 爬楼梯
  • 原文地址:https://www.cnblogs.com/lingluo2017/p/8540275.html
Copyright © 2011-2022 走看看