zoukankan      html  css  js  c++  java
  • 大数据笔记(十二)——使用MRUnit进行单元测试

    package demo.wc;
    
    import java.util.ArrayList;
    import java.util.List;
    
    import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mrunit.mapreduce.MapDriver;
    import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
    import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
    import org.junit.Test;
    
    public class MRUnitWordCount {
    
        @Test
        public void testMapper() throws Exception{
            //设置一个环境变量(没有可能会报错)
            System.setProperty("hadoop.home.dir", "D:\temp\hadoop-2.4.1\hadoop-2.4.1");
            
            //创建一个测试对象
            WordCountMapper mapper = new WordCountMapper();
            
            //创建一个MapDriver进行单元测试
            MapDriver<LongWritable, Text, Text, IntWritable> driver = new MapDriver<>(mapper);
            
            //ָ指定Map的输入值: k1  v1
            driver.withInput(new LongWritable(1), new Text("I love Beijing"));
            
            //ָ指定Map的输出值:k2   v2  ----> 期望值
            driver.withOutput(new Text("I"), new IntWritable(1))
                  .withOutput(new Text("love"), new IntWritable(1))
                  .withOutput(new Text("Beijing"), new IntWritable(1));
            
            //ִ执行单元测试,对比 期望的结果和实际的结果
            driver.runTest();
        }
        
        @Test
        public void testReducer() throws Exception{
            //设置一个环境变量
            System.setProperty("hadoop.home.dir", "D:\temp\hadoop-2.4.1\hadoop-2.4.1");
            
            //创建一个测试对象
            WordCountReducer reducer = new WordCountReducer();
            
            //创建一个ReduceDriver进行单元测试
            ReduceDriver<Text, IntWritable, Text, IntWritable> driver = new ReduceDriver<>(reducer);
            
            //构造v3:List
            List<IntWritable> value3 = new ArrayList<>();
            value3.add(new IntWritable(1));
            value3.add(new IntWritable(1));
            value3.add(new IntWritable(1));
            
            
            //指定reducer的输入
            driver.withInput(new Text("Beijing"), value3);
            
            
            //指定reducer的输出
            driver.withOutput(new Text("Beijing"), new IntWritable(3));
            
            
            //ִ执行测试
            driver.runTest();
        }
        
        @Test
        public void testJob() throws Exception{
            //设置一个环境变量
            System.setProperty("hadoop.home.dir", "D:\temp\hadoop-2.4.1\hadoop-2.4.1");
            
            //创建一个测试对象
            WordCountMapper mapper = new WordCountMapper();        
            WordCountReducer reducer = new WordCountReducer();        
            
            //创建一个Driver
            //MapReduceDriver<K1, V1, K2, V2, K4, V4>
            MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable>
                    driver = new MapReduceDriver<>(mapper,reducer);
            
            //指定Map输入的数据
            driver.withInput(new LongWritable(1), new Text("I love Beijing"))
                  .withInput(new LongWritable(4), new Text("I love China"))
                  .withInput(new LongWritable(7), new Text("Beijing is the capital of China"));
            
            //ָ指定Reducer的输出
    //        driver.withOutput(new Text("I"), new IntWritable(2))
    //              .withOutput(new Text("love"), new IntWritable(2))
    //              .withOutput(new Text("Beijing"), new IntWritable(2))
    //              .withOutput(new Text("China"), new IntWritable(2))
    //              .withOutput(new Text("is"), new IntWritable(1))
    //              .withOutput(new Text("the"), new IntWritable(1))
    //              .withOutput(new Text("capital"), new IntWritable(1))
    //              .withOutput(new Text("of"), new IntWritable(1));
            
            //指定Reducer的输出(默认排序规则)
            driver.withOutput(new Text("Beijing"), new IntWritable(2))
                  .withOutput(new Text("China"), new IntWritable(2))
                  .withOutput(new Text("I"), new IntWritable(2))
                  .withOutput(new Text("capital"), new IntWritable(1))
                  .withOutput(new Text("is"), new IntWritable(1))
                  .withOutput(new Text("love"), new IntWritable(2))
                  .withOutput(new Text("of"), new IntWritable(1))
                  .withOutput(new Text("the"), new IntWritable(1));
            
            driver.runTest();
        }
    }
  • 相关阅读:
    tcp/ip网络分层 紫翼龙王
    软件防火墙之iptables/netfilter概念篇(一) 紫翼龙王
    linux cpu负载详解 紫翼龙王
    ThinkInJava4读书笔记之第五章隐藏实施过程
    ThinkInJava4读书笔记之第六章类继承
    工具类sql操作工具类
    python 面向对象
    python 初识
    python 函数,内置函数
    python 正则,os,sys,hashlib模块
  • 原文地址:https://www.cnblogs.com/lingluo2017/p/8540275.html
Copyright © 2011-2022 走看看