zoukankan      html  css  js  c++  java
  • 使用MRUnit和TestNG进行单元测试

    MRUnit

      MRUnit是一款由Couldera公司开发的专门针对Hadoop中编写MapReduce单元测试的框架。

     

    定义Map逻辑

    import java.io.IOException;
    
    import org.apache.hadoop.io.*;
    import org.apache.hadoop.mapreduce.Mapper;
    
    
    public class WordMapper extends Mapper<LongWritable, Text, Text, Text> 
    {
        @Override
        public void map(LongWritable key, Text value, Context context) throws InterruptedException, IOException 
        {
            String[] line = value.toString().split(" ");
            context.write(new Text(line[0]), new Text(line[1]));
        }
    }

    定义Reduce逻辑

    import java.io.IOException;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Reducer;
    
    
    public class WordReducer extends Reducer<Text, Text, Text, Text> 
    {
        public void reduce(Text key, Iterable<Text> values, Context context) throws InterruptedException, IOException 
        {
            for (Text value : values) {
                context.write(value, key);
            }
        }
    }

    单元测试示例

    import java.util.*;
    
    import org.apache.hadoop.io.*;
    import org.apache.hadoop.mapreduce.*;
    import org.apache.hadoop.mrunit.mapreduce.*;
    import org.apache.hadoop.mrunit.types.Pair;
    import org.testng.annotations.*;
    
    
    public class UserForAreaDistributeTest 
    {
        private Mapper<LongWritable, Text, Text, Text> mapper;
        private MapDriver<LongWritable, Text, Text, Text> mapDriver;
        
        private Reducer<Text, Text, Text, Text> reducer;
        private ReduceDriver<Text, Text, Text, Text> reduceDriver;
        
        
        @BeforeClass
        public void setUp()
        {
            mapper = new WordMapper();
            mapDriver = new MapDriver(mapper);
            
            reducer = new WordReducer();
            reduceDriver = new ReduceDriver(reducer);
        }
        
        @Test
        public void testMap() 
        {
            mapDriver.withInput(new Pair(new LongWritable(1L), new Text("abc def")));
            mapDriver.withOutput(new Text("abc"), new Text("def"));
            mapDriver.runTest();
        }
        
        @Test
        public void testReduce()
        {
            List<Text> list = new ArrayList<Text>();
            list.add(new Text("value1"));
            list.add(new Text("value2"));
            
            reduceDriver.withInput(new Text("key"), list);
            reduceDriver.withOutput(new Text("value1"), new Text("key"));
            reduceDriver.withOutput(new Text("value2"), new Text("key"));
            reduceDriver.runTest();
        }
    }
  • 相关阅读:
    nginx proxy_cache 缓存配置[转]
    MongoDB使用小结:一些常用操作分享
    PHP读取Mongodb数据报错,Cannot natively represent the long 8331412483000 on this platform
    MongoDB 学习笔记(python操作)
    Python 中 os.path模板
    Python 优雅的操作字典【转】
    MongoDB 一对多关系建模
    nginx之location配置
    MongoDB安装Windows服务
    .net4.0 请求HTTPS出错:未能创建 SSL/TLS 安全通道
  • 原文地址:https://www.cnblogs.com/rilley/p/2795780.html
Copyright © 2011-2022 走看看