zoukankan      html  css  js  c++  java

  • mapreduce作业单元测试

    用官方提供的MRunit包进行对MapReducer作业进行测试

    http://mrunit.apache.org/

    先要下载对应的测试包，分为1.0和2.0的hadoop版本

    来自：

    https://cwiki.apache.org/confluence/display/MRUNIT/MRUnit+Tutorial

    Mapper

    Java代码 复制代码 收藏代码
    1. import org.apache.hadoop.io.IntWritable;   
    2. import org.apache.hadoop.io.LongWritable;   
    3. import org.apache.hadoop.io.Text;   
    4. import org.apache.hadoop.mapreduce.Mapper;   
    5.   
    6. public class SMSCDRMapper extends Mapper<LongWritable, Text, Text, IntWritable>   
    7. {   
    8.   
    9.     private Text status = new Text();   
    10.   
    11.     private final static IntWritable addOne = new IntWritable(1);   
    12.   
    13.     @Override  
    14.     protected void map(LongWritable key, Text value, Context context) throws java.io.IOException, InterruptedException   
    15.     {   
    16.   
    17.         //655209;1;796764372490213;804422938115889;6 is the Sample record format   
    18.         String[] line = value.toString().split(";");   
    19.         // If record is of SMS CDR   
    20.         if (Integer.parseInt(line[1]) == 1)   
    21.         {   
    22.             status.set(line[4]);   
    23.             context.write(status, addOne);   
    24.         }   
    25.     }   
    26.   
    27. }  
    import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class SMSCDRMapper extends Mapper<LongWritable, Text, Text, IntWritable>
{

    private Text status = new Text();

    private final static IntWritable addOne = new IntWritable(1);

    @Override
    protected void map(LongWritable key, Text value, Context context) throws java.io.IOException, InterruptedException
    {

        //655209;1;796764372490213;804422938115889;6 is the Sample record format
        String[] line = value.toString().split(";");
        // If record is of SMS CDR
        if (Integer.parseInt(line[1]) == 1)
        {
            status.set(line[4]);
            context.write(status, addOne);
        }
    }

}

    Reducer:

    Java代码 复制代码 收藏代码
    1. import java.util.List;   
    2.   
    3. import org.apache.hadoop.io.IntWritable;   
    4. import org.apache.hadoop.io.LongWritable;   
    5. import org.apache.hadoop.io.Text;   
    6. import org.apache.hadoop.mrunit.mapreduce.MapDriver;   
    7. import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;   
    8. import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;   
    9. import org.junit.Before;   
    10. import org.junit.Test;   
    11.   
    12. import flex.messaging.io.ArrayList;   
    13.   
    14. public class SMSCDRMapperReducerTest   
    15. {   
    16.     MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;   
    17.     ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;   
    18.     MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver;   
    19.     @Before  
    20.     public void setup()   
    21.     {   
    22.         SMSCDRMapper mapper = new SMSCDRMapper();   
    23.         SMSCDRReducer reducer = new SMSCDRReducer();   
    24.         mapDriver = MapDriver.newMapDriver(mapper);   
    25.            
    26.         reduceDriver = ReduceDriver.newReduceDriver(reducer);   
    27.         mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);   
    28.     }   
    29.        
    30.     @Test  
    31.     public void testMapper()   
    32.     {   
    33.         mapDriver.withInput(new LongWritable(), new Text(   
    34.         "655209;1;796764372490213;804422938115889;6"));   
    35.         mapDriver.withOutput(new Text("6"), new IntWritable(1));   
    36.         mapDriver.runTest();   
    37.     }   
    38.        
    39.     @Test  
    40.     public void testReducer()   
    41.     {   
    42.         List<IntWritable> values = new ArrayList();   
    43.         values.add(new IntWritable(1));   
    44.         values.add(new IntWritable(1));   
    45.         reduceDriver.withInput(new Text("6"), values);   
    46.         reduceDriver.withOutput(new Text("6"), new IntWritable(2));   
    47.         reduceDriver.runTest();   
    48.     }   
    49. }  
    import java.util.List;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
import org.junit.Before;
import org.junit.Test;

import flex.messaging.io.ArrayList;

public class SMSCDRMapperReducerTest
{
    MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;
    ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;
    MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver;
    @Before
    public void setup()
    {
        SMSCDRMapper mapper = new SMSCDRMapper();
        SMSCDRReducer reducer = new SMSCDRReducer();
        mapDriver = MapDriver.newMapDriver(mapper);
        
        reduceDriver = ReduceDriver.newReduceDriver(reducer);
        mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
    }
    
    @Test
    public void testMapper()
    {
        mapDriver.withInput(new LongWritable(), new Text(
        "655209;1;796764372490213;804422938115889;6"));
        mapDriver.withOutput(new Text("6"), new IntWritable(1));
        mapDriver.runTest();
    }
    
    @Test
    public void testReducer()
    {
        List<IntWritable> values = new ArrayList();
        values.add(new IntWritable(1));
        values.add(new IntWritable(1));
        reduceDriver.withInput(new Text("6"), values);
        reduceDriver.withOutput(new Text("6"), new IntWritable(2));
        reduceDriver.runTest();
    }
}

    Test测试类

    Java代码 复制代码 收藏代码
    1. import java.util.List;   
    2.   
    3. import org.apache.hadoop.io.IntWritable;   
    4. import org.apache.hadoop.io.LongWritable;   
    5. import org.apache.hadoop.io.Text;   
    6. import org.apache.hadoop.mrunit.mapreduce.MapDriver;   
    7. import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;   
    8. import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;   
    9. import org.junit.Before;   
    10. import org.junit.Test;   
    11.   
    12. import flex.messaging.io.ArrayList;   
    13.   
    14. public class SMSCDRMapperReducerTest   
    15. {   
    16.     MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;   
    17.     ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;   
    18.     MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver;   
    19.     @Before  
    20.     public void setup()   
    21.     {   
    22.         SMSCDRMapper mapper = new SMSCDRMapper();   
    23.         SMSCDRReducer reducer = new SMSCDRReducer();   
    24.         mapDriver = MapDriver.newMapDriver(mapper);   
    25.            
    26.         reduceDriver = ReduceDriver.newReduceDriver(reducer);   
    27.         mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);   
    28.     }   
    29.        
    30.     @Test  
    31.     public void testMapper()   
    32.     {   
    33.         mapDriver.withInput(new LongWritable(), new Text(   
    34.         "655209;1;796764372490213;804422938115889;6"));   
    35.         mapDriver.withOutput(new Text("6"), new IntWritable(1));   
    36.         mapDriver.runTest();   
    37.     }   
    38.        
    39.     @Test  
    40.     public void testReducer()   
    41.     {   
    42.         List<IntWritable> values = new ArrayList();   
    43.         values.add(new IntWritable(1));   
    44.         values.add(new IntWritable(1));   
    45.         reduceDriver.withInput(new Text("6"), values);   
    46.         reduceDriver.withOutput(new Text("6"), new IntWritable(2));   
    47.         reduceDriver.runTest();   
    48.     }   
    49. }  
    import java.util.List;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
import org.junit.Before;
import org.junit.Test;

import flex.messaging.io.ArrayList;

public class SMSCDRMapperReducerTest
{
    MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;
    ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;
    MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver;
    @Before
    public void setup()
    {
        SMSCDRMapper mapper = new SMSCDRMapper();
        SMSCDRReducer reducer = new SMSCDRReducer();
        mapDriver = MapDriver.newMapDriver(mapper);
        
        reduceDriver = ReduceDriver.newReduceDriver(reducer);
        mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
    }
    
    @Test
    public void testMapper()
    {
        mapDriver.withInput(new LongWritable(), new Text(
        "655209;1;796764372490213;804422938115889;6"));
        mapDriver.withOutput(new Text("6"), new IntWritable(1));
        mapDriver.runTest();
    }
    
    @Test
    public void testReducer()
    {
        List<IntWritable> values = new ArrayList();
        values.add(new IntWritable(1));
        values.add(new IntWritable(1));
        reduceDriver.withInput(new Text("6"), values);
        reduceDriver.withOutput(new Text("6"), new IntWritable(2));
        reduceDriver.runTest();
    }
}

    转自http://piaoling.iteye.com/blog/1672683
  • 相关阅读:
    [置顶] 呼~~~~--历时几个星期终于搞好了HTTPS协议---阿里云
    云数据库连接权限等
    Developer连接Oracle报错“ORA-12541
    VUE项目使用
    数据库优化面试内容
    CentOS配置svn
    老年痴呆博客引导
    日常问题记录
    Spring项目定时任务
    程序员逻辑思维题解答；
  • 原文地址：https://www.cnblogs.com/xd502djj/p/3011240.html
Copyright © 2011-2022 走看看