zoukankan      html  css  js  c++  java

  • mapreduce作业单元测试

    用官方提供的MRunit包进行对MapReducer作业进行测试

    http://mrunit.apache.org/

    先要下载对应的测试包，分为1.0和2.0的hadoop版本

    来自：

    https://cwiki.apache.org/confluence/display/MRUNIT/MRUnit+Tutorial

    Mapper

    Java代码 复制代码 收藏代码
    1. import org.apache.hadoop.io.IntWritable;   
    2. import org.apache.hadoop.io.LongWritable;   
    3. import org.apache.hadoop.io.Text;   
    4. import org.apache.hadoop.mapreduce.Mapper;   
    5.   
    6. public class SMSCDRMapper extends Mapper<LongWritable, Text, Text, IntWritable>   
    7. {   
    8.   
    9.     private Text status = new Text();   
    10.   
    11.     private final static IntWritable addOne = new IntWritable(1);   
    12.   
    13.     @Override  
    14.     protected void map(LongWritable key, Text value, Context context) throws java.io.IOException, InterruptedException   
    15.     {   
    16.   
    17.         //655209;1;796764372490213;804422938115889;6 is the Sample record format   
    18.         String[] line = value.toString().split(";");   
    19.         // If record is of SMS CDR   
    20.         if (Integer.parseInt(line[1]) == 1)   
    21.         {   
    22.             status.set(line[4]);   
    23.             context.write(status, addOne);   
    24.         }   
    25.     }   
    26.   
    27. }  
    import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class SMSCDRMapper extends Mapper<LongWritable, Text, Text, IntWritable>
{

    private Text status = new Text();

    private final static IntWritable addOne = new IntWritable(1);

    @Override
    protected void map(LongWritable key, Text value, Context context) throws java.io.IOException, InterruptedException
    {

        //655209;1;796764372490213;804422938115889;6 is the Sample record format
        String[] line = value.toString().split(";");
        // If record is of SMS CDR
        if (Integer.parseInt(line[1]) == 1)
        {
            status.set(line[4]);
            context.write(status, addOne);
        }
    }

}

    Reducer:

    Java代码 复制代码 收藏代码
    1. import java.util.List;   
    2.   
    3. import org.apache.hadoop.io.IntWritable;   
    4. import org.apache.hadoop.io.LongWritable;   
    5. import org.apache.hadoop.io.Text;   
    6. import org.apache.hadoop.mrunit.mapreduce.MapDriver;   
    7. import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;   
    8. import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;   
    9. import org.junit.Before;   
    10. import org.junit.Test;   
    11.   
    12. import flex.messaging.io.ArrayList;   
    13.   
    14. public class SMSCDRMapperReducerTest   
    15. {   
    16.     MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;   
    17.     ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;   
    18.     MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver;   
    19.     @Before  
    20.     public void setup()   
    21.     {   
    22.         SMSCDRMapper mapper = new SMSCDRMapper();   
    23.         SMSCDRReducer reducer = new SMSCDRReducer();   
    24.         mapDriver = MapDriver.newMapDriver(mapper);   
    25.            
    26.         reduceDriver = ReduceDriver.newReduceDriver(reducer);   
    27.         mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);   
    28.     }   
    29.        
    30.     @Test  
    31.     public void testMapper()   
    32.     {   
    33.         mapDriver.withInput(new LongWritable(), new Text(   
    34.         "655209;1;796764372490213;804422938115889;6"));   
    35.         mapDriver.withOutput(new Text("6"), new IntWritable(1));   
    36.         mapDriver.runTest();   
    37.     }   
    38.        
    39.     @Test  
    40.     public void testReducer()   
    41.     {   
    42.         List<IntWritable> values = new ArrayList();   
    43.         values.add(new IntWritable(1));   
    44.         values.add(new IntWritable(1));   
    45.         reduceDriver.withInput(new Text("6"), values);   
    46.         reduceDriver.withOutput(new Text("6"), new IntWritable(2));   
    47.         reduceDriver.runTest();   
    48.     }   
    49. }  
    import java.util.List;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
import org.junit.Before;
import org.junit.Test;

import flex.messaging.io.ArrayList;

public class SMSCDRMapperReducerTest
{
    MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;
    ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;
    MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver;
    @Before
    public void setup()
    {
        SMSCDRMapper mapper = new SMSCDRMapper();
        SMSCDRReducer reducer = new SMSCDRReducer();
        mapDriver = MapDriver.newMapDriver(mapper);
        
        reduceDriver = ReduceDriver.newReduceDriver(reducer);
        mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
    }
    
    @Test
    public void testMapper()
    {
        mapDriver.withInput(new LongWritable(), new Text(
        "655209;1;796764372490213;804422938115889;6"));
        mapDriver.withOutput(new Text("6"), new IntWritable(1));
        mapDriver.runTest();
    }
    
    @Test
    public void testReducer()
    {
        List<IntWritable> values = new ArrayList();
        values.add(new IntWritable(1));
        values.add(new IntWritable(1));
        reduceDriver.withInput(new Text("6"), values);
        reduceDriver.withOutput(new Text("6"), new IntWritable(2));
        reduceDriver.runTest();
    }
}

    Test测试类

    Java代码 复制代码 收藏代码
    1. import java.util.List;   
    2.   
    3. import org.apache.hadoop.io.IntWritable;   
    4. import org.apache.hadoop.io.LongWritable;   
    5. import org.apache.hadoop.io.Text;   
    6. import org.apache.hadoop.mrunit.mapreduce.MapDriver;   
    7. import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;   
    8. import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;   
    9. import org.junit.Before;   
    10. import org.junit.Test;   
    11.   
    12. import flex.messaging.io.ArrayList;   
    13.   
    14. public class SMSCDRMapperReducerTest   
    15. {   
    16.     MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;   
    17.     ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;   
    18.     MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver;   
    19.     @Before  
    20.     public void setup()   
    21.     {   
    22.         SMSCDRMapper mapper = new SMSCDRMapper();   
    23.         SMSCDRReducer reducer = new SMSCDRReducer();   
    24.         mapDriver = MapDriver.newMapDriver(mapper);   
    25.            
    26.         reduceDriver = ReduceDriver.newReduceDriver(reducer);   
    27.         mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);   
    28.     }   
    29.        
    30.     @Test  
    31.     public void testMapper()   
    32.     {   
    33.         mapDriver.withInput(new LongWritable(), new Text(   
    34.         "655209;1;796764372490213;804422938115889;6"));   
    35.         mapDriver.withOutput(new Text("6"), new IntWritable(1));   
    36.         mapDriver.runTest();   
    37.     }   
    38.        
    39.     @Test  
    40.     public void testReducer()   
    41.     {   
    42.         List<IntWritable> values = new ArrayList();   
    43.         values.add(new IntWritable(1));   
    44.         values.add(new IntWritable(1));   
    45.         reduceDriver.withInput(new Text("6"), values);   
    46.         reduceDriver.withOutput(new Text("6"), new IntWritable(2));   
    47.         reduceDriver.runTest();   
    48.     }   
    49. }  
    import java.util.List;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
import org.junit.Before;
import org.junit.Test;

import flex.messaging.io.ArrayList;

public class SMSCDRMapperReducerTest
{
    MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;
    ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;
    MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver;
    @Before
    public void setup()
    {
        SMSCDRMapper mapper = new SMSCDRMapper();
        SMSCDRReducer reducer = new SMSCDRReducer();
        mapDriver = MapDriver.newMapDriver(mapper);
        
        reduceDriver = ReduceDriver.newReduceDriver(reducer);
        mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
    }
    
    @Test
    public void testMapper()
    {
        mapDriver.withInput(new LongWritable(), new Text(
        "655209;1;796764372490213;804422938115889;6"));
        mapDriver.withOutput(new Text("6"), new IntWritable(1));
        mapDriver.runTest();
    }
    
    @Test
    public void testReducer()
    {
        List<IntWritable> values = new ArrayList();
        values.add(new IntWritable(1));
        values.add(new IntWritable(1));
        reduceDriver.withInput(new Text("6"), values);
        reduceDriver.withOutput(new Text("6"), new IntWritable(2));
        reduceDriver.runTest();
    }
}

    转自http://piaoling.iteye.com/blog/1672683
  • 相关阅读:
    《算法竞赛入门经典》 例题35 生成元 （Digit Generator, ACM ICPC Seoul 2005,UVa）
    《算法竞赛入门经典》 例题35 生成元 （Digit Generator, ACM ICPC Seoul 2005,UVa）
    《算法竞赛入门经典》 例题35 生成元 （Digit Generator, ACM ICPC Seoul 2005,UVa）
    SVN分支
    SVN分支
    SVN 版本回退
    SVN 版本回退
    如何在excel中取消合并单元格后内容自动填充？
    如何在excel中取消合并单元格后内容自动填充？
    如何让自己像打王者荣耀一样发了疯、拼了命的学习？
  • 原文地址：https://www.cnblogs.com/xd502djj/p/3011240.html
Copyright © 2011-2022 走看看