MapReduce的SequenceFileInputFormat使用

package com.mengyao.hadoop.mapreduce;

import java.io.IOException;
import java.util.Iterator;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

/**
 * 使用SequenceFileInputFormat处理HDFS上的SequenceFile类型文件，输出为普通文件
 *
 * @author mengyao
 *
 */
public class SequenceFileInputFormatApp extends Configured implements Tool  {

    static class SequenceFileInputFormatMapper extends Mapper<IntWritable, Text, NullWritable, Text> {
        
        private NullWritable outputKey;
        
        @Override
        protected void setup(Context context)
                throws IOException, InterruptedException {
            this.outputKey = NullWritable.get();
        }
        
        @Override
        protected void map(IntWritable key, Text value, Context context)
                throws IOException, InterruptedException {
            context.write(outputKey, value);
        }
    }
    
    static class SequenceFileInputFormatReducer extends Reducer<NullWritable, Text, NullWritable, Text> {
        
        private NullWritable outputKey;
        
        @Override
        protected void setup(Context context)
                throws IOException, InterruptedException {
            this.outputKey = NullWritable.get();
        }
        
        @Override
        protected void reduce(NullWritable key, Iterable<Text> value, Context context)
                throws IOException, InterruptedException {
            Iterator<Text> iterator = value.iterator();
            while (iterator.hasNext()) {
                context.write(outputKey, iterator.next());
            }
        }
    }

    @Override
    public int run(String[] args) throws Exception {
        Job job = Job.getInstance(getConf(), SequenceFileInputFormatApp.class.getSimpleName());
        job.setJarByClass(SequenceFileInputFormatApp.class);
        
        job.setInputFormatClass(SequenceFileInputFormat.class);
        FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));
        
        job.setMapperClass(SequenceFileInputFormatMapper.class);
        job.setMapOutputKeyClass(NullWritable.class);
        job.setMapOutputValueClass(Text.class);
        
        job.setReducerClass(SequenceFileInputFormatReducer.class);
        job.setOutputKeyClass(NullWritable.class);
        job.setOutputValueClass(Text.class);
        
        return job.waitForCompletion(true)?0:1;
    }
    
    public static int createJob(String[] args) {
        Configuration conf = new Configuration();
        conf.set("dfs.datanode.socket.write.timeout", "7200000");
        conf.set("mapreduce.input.fileinputformat.split.minsize", "268435456");
        conf.set("mapreduce.input.fileinputformat.split.maxsize", "536870912");
        conf.set("mapreduce.job.jvm.numtasks", "-1");        
        conf.set("mapreduce.map.speculative", "false");        
        conf.set("mapreduce.reduce.speculative", "false");    
        conf.set("mapreduce.map.maxattempts", "4");            
        conf.set("mapreduce.reduce.maxattempts", "4");        
        conf.set("mapreduce.map.skip.maxrecords", "0");
        int status = 0;
        
        try {
            status = ToolRunner.run(conf, new SequenceFileInputFormatApp(), args);
        } catch (Exception e) {
            e.printStackTrace();
        }
        
        return status;
    }
    
    public static void main(String[] args) {
        args = new String[]{"/mapreduces/seqfile/book1.txt", "/mapreduces/sequencefileinputformat"};
        if (args.length!=2) {
            System.out.println("Usage: "+SequenceFileInputFormatApp.class.getName()+" Input paramters <INPUT_PATH> <OUTPUT_PATH>");
        } else {
            int status = createJob(args);
            System.exit(status);
        }
    }
}