zoukankan      html  css  js  c++  java
  • 向MapReduce转换:通过部分成绩计算矩阵乘法

    代码共分为四部分:


    <strong><span style="font-size:18px;">/***
     * @author YangXin
     * @info 封装共现关系列
     */
    package unitSix;
    import java.io.IOException;
    import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.mahout.cf.taste.hadoop.item.VectorOrPrefWritable;
    import org.apache.mahout.math.VectorWritable;
    
    public class CooccurrenceColumnWrapperMapper extends Mapper<IntWritable, VectorWritable, IntWritable, VectorOrPrefWritable>{
    	public void map(IntWritable key, VectorWritable value, Context context) throws IOException, InterruptedException{
    		context.write(key, new VectorOrPrefWritable(value.get()));
    	}
    }
    </span></strong>

    <strong><span style="font-size:18px;">/***
     * @author YangXin
     * @info 切割用户数量
     */
    package unitSix;
    
    import java.io.IOException;
    import java.util.Iterator;
    import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.mahout.cf.taste.hadoop.item.VectorOrPrefWritable;
    import org.apache.mahout.math.VarLongWritable;
    import org.apache.mahout.math.Vector;
    import org.apache.mahout.math.VectorWritable;
    
    public class UserVectorSplitterMapper extends Mapper<VarLongWritable, VectorWritable, IntWritable, VectorOrPrefWritable>{
    	public void map(VarLongWritable key, VectorWritable value, Context context) throws IOException, InterruptedException{
    		long userID = key.get();
    		Vector userVector = value.get();
    		Iterator<Vector.Element> it = userVector.nonZeroes().iterator();
    		IntWritable itemIndexWritable = new IntWritable();
    		while(it.hasNext()){
    			Vector.Element e = it.next();
    			int itemIndex = e.index();
    			float preferenceValue = (float)e.get();
    			itemIndexWritable.set(itemIndex);
    			context.write(itemIndexWritable, new VectorOrPrefWritable(userID, preferenceValue));
    		}
    	}
    }</span></strong>


    <strong><span style="font-size:18px;">/***
     * @author YangXin
     * @info 计算部分推荐向量
     */
    package unitSix;
    import java.io.IOException;
    import java.util.List;
    import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.mahout.cf.taste.hadoop.item.VectorAndPrefsWritable;
    import org.apache.mahout.math.VarLongWritable;
    import org.apache.mahout.math.Vector;
    import org.apache.mahout.math.VectorWritable;
    
    public class PartialMultiplyMapper extends Mapper<IntWritable, VectorAndPrefsWritable, VarLongWritable, VectorWritable>{
    	public void map(IntWritable key, VectorAndPrefsWritable vectorAndPrefsWritable, Context context) throws IOException, InterruptedException{
    		Vector cooccurrenceColumn = vectorAndPrefsWritable.getVector();
    		List<Long> userIDs = vectorAndPrefsWritable.getUserIDs();
    		List<Float> prefValues = vectorAndPrefsWritable.getValues();
    		for(int i = 0; i < userIDs.size(); i++){
    			long userID = userIDs.get(i);
    			float prefValue = prefValues.get(i);
    			Vector partialProduct = cooccurrenceColumn.times(prefValue);
    			context.write(new VarLongWritable(userID), new VectorWritable(partialProduct));;
    		}
    	}
    }
    </span></strong>


    <strong><span style="font-size:18px;">/***
     * @author YangXin
     * @info 实现部分成绩的combiner
     */
    package unitSix;
    
    import java.io.IOException;
    
    import org.apache.hadoop.mapreduce.Reducer;
    import org.apache.mahout.math.VarLongWritable;
    import org.apache.mahout.math.Vector;
    import org.apache.mahout.math.VectorWritable;
    
    public class AggregateCombiner extends Reducer<VarLongWritable, VectorWritable, VarLongWritable, VectorWritable>{	
    	public void reduce(VarLongWritable key, Iterable<VectorWritable> values, Context context) throws IOException, InterruptedException{
    		Vector partial = null;
    		for(VectorWritable vectorWritable : values){
    			partial = partial == null ? vectorWritable.get() : partial.plus(vectorWritable.get());
    		}
    		context.write(key, new VectorWritable(partial));
    	}
    }
    </span></strong>




  • 相关阅读:
    ZOJ
    CodeForces
    模板
    前门
    错误记录
    2021/1/10例会 academy of management journal 2014vol 57 No.2,484-514
    Day7下
    Day7上
    Day6 下(
    Day6上 括号匹配专项
  • 原文地址:https://www.cnblogs.com/tlnshuju/p/7213931.html
Copyright © 2011-2022 走看看