mahout推荐引擎使用hadoop（一） 生成偏好矩阵

 if (shouldRunNextPhase(parsedArgs, currentPhase)) {
      ToolRunner.run(getConf(), new PreparePreferenceMatrixJob(), new String[]{
          "--input", getInputPath().toString(),//输入路径
          "--output", prepPath.toString(),//输出路径
          "--maxPrefsPerUser", String.valueOf(maxPrefsPerUser),
          "--minPrefsPerUser", String.valueOf(minPrefsPerUser),
          "--booleanData", String.valueOf(booleanData),//是否为boolean型的偏好数据
          "--tempDir", getTempPath().toString() });
    }

下边详细的分析一下PreparePreferenceMatrixJob类的实现，主要是通过三个MapReduce来完成的：

public class PreparePreferenceMatrixJob extends AbstractJob {

@Override
  public int run(String[] args) throws Exception {
//第一个MapReduce
Job itemIDIndex = prepareJob(getInputPath(), getOutputPath(ITEMID_INDEX), TextInputFormat.class,
            ItemIDIndexMapper.class, VarIntWritable.class, VarLongWritable.class, ItemIDIndexReducer.class,
            VarIntWritable.class, VarLongWritable.class, SequenceFileOutputFormat.class);
//第二个MapReduce
 Job toUserVectors = prepareJob(getInputPath(), getOutputPath(USER_VECTORS), TextInputFormat.class,
            ToItemPrefsMapper.class, VarLongWritable.class, booleanData ? VarLongWritable.class : EntityPrefWritable.class,
            ToUserVectorsReducer.class, VarLongWritable.class, VectorWritable.class, SequenceFileOutputFormat.class);
//第三个MapReduce
 Job toItemVectors = prepareJob(getOutputPath(USER_VECTORS), getOutputPath(RATING_MATRIX),
            ToItemVectorsMapper.class, IntWritable.class, VectorWritable.class, ToItemVectorsReducer.class,
            IntWritable.class, VectorWritable.class);
    toItemVectors.setCombinerClass(ToItemVectorsReducer.class);

}

首先说一下prepareJob()函数，他的参数列表

   （输入路径， 输出路径， Mapper类，Mapper的key，Mapper的value，reducer类，reducer的key，reducer的value)

至于这函数里边的实现就不用看了，反正就是执行了一个MapReducer操作。

下边挨个分析这三个MapReducer的具体计算过程：

(1) Mapper :  ItemIDIndexMapper类，将类型为Long的itemid转换成int类型的itemid_index，并输出<itemid_index, itemid>这样将所有的item先进行split

public final class ItemIDIndexMapper extends
    Mapper<LongWritable,Text, VarIntWritable, VarLongWritable>{

 @Override
  protected void map(LongWritable key,
                     Text value,
                     Context context) throws IOException, InterruptedException {
    String[] tokens = TasteHadoopUtils.splitPrefTokens(value.toString());
    long itemID = Long.parseLong(tokens[transpose ? 0 : 1]);
    //将long类型itemid转换成int类型的itemid_index
    int index = TasteHadoopUtils.idToIndex(itemID);
    //输出（itemid_index, itemid）类型
    context.write(new VarIntWritable(index), new VarLongWritable(itemID));
  }  

}

 Reducer : temIDIndexReducer类，对每一个itemid_index下的所有的itemid取最小值，并输出<itemid_index, minimumItemid>

public final class ItemIDIndexReducer extends
    Reducer<VarIntWritable, VarLongWritable, VarIntWritable,VarLongWritable> {
 
  //对每一个itemid_index下的所有的itemid取最小值，并输出<itemid_index, minimumItemid>
  @Override
  protected void reduce(VarIntWritable index,
                        Iterable<VarLongWritable> possibleItemIDs,
                        Context context) throws IOException, InterruptedException {
    long minimumItemID = Long.MAX_VALUE;
    for (VarLongWritable varLongWritable : possibleItemIDs) {
      long itemID = varLongWritable.get();
      if (itemID < minimumItemID) {
        minimumItemID = itemID;
      }
    }
    if (minimumItemID != Long.MAX_VALUE) {
      context.write(index, new VarLongWritable(minimumItemID));
    }
  }
  
}

 （2）Mapper :  ToItemPrefsMapper类，继承于ToEntityPrefsMapper类，作用是从文件中读取数据，并以

  userid，<itemid, pref>  的形式作为reducer的输出

public final class ToItemPrefsMapper extends ToEntityPrefsMapper {
  public ToItemPrefsMapper() {
    super(false);
  }  
}

public abstract class ToEntityPrefsMapper extends
    Mapper<LongWritable,Text, VarLongWritable,VarLongWritable> {
@Override
  public void map(LongWritable key,
                  Text value,
                  Context context) throws IOException, InterruptedException {
    String[] tokens = DELIMITER.split(value.toString());
    long userID = Long.parseLong(tokens[0]);
    long itemID = Long.parseLong(tokens[1]);
    if (itemKey ^ transpose) {
      // If using items as keys, and not transposing items and users, then users are items!
      // Or if not using items as keys (users are, as usual), but transposing items and users,
      // then users are items! Confused?
      //如果设置偏转，那么就是将userid和itemid的位置互换，也就是基于user的CF了
      long temp = userID;
      userID = itemID;
      itemID = temp;
    }
    if (booleanData) {
      context.write(new VarLongWritable(userID), new VarLongWritable(itemID));
    } else {
      float prefValue = tokens.length > 2 ? Float.parseFloat(tokens[2]) + ratingShift : 1.0f;
      //输出（userid，<itemid, pref> )类型
      context.write(new VarLongWritable(userID), new EntityPrefWritable(itemID, prefValue));
    }
  }

}

 Reducer : ToUserVectorsReducer类，收集同一个userid下的<itemid, pref>对，并将itemid映射成itemid_index，然后和pref组成<itemid_index, pref>对，并将同一个userid下的所有<itemid_index, pref>对保存到一个vecotr（RandomAccessSparseVector类型的）中，并输出 (userid，vector<itemid_index, pref>) 类型的结果

public final class ToUserVectorsReducer extends
    Reducer<VarLongWritable,VarLongWritable,VarLongWritable,VectorWritable> {

  @Override
  protected void reduce(VarLongWritable userID,
                        Iterable<VarLongWritable> itemPrefs,
                        Context context) throws IOException, InterruptedException {
    Vector userVector = new RandomAccessSparseVector(Integer.MAX_VALUE, 100);
    for (VarLongWritable itemPref : itemPrefs) {
      //将long类型的itemid映射成int类型的itemid_index
      int index = TasteHadoopUtils.idToIndex(itemPref.get());
      float value = itemPref instanceof EntityPrefWritable ? ((EntityPrefWritable) itemPref).getPrefValue() : 1.0f;
      userVector.set(index, value);
    }

    if (userVector.getNumNondefaultElements() >= minPreferences) {
      VectorWritable vw = new VectorWritable(userVector);
      vw.setWritesLaxPrecision(true);
      context.getCounter(Counters.USERS).increment(1);
      context.write(userID, vw);
    }
  }
  
}

（3）Mapper ：ToItemVectorsMapper类，将上边（2）中的输出作为Mapper的输入，将Long类型userid映射为int类型的userid_index，并输出( itemid_index, <userid_index, pref> )类型

public class ToItemVectorsMapper
    extends Mapper<VarLongWritable,VectorWritable,IntWritable,VectorWritable> {

  @Override
  protected void map(VarLongWritable rowIndex, VectorWritable vectorWritable, Context ctx)
      throws IOException, InterruptedException {
    Vector userRatings = vectorWritable.get();

    int numElementsBeforeSampling = userRatings.getNumNondefaultElements();
    userRatings = Vectors.maybeSample(userRatings, sampleSize);
    int numElementsAfterSampling = userRatings.getNumNondefaultElements();

    int column = TasteHadoopUtils.idToIndex(rowIndex.get());
    VectorWritable itemVector = new VectorWritable(new RandomAccessSparseVector(Integer.MAX_VALUE, 1));
    itemVector.setWritesLaxPrecision(true);

    Iterator<Vector.Element> iterator = userRatings.iterateNonZero();
    while (iterator.hasNext()) {
      Vector.Element elem = iterator.next();
      //这里vector.setQuick()是将键值对插入到一个map里，因为column不变，所以每次替换value
      itemVector.get().setQuick(column, elem.get());
      //输出( itemid_index, <userid_index, pref> )
      ctx.write(new IntWritable(elem.index()), itemVector);
    }

    ctx.getCounter(Elements.USER_RATINGS_USED).increment(numElementsAfterSampling);
    ctx.getCounter(Elements.USER_RATINGS_NEGLECTED).increment(numElementsBeforeSampling - numElementsAfterSampling);
  }

}

 Reducer ：ToItemVectorsReducer类，将同一个itemid_index下的<userid_index, pref>对收集到一起，保存在一个vector中，并输出 ( itemid_index，vector<userid_index, pref> ) 类型

public class ToItemVectorsReducer extends Reducer<IntWritable,VectorWritable,IntWritable,VectorWritable> {

  @Override
  protected void reduce(IntWritable row, Iterable<VectorWritable> vectors, Context ctx)
      throws IOException, InterruptedException {
    //合并同一个itemid_index下的<userid_index, pref>对
    VectorWritable vectorWritable = VectorWritable.merge(vectors.iterator());
    vectorWritable.setWritesLaxPrecision(true);
    ctx.write(row, vectorWritable);
  }
}

 至此，在PreparePreferenceMatrixJob中的所有MapReduce计算都完成了，最终我们看到，生成的是一个

itemid_index,  vector<userid_index, pref>   的偏好矩阵