在Hadoop上使用集算器高效开发

在Hadoop上使用集算器高效开发Hadoop是优秀的大数据解决方案，低成本和高扩展性是用户选择它的主要原因，但它低下的开发效率也常被用户所诟病。

Hadoop使用MapReduce框架进行大数据的开发和计算，进行简单的计算时问题并不会显露出来。

一旦计算稍有难度，低下的开发效率就会产生严重的影响。

这其中就包括了“关联计算”这种最常见的计算。

例如：HDFS上有2个文件，分别是客户信息和订单信息，customerID是它们之间的关联字段。

如何进行关联计算，以便将客户名称添加到订单列表中？一般方法是：输入2个源文件。

根据文件名在Map中处理每条数据，如果是Order，则在foreign key 上加标记”O”，形成combined key；如果是Customer则做标记”C”。

Map之后的数据按照key分区，再按照combined key分组排序。

最后在reduce中合并结果再输出。

以下的代码据说比较常见：public static class JMapper extends Mapper<LongWritable, Text, TextPair, Text> {//mark every row with "O" or "C" according to file name@Overrideprotected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {String pathName = ((FileSplit) context.getInputSplit()).getPath().toString();if (pathName.contains("order.txt")) {//identify order by file nameString values[] = value.toString().split("\t");TextPair tp = new TextPair(new Text(values[1]), new Text("O"));//mark with "O"context.write(tp, new Text(values[0] + "\t" + values[2]));}if (pathName.contains("customer.txt")) {//identify customer by file nameString values[] = value.toString().split("\t");TextPair tp = new TextPair(new Text(values[0]), new Text("C"));//mark with "C"context.write(tp, new Text(values[1]));}}}public static class JPartitioner extends Partitioner<TextPair, Text> {//partition by key, i.e. customerID@Overridepublic int getPartition(TextPair key, T ext value, int numParititon) {return Math.abs(key.getFirst().hashCode() * 127) % numParititon;}}public static class JComparator extends WritableComparator {//group by muti-keypublic JComparator() {super(TextPair.class, true);}@SuppressWarnings("unchecked")public int compare(WritableComparable a, WritableComparable b) {TextPair t1 = (TextPair) a;TextPair t2 = (TextPair) b;return t1.getFirst().compareTo(t2.getFirst());}}public static class JReduce extends Reducer<TextPair, Text, Text, T ext> {//merge and outputprotected void reduce(TextPair key, Iterable<Text> values, Context context) throws IOException,InterruptedException {Text pid = key.getFirst();String desc = values.iterator().next().toString();while (values.iterator().hasNext()) {context.write(pid, new Text(values.iterator().next().toString() + "\t" + desc));}}}public class TextPair implements WritableComparable<TextPair> {//make muti-keyprivate Text first;private Text second;public TextPair() {set(new Text(), new Text());}public TextPair(String first, String second) {set(new Text(first), new Text(second));}public TextPair(Text first, Text second) {set(first, second);}public void set(Text first, Text second) {this.first = first;this.second = second;}public Text getFirst() {return first;}public Text getSecond() {return second;}public void write(DataOutput out) throws IOException {first.write(out);second.write(out);}public void readFields(DataInput in) throws IOException {first.readFields(in);second.readFields(in);}public int compareTo(TextPair tp) {int cmp = pareTo(tp.first);if (cmp != 0) {return cmp;}return pareTo(tp.second);}}public static void main(String agrs[]) throws IOException, InterruptedException, ClassNotFoundException { //job entranceConfiguration conf = new Configuration();GenericOptionsParser parser = new GenericOptionsParser(conf, agrs);String[] otherArgs = parser.getRemainingArgs();if (agrs.length < 3) {System.err.println("Usage: J <in_path_one> <in_path_two> <output>");System.exit(2);}Job job = new Job(conf, "J");job.setJarByClass(J.class);//Join classjob.setMapperClass(JMapper.class);//Map classjob.setMapOutputKeyClass(TextPair.class);//Map output key classjob.setMapOutputValueClass(Text.class);//Map output value classjob.setPartitionerClass(JPartitioner.class);//partition classjob.setGroupingComparatorClass(JComparator.class);//condition group class after partitionjob.setReducerClass(Example_Join_01_Reduce.class);//reduce classjob.setOutputKeyClass(Text.class);//reduce output key classjob.setOutputValueClass(Text.class);//reduce ouput value classFileInputFormat.addInputPath(job, new Path(otherArgs[0]));//one of source filesFileInputFormat.addInputPath(job, new Path(otherArgs[1]));//another fileFileOutputFormat.setOutputPath(job, new Path(otherArgs[2]));//output pathSystem.exit(job.waitForCompletion(true) ? 0 : 1);//run untill job ends}可以看到，为了实现关联计算，程序员往往不能直接使用原始数据，而是要书写复杂的代码处理标记，并绕过MapReduce原本的架构，最后从底层设计并计算数据之间的关联关系。