educoder平台MapReduce基础实战

MapReduce第1关：成绩统计过关代码:import java.io.IOException;import java.util.StringTokenizer;import java.io.IOException;import java.util.StringTokenizer;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.*;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.util.GenericOptionsParser;public class WordCount {Beginpublic static class TokenizerMapper extends Mapper<LongWritable, Text, Text, IntWritable> { private final static IntWritable one = new IntWritable(1);private Text word = new Text();private int maxValue = 0;public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {StringTokenizer itr = new StringTokenizer(value.toString(),"\n");while (itr.hasMoreTokens()) {String[] str = itr.nextToken().split("");String name = str[0];one.set(Integer.parseInt(str[1]));word.set(name);context.write(word,one);)//context.write(word,one);))public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> { private IntWritable result = new IntWritable();public void reduce(Text key, Iterable<IntWritable> values, Context context)throws IOException, InterruptedException {int maxAge = 0;int age = 0;for (IntWritable intWritable : values) {maxAge = Math.max(maxAge, intWritable.get());)result.set(maxAge);context.write(key, result);))public static void main(String[] args) throws Exception {Configuration conf = new Configuration();Job job = new Job(conf, "word count");job.setJarByClass(WordCount.class);job.setMapperClass(TokenizerMapper.class);job.setCombinerClass(IntSumReducer.class);job.setReducerClass(IntSumReducer.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(IntWritable.class);String inputfile = "/user/test/input";String outputFile = "/user/test/output/";FileInputFormat.addInputPath(job, new Path(inputfile));FileOutputFormat.setOutputPath(job, new Path(outputFile));job.waitForCompletion(true);End))命令行touch file01 echo Hello World Bye Worldcat file01echo Hello World Bye World >file01cat file01touch file02echo Hello Hadoop Goodbye Hadoop >file02 cat file02start-dfs.shhadoop fs -mkdir /usrhadoop fs -mkdir /usr/inputhadoop fs -ls /usr/outputhadoop fs -ls /hadoop fs -ls /usrhadoop fs -put file01 /usr/inputhadoop fs -put file02 /usr/inputhadoop fs -ls /usr/input测评MapReduce第2关：文件内容合并去重代码import java.io.IOException;import java.util.*;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.*;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.util.GenericOptionsParser;public class Merge {/*** @param args对A,B两个文件进行合并,并剔除其中重复的内容,得到一个新的输出文件C /〃在这重载map函数,直接将输入中的value复制到输出数据的key上注意在map方法中要抛出异常：throws IOException,InterruptedException/********** Begin **********/public static class Map extends Mapper<LongWritable, Text, Text, Text >{protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, Text>.Context context)throws IOException, InterruptedException {String str = value.toString();String[] data = str.split("");Text t1= new Text(data[0]);Text t2 = new Text(data[1]);context.write(t1,t2);))/********** End **********/〃在这重载reduce函数,直接将输入中的key复制到输出数据的key上注意在reduce 方法上要抛出异常：throws IOException,InterruptedException/********** Begin **********/public static class Reduce extends Reducer<Text, Text, Text, Text>{protected void reduce(Text key, Iterable<Text> values, Reducer<Text, Text, Text, Text>.Context context)throws IOException, InterruptedException {List<String> list = new ArrayList<>();for (Text text : values) {String str = text.toString();if(!list.contains(str)){ list.add(str);))Collections.sort(list);for (String text : list) { context.write(key, new Text(text));))/********** End **********/ )public static void main(String[] args) throws Exception{Configuration conf = new Configuration();Job job = new Job(conf, "word count");job.setJarByClass(Merge.class);job.setMapperClass(Map.class);job.setCombinerClass(Reduce.class);job.setReducerClass(Reduce.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(Text.class);String inputPath = "/user/tmp/input/"; 〃在这里设置输入路径String outputPath ="/user/tmp/output/"; //在这里设置输出路径FileInputFormat.addInputPath(job, newPath(inputPath));FileOutputFormat.setOutputPath(job, new Path(outputPath));System.exit(job.waitForCompletion(true) ? 0 : 1);) ) 测评MapReduce第3关：信息挖掘-挖掘父子关系代码import java.io.IOException;import java.util.*;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; importorg.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.util.GenericOptionsParser;public class simple_data_mining {public static int time = 0;/***@param args*输入一个child-parent的表格*输出一个表达grandchild-grandparent关系的表格*///Map将输入文件根据空格分割成child和parent,然后正序输出一次作为右表,反序输出一次作为左表,需要注意的是在输出的value中必须加上左右表区别标志public static class Map extends Mapper<Object, Text, Text, Text>{public void map(Object key, Text value, Context context) throws IOException,InterruptedException{BeginString child_name = new String();String parent_name = new String();String relation_type = new String();String line = value.toString();int i = 0;while(line.charAt(i) != ' '){i++;)String[] values = {line.substring(0,i),line.substring(i+1)};if(values[0] pareTo("child") != 0){child_name = values[0];parent_name = values[1];relation_type = "1";//左右表区分标志context.write(newText(values[1]),newText(relation_type+"+"+child_name+"+"+parent_name));//左表relation_type = "2";context.write(newText(values[0]),newText(relation_type+"+"+child_name+"+"+parent_name));//右表}/********** End **********/public static class Reduce extends Reducer<Text, Text, Text, Text>{public void reduce(Text key, Iterable<Text> values,Context context)throws IOException,InterruptedException{Beginif(time == 0){〃输出表头context.write(new Text("grand_child"), new Text("grand_parent"));time++;)int grand_child_num = 0;String grand_child[] = new String[10];int grand_parent_num = 0;String grand_parent[]= new String[10];Iterator ite = values.iterator();while(ite.hasNext()){String record = ite.next().toString();int len = record.length();int i = 2;if(len == 0) continue;char relation_type = record.charAt(0);String child_name = new String();String parent_name = new String();//获取value-list 中value 的childwhile(record.charAt(i) != '+'){child_name = child_name + record.charAt(i); i++;) i=i+1;//获取value-list 中value 的parentwhile(i<len){parent_name = parent_name+record.charAt(i);i++;)//左表,取出child 放入grand_childif(relation_type == '1'){grand_child[grand_child_num] = child_name;grand_child_num++;)else{//右表,取出parent 放入grand_parentgrand_parent[grand_parent_num] = parent_name;grand_parent_num++;))if(grand_parent_num != 0 && grand_child_num != 0 ){for(int m = 0;m<grand_child_num;m++){for(int n=0;n<grand_parent_num;n++){context.write(newText(grand_child[m]), new Text(grand_parent[n]));〃输出结果)))/********** End **********/))public static void main(String[] args) throws Exception{// TODO Auto-generated method stubConfiguration conf = new Configuration();Job job = Job.getInstance(conf,"Single table join");job.setJarByClass(simple_data_mining.class);job.setMapperClass(Map.class);job.setReducerClass(Reduce.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(Text.class);String inputPath = "/user/reduce/input";//设置输入路径String outputPath = "/user/reduce/output";//设置输出路径FileInputFormat.addInputPath(job, new Path(inputPath));FileOutputFormat.setOutputPath(job, new Path(outputPath));System.exit(job.waitForCompletion(true) ? 0 : 1);) ) 测评。