From ecec9dfeb69e04f3d6fcd6a7f3a521a4d047f7b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=98=BF=E6=B4=AA?= Date: Wed, 19 Dec 2018 13:08:58 +0800 Subject: [PATCH] =?UTF-8?q?Update=20=E5=80=92=E6=8E=92=E7=B4=A2=E5=BC=95.m?= =?UTF-8?q?d?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...22\346\216\222\347\264\242\345\274\225.md" | 207 +++++++++--------- 1 file changed, 104 insertions(+), 103 deletions(-) diff --git "a/MapReduce/\345\200\222\346\216\222\347\264\242\345\274\225.md" "b/MapReduce/\345\200\222\346\216\222\347\264\242\345\274\225.md" index f4fea53..19ad9ef 100644 --- "a/MapReduce/\345\200\222\346\216\222\347\264\242\345\274\225.md" +++ "b/MapReduce/\345\200\222\346\216\222\347\264\242\345\274\225.md" @@ -8,111 +8,112 @@ >InversIndex类 - package com.mr.ii.action; - - import java.io.IOException; - - import org.apache.hadoop.conf.Configuration; - import org.apache.hadoop.fs.Path; - import org.apache.hadoop.io.LongWritable; - import org.apache.hadoop.io.NullWritable; - import org.apache.hadoop.io.Text; - import org.apache.hadoop.mapreduce.Job; - import org.apache.hadoop.mapreduce.Mapper; - import org.apache.hadoop.mapreduce.Reducer; - import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; - import org.apache.hadoop.mapreduce.lib.input.FileSplit; - import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; - - public class InversIndex { - - public static void main(String[] args) throws Exception { - // TODO Auto-generated method stub - Job job=Job.getInstance(new Configuration()); - - job.setJarByClass(InversIndex.class); - - job.setMapperClass(InversMapper.class); - job.setMapOutputKeyClass(Text.class); - job.setMapOutputValueClass(Text.class); - FileInputFormat.setInputPaths(job, new Path(args[0])); - - job.setReducerClass(InversReducer.class); - job.setOutputKeyClass(Text.class); - job.setOutputValueClass(Text.class); - FileOutputFormat.setOutputPath(job, new Path(args[1])); - - job.setCombinerClass(InversConbiner.class); - - job.waitForCompletion(true); - } - - - public static class InversMapper extends Mapper{ - - private Text k2=new Text(); - private Text v2=new Text(); - - @Override - protected void map(LongWritable key, Text value, Mapper.Context context) - throws IOException, InterruptedException { - // TODO Auto-generated method stub - String hang=value.toString(); - String[] values=hang.split("\t"); - - for(String string : values){ - FileSplit in=(FileSplit) context.getInputSplit(); - Path path=in.getPath(); - String fileName=path.getName(); - - k2.set(string+"->"+ fileName); - v2.set("1"); - context.write(k2, v2); - } - } - } - - public static class InversConbiner extends Reducer{ - - private Text k22=new Text(); - private Text v22=new Text(); - @Override - protected void reduce(Text k2, Iterable v2, Reducer.Context context) - throws IOException, InterruptedException { - String keyAndName = k2.toString(); - String[] strings=keyAndName.split("->"); - String key = strings[0]; - String fileName = strings[1]; - - long sum = 0; - - for(Text text : v2){ - sum += Long.parseLong(text.toString()); - } - k22.set(key); - v22.set(fileName +"->"+ sum); - - context.write(k22, v22); - } - } - - public static class InversReducer extends Reducer{ - - private Text v3=new Text(); - @Override - protected void reduce(Text k2, Iterable v2, Reducer.Context context) - throws IOException, InterruptedException { - String sum =""; - for(Text text : v2){ - sum += text.toString() + "\t"; - } - - v3.set(sum); - context.write(k2, v3); - } - } +```java +package com.mr.ii.action; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.FileSplit; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; + +public class InversIndex { + + public static void main(String[] args) throws Exception { + // TODO Auto-generated method stub + Job job=Job.getInstance(new Configuration()); + + job.setJarByClass(InversIndex.class); + + job.setMapperClass(InversMapper.class); + job.setMapOutputKeyClass(Text.class); + job.setMapOutputValueClass(Text.class); + FileInputFormat.setInputPaths(job, new Path(args[0])); + + job.setReducerClass(InversReducer.class); + job.setOutputKeyClass(Text.class); + job.setOutputValueClass(Text.class); + FileOutputFormat.setOutputPath(job, new Path(args[1])); + + job.setCombinerClass(InversConbiner.class); + + job.waitForCompletion(true); + } + + + public static class InversMapper extends Mapper{ + + private Text k2=new Text(); + private Text v2=new Text(); + + @Override + protected void map(LongWritable key, Text value, Mapper.Context context) + throws IOException, InterruptedException { + // TODO Auto-generated method stub + String hang=value.toString(); + String[] values=hang.split("\t"); + + for(String string : values){ + FileSplit in=(FileSplit) context.getInputSplit(); + Path path=in.getPath(); + String fileName=path.getName(); + + k2.set(string+"->"+ fileName); + v2.set("1"); + context.write(k2, v2); + } + } + } + + public static class InversConbiner extends Reducer{ + + private Text k22=new Text(); + private Text v22=new Text(); + @Override + protected void reduce(Text k2, Iterable v2, Reducer.Context context) + throws IOException, InterruptedException { + String keyAndName = k2.toString(); + String[] strings=keyAndName.split("->"); + String key = strings[0]; + String fileName = strings[1]; + + long sum = 0; + + for(Text text : v2){ + sum += Long.parseLong(text.toString()); + } + k22.set(key); + v22.set(fileName +"->"+ sum); + + context.write(k22, v22); + } + } + + public static class InversReducer extends Reducer{ + + private Text v3=new Text(); + @Override + protected void reduce(Text k2, Iterable v2, Reducer.Context context) + throws IOException, InterruptedException { + String sum =""; + for(Text text : v2){ + sum += text.toString() + "\t"; } + v3.set(sum); + context.write(k2, v3); + } + } +} +``` ### 三、数据 >a.txt