访问计数成功

lkfs · Sep 27, 2020 · eed1529 · eed1529
1 parent 1b467f0
commit eed1529
Show file tree

Hide file tree

Showing 3 changed files with 115 additions and 0 deletions.
diff --git a/src/main/java/net/venusoft/count/AccessCountDriver.java b/src/main/java/net/venusoft/count/AccessCountDriver.java
@@ -0,0 +1,40 @@
+package net.venusoft.count;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+
+import java.io.IOException;
+
+/**
+ * @author lk
+ * @date 2020/9/27 15:12
+ */
+public class AccessCountDriver {
+    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
+        System.setProperty("HADOOP_USER_NAME", "hadoop");
+        String inputPath = "hdfs://hadoop01:9000/input";
+        String outputPath = "hdfs://hadoop01:9000/output1_3";
+
+        Configuration conf = new Configuration();
+        Job job2 = Job.getInstance(conf);
+        job2.setJarByClass(AccessCountDriver.class);
+
+        job2.setMapperClass(AccessCountMapper.class);
+        job2.setReducerClass(AccessCountReducer.class);
+
+        job2.setMapOutputKeyClass(Text.class);
+        job2.setMapOutputValueClass(IntWritable.class);
+        job2.setOutputKeyClass(Text.class);
+        job2.setOutputValueClass(IntWritable.class);
+
+        FileInputFormat.addInputPath(job2, new Path(inputPath));
+        FileOutputFormat.setOutputPath(job2, new Path(outputPath));
+
+        job2.waitForCompletion(true);
+    }
+}
diff --git a/src/main/java/net/venusoft/count/AccessCountMapper.java b/src/main/java/net/venusoft/count/AccessCountMapper.java
@@ -0,0 +1,52 @@
+package net.venusoft.count;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Mapper;
+
+import java.io.IOException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * @author lk
+ * @date 2020/9/27 14:54
+ */
+@Slf4j
+public class AccessCountMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
+    //private Pattern requesPattern = Pattern.compile("\\[(.*?)\\].*\\[Request\\]\\s*?(LOG\\d*)\\s*(.*?)\\([0-9\\.]*\\)");
+    Pattern requesPattern = Pattern.compile("(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3})\\s*\\[(\\d*)\\].*path:\\[(.+?)\\] took time");
+    @Override
+    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
+        String line = value.toString();
+        Matcher matcherRequest = requesPattern.matcher(line);
+        if(matcherRequest.find()){
+            String time = matcherRequest.group(1);
+            String logId = matcherRequest.group(2);
+            String path = matcherRequest.group(3);
+
+            log.info("find request time = {}, logId = {},  path = {}", time, logId, path);
+            context.write(new Text(path), new IntWritable(1));
+        }
+    }
+
+    public static void main(String[] args) {
+        String s = "2020-09-12 09:06:41.684 [1599872801593] [http-nio-8089-exec-8] INFO  com.qrqy.ibd.config.AopConfig - class:[GetConfigController],method:[getConfig],path:[/getConfig] took time:[91ms],response ";
+        Pattern requesPattern = Pattern.compile("(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3})\\s*\\[(\\d*)\\].*path:\\[(.*?)\\]");
+        Matcher matcherRequest = requesPattern.matcher(s);
+        if(matcherRequest.find()){
+
+            String time = matcherRequest.group(1);
+            String logId = matcherRequest.group(2);
+            String path = matcherRequest.group(3);
+            log.info("found");
+        }
+        else{
+
+            log.error("not found");
+        }
+
+    }
+}
diff --git a/src/main/java/net/venusoft/count/AccessCountReducer.java b/src/main/java/net/venusoft/count/AccessCountReducer.java
@@ -0,0 +1,23 @@
+package net.venusoft.count;
+
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Reducer;
+
+import java.io.IOException;
+
+
+/**
+ * @author lk
+ * @date 2020/9/27 15:09
+ */
+public class AccessCountReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
+    @Override
+    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
+        int sum = 0;
+        for(IntWritable i:values){
+            sum += i.get();
+        }
+        context.write(key, new IntWritable(sum));
+    }
+}