Merge pull request protocolbuffers#4302 from BSBandme/down_integ_benc…

…hmark Sync internal benchmark changes
shields · Feb 9, 2018 · 96b535c · 96b535c
2 parents 4da7706 + 6cd4ec4
commit 96b535c
Show file tree

Hide file tree

Showing 17 changed files with 89 additions and 30 deletions.
diff --git a/benchmarks/Makefile.am b/benchmarks/Makefile.am
@@ -104,7 +104,7 @@ initialize_submodule:
 	oldpwd=`pwd`
 	cd $(top_srcdir)/third_party
 	git submodule update --init -r
-	cd $(top_srcdir)/third_party/benchmark && cmake -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_GTEST_TESTS=OFF && make
+	cd $(top_srcdir)/third_party/benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make
 	cd $$oldpwd
 	touch initialize_submodule
 
@@ -137,7 +137,7 @@ cpp: protoc_middleman protoc_middleman2 cpp-benchmark initialize_submodule
 
 java_benchmark_testing_files =                                      \
 	java/src/main/java/com/google/protobuf/ProtoBench.java            \
-	java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java
+	java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java
 
 javac_middleman: $(java_benchmark_testing_files) protoc_middleman protoc_middleman2
 	cp -r java tmp && cd tmp/java &&  mvn clean compile assembly:single

diff --git a/benchmarks/cpp_benchmark.cc b/benchmarks/cpp_benchmark.cc
@@ -238,6 +238,7 @@ void RegisterBenchmarks(const std::string& dataset_bytes) {
 }
 
 int main(int argc, char *argv[]) {
+  ::benchmark::Initialize(&argc, argv);
   if (argc == 1) {
     std::cerr << "Usage: ./cpp-benchmark <input data>" << std::endl;
     std::cerr << "input data is in the format of \"benchmarks.proto\""
@@ -249,6 +250,5 @@ int main(int argc, char *argv[]) {
     }
   }
 
-  ::benchmark::Initialize(&argc, argv);
   ::benchmark::RunSpecifiedBenchmarks();
 }
diff --git a/benchmarks/datasets/google_message3/benchmark_message3.proto b/benchmarks/datasets/google_message3/benchmark_message3.proto
@@ -10,6 +10,7 @@ import "datasets/google_message3/benchmark_message3_8.proto";
 package benchmarks.google_message3;
 
 option cc_enable_arenas = true;
+option java_package = "com.google.protobuf.benchmarks";
 
 message GoogleMessage3 {
   optional .benchmarks.google_message3.Message37487 field37519 = 2;

diff --git a/benchmarks/datasets/google_message3/benchmark_message3_1.proto b/benchmarks/datasets/google_message3/benchmark_message3_1.proto
@@ -8,6 +8,7 @@ import "datasets/google_message3/benchmark_message3_8.proto";
 package benchmarks.google_message3;
 
 option cc_enable_arenas = true;
+option java_package = "com.google.protobuf.benchmarks";
 
 message Message34390 {
   repeated .benchmarks.google_message3.Message34387 field34452 = 1;

diff --git a/benchmarks/datasets/google_message3/benchmark_message3_2.proto b/benchmarks/datasets/google_message3/benchmark_message3_2.proto
@@ -8,6 +8,7 @@ import "datasets/google_message3/benchmark_message3_8.proto";
 package benchmarks.google_message3;
 
 option cc_enable_arenas = true;
+option java_package = "com.google.protobuf.benchmarks";
 
 message Message22853 {
   optional .benchmarks.google_message3.Enum22854 field22869 = 1;

diff --git a/benchmarks/datasets/google_message3/benchmark_message3_3.proto b/benchmarks/datasets/google_message3/benchmark_message3_3.proto
@@ -7,6 +7,7 @@ import "datasets/google_message3/benchmark_message3_8.proto";
 package benchmarks.google_message3;
 
 option cc_enable_arenas = true;
+option java_package = "com.google.protobuf.benchmarks";
 
 message Message35546 {
   optional int64 field35556 = 1;

diff --git a/benchmarks/datasets/google_message3/benchmark_message3_4.proto b/benchmarks/datasets/google_message3/benchmark_message3_4.proto
@@ -7,6 +7,7 @@ import "datasets/google_message3/benchmark_message3_8.proto";
 package benchmarks.google_message3;
 
 option cc_enable_arenas = true;
+option java_package = "com.google.protobuf.benchmarks";
 
 message Message24346 {
 }

diff --git a/benchmarks/datasets/google_message3/benchmark_message3_5.proto b/benchmarks/datasets/google_message3/benchmark_message3_5.proto
@@ -6,6 +6,7 @@ import "datasets/google_message3/benchmark_message3_8.proto";
 package benchmarks.google_message3;
 
 option cc_enable_arenas = true;
+option java_package = "com.google.protobuf.benchmarks";
 
 message Message24377 {
 }

diff --git a/benchmarks/datasets/google_message3/benchmark_message3_6.proto b/benchmarks/datasets/google_message3/benchmark_message3_6.proto
@@ -5,6 +5,7 @@ import "datasets/google_message3/benchmark_message3_8.proto";
 package benchmarks.google_message3;
 
 option cc_enable_arenas = true;
+option java_package = "com.google.protobuf.benchmarks";
 
 message Message10576 {
 }

diff --git a/benchmarks/datasets/google_message3/benchmark_message3_7.proto b/benchmarks/datasets/google_message3/benchmark_message3_7.proto
@@ -3,6 +3,7 @@ syntax = "proto2";
 package benchmarks.google_message3;
 
 option cc_enable_arenas = true;
+option java_package = "com.google.protobuf.benchmarks";
 
 message Message11018 {
 }

diff --git a/benchmarks/datasets/google_message3/benchmark_message3_8.proto b/benchmarks/datasets/google_message3/benchmark_message3_8.proto
@@ -3,6 +3,7 @@ syntax = "proto2";
 package benchmarks.google_message3;
 
 option cc_enable_arenas = true;
+option java_package = "com.google.protobuf.benchmarks";
 
 enum Enum720 {
   ENUM_VALUE721 = 1;

diff --git a/benchmarks/datasets/google_message4/benchmark_message4.proto b/benchmarks/datasets/google_message4/benchmark_message4.proto
@@ -6,6 +6,7 @@ import "datasets/google_message4/benchmark_message4_3.proto";
 package benchmarks.google_message4;
 
 option cc_enable_arenas = true;
+option java_package = "com.google.protobuf.benchmarks";
 
 message GoogleMessage4 {
   optional int32 field37503 = 1;

diff --git a/benchmarks/datasets/google_message4/benchmark_message4_1.proto b/benchmarks/datasets/google_message4/benchmark_message4_1.proto
@@ -5,6 +5,7 @@ import "datasets/google_message4/benchmark_message4_3.proto";
 package benchmarks.google_message4;
 
 option cc_enable_arenas = true;
+option java_package = "com.google.protobuf.benchmarks";
 
 message Message2463 {
   repeated .benchmarks.google_message4.Message2462 field2498 = 1;

diff --git a/benchmarks/datasets/google_message4/benchmark_message4_2.proto b/benchmarks/datasets/google_message4/benchmark_message4_2.proto
@@ -4,6 +4,7 @@ import "datasets/google_message4/benchmark_message4_3.proto";
 package benchmarks.google_message4;
 
 option cc_enable_arenas = true;
+option java_package = "com.google.protobuf.benchmarks";
 
 message Message12774 {
   optional uint32 field12777 = 1;

diff --git a/benchmarks/datasets/google_message4/benchmark_message4_3.proto b/benchmarks/datasets/google_message4/benchmark_message4_3.proto
@@ -3,6 +3,7 @@ syntax = "proto2";
 package benchmarks.google_message4;
 
 option cc_enable_arenas = true;
+option java_package = "com.google.protobuf.benchmarks";
 
 enum UnusedEnum {
   UNUSED_ENUM_VALUE1 = 0;

diff --git a/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java b/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java
@@ -97,7 +97,7 @@ public static boolean runTest(String file) {
       return false;
     }
     argsList.add("-DdataFile=" + file);
-    argsList.add("com.google.protobuf.ProtoBenchCaliper");
+    argsList.add("com.google.protobuf.ProtoCaliperBenchmark");
 
     try {
       String args[] = new String[argsList.size()];
@@ -142,19 +142,14 @@ public static boolean runTest(String file) {
   private static List<String> getCaliperOption(final BenchmarkDataset benchmarkDataset) {
     List<String> temp = new ArrayList<String>();
     if (benchmarkDataset.getMessageName().equals("benchmarks.proto3.GoogleMessage1")) {
-      temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE1_PROTO3");
     } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage1")) {
-      temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE1_PROTO2");
     } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage2")) {
-      temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE2");
     } else if (benchmarkDataset.getMessageName().
         equals("benchmarks.google_message3.GoogleMessage3")) {
-      temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE3");
       temp.add("-Cinstrument.runtime.options.timingInterval=3000ms");
       temp.add("-Cinstrument.runtime.options.measurements=20");
     } else if (benchmarkDataset.getMessageName().
         equals("benchmarks.google_message4.GoogleMessage4")) {
-      temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE4");
       temp.add("-Cinstrument.runtime.options.timingInterval=1500ms");
       temp.add("-Cinstrument.runtime.options.measurements=20");
     } else {
@@ -172,6 +167,9 @@ private static List<String> getCaliperOption(final BenchmarkDataset benchmarkDat
   }
 
   public static byte[] readAllBytes(String filename) throws IOException {
+    if (filename.equals("")) {
+      return new byte[0];
+    }
     RandomAccessFile file = new RandomAccessFile(new File(filename), "r");
     byte[] content = new byte[(int) file.length()];
     file.readFully(content);

diff --git a/...om/google/protobuf/ProtoBenchCaliper.java → ...oogle/protobuf/ProtoCaliperBenchmark.java b/...om/google/protobuf/ProtoBenchCaliper.java → ...oogle/protobuf/ProtoCaliperBenchmark.java
@@ -13,14 +13,16 @@
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.BufferedWriter;
+import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.io.RandomAccessFile;
 import java.util.ArrayList;
 import java.util.List;
 
-public class ProtoBenchCaliper {
+public class ProtoCaliperBenchmark {
   public enum BenchmarkMessageType {
     GOOGLE_MESSAGE1_PROTO3 {
       @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
@@ -49,45 +51,44 @@ Message getDefaultInstance() {
       @Override
       ExtensionRegistry getExtensionRegistry() {
         ExtensionRegistry extensions = ExtensionRegistry.newInstance();
-        benchmarks.google_message3.BenchmarkMessage38.registerAllExtensions(extensions);
-        benchmarks.google_message3.BenchmarkMessage37.registerAllExtensions(extensions);
-        benchmarks.google_message3.BenchmarkMessage36.registerAllExtensions(extensions);
-        benchmarks.google_message3.BenchmarkMessage35.registerAllExtensions(extensions);
-        benchmarks.google_message3.BenchmarkMessage34.registerAllExtensions(extensions);
-        benchmarks.google_message3.BenchmarkMessage33.registerAllExtensions(extensions);
-        benchmarks.google_message3.BenchmarkMessage32.registerAllExtensions(extensions);
-        benchmarks.google_message3.BenchmarkMessage31.registerAllExtensions(extensions);
-        benchmarks.google_message3.BenchmarkMessage3.registerAllExtensions(extensions);
+        com.google.protobuf.benchmarks.BenchmarkMessage38.registerAllExtensions(extensions);
+        com.google.protobuf.benchmarks.BenchmarkMessage37.registerAllExtensions(extensions);
+        com.google.protobuf.benchmarks.BenchmarkMessage36.registerAllExtensions(extensions);
+        com.google.protobuf.benchmarks.BenchmarkMessage35.registerAllExtensions(extensions);
+        com.google.protobuf.benchmarks.BenchmarkMessage34.registerAllExtensions(extensions);
+        com.google.protobuf.benchmarks.BenchmarkMessage33.registerAllExtensions(extensions);
+        com.google.protobuf.benchmarks.BenchmarkMessage32.registerAllExtensions(extensions);
+        com.google.protobuf.benchmarks.BenchmarkMessage31.registerAllExtensions(extensions);
+        com.google.protobuf.benchmarks.BenchmarkMessage3.registerAllExtensions(extensions);
         return extensions;
       }
       @Override
       Message getDefaultInstance() {
-        return benchmarks.google_message3.BenchmarkMessage3.GoogleMessage3.getDefaultInstance();
+        return com.google.protobuf.benchmarks.BenchmarkMessage3.GoogleMessage3.getDefaultInstance();
       }
     },
     GOOGLE_MESSAGE4 {
       @Override
       ExtensionRegistry getExtensionRegistry() {
         ExtensionRegistry extensions = ExtensionRegistry.newInstance();
-        benchmarks.google_message4.BenchmarkMessage43.registerAllExtensions(extensions);
-        benchmarks.google_message4.BenchmarkMessage42.registerAllExtensions(extensions);
-        benchmarks.google_message4.BenchmarkMessage41.registerAllExtensions(extensions);
-        benchmarks.google_message4.BenchmarkMessage4.registerAllExtensions(extensions);
+        com.google.protobuf.benchmarks.BenchmarkMessage43.registerAllExtensions(extensions);
+        com.google.protobuf.benchmarks.BenchmarkMessage42.registerAllExtensions(extensions);
+        com.google.protobuf.benchmarks.BenchmarkMessage41.registerAllExtensions(extensions);
+        com.google.protobuf.benchmarks.BenchmarkMessage4.registerAllExtensions(extensions);
         return extensions;
       }
       @Override
       Message getDefaultInstance() {
-        return benchmarks.google_message4.BenchmarkMessage4.GoogleMessage4.getDefaultInstance();
+        return com.google.protobuf.benchmarks.BenchmarkMessage4.GoogleMessage4.getDefaultInstance();
       }
     };
 
     abstract ExtensionRegistry getExtensionRegistry();
     abstract Message getDefaultInstance();
   }
-
-  @Param 
+
   private BenchmarkMessageType benchmarkMessageType;
-  @Param
+  @Param("")
   private String dataFile;
 
   private byte[] inputData;
@@ -99,11 +100,39 @@ Message getDefaultInstance() {
   private List<ByteString> inputStringList;
   private List<Message> sampleMessageList;
   private long counter;
+
+  private BenchmarkMessageType getMessageType() throws IOException {
+    if (benchmarkDataset.getMessageName().equals("benchmarks.proto3.GoogleMessage1")) {
+      return BenchmarkMessageType.GOOGLE_MESSAGE1_PROTO3;
+    } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage1")) {
+      return BenchmarkMessageType.GOOGLE_MESSAGE1_PROTO2;
+    } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage2")) {
+      return BenchmarkMessageType.GOOGLE_MESSAGE2;
+    } else if (benchmarkDataset.getMessageName().
+        equals("benchmarks.google_message3.GoogleMessage3")) {
+      return BenchmarkMessageType.GOOGLE_MESSAGE3;
+    } else if (benchmarkDataset.getMessageName().
+        equals("benchmarks.google_message4.GoogleMessage4")) {
+      return BenchmarkMessageType.GOOGLE_MESSAGE4;
+    } else {
+      throw new IllegalStateException("Invalid DataFile! There's no testing message named "
+          + benchmarkDataset.getMessageName());
+    }
+  }
 
   @BeforeExperiment
   void setUp() throws IOException {
-    inputData = ProtoBench.readAllBytes(dataFile);
-    benchmarkDataset = BenchmarkDataset.parseFrom(inputData);
+    if (!dataFile.equals("")) {
+      RandomAccessFile file = new RandomAccessFile(new File(dataFile), "r");
+      inputData = new byte[(int) file.length()];
+      file.readFully(inputData);
+      benchmarkDataset = BenchmarkDataset.parseFrom(inputData);
+      benchmarkMessageType = getMessageType();
+    } else {
+      inputData = new byte[0];
+      benchmarkDataset = BenchmarkDataset.parseFrom(inputData);
+      benchmarkMessageType = BenchmarkMessageType.GOOGLE_MESSAGE2;
+    }
     defaultMessage = benchmarkMessageType.getDefaultInstance();
     extensions = benchmarkMessageType.getExtensionRegistry();
     inputDataList = new ArrayList<byte[]>();
@@ -127,6 +156,9 @@ void setUp() throws IOException {
 
   @Benchmark
   void serializeToByteString(int reps) throws IOException {
+    if (sampleMessageList.size() == 0) {
+      return;
+    }
     for (int i = 0; i < reps; i++) {
       sampleMessageList.get((int) (counter % sampleMessageList.size())).toByteString();
       counter++;
@@ -135,6 +167,9 @@ void serializeToByteString(int reps) throws IOException {
 
   @Benchmark
   void serializeToByteArray(int reps) throws IOException {
+    if (sampleMessageList.size() == 0) {
+      return;
+    }
     for (int i = 0; i < reps; i++) {
       sampleMessageList.get((int) (counter % sampleMessageList.size())).toByteArray();
       counter++;
@@ -143,6 +178,9 @@ void serializeToByteArray(int reps) throws IOException {
 
   @Benchmark
   void serializeToMemoryStream(int reps) throws IOException {
+    if (sampleMessageList.size() == 0) {
+      return;
+    }
     for (int i = 0; i < reps; i++) {
       ByteArrayOutputStream output = new ByteArrayOutputStream();
       sampleMessageList.get((int) (counter % sampleMessageList.size())).writeTo(output);
@@ -152,6 +190,9 @@ void serializeToMemoryStream(int reps) throws IOException {
 
   @Benchmark
   void deserializeFromByteString(int reps) throws IOException {
+    if (inputStringList.size() == 0) {
+      return;
+    }
     for (int i = 0; i < reps; i++) {
       benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom(
           inputStringList.get((int) (counter % inputStringList.size())), extensions);
@@ -161,6 +202,9 @@ void deserializeFromByteString(int reps) throws IOException {
 
   @Benchmark
   void deserializeFromByteArray(int reps) throws IOException {
+    if (inputDataList.size() == 0) {
+      return;
+    }
     for (int i = 0; i < reps; i++) {
       benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom(
           inputDataList.get((int) (counter % inputDataList.size())), extensions);
@@ -170,6 +214,9 @@ void deserializeFromByteArray(int reps) throws IOException {
 
   @Benchmark
   void deserializeFromMemoryStream(int reps) throws IOException {
+    if (inputStreamList.size() == 0) {
+      return;
+    }
     for (int i = 0; i < reps; i++) {
       benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom(
           inputStreamList.get((int) (counter % inputStreamList.size())), extensions);
@@ -195,3 +242,4 @@ void checkCounter() throws IOException {
     }
   }
 }
+