Merge pull request protocolbuffers#3770 from pherl/3.5-integrate

Integrate google internal changes for the up coming 3.5 release.

BUILD

@@ -541,6 +541,7 @@ cc_test(
         "src/google/protobuf/any_test.cc",
         "src/google/protobuf/arena_unittest.cc",
         "src/google/protobuf/arenastring_unittest.cc",
+        "src/google/protobuf/compiler/annotation_test_util.cc",
         "src/google/protobuf/compiler/command_line_interface_unittest.cc",
         "src/google/protobuf/compiler/cpp/cpp_bootstrap_unittest.cc",
         "src/google/protobuf/compiler/cpp/cpp_move_unittest.cc",

cmake/tests.cmake

@@ -121,6 +121,7 @@ set(tests_files
   ${protobuf_source_dir}/src/google/protobuf/any_test.cc
   ${protobuf_source_dir}/src/google/protobuf/arena_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/arenastring_unittest.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/annotation_test_util.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/command_line_interface_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_bootstrap_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_move_unittest.cc

conformance/failure_list_js.txt

@@ -1,5 +1,3 @@
-Required.Proto3.ProtobufInput.RepeatedScalarSelectsLast.INT32.ProtobufOutput
-Required.Proto3.ProtobufInput.RepeatedScalarSelectsLast.UINT32.ProtobufOutput
 Required.Proto3.ProtobufInput.ValidDataRepeated.BOOL.ProtobufOutput
 Required.Proto3.ProtobufInput.ValidDataRepeated.DOUBLE.ProtobufOutput
 Required.Proto3.ProtobufInput.ValidDataRepeated.FIXED32.ProtobufOutput
@@ -13,7 +11,3 @@ Required.Proto3.ProtobufInput.ValidDataRepeated.SINT32.ProtobufOutput
 Required.Proto3.ProtobufInput.ValidDataRepeated.SINT64.ProtobufOutput
 Required.Proto3.ProtobufInput.ValidDataRepeated.UINT32.ProtobufOutput
 Required.Proto3.ProtobufInput.ValidDataRepeated.UINT64.ProtobufOutput
-Required.Proto2.ProtobufInput.RepeatedScalarSelectsLast.INT32.ProtobufOutput
-Required.Proto2.ProtobufInput.RepeatedScalarSelectsLast.UINT32.ProtobufOutput
-Required.Proto2.ProtobufInput.ValidDataRepeated.INT32.ProtobufOutput
-Required.Proto2.ProtobufInput.ValidDataRepeated.UINT32.ProtobufOutput

conformance/failure_list_python.txt

@@ -7,7 +7,6 @@ Recommended.Proto3.JsonInput.FloatFieldInfinityNotQuoted
 Recommended.Proto3.JsonInput.FloatFieldNanNotQuoted
 Recommended.Proto3.JsonInput.FloatFieldNegativeInfinityNotQuoted
 Required.Proto3.JsonInput.DoubleFieldTooSmall
-Required.Proto3.JsonInput.EnumFieldUnknownValue.Validator
 Required.Proto3.JsonInput.FloatFieldTooLarge
 Required.Proto3.JsonInput.FloatFieldTooSmall
 Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool

conformance/failure_list_python_cpp.txt

@@ -16,7 +16,6 @@ Recommended.Proto3.JsonInput.FloatFieldInfinityNotQuoted
 Recommended.Proto3.JsonInput.FloatFieldNanNotQuoted
 Recommended.Proto3.JsonInput.FloatFieldNegativeInfinityNotQuoted
 Required.Proto3.JsonInput.DoubleFieldTooSmall
-Required.Proto3.JsonInput.EnumFieldUnknownValue.Validator
 Required.Proto3.JsonInput.FloatFieldTooLarge
 Required.Proto3.JsonInput.FloatFieldTooSmall
 Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool

csharp/src/Google.Protobuf/Reflection/Descriptor.cs

@@ -3478,7 +3478,8 @@ public string PhpNamespace {
         = pb::FieldCodec.ForMessage(7994, global::Google.Protobuf.Reflection.UninterpretedOption.Parser);
     private readonly pbc::RepeatedField<global::Google.Protobuf.Reflection.UninterpretedOption> uninterpretedOption_ = new pbc::RepeatedField<global::Google.Protobuf.Reflection.UninterpretedOption>();
     /// <summary>
-    /// The parser stores options it doesn't recognize here. See above.
+    /// The parser stores options it doesn't recognize here.
+    /// See the documentation for the "Options" section above.
     /// </summary>
     [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
     public pbc::RepeatedField<global::Google.Protobuf.Reflection.UninterpretedOption> UninterpretedOption {

csharp/src/Google.Protobuf/WellKnownTypes/FieldMask.cs

@@ -239,6 +239,12 @@ static FieldMaskReflection() {
   ///
   /// Note that oneof type names ("test_oneof" in this case) cannot be used in
   /// paths.
+  ///
+  /// ## Field Mask Verification
+  ///
+  /// The implementation of the all the API methods, which have any FieldMask type
+  /// field in the request, should verify the included field paths, and return
+  /// `INVALID_ARGUMENT` error if any path is duplicated or unmappable.
   /// </summary>
   public sealed partial class FieldMask : pb::IMessage<FieldMask> {
     private static readonly pb::MessageParser<FieldMask> _parser = new pb::MessageParser<FieldMask>(() => new FieldMask());

java/core/src/main/java/com/google/protobuf/AbstractMessage.java

@@ -380,6 +380,10 @@ protected BuilderType internalMergeFrom(AbstractMessageLite other) {
 
     @Override
     public BuilderType mergeFrom(final Message other) {
+      return mergeFrom(other, other.getAllFields());
+    }
+
+    BuilderType mergeFrom(final Message other, Map<FieldDescriptor, Object> allFields) {
       if (other.getDescriptorForType() != getDescriptorForType()) {
         throw new IllegalArgumentException(
           "mergeFrom(Message) can only merge messages of the same type.");
@@ -394,8 +398,7 @@ public BuilderType mergeFrom(final Message other) {
       // TODO(kenton):  Provide a function somewhere called makeDeepCopy()
       //   which allows people to make secure deep copies of messages.
 
-      for (final Map.Entry<FieldDescriptor, Object> entry :
-           other.getAllFields().entrySet()) {
+      for (final Map.Entry<FieldDescriptor, Object> entry : allFields.entrySet()) {
         final FieldDescriptor field = entry.getKey();
         if (field.isRepeated()) {
           for (final Object element : (List)entry.getValue()) {

java/core/src/main/java/com/google/protobuf/CodedInputStream.java

@@ -413,8 +413,7 @@ public final int setSizeLimit(final int limit) {
 
   private boolean explicitDiscardUnknownFields = false;
 
-  /** TODO(liujisi): flip the default.*/
-  private static volatile boolean proto3DiscardUnknownFieldsDefault = true;
+  private static volatile boolean proto3DiscardUnknownFieldsDefault = false;
 
   static void setProto3DiscardUnknownsByDefaultForTest() {
     proto3DiscardUnknownFieldsDefault = true;

java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java

@@ -31,7 +31,6 @@
 package com.google.protobuf;
 
 import com.google.protobuf.AbstractMessageLite.Builder.LimitedInputStream;
-import com.google.protobuf.GeneratedMessageLite.EqualsVisitor.NotEqualsException;
 import com.google.protobuf.Internal.BooleanList;
 import com.google.protobuf.Internal.DoubleList;
 import com.google.protobuf.Internal.EnumLiteMap;
@@ -52,6 +51,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
 /**
  * Lite version of {@link GeneratedMessage}.
@@ -62,6 +62,12 @@ public abstract class GeneratedMessageLite<
     MessageType extends GeneratedMessageLite<MessageType, BuilderType>,
     BuilderType extends GeneratedMessageLite.Builder<MessageType, BuilderType>>
         extends AbstractMessageLite<MessageType, BuilderType> {
+  // BEGIN REGULAR
+  static final boolean ENABLE_EXPERIMENTAL_RUNTIME_AT_BUILD_TIME = false;
+  // END REGULAR
+  // BEGIN EXPERIMENTAL
+  // static final boolean ENABLE_EXPERIMENTAL_RUNTIME_AT_BUILD_TIME = true;
+  // END EXPERIMENTAL
 
   /** For use by generated code only. Lazily initialized to reduce allocations. */
   protected UnknownFieldSetLite unknownFields = UnknownFieldSetLite.getDefaultInstance();
@@ -110,12 +116,19 @@ public int hashCode() {
     if (memoizedHashCode != 0) {
       return memoizedHashCode;
     }
+    // BEGIN EXPERIMENTAL
+    // memoizedHashCode = Protobuf.getInstance().schemaFor(this).hashCode(this);
+    // return memoizedHashCode;
+    // END EXPERIMENTAL
+    // BEGIN REGULAR
     HashCodeVisitor visitor = new HashCodeVisitor();
     visit(visitor, (MessageType) this);
     memoizedHashCode = visitor.hashCode;
     return memoizedHashCode;
+    // END REGULAR
   }
 
+  // BEGIN REGULAR
   @SuppressWarnings("unchecked") // Guaranteed by runtime
   int hashCode(HashCodeVisitor visitor) {
     if (memoizedHashCode == 0) {
@@ -127,6 +140,7 @@ int hashCode(HashCodeVisitor visitor) {
     }
     return memoizedHashCode;
   }
+  // END REGULAR
 
   @SuppressWarnings("unchecked") // Guaranteed by isInstance + runtime
   @Override
@@ -139,18 +153,22 @@ public boolean equals(Object other) {
       return false;
     }
 
+    // BEGIN EXPERIMENTAL
+    // return Protobuf.getInstance().schemaFor(this).equals(this, (MessageType) other);
+    // END EXPERIMENTAL
+    // BEGIN REGULAR
 
     try {
       visit(EqualsVisitor.INSTANCE, (MessageType) other);
-    } catch (NotEqualsException e) {
+    } catch (EqualsVisitor.NotEqualsException e) {
       return false;
     }
     return true;
+    // END REGULAR
   }
 
-  /**
-   * Same as {@link #equals(Object)} but throws {@code NotEqualsException}.
-   */
+  // BEGIN REGULAR
+  /** Same as {@link #equals(Object)} but throws {@code NotEqualsException}. */
   @SuppressWarnings("unchecked") // Guaranteed by isInstance + runtime
   boolean equals(EqualsVisitor visitor, MessageLite other) {
     if (this == other) {
@@ -164,14 +182,13 @@ boolean equals(EqualsVisitor visitor, MessageLite other) {
     visit(visitor, (MessageType) other);
     return true;
   }
+  // END REGULAR
 
   // The general strategy for unknown fields is to use an UnknownFieldSetLite that is treated as
   // mutable during the parsing constructor and immutable after. This allows us to avoid
   // any unnecessary intermediary allocations while reducing the generated code size.
 
-  /**
-   * Lazily initializes unknown fields.
-   */
+  /** Lazily initializes unknown fields. */
   private final void ensureUnknownFieldsInitialized() {
     if (unknownFields == UnknownFieldSetLite.getDefaultInstance()) {
       unknownFields = UnknownFieldSetLite.newInstance();
@@ -218,6 +235,20 @@ protected void makeImmutable() {
     unknownFields.makeImmutable();
   }
 
+  protected final <
+    MessageType extends GeneratedMessageLite<MessageType, BuilderType>,
+    BuilderType extends GeneratedMessageLite.Builder<MessageType, BuilderType>>
+        BuilderType createBuilder() {
+    return (BuilderType) dynamicMethod(MethodToInvoke.NEW_BUILDER);
+  }
+
+  protected final <
+    MessageType extends GeneratedMessageLite<MessageType, BuilderType>,
+    BuilderType extends GeneratedMessageLite.Builder<MessageType, BuilderType>>
+        BuilderType createBuilder(MessageType prototype) {
+    return ((BuilderType) createBuilder()).mergeFrom(prototype);
+  }
+
   @Override
   public final boolean isInitialized() {
     return isInitialized((MessageType) this, Boolean.TRUE);
@@ -238,11 +269,13 @@ public final BuilderType toBuilder() {
    * For use by generated code only.
    */
   public static enum MethodToInvoke {
-    // Rely on/modify instance state
     IS_INITIALIZED,
+    // BEGIN REGULAR
+    VISIT,
+    // END REGULAR
+    // Rely on/modify instance state
     GET_MEMOIZED_IS_INITIALIZED,
     SET_MEMOIZED_IS_INITIALIZED,
-    VISIT,
     MERGE_FROM_STREAM,
     MAKE_IMMUTABLE,
 
@@ -299,10 +332,13 @@ protected Object dynamicMethod(MethodToInvoke method) {
     return dynamicMethod(method, null, null);
   }
 
+  // BEGIN REGULAR
   void visit(Visitor visitor, MessageType other) {
     dynamicMethod(MethodToInvoke.VISIT, visitor, other);
     unknownFields = visitor.visitUnknownFields(unknownFields, other.unknownFields);
   }
+  // END REGULAR
+
 
 
   /**
@@ -399,7 +435,12 @@ public BuilderType mergeFrom(MessageType message) {
     }
 
     private void mergeFromInstance(MessageType dest, MessageType src) {
+      // BEGIN EXPERIMENTAL
+      // Protobuf.getInstance().schemaFor(dest).mergeFrom(dest, src);
+      // END EXPERIMENTAL
+      // BEGIN REGULAR
       dest.visit(MergeFromVisitor.INSTANCE, src);
+      // END REGULAR
     }
 
     @Override
@@ -477,11 +518,13 @@ protected final void mergeExtensionFields(final MessageType other) {
       extensions.mergeFrom(((ExtendableMessage) other).extensions);
     }
 
+    // BEGIN REGULAR
     @Override
     final void visit(Visitor visitor, MessageType other) {
       super.visit(visitor, other);
       extensions = visitor.visitExtensions(extensions, other.extensions);
     }
+    // END REGULAR
 
     /**
      * Parse an unknown field or an extension. For use by generated code only.
@@ -494,7 +537,8 @@ protected <MessageType extends MessageLite> boolean parseUnknownField(
         MessageType defaultInstance,
         CodedInputStream input,
         ExtensionRegistryLite extensionRegistry,
-        int tag) throws IOException {
+        int tag)
+        throws IOException {
       int fieldNumber = WireFormat.getTagFieldNumber(tag);
 
       // TODO(dweis): How much bytecode would be saved by not requiring the generated code to
@@ -1716,6 +1760,7 @@ public T parsePartialFrom(CodedInputStream input, ExtensionRegistryLite extensio
     return message;
   }
 
+  // BEGIN REGULAR
   /**
    * An abstract visitor that the generated code calls into that we use to implement various
    * features. Fields that are not members of oneofs are always visited. Members of a oneof are only
@@ -2401,4 +2446,5 @@ public <K, V> MapFieldLite<K, V> visitMap(MapFieldLite<K, V> mine, MapFieldLite<
       return mine;
     }
   }
+  // END REGULAR
 }

java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java

@@ -358,6 +358,10 @@ protected static <M extends Message> M parseDelimitedWithIOException(Parser<M> p
       throw e.unwrapIOException();
     }
   }
+
+  protected static boolean canUseUnsafe() {
+    return UnsafeUtil.hasUnsafeArrayOperations() && UnsafeUtil.hasUnsafeByteBufferOperations();
+  }
 
   @Override
   public void writeTo(final CodedOutputStream output) throws IOException {
@@ -655,6 +659,7 @@ public BuilderType mergeUnknownFields(
                        .build());
     }
 
+
     @Override
     public boolean isInitialized() {
       for (final FieldDescriptor field : getDescriptorForType().getFields()) {
@@ -2853,3 +2858,4 @@ private static <K, V> void serializeMapTo(
     }
   }
 }
+

java/core/src/main/java/com/google/protobuf/Message.java

@@ -125,7 +125,7 @@ interface Builder extends MessageLite.Builder, MessageOrBuilder {
      *   it is merged into the corresponding sub-message of this message
      *   using the same merging rules.<br>
      * * For repeated fields, the elements in {@code other} are concatenated
-     *   with the elements in this message.
+     *   with the elements in this message.<br>
      * * For oneof groups, if the other message has one of the fields set,
      *   the group of this message is cleared and replaced by the field
      *   of the other message, so that the oneof constraint is preserved.

java/core/src/test/java/com/google/protobuf/FieldPresenceTest.java

@@ -404,34 +404,4 @@ public void testIsInitialized() throws Exception {
     assertTrue(builder.buildPartial().isInitialized());
   }
 
-
-  // Test that unknown fields are dropped.
-  public void testUnknownFields() throws Exception {
-    TestAllTypes.Builder builder = TestAllTypes.newBuilder();
-    builder.setOptionalInt32(1234);
-    builder.addRepeatedInt32(5678);
-    TestAllTypes message = builder.build();
-    ByteString data = message.toByteString();
-
-    TestOptionalFieldsOnly optionalOnlyMessage =
-        TestOptionalFieldsOnly.parseFrom(data);
-    // UnknownFieldSet should be empty.
-    assertEquals(
-        0, optionalOnlyMessage.getUnknownFields().toByteString().size());
-    assertEquals(1234, optionalOnlyMessage.getOptionalInt32());
-    message = TestAllTypes.parseFrom(optionalOnlyMessage.toByteString());
-    assertEquals(1234, message.getOptionalInt32());
-    // The repeated field is discarded because it's unknown to the optional-only
-    // message.
-    assertEquals(0, message.getRepeatedInt32Count());
-
-    DynamicMessage dynamicOptionalOnlyMessage =
-        DynamicMessage.getDefaultInstance(
-            TestOptionalFieldsOnly.getDescriptor())
-        .getParserForType().parseFrom(data);
-    assertEquals(
-        0, dynamicOptionalOnlyMessage.getUnknownFields().toByteString().size());
-    assertEquals(optionalOnlyMessage.toByteString(),
-        dynamicOptionalOnlyMessage.toByteString());
-  }
 }

java/core/src/test/java/com/google/protobuf/LiteTest.java

@@ -1453,6 +1453,15 @@ public void testSanityCopyOnWrite() throws InvalidProtocolBufferException {
             UnittestLite.optionalFixed32ExtensionLite));
   }
 
+  public void testBuilderMergeFromNull() throws Exception {
+    try {
+      TestAllTypesLite.newBuilder().mergeFrom((TestAllTypesLite) null);
+      fail("Expected exception");
+    } catch (NullPointerException e) {
+      // Pass.
+    }
+  }
+
   // Builder.mergeFrom() should keep existing extensions.
   public void testBuilderMergeFromWithExtensions() throws Exception {
     TestAllExtensionsLite message =