Merge branch 'sync-piper' into sync-stage

conformance/failure_list_python.txt

@@ -1,30 +0,0 @@
-Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput
-Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput
-Recommended.Proto3.JsonInput.DoubleFieldInfinityNotQuoted
-Recommended.Proto3.JsonInput.DoubleFieldNanNotQuoted
-Recommended.Proto3.JsonInput.DoubleFieldNegativeInfinityNotQuoted
-Recommended.Proto3.JsonInput.FloatFieldInfinityNotQuoted
-Recommended.Proto3.JsonInput.FloatFieldNanNotQuoted
-Recommended.Proto3.JsonInput.FloatFieldNegativeInfinityNotQuoted
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.BOOL[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.BYTES[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.DOUBLE[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.ENUM[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.FIXED32[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.FIXED64[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.FLOAT[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.INT32[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.INT32[6].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.INT64[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.SFIXED32[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.SFIXED64[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.SINT32[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.SINT64[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.STRING[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.UINT32[0].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.UINT32[5].ProtobufOutput
-Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.UINT64[0].ProtobufOutput
-Required.Proto3.JsonInput.DoubleFieldTooSmall
-Required.Proto3.JsonInput.FloatFieldTooLarge
-Required.Proto3.JsonInput.FloatFieldTooSmall
-Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool

conformance/failure_list_python_cpp.txt

@@ -6,16 +6,3 @@
 #
 # TODO(haberman): insert links to corresponding bugs tracking the issue.
 # Should we use GitHub issues or the Google-internal bug tracker?
-
-Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput
-Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput
-Recommended.Proto3.JsonInput.DoubleFieldInfinityNotQuoted
-Recommended.Proto3.JsonInput.DoubleFieldNanNotQuoted
-Recommended.Proto3.JsonInput.DoubleFieldNegativeInfinityNotQuoted
-Recommended.Proto3.JsonInput.FloatFieldInfinityNotQuoted
-Recommended.Proto3.JsonInput.FloatFieldNanNotQuoted
-Recommended.Proto3.JsonInput.FloatFieldNegativeInfinityNotQuoted
-Required.Proto3.JsonInput.DoubleFieldTooSmall
-Required.Proto3.JsonInput.FloatFieldTooLarge
-Required.Proto3.JsonInput.FloatFieldTooSmall
-Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool

java/util/src/main/java/com/google/protobuf/util/FieldMaskTree.java

@@ -30,6 +30,7 @@
 
 package com.google.protobuf.util;
 
+import com.google.common.base.Splitter;
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
 import com.google.protobuf.Descriptors.Descriptor;
 import com.google.protobuf.Descriptors.FieldDescriptor;
@@ -66,7 +67,7 @@ final class FieldMaskTree {
   private static final String FIELD_PATH_SEPARATOR_REGEX = "\\.";
 
   private static final class Node {
-    final SortedMap<String, Node> children = new TreeMap<String, Node>();
+    final SortedMap<String, Node> children = new TreeMap<>();
   }
 
   private final Node root = new Node();
@@ -97,6 +98,7 @@ public String toString() {
    * to add is a sub-path of an existing leaf node, nothing will be changed in the tree.
    */
   @CanIgnoreReturnValue
+  @SuppressWarnings("StringSplitter")
   FieldMaskTree addFieldPath(String path) {
     String[] parts = path.split(FIELD_PATH_SEPARATOR_REGEX);
     if (parts.length == 0) {
@@ -134,14 +136,53 @@ FieldMaskTree mergeFromFieldMask(FieldMask mask) {
     return this;
   }
 
+  /**
+   * Remove {@code path} from the tree.
+   *
+   * <p>When removing a field path from the tree, all sub-paths will be removed. That is, after
+   * removing "foo.bar" from the tree, "foo.bar.baz" will be removed. Likewise, if the field path to
+   * remove is a non-exist sub-path, nothing will be changed.
+   */
+  @CanIgnoreReturnValue
+  FieldMaskTree removeFieldPath(String path) {
+    List<String> parts = Splitter.onPattern(FIELD_PATH_SEPARATOR_REGEX).splitToList(path);
+    if (parts.isEmpty()) {
+      return this;
+    }
+    Node node = root;
+    for (int i = 0; i < parts.size(); i++) {
+      String key = parts.get(i);
+      if (!node.children.containsKey(key)) {
+        // Path does not exist.
+        return this;
+      }
+      if (i == parts.size() - 1) {
+        // Remove path.
+        node.children.remove(key);
+        return this;
+      }
+      node = node.children.get(key);
+    }
+    return this;
+  }
+
+  /** Removes all field paths in {@code mask} from this tree. */
+  @CanIgnoreReturnValue
+  FieldMaskTree removeFromFieldMask(FieldMask mask) {
+    for (String path : mask.getPathsList()) {
+      removeFieldPath(path);
+    }
+    return this;
+  }
+
   /**
    * Converts this tree to a FieldMask.
    */
   FieldMask toFieldMask() {
     if (root.children.isEmpty()) {
       return FieldMask.getDefaultInstance();
     }
-    List<String> paths = new ArrayList<String>();
+    List<String> paths = new ArrayList<>();
     getFieldPaths(root, "", paths);
     return FieldMask.newBuilder().addAllPaths(paths).build();
   }
@@ -186,7 +227,7 @@ void intersectFieldPath(String path, FieldMaskTree output) {
     }
     // We found a matching node for the path. All leaf children of this matching
     // node is in the intersection.
-    List<String> paths = new ArrayList<String>();
+    List<String> paths = new ArrayList<>();
     getFieldPaths(node, path, paths);
     for (String value : paths) {
       output.addFieldPath(value);

java/util/src/main/java/com/google/protobuf/util/FieldMaskUtil.java

@@ -276,6 +276,16 @@ public static FieldMask union(
     return maskTree.toFieldMask();
   }
 
+  /** Subtracts {@code secondMask} and {@code otherMasks} from {@code firstMask}. */
+  public static FieldMask subtract(
+      FieldMask firstMask, FieldMask secondMask, FieldMask... otherMasks) {
+    FieldMaskTree maskTree = new FieldMaskTree(firstMask).removeFromFieldMask(secondMask);
+    for (FieldMask mask : otherMasks) {
+      maskTree.removeFromFieldMask(mask);
+    }
+    return maskTree.toFieldMask();
+  }
+
   /**
    * Calculates the intersection of two FieldMasks.
    */

java/util/src/test/java/com/google/protobuf/util/FieldMaskTreeTest.java

@@ -30,6 +30,8 @@
 
 package com.google.protobuf.util;
 
+import static com.google.common.truth.Truth.assertThat;
+
 import com.google.protobuf.DynamicMessage;
 import com.google.protobuf.Message;
 import com.google.protobuf.UninitializedMessageException;
@@ -43,9 +45,9 @@
 public class FieldMaskTreeTest extends TestCase {
   public void testAddFieldPath() throws Exception {
     FieldMaskTree tree = new FieldMaskTree();
-    assertEquals("", tree.toString());
+    assertThat(tree.toString()).isEmpty();
     tree.addFieldPath("");
-    assertEquals("", tree.toString());
+    assertThat(tree.toString()).isEmpty();
     // New branch.
     tree.addFieldPath("foo");
     assertEquals("foo", tree.toString());
@@ -73,15 +75,45 @@ public void testMergeFromFieldMask() throws Exception {
     assertEquals("bar,foo", tree.toString());
   }
 
+  public void testRemoveFieldPath() throws Exception {
+    String initialTreeString = "bar.baz,bar.quz.bar,foo";
+    FieldMaskTree tree = new FieldMaskTree(FieldMaskUtil.fromString(initialTreeString));
+    // Empty path.
+    tree.removeFieldPath("");
+    assertEquals(initialTreeString, tree.toString());
+    // Non-exist sub-path of an existing leaf.
+    tree.removeFieldPath("foo.bar");
+    assertEquals(initialTreeString, tree.toString());
+    // Non-exist path.
+    tree.removeFieldPath("bar.foo");
+    assertEquals(initialTreeString, tree.toString());
+    // Match an existing leaf node.
+    tree.removeFieldPath("foo");
+    assertEquals("bar.baz,bar.quz.bar", tree.toString());
+    // Match sub-path of an existing leaf node.
+    tree.removeFieldPath("bar.quz.bar");
+    assertEquals("bar.baz,bar.quz", tree.toString());
+    // Match a non-leaf node.
+    tree.removeFieldPath("bar");
+    assertThat(tree.toString()).isEmpty();
+  }
+
+  public void testRemoveFromFieldMask() throws Exception {
+    FieldMaskTree tree = new FieldMaskTree(FieldMaskUtil.fromString("foo,bar.baz,bar.quz"));
+    assertEquals("bar.baz,bar.quz,foo", tree.toString());
+    tree.removeFromFieldMask(FieldMaskUtil.fromString("foo.bar,bar"));
+    assertEquals("foo", tree.toString());
+  }
+
   public void testIntersectFieldPath() throws Exception {
     FieldMaskTree tree = new FieldMaskTree(FieldMaskUtil.fromString("foo,bar.baz,bar.quz"));
     FieldMaskTree result = new FieldMaskTree();
     // Empty path.
     tree.intersectFieldPath("", result);
-    assertEquals("", result.toString());
+    assertThat(result.toString()).isEmpty();
     // Non-exist path.
     tree.intersectFieldPath("quz", result);
-    assertEquals("", result.toString());
+    assertThat(result.toString()).isEmpty();
     // Sub-path of an existing leaf.
     tree.intersectFieldPath("foo.bar", result);
     assertEquals("foo.bar", result.toString());

java/util/src/test/java/com/google/protobuf/util/FieldMaskUtilTest.java

@@ -224,6 +224,24 @@ public void testUnion_usingVarArgs() throws Exception {
     assertEquals("bar,foo", FieldMaskUtil.toString(result));
   }
 
+  public void testSubstract() throws Exception {
+    // Only test a simple case here and expect
+    // {@link FieldMaskTreeTest#testRemoveFieldPath} to cover all scenarios.
+    FieldMask mask1 = FieldMaskUtil.fromString("foo,bar.baz,bar.quz");
+    FieldMask mask2 = FieldMaskUtil.fromString("foo.bar,bar");
+    FieldMask result = FieldMaskUtil.subtract(mask1, mask2);
+    assertEquals("foo", FieldMaskUtil.toString(result));
+  }
+
+  public void testSubstract_usingVarArgs() throws Exception {
+    FieldMask mask1 = FieldMaskUtil.fromString("foo,bar.baz,bar.quz.bar");
+    FieldMask mask2 = FieldMaskUtil.fromString("foo.bar,bar.baz.quz");
+    FieldMask mask3 = FieldMaskUtil.fromString("bar.quz");
+    FieldMask mask4 = FieldMaskUtil.fromString("foo,bar.baz");
+    FieldMask result = FieldMaskUtil.subtract(mask1, mask2, mask3, mask4);
+    assertEquals("bar", FieldMaskUtil.toString(result));
+  }
+
   public void testIntersection() throws Exception {
     // Only test a simple case here and expect
     // {@link FieldMaskTreeTest#testIntersectFieldPath} to cover all scenarios.

python/google/protobuf/internal/decoder.py

@@ -209,7 +209,8 @@ def _SimpleDecoder(wire_type, decode_value):
         _DecodeVarint()
   """
 
-  def SpecificDecoder(field_number, is_repeated, is_packed, key, new_default):
+  def SpecificDecoder(field_number, is_repeated, is_packed, key, new_default,
+                      clear_if_default=False):
     if is_packed:
       local_DecodeVarint = _DecodeVarint
       def DecodePackedField(buffer, pos, end, message, field_dict):
@@ -249,10 +250,13 @@ def DecodeRepeatedField(buffer, pos, end, message, field_dict):
       return DecodeRepeatedField
     else:
       def DecodeField(buffer, pos, end, message, field_dict):
-        (field_dict[key], pos) = decode_value(buffer, pos)
+        (new_value, pos) = decode_value(buffer, pos)
         if pos > end:
-          del field_dict[key]  # Discard corrupt value.
           raise _DecodeError('Truncated message.')
+        if clear_if_default and not new_value:
+          field_dict.pop(key, None)
+        else:
+          field_dict[key] = new_value
         return pos
       return DecodeField
 
@@ -383,7 +387,9 @@ def InnerDecode(buffer, pos):
   return _SimpleDecoder(wire_format.WIRETYPE_FIXED64, InnerDecode)
 
 
-def EnumDecoder(field_number, is_repeated, is_packed, key, new_default):
+def EnumDecoder(field_number, is_repeated, is_packed, key, new_default,
+                clear_if_default=False):
+  """Returns a decoder for enum field."""
   enum_type = key.enum_type
   if is_packed:
     local_DecodeVarint = _DecodeVarint
@@ -498,6 +504,9 @@ def DecodeField(buffer, pos, end, message, field_dict):
       (enum_value, pos) = _DecodeSignedVarint32(buffer, pos)
       if pos > end:
         raise _DecodeError('Truncated message.')
+      if clear_if_default and not enum_value:
+        field_dict.pop(key, None)
+        return pos
       # pylint: disable=protected-access
       if enum_value in enum_type.values_by_number:
         field_dict[key] = enum_value
@@ -550,7 +559,7 @@ def DecodeField(buffer, pos, end, message, field_dict):
 
 
 def StringDecoder(field_number, is_repeated, is_packed, key, new_default,
-                  is_strict_utf8=False):
+                  is_strict_utf8=False, clear_if_default=False):
   """Returns a decoder for a string field."""
 
   local_DecodeVarint = _DecodeVarint
@@ -604,12 +613,16 @@ def DecodeField(buffer, pos, end, message, field_dict):
       new_pos = pos + size
       if new_pos > end:
         raise _DecodeError('Truncated string.')
-      field_dict[key] = _ConvertToUnicode(buffer[pos:new_pos])
+      if clear_if_default and not size:
+        field_dict.pop(key, None)
+      else:
+        field_dict[key] = _ConvertToUnicode(buffer[pos:new_pos])
       return new_pos
     return DecodeField
 
 
-def BytesDecoder(field_number, is_repeated, is_packed, key, new_default):
+def BytesDecoder(field_number, is_repeated, is_packed, key, new_default,
+                 clear_if_default=False):
   """Returns a decoder for a bytes field."""
 
   local_DecodeVarint = _DecodeVarint
@@ -641,7 +654,10 @@ def DecodeField(buffer, pos, end, message, field_dict):
       new_pos = pos + size
       if new_pos > end:
         raise _DecodeError('Truncated string.')
-      field_dict[key] = buffer[pos:new_pos].tobytes()
+      if clear_if_default and not size:
+        field_dict.pop(key, None)
+      else:
+        field_dict[key] = buffer[pos:new_pos].tobytes()
       return new_pos
     return DecodeField
 

python/google/protobuf/internal/encoder.py

@@ -372,14 +372,15 @@ def FieldSize(map_value):
 def _VarintEncoder():
   """Return an encoder for a basic varint value (does not include tag)."""
 
+  local_int2byte = six.int2byte
   def EncodeVarint(write, value, unused_deterministic=None):
     bits = value & 0x7f
     value >>= 7
     while value:
-      write(six.int2byte(0x80|bits))
+      write(local_int2byte(0x80|bits))
       bits = value & 0x7f
       value >>= 7
-    return write(six.int2byte(bits))
+    return write(local_int2byte(bits))
 
   return EncodeVarint
 
@@ -388,16 +389,17 @@ def _SignedVarintEncoder():
   """Return an encoder for a basic signed varint value (does not include
   tag)."""
 
+  local_int2byte = six.int2byte
   def EncodeSignedVarint(write, value, unused_deterministic=None):
     if value < 0:
       value += (1 << 64)
     bits = value & 0x7f
     value >>= 7
     while value:
-      write(six.int2byte(0x80|bits))
+      write(local_int2byte(0x80|bits))
       bits = value & 0x7f
       value >>= 7
-    return write(six.int2byte(bits))
+    return write(local_int2byte(bits))
 
   return EncodeSignedVarint