Skip to content
Permalink
Browse files

Down Integrate Internal Changes (#6634)

* Down integrate to GitHub

* Remove unintentional double assign

* Change ByteCountInt64 to int64_t

* Revert changes in binary_conformance_test.cc

* Revert unnecessary changes
  • Loading branch information...
TeBoring committed Sep 12, 2019
1 parent 580f585 commit 763c3588740b97e8e80b1b1a1a2dc4f417647133
Showing with 1,048 additions and 652 deletions.
  1. +1 −1 java/core/src/main/java/com/google/protobuf/AbstractMessage.java
  2. +16 −1 java/core/src/main/java/com/google/protobuf/ByteString.java
  3. +2 −2 java/core/src/main/java/com/google/protobuf/FieldSet.java
  4. +2 −2 java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java
  5. +5 −5 java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java
  6. +1 −1 java/core/src/main/java/com/google/protobuf/MapEntry.java
  7. +1 −1 java/core/src/main/java/com/google/protobuf/MapFieldLite.java
  8. +19 −0 java/core/src/test/java/com/google/protobuf/ByteStringTest.java
  9. +8 −0 java/util/src/main/java/com/google/protobuf/util/JsonFormat.java
  10. +16 −0 java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java
  11. +8 −2 js/binary/decoder_test.js
  12. +8 −4 js/binary/encoder.js
  13. +19 −2 js/binary/utils.js
  14. +29 −2 js/binary/utils_test.js
  15. +47 −0 js/binary/writer_test.js
  16. +14 −3 js/compatibility_tests/v3.1.0/binary/decoder_test.js
  17. +0 −1 js/message_test.js
  18. +10 −7 python/google/protobuf/internal/containers.py
  19. +13 −0 python/google/protobuf/internal/message_test.py
  20. +0 −7 python/google/protobuf/internal/python_message.py
  21. +45 −0 python/google/protobuf/internal/text_format_test.py
  22. +18 −0 python/google/protobuf/message.py
  23. +14 −59 python/google/protobuf/pyext/message.cc
  24. +38 −11 python/google/protobuf/text_format.py
  25. +9 −9 src/google/protobuf/api.pb.cc
  26. +1 −8 src/google/protobuf/arena.h
  27. +4 −4 src/google/protobuf/compiler/command_line_interface.cc
  28. +2 −2 src/google/protobuf/compiler/cpp/cpp_enum_field.cc
  29. +1 −1 src/google/protobuf/compiler/cpp/cpp_generator.h
  30. +1 −1 src/google/protobuf/compiler/cpp/cpp_message.cc
  31. +3 −3 src/google/protobuf/compiler/cpp/cpp_message_field.cc
  32. +2 −2 src/google/protobuf/compiler/cpp/cpp_primitive_field.cc
  33. +5 −5 src/google/protobuf/compiler/parser_unittest.cc
  34. +6 −6 src/google/protobuf/compiler/plugin.pb.cc
  35. +16 −13 src/google/protobuf/descriptor.cc
  36. +84 −84 src/google/protobuf/descriptor.pb.cc
  37. +2 −2 src/google/protobuf/duration.pb.cc
  38. +7 −9 src/google/protobuf/dynamic_message.h
  39. +12 −12 src/google/protobuf/extension_set.cc
  40. +6 −4 src/google/protobuf/generated_message_reflection.cc
  41. +6 −6 src/google/protobuf/io/coded_stream.cc
  42. +12 −10 src/google/protobuf/io/coded_stream.h
  43. +8 −8 src/google/protobuf/io/coded_stream_unittest.cc
  44. +2 −2 src/google/protobuf/io/gzip_stream.cc
  45. +3 −2 src/google/protobuf/io/gzip_stream.h
  46. +1 −0 src/google/protobuf/io/printer.h
  47. +1 −0 src/google/protobuf/io/tokenizer.h
  48. +4 −4 src/google/protobuf/io/tokenizer_unittest.cc
  49. +1 −0 src/google/protobuf/io/zero_copy_stream.h
  50. +5 −5 src/google/protobuf/io/zero_copy_stream_impl.cc
  51. +6 −5 src/google/protobuf/io/zero_copy_stream_impl.h
  52. +6 −6 src/google/protobuf/io/zero_copy_stream_impl_lite.cc
  53. +7 −6 src/google/protobuf/io/zero_copy_stream_impl_lite.h
  54. +1 −1 src/google/protobuf/map_entry_lite.h
  55. +35 −25 src/google/protobuf/map_field.cc
  56. +11 −10 src/google/protobuf/map_field.h
  57. +15 −0 src/google/protobuf/map_test.cc
  58. +3 −3 src/google/protobuf/map_type_handler.h
  59. +6 −6 src/google/protobuf/message.h
  60. +1 −1 src/google/protobuf/message_lite.cc
  61. +4 −6 src/google/protobuf/message_unittest.inc
  62. +8 −9 src/google/protobuf/no_field_presence_test.cc
  63. +4 −7 src/google/protobuf/proto3_arena_unittest.cc
  64. +0 −2 src/google/protobuf/repeated_field.h
  65. +6 −6 src/google/protobuf/struct.pb.cc
  66. +9 −6 src/google/protobuf/text_format.h
  67. +2 −2 src/google/protobuf/timestamp.pb.cc
  68. +17 −17 src/google/protobuf/type.pb.cc
  69. +10 −0 src/google/protobuf/unittest.proto
  70. +10 −0 src/google/protobuf/unittest_proto3_arena.proto
  71. +1 −0 src/google/protobuf/util/delimited_message_util.h
  72. +0 −3 src/google/protobuf/util/internal/datapiece.cc
  73. +19 −20 src/google/protobuf/util/internal/default_value_objectwriter.cc
  74. +22 −0 src/google/protobuf/util/internal/json_escaping.cc
  75. +5 −0 src/google/protobuf/util/internal/json_escaping.h
  76. +5 −9 src/google/protobuf/util/internal/json_objectwriter.cc
  77. +48 −6 src/google/protobuf/util/internal/json_objectwriter.h
  78. +54 −13 src/google/protobuf/util/internal/json_stream_parser.cc
  79. +4 −0 src/google/protobuf/util/internal/json_stream_parser.h
  80. +39 −11 src/google/protobuf/util/internal/json_stream_parser_test.cc
  81. +26 −29 src/google/protobuf/util/internal/proto_writer.cc
  82. +48 −50 src/google/protobuf/util/internal/protostream_objectsource.cc
  83. +2 −3 src/google/protobuf/util/internal/protostream_objectwriter.cc
  84. +7 −7 src/google/protobuf/util/internal/protostream_objectwriter_test.cc
  85. +12 −11 src/google/protobuf/util/internal/utility.cc
  86. +6 −6 src/google/protobuf/util/internal/utility.h
  87. +3 −3 src/google/protobuf/util/json_util_test.cc
  88. +9 −9 src/google/protobuf/util/message_differencer_unittest.cc
  89. +11 −11 src/google/protobuf/wire_format.cc
  90. +1 −1 src/google/protobuf/wire_format_lite.h
  91. +2 −3 src/google/protobuf/wire_format_unittest.cc
  92. +7 −7 src/google/protobuf/wrappers.pb.cc
  93. +18 −19 update_compatibility_version.py
@@ -193,7 +193,7 @@ private static Map convertMapEntryListToMap(List list) {
if (list.isEmpty()) {
return Collections.emptyMap();
}
Map result = new HashMap();
Map result = new HashMap<>();
Iterator iterator = list.iterator();
Message entry = (Message) iterator.next();
Descriptors.Descriptor descriptor = entry.getDescriptorForType();
@@ -30,6 +30,10 @@

package com.google.protobuf;

import static com.google.protobuf.TextFormatEscaper.escapeBytes;
import static java.lang.Integer.toHexString;
import static java.lang.System.identityHashCode;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -49,6 +53,7 @@
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.NoSuchElementException;

/**
@@ -1268,7 +1273,17 @@ static int checkRange(int startIndex, int endIndex, int size) {
@Override
public final String toString() {
return String.format(
"<ByteString@%s size=%d>", Integer.toHexString(System.identityHashCode(this)), size());
Locale.ROOT,
"<ByteString@%s size=%d contents=\"%s\">",
toHexString(identityHashCode(this)),
size(),
truncateAndEscapeForDisplay());
}

private String truncateAndEscapeForDisplay() {
final int limit = 50;

return size() <= limit ? escapeBytes(this) : escapeBytes(substring(0, limit - 3)) + "...";
}

/**
@@ -526,7 +526,7 @@ private void mergeFromField(final Map.Entry<T, Object> entry) {
if (descriptor.isRepeated()) {
Object value = getField(descriptor);
if (value == null) {
value = new ArrayList();
value = new ArrayList<>();
}
for (Object element : (List) otherValue) {
((List) value).add(cloneIfMutable(element));
@@ -1266,7 +1266,7 @@ private void mergeFromField(final Map.Entry<T, Object> entry) {
if (descriptor.isRepeated()) {
Object value = getField(descriptor);
if (value == null) {
value = new ArrayList();
value = new ArrayList<>();
}
for (Object element : (List) otherValue) {
((List) value).add(FieldSet.cloneIfMutable(element));
@@ -1238,7 +1238,7 @@ public MessageLite getMessageDefaultInstance() {
Object fromFieldSetType(final Object value) {
if (descriptor.isRepeated()) {
if (descriptor.getLiteJavaType() == WireFormat.JavaType.ENUM) {
final List result = new ArrayList();
final List result = new ArrayList<>();
for (final Object element : (List) value) {
result.add(singularFromFieldSetType(element));
}
@@ -1263,7 +1263,7 @@ Object singularFromFieldSetType(final Object value) {
Object toFieldSetType(final Object value) {
if (descriptor.isRepeated()) {
if (descriptor.getLiteJavaType() == WireFormat.JavaType.ENUM) {
final List result = new ArrayList();
final List result = new ArrayList<>();
for (final Object element : (List) value) {
result.add(singularToFieldSetType(element));
}
@@ -158,7 +158,7 @@ protected void mergeFromAndMakeImmutableInternal(

/**
* Internal helper to return a modifiable map containing all the fields.
* The returned Map is modifialbe so that the caller can add additional
* The returned Map is modifiable so that the caller can add additional
* extension fields to implement {@link #getAllFields()}.
*
* @param getBytesForString whether to generate ByteString for string fields
@@ -2853,7 +2853,7 @@ private Message coerceType(Message value) {
@Override
@SuppressWarnings("unchecked")
public Object get(GeneratedMessageV3 message) {
List result = new ArrayList();
List result = new ArrayList<>();
for (int i = 0; i < getRepeatedCount(message); i++) {
result.add(getRepeated(message, i));
}
@@ -2863,7 +2863,7 @@ public Object get(GeneratedMessageV3 message) {
@Override
@SuppressWarnings("unchecked")
public Object get(Builder builder) {
List result = new ArrayList();
List result = new ArrayList<>();
for (int i = 0; i < getRepeatedCount(builder); i++) {
result.add(getRepeated(builder, i));
}
@@ -3068,7 +3068,7 @@ public void set(final Builder builder, final Object value) {
@Override
@SuppressWarnings("unchecked")
public Object get(final GeneratedMessageV3 message) {
final List newList = new ArrayList();
final List newList = new ArrayList<>();
final int size = getRepeatedCount(message);
for (int i = 0; i < size; i++) {
newList.add(getRepeated(message, i));
@@ -3079,7 +3079,7 @@ public Object get(final GeneratedMessageV3 message) {
@Override
@SuppressWarnings("unchecked")
public Object get(final GeneratedMessageV3.Builder builder) {
final List newList = new ArrayList();
final List newList = new ArrayList<>();
final int size = getRepeatedCount(builder);
for (int i = 0; i < size; i++) {
newList.add(getRepeated(builder, i));
@@ -439,7 +439,7 @@ public UnknownFieldSet getUnknownFields() {
@Override
@SuppressWarnings("unchecked")
public Builder<K, V> clone() {
return new Builder(metadata, key, value, hasKey, hasValue);
return new Builder<>(metadata, key, value, hasKey, hasValue);
}
}

@@ -58,7 +58,7 @@ private MapFieldLite(Map<K, V> mapData) {
}

@SuppressWarnings({"rawtypes", "unchecked"})
private static final MapFieldLite EMPTY_MAP_FIELD = new MapFieldLite();
private static final MapFieldLite EMPTY_MAP_FIELD = new MapFieldLite<>();

static {
EMPTY_MAP_FIELD.makeImmutable();
@@ -486,6 +486,25 @@ public void testToStringUtf8() {
"copyToStringUtf8 must respect the charset", testString, byteString.toStringUtf8());
}

public void testToString() {
String toString =
ByteString.copyFrom("Here are some bytes: \t\u00a1".getBytes(Internal.UTF_8)).toString();
assertTrue(toString, toString.contains("size=24"));
assertTrue(toString, toString.contains("contents=\"Here are some bytes: \\t\\302\\241\""));
}

public void testToString_long() {
String toString =
ByteString.copyFrom(
"123456789012345678901234567890123456789012345678901234567890"
.getBytes(Internal.UTF_8))
.toString();
assertTrue(toString, toString.contains("size=60"));
assertTrue(
toString,
toString.contains("contents=\"12345678901234567890123456789012345678901234567...\""));
}

public void testNewOutput_InitialCapacity() throws IOException {
byte[] bytes = getTestBytes();
ByteString.Output output = ByteString.newOutput(bytes.length + 100);
@@ -1945,6 +1945,14 @@ private Object parseFieldValue(FieldDescriptor field, JsonElement json, Message.
return field.getEnumType().findValueByNumber(0);
}
return null;
} else if (json instanceof JsonObject) {
if (field.getType() != FieldDescriptor.Type.MESSAGE
&& field.getType() != FieldDescriptor.Type.GROUP) {
// If the field type is primitive, but the json type is JsonObject rather than
// JsonElement, throw a type mismatch error.
throw new InvalidProtocolBufferException(
String.format("Invalid value: %s for expected type: %s", json, field.getType()));
}
}
switch (field.getType()) {
case INT32:
@@ -1723,6 +1723,22 @@ public int read() throws IOException {
}
}

// Test that an error is thrown if a nested JsonObject is parsed as a primitive field.
public void testJsonObjectForPrimitiveField() throws Exception {
TestAllTypes.Builder builder = TestAllTypes.newBuilder();
try {
mergeFromJson(
"{\n"
+ " \"optionalString\": {\n"
+ " \"invalidNestedString\": \"Hello world\"\n"
+ " }\n"
+ "}\n",
builder);
} catch (InvalidProtocolBufferException e) {
// Expected.
}
}

public void testSortedMapKeys() throws Exception {
TestMap.Builder mapBuilder = TestMap.newBuilder();
mapBuilder.putStringToInt32Map("\ud834\udd20", 3); // utf-8 F0 9D 84 A0
@@ -141,8 +141,14 @@ function doTestSignedValue(readValue,
}

// Encoding values outside the valid range should assert.
assertThrows(function() {writeValue.call(encoder, lowerLimit * 1.1);});
assertThrows(function() {writeValue.call(encoder, upperLimit * 1.1);});
var pastLowerLimit = lowerLimit * 1.1;
var pastUpperLimit = upperLimit * 1.1;
if (pastLowerLimit !== -Infinity) {
expect(() => void writeValue.call(encoder, pastLowerLimit)).toThrow();
}
if (pastUpperLimit !== Infinity) {
expect(() => void writeValue.call(encoder, pastUpperLimit)).toThrow();
}
}

describe('binaryDecoderTest', function() {
@@ -382,8 +382,10 @@ jspb.BinaryEncoder.prototype.writeInt64String = function(value) {
* @param {number} value The value to write.
*/
jspb.BinaryEncoder.prototype.writeFloat = function(value) {
goog.asserts.assert((value >= -jspb.BinaryConstants.FLOAT32_MAX) &&
(value <= jspb.BinaryConstants.FLOAT32_MAX));
goog.asserts.assert(
value === Infinity || value === -Infinity || isNaN(value) ||
((value >= -jspb.BinaryConstants.FLOAT32_MAX) &&
(value <= jspb.BinaryConstants.FLOAT32_MAX)));
jspb.utils.splitFloat32(value);
this.writeUint32(jspb.utils.split64Low);
};
@@ -395,8 +397,10 @@ jspb.BinaryEncoder.prototype.writeFloat = function(value) {
* @param {number} value The value to write.
*/
jspb.BinaryEncoder.prototype.writeDouble = function(value) {
goog.asserts.assert((value >= -jspb.BinaryConstants.FLOAT64_MAX) &&
(value <= jspb.BinaryConstants.FLOAT64_MAX));
goog.asserts.assert(
value === Infinity || value === -Infinity || isNaN(value) ||
((value >= -jspb.BinaryConstants.FLOAT64_MAX) &&
(value <= jspb.BinaryConstants.FLOAT64_MAX)));
jspb.utils.splitFloat64(value);
this.writeUint32(jspb.utils.split64Low);
this.writeUint32(jspb.utils.split64High);
@@ -255,8 +255,25 @@ jspb.utils.splitFloat64 = function(value) {
return;
}

var exp = Math.floor(Math.log(value) / Math.LN2);
if (exp == 1024) exp = 1023;
// Compute the least significant exponent needed to represent the magnitude of
// the value by repeadly dividing/multiplying by 2 until the magnitude
// crosses 2. While tempting to use log math to find the exponent, at the
// bounadaries of precision, the result can be off by one.
var maxDoubleExponent = 1023;
var minDoubleExponent = -1022;
var x = value;
var exp = 0;
if (x >= 2) {
while (x >= 2 && exp < maxDoubleExponent) {
exp++;
x = x / 2;
}
} else {
while (x < 1 && exp > minDoubleExponent) {
x = x * 2;
exp--;
}
}
var mant = value * Math.pow(2, -exp);

var mantHigh = (mant * jspb.BinaryConstants.TWO_TO_20) & 0xFFFFF;
@@ -341,6 +341,8 @@ describe('binaryUtilsTest', function() {
var f32_eps = jspb.BinaryConstants.FLOAT32_EPS;
var f32_min = jspb.BinaryConstants.FLOAT32_MIN;
var f32_max = jspb.BinaryConstants.FLOAT32_MAX;
var f32_max_safe_int = jspb.utils.joinFloat32(0x4b7fffff, 0);
var f32_pi = Math.fround(Math.PI);

// NaN.
jspb.utils.splitFloat32(NaN);
@@ -382,6 +384,13 @@ describe('binaryUtilsTest', function() {
test(f32_max, 0x7F7FFFFF);
test(-f32_max, 0xFF7FFFFF);

// Positive and negative max_safe_int.
test(f32_max_safe_int, 0x4B7FFFFF);
test(-f32_max_safe_int, 0xCB7FFFFF);

// Pi.
test(f32_pi, 0x40490fdb);

// Various positive values.
var cursor = f32_eps * 10;
while (cursor != Infinity) {
@@ -420,10 +429,12 @@ describe('binaryUtilsTest', function() {
function test(x, opt_highBits, opt_lowBits) {
jspb.utils.splitFloat64(x);
if (goog.isDef(opt_highBits)) {
if (opt_highBits != jspb.utils.split64High) throw 'fail!';
var split64High = jspb.utils.split64High;
expect(opt_highBits.toString(16)).toEqual(split64High.toString(16));
}
if (goog.isDef(opt_lowBits)) {
if (opt_lowBits != jspb.utils.split64Low) throw 'fail!';
var split64Low = jspb.utils.split64Low;
expect(opt_lowBits.toString(16)).toEqual(split64Low.toString(16));
}
expect(
jspb.utils.joinFloat64(jspb.utils.split64Low, jspb.utils.split64High))
@@ -438,6 +449,9 @@ describe('binaryUtilsTest', function() {
test(0, 0x00000000, 0x00000000);
test(-0, 0x80000000, 0x00000000);

test(1, 0x3FF00000, 0x00000000);
test(2, 0x40000000, 0x00000000);

// Positive and negative epsilon.
test(f64_eps, 0x00000000, 0x00000001);
test(-f64_eps, 0x80000000, 0x00000001);
@@ -450,6 +464,19 @@ describe('binaryUtilsTest', function() {
test(f64_max, 0x7FEFFFFF, 0xFFFFFFFF);
test(-f64_max, 0xFFEFFFFF, 0xFFFFFFFF);

test(Number.MAX_SAFE_INTEGER, 0x433FFFFF, 0xFFFFFFFF);
test(Number.MIN_SAFE_INTEGER, 0xC33FFFFF, 0xFFFFFFFF);

// Test various edge cases with mantissa of all 1, all 0, or just the
// highest or lowest significant bit.
test(4503599627370497, 0x43300000, 0x00000001);
test(6755399441055744, 0x43380000, 0x00000000);
test(1.348269851146737e+308, 0x7FE80000, 0x00000000);
test(1.9999999999999998, 0x3FFFFFFF, 0xFFFFFFFF);
test(2.225073858507201e-308, 0x000FFFFF, 0xFFFFFFFF);
test(Math.PI, 0x400921fb, 0x54442d18);
test(jspb.BinaryConstants.FLOAT32_MIN, 0x38100000, 0x00000000);

// Various positive values.
var cursor = f64_eps * 10;
while (cursor != Infinity) {

0 comments on commit 763c358

Please sign in to comment.
You can’t perform that action at this time.