Merge pull request #4387 from acozzette/down-integrate

Integrated internal changes from Google
protocolbuffers · Mar 20, 2018 · ed4321d · ed4321d
2 parents e436ee0 + 11d26ce
commit ed4321d
Show file tree

Hide file tree

Showing 312 changed files with 15,463 additions and 14,342 deletions.
diff --git a/Makefile.am b/Makefile.am
@@ -354,6 +354,7 @@ java_EXTRA_DIST=
   java/core/src/test/proto/com/google/protobuf/map_for_proto2_lite_test.proto      \
   java/core/src/test/proto/com/google/protobuf/map_for_proto2_test.proto           \
   java/core/src/test/proto/com/google/protobuf/map_initialization_order_test.proto \
+  java/core/src/test/proto/com/google/protobuf/map_lite_test.proto                 \
   java/core/src/test/proto/com/google/protobuf/map_test.proto                      \
   java/core/src/test/proto/com/google/protobuf/multiple_files_test.proto           \
   java/core/src/test/proto/com/google/protobuf/nested_builders_test.proto          \
@@ -808,6 +809,7 @@ python_EXTRA_DIST=                                                           \
   python/google/protobuf/internal/more_extensions.proto                      \
   python/google/protobuf/internal/more_extensions_dynamic.proto              \
   python/google/protobuf/internal/more_messages.proto                        \
+  python/google/protobuf/internal/no_package.proto                           \
   python/google/protobuf/internal/packed_field_test.proto                    \
   python/google/protobuf/internal/proto_builder_test.py                      \
   python/google/protobuf/internal/python_message.py                          \
@@ -861,6 +863,7 @@ python_EXTRA_DIST=                                                           \
   python/google/protobuf/pyext/repeated_scalar_container.h                   \
   python/google/protobuf/pyext/safe_numerics.h                               \
   python/google/protobuf/pyext/scoped_pyobject_ptr.h                         \
+  python/google/protobuf/pyext/thread_unsafe_shared_ptr.h                    \
   python/google/protobuf/reflection.py                                       \
   python/google/protobuf/service.py                                          \
   python/google/protobuf/service_reflection.py                               \

diff --git a/appveyor.yml b/appveyor.yml
@@ -10,6 +10,7 @@ configuration:
 environment:
   matrix:
     - language: cpp
+      image: Visual Studio 2015
       BUILD_DLL: ON
       UNICODE: ON
 
@@ -32,8 +33,8 @@ install:
   - move gtest gmock
 
 before_build:
-  - if %platform%==Win32 set generator=Visual Studio 12
-  - if %platform%==Win64 set generator=Visual Studio 12 Win64
+  - if %platform%==Win32 set generator=Visual Studio 14
+  - if %platform%==Win64 set generator=Visual Studio 14 Win64
   - if %platform%==Win32 set vcplatform=Win32
   - if %platform%==Win64 set vcplatform=x64
 

diff --git a/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java b/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java
@@ -243,3 +243,4 @@ void checkCounter() throws IOException {
   }
 }
 
+
diff --git a/benchmarks/py_benchmark.py b/benchmarks/py_benchmark.py
@@ -115,3 +115,4 @@ def run_benchmark(self, test_method_args='', setup_method_args=''):
 if __name__ == "__main__":
   for i in range(2, len(sys.argv)):
     run_one_test(sys.argv[i])
+
diff --git a/cmake/extract_includes.bat.in b/cmake/extract_includes.bat.in
@@ -79,20 +79,6 @@ copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\repeated_field.h" inc
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\service.h" include\google\protobuf\service.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\source_context.pb.h" include\google\protobuf\source_context.pb.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\struct.pb.h" include\google\protobuf\struct.pb.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomic_sequence_num.h" include\google\protobuf\stubs\atomic_sequence_num.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops.h" include\google\protobuf\stubs\atomicops.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_arm64_gcc.h" include\google\protobuf\stubs\atomicops_internals_arm64_gcc.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_arm_gcc.h" include\google\protobuf\stubs\atomicops_internals_arm_gcc.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_arm_qnx.h" include\google\protobuf\stubs\atomicops_internals_arm_qnx.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_generic_c11_atomic.h" include\google\protobuf\stubs\atomicops_internals_generic_c11_atomic.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_generic_gcc.h" include\google\protobuf\stubs\atomicops_internals_generic_gcc.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_mips_gcc.h" include\google\protobuf\stubs\atomicops_internals_mips_gcc.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_power.h" include\google\protobuf\stubs\atomicops_internals_power.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_ppc_gcc.h" include\google\protobuf\stubs\atomicops_internals_ppc_gcc.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_solaris.h" include\google\protobuf\stubs\atomicops_internals_solaris.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_tsan.h" include\google\protobuf\stubs\atomicops_internals_tsan.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_x86_gcc.h" include\google\protobuf\stubs\atomicops_internals_x86_gcc.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_x86_msvc.h" include\google\protobuf\stubs\atomicops_internals_x86_msvc.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\bytestream.h" include\google\protobuf\stubs\bytestream.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\callback.h" include\google\protobuf\stubs\callback.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\casts.h" include\google\protobuf\stubs\casts.h
@@ -105,8 +91,6 @@ copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\mutex.h" includ
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\once.h" include\google\protobuf\stubs\once.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\platform_macros.h" include\google\protobuf\stubs\platform_macros.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\port.h" include\google\protobuf\stubs\port.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\scoped_ptr.h" include\google\protobuf\stubs\scoped_ptr.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\shared_ptr.h" include\google\protobuf\stubs\shared_ptr.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\singleton.h" include\google\protobuf\stubs\singleton.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\status.h" include\google\protobuf\stubs\status.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\stl_util.h" include\google\protobuf\stubs\stl_util.h

diff --git a/cmake/libprotobuf-lite.cmake b/cmake/libprotobuf-lite.cmake
@@ -10,13 +10,10 @@ set(libprotobuf_lite_files
   ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream_impl_lite.cc
   ${protobuf_source_dir}/src/google/protobuf/message_lite.cc
   ${protobuf_source_dir}/src/google/protobuf/repeated_field.cc
-  ${protobuf_source_dir}/src/google/protobuf/stubs/atomicops_internals_x86_gcc.cc
-  ${protobuf_source_dir}/src/google/protobuf/stubs/atomicops_internals_x86_msvc.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/bytestream.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/common.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/int128.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/io_win32.cc
-  ${protobuf_source_dir}/src/google/protobuf/stubs/once.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/status.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/statusor.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/stringpiece.cc
@@ -38,7 +35,6 @@ set(libprotobuf_lite_includes
   ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream_impl_lite.h
   ${protobuf_source_dir}/src/google/protobuf/message_lite.h
   ${protobuf_source_dir}/src/google/protobuf/repeated_field.h
-  ${protobuf_source_dir}/src/google/protobuf/stubs/atomicops_internals_x86_msvc.h
   ${protobuf_source_dir}/src/google/protobuf/stubs/bytestream.h
   ${protobuf_source_dir}/src/google/protobuf/stubs/common.h
   ${protobuf_source_dir}/src/google/protobuf/stubs/int128.h

diff --git a/cmake/tests.cmake b/cmake/tests.cmake
@@ -155,15 +155,15 @@ set(tests_files
   ${protobuf_source_dir}/src/google/protobuf/preserve_unknown_enum_test.cc
   ${protobuf_source_dir}/src/google/protobuf/proto3_arena_lite_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/proto3_arena_unittest.cc
-  ${protobuf_source_dir}/src/google/protobuf/proto3_lite_unittest.cc
+# TODO(b/74491957) Make this unittest work
+#  ${protobuf_source_dir}/src/google/protobuf/proto3_lite_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/reflection_ops_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/repeated_field_reflection_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/repeated_field_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/bytestream_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/common_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/int128_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/io_win32_unittest.cc
-  ${protobuf_source_dir}/src/google/protobuf/stubs/once_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/status_test.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/statusor_test.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/stringpiece_unittest.cc
@@ -172,7 +172,6 @@ set(tests_files
   ${protobuf_source_dir}/src/google/protobuf/stubs/strutil_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/template_util_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/time_test.cc
-  ${protobuf_source_dir}/src/google/protobuf/stubs/type_traits_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/text_format_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/unknown_field_set_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/util/delimited_message_util_test.cc

diff --git a/conformance/conformance_cpp.cc b/conformance/conformance_cpp.cc
@@ -45,7 +45,6 @@ using google::protobuf::Descriptor;
 using google::protobuf::DescriptorPool;
 using google::protobuf::Message;
 using google::protobuf::MessageFactory;
-using google::protobuf::internal::scoped_ptr;
 using google::protobuf::util::BinaryToJsonString;
 using google::protobuf::util::JsonToBinaryString;
 using google::protobuf::util::NewTypeResolverForDescriptorPool;

diff --git a/conformance/conformance_test.cc b/conformance/conformance_test.cc
@@ -1910,6 +1910,10 @@ bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
   {
     TestAllTypesProto3 messageProto3;
     TestAllTypesProto2 messageProto2;
+    //TODO(yilunchong): update this behavior when unknown field's behavior
+    // changed in open source. Also delete
+    // Required.Proto3.ProtobufInput.UnknownVarint.ProtobufOutput
+    // from failure list of python_cpp python java
     TestUnknownMessage(messageProto3, true);
     TestUnknownMessage(messageProto2, false);
   }

diff --git a/conformance/conformance_test.h b/conformance/conformance_test.h
@@ -256,8 +256,7 @@ class ConformanceTestSuite {
   // The set of tests that the testee opted out of;
   std::set<std::string> skipped_;
 
-  google::protobuf::internal::scoped_ptr<google::protobuf::util::TypeResolver>
-      type_resolver_;
+  std::unique_ptr<google::protobuf::util::TypeResolver> type_resolver_;
   std::string type_url_;
 };
 

diff --git a/conformance/conformance_test_runner.cc b/conformance/conformance_test_runner.cc
@@ -68,7 +68,6 @@
 
 using conformance::ConformanceRequest;
 using conformance::ConformanceResponse;
-using google::protobuf::internal::scoped_array;
 using google::protobuf::StringAppendF;
 using std::string;
 using std::vector;
@@ -183,7 +182,7 @@ class ForkPipeRunner : public google::protobuf::ConformanceTestRunner {
       CHECK_SYSCALL(close(toproc_pipe_fd[1]));
       CHECK_SYSCALL(close(fromproc_pipe_fd[0]));
 
-      scoped_array<char> executable(new char[executable_.size() + 1]);
+      std::unique_ptr<char[]> executable(new char[executable_.size() + 1]);
       memcpy(executable.get(), executable_.c_str(), executable_.size());
       executable[executable_.size()] = '\0';
 

diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/FieldMask.cs b/csharp/src/Google.Protobuf/WellKnownTypes/FieldMask.cs
@@ -244,8 +244,8 @@ public static partial class FieldMaskReflection {
   ///
   /// ## Field Mask Verification
   ///
-  /// The implementation of the all the API methods, which have any FieldMask type
-  /// field in the request, should verify the included field paths, and return
+  /// The implementation of any API method which has a FieldMask type field in the
+  /// request should verify the included field paths, and return an
   /// `INVALID_ARGUMENT` error if any path is duplicated or unmappable.
   /// </summary>
   public sealed partial class FieldMask : pb::IMessage<FieldMask> {

diff --git a/java/core/generate-test-sources-build.xml b/java/core/generate-test-sources-build.xml
@@ -39,6 +39,7 @@
         <arg value="${test.proto.dir}/com/google/protobuf/field_presence_test.proto"/>
         <arg value="${test.proto.dir}/com/google/protobuf/map_for_proto2_lite_test.proto"/>
         <arg value="${test.proto.dir}/com/google/protobuf/map_for_proto2_test.proto"/>
+        <arg value="${test.proto.dir}/com/google/protobuf/map_lite_test.proto"/>
         <arg value="${test.proto.dir}/com/google/protobuf/map_test.proto"/>
         <arg value="${test.proto.dir}/com/google/protobuf/map_initialization_order_test.proto"/>
     </exec>

diff --git a/java/core/src/main/java/com/google/protobuf/AbstractMessage.java b/java/core/src/main/java/com/google/protobuf/AbstractMessage.java
@@ -124,6 +124,16 @@ public void writeTo(final CodedOutputStream output) throws IOException {
 
   protected int memoizedSize = -1;
 
+  @Override
+  int getMemoizedSerializedSize() {
+    return memoizedSize;
+  }
+
+  @Override
+  void setMemoizedSerializedSize(int size) {
+    memoizedSize = size;
+  }
+
   @Override
   public int getSerializedSize() {
     int size = memoizedSize;

diff --git a/java/core/src/main/java/com/google/protobuf/AbstractMessageLite.java b/java/core/src/main/java/com/google/protobuf/AbstractMessageLite.java
@@ -99,6 +99,16 @@ public void writeDelimitedTo(final OutputStream output) throws IOException {
     codedOutput.flush();
   }
 
+  // We'd like these to be abstract but some folks are extending this class directly. They shouldn't
+  // be doing that and they should feel bad.
+  int getMemoizedSerializedSize() {
+    throw new UnsupportedOperationException();
+  }
+
+  void setMemoizedSerializedSize(int size) {
+    throw new UnsupportedOperationException();
+  }
+
 
   /**
    * Package private helper method for AbstractParser to create

diff --git a/java/core/src/main/java/com/google/protobuf/BooleanArrayList.java b/java/core/src/main/java/com/google/protobuf/BooleanArrayList.java
@@ -81,6 +81,18 @@ private BooleanArrayList(boolean[] other, int size) {
     this.size = size;
   }
 
+  @Override
+  protected void removeRange(int fromIndex, int toIndex) {
+    ensureIsMutable();
+    if (toIndex < fromIndex) {
+      throw new IndexOutOfBoundsException("toIndex < fromIndex");
+    }
+
+    System.arraycopy(array, toIndex, array, fromIndex, size - toIndex);
+    size -= (toIndex - fromIndex);
+    modCount++;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {
@@ -246,7 +258,9 @@ public Boolean remove(int index) {
     ensureIsMutable();
     ensureIndexInRange(index);
     boolean value = array[index];
-    System.arraycopy(array, index + 1, array, index, size - index);
+    if (index < size - 1) {
+      System.arraycopy(array, index + 1, array, index, size - index);
+    }
     size--;
     modCount++;
     return value;

diff --git a/java/core/src/main/java/com/google/protobuf/CodedInputStream.java b/java/core/src/main/java/com/google/protobuf/CodedInputStream.java
@@ -66,11 +66,9 @@ public abstract class CodedInputStream {
 
   /**
    * Whether to enable our custom UTF-8 decode codepath which does not use {@link StringCoding}.
-   * Enabled by default, disable by setting
-   * {@code -Dcom.google.protobuf.enableCustomutf8Decode=false} in JVM args.
+   * Currently disabled.
    */
-  private static final boolean ENABLE_CUSTOM_UTF8_DECODE
-      = !"false".equals(System.getProperty("com.google.protobuf.enableCustomUtf8Decode"));
+  private static final boolean ENABLE_CUSTOM_UTF8_DECODE = false;
 
   /** Visible for subclasses. See setRecursionLimit() */
   int recursionDepth;