Merge remote-tracking branch 'origin/3.5.x' into master

CHANGES.txt

@@ -1,3 +1,34 @@
+2017-12-20 version 3.5.1 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
+  Planned Future Changes
+  * Make C++ implementation C++11 only: we plan to require C++11 to build
+    protobuf code starting from 3.6.0 release. Please join this github issue:
+    https://github.com/google/protobuf/issues/2780 to provide your feedback.
+
+  protoc
+  * Fixed a bug introduced in 3.5.0 and protoc in Windows now accepts non-ascii
+    characters in paths again.
+
+  C++
+  * Removed several usages of C++11 features in the code base.
+  * Fixed some compiler warnings.
+
+  PHP
+  * Fixed memory leak in C-extension implementation.
+  * Added discardUnknokwnFields API.
+  * Removed duplicatd typedef in C-extension headers.
+  * Avoided calling private php methods (timelib_update_ts).
+  * Fixed Any.php to use fully-qualified name for DescriptorPool.
+
+  Ruby
+  * Added Google_Protobuf_discard_unknown for discarding unknown fields in
+    messages.
+
+  C#
+  * Unknown fields are now preserved by default.
+  * Floating point values are now bitwise compared, affecting message equality
+    check and Contains() API in map and repeated fields.
+
+
 2017-11-13 version 3.5.0 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
   Planned Future Changes
   * Make C++ implementation C++11 only: we plan to require C++11 to build

Makefile.am

@@ -926,6 +926,7 @@ ruby_EXTRA_DIST=                                                             \
   ruby/src/main/java/google/ProtobufJavaService.java                         \
   ruby/src/main/sentinel.proto                                               \
   ruby/tests/basic.rb                                                        \
+  ruby/tests/encode_decode_test.rb                                           \
   ruby/tests/gc_test.rb                                                      \
   ruby/tests/repeated_field_test.rb                                          \
   ruby/tests/stress.rb                                                       \

Protobuf.podspec

@@ -5,7 +5,7 @@
 # dependent projects use the :git notation to refer to the library.
 Pod::Spec.new do |s|
   s.name     = 'Protobuf'
-  s.version  = '3.5.0'
+  s.version  = '3.5.1'
   s.summary  = 'Protocol Buffers v.3 runtime library for Objective-C.'
   s.homepage = 'https://github.com/google/protobuf'
   s.license  = '3-Clause BSD License'

configure.ac

@@ -17,7 +17,7 @@ AC_PREREQ(2.59)
 # In the SVN trunk, the version should always be the next anticipated release
 # version with the "-pre" suffix.  (We used to use "-SNAPSHOT" but this pushed
 # the size of one file name in the dist tarfile over the 99-char limit.)
-AC_INIT([Protocol Buffers],[3.5.0],[protobuf@googlegroups.com],[protobuf])
+AC_INIT([Protocol Buffers],[3.5.1],[protobuf@googlegroups.com],[protobuf])
 
 AM_MAINTAINER_MODE([enable])
 

csharp/Google.Protobuf.Tools.nuspec

@@ -5,7 +5,7 @@
     <title>Google Protocol Buffers tools</title>
     <summary>Tools for Protocol Buffers - Google's data interchange format.</summary>
     <description>See project site for more info.</description>
-    <version>3.5.0</version>
+    <version>3.5.1</version>
     <authors>Google Inc.</authors>
     <owners>protobuf-packages</owners>
     <licenseUrl>https://github.com/google/protobuf/blob/master/LICENSE</licenseUrl>

csharp/src/Google.Protobuf/Google.Protobuf.csproj

@@ -4,7 +4,7 @@
     <Description>C# runtime library for Protocol Buffers - Google's data interchange format.</Description>
     <Copyright>Copyright 2015, Google Inc.</Copyright>
     <AssemblyTitle>Google Protocol Buffers</AssemblyTitle>
-    <VersionPrefix>3.5.0</VersionPrefix>
+    <VersionPrefix>3.5.1</VersionPrefix>
     <Authors>Google Inc.</Authors>
     <TargetFrameworks>netstandard1.0;net45</TargetFrameworks>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>

java/core/pom.xml

@@ -6,7 +6,7 @@
   <parent>
     <groupId>com.google.protobuf</groupId>
     <artifactId>protobuf-parent</artifactId>
-    <version>3.5.0</version>
+    <version>3.5.1</version>
   </parent>
 
   <artifactId>protobuf-java</artifactId>

java/pom.xml

@@ -11,7 +11,7 @@
 
   <groupId>com.google.protobuf</groupId>
   <artifactId>protobuf-parent</artifactId>
-  <version>3.5.0</version>
+  <version>3.5.1</version>
   <packaging>pom</packaging>
 
   <name>Protocol Buffers [Parent]</name>

java/util/pom.xml

@@ -6,7 +6,7 @@
   <parent>
     <groupId>com.google.protobuf</groupId>
     <artifactId>protobuf-parent</artifactId>
-    <version>3.5.0</version>
+    <version>3.5.1</version>
   </parent>
 
   <artifactId>protobuf-java-util</artifactId>

js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "google-protobuf",
-  "version": "3.5.0",
+  "version": "3.5.1",
   "description": "Protocol Buffers for JavaScript",
   "main": "google-protobuf.js",
   "files": [

php/ext/google/protobuf/def.c

@@ -249,9 +249,11 @@ PHP_METHOD(Descriptor, getField) {
     MAKE_STD_ZVAL(field_hashtable_value);
     ZVAL_OBJ(field_hashtable_value, field_descriptor_type->create_object(
                                         field_descriptor_type TSRMLS_CC));
+    Z_DELREF_P(field_hashtable_value);
 #else
     field_hashtable_value =
         field_descriptor_type->create_object(field_descriptor_type TSRMLS_CC);
+    --GC_REFCOUNT(field_hashtable_value);
 #endif
     FieldDescriptor *field_php =
         UNBOX_HASHTABLE_VALUE(FieldDescriptor, field_hashtable_value);

php/ext/google/protobuf/encode_decode.c

@@ -1402,7 +1402,6 @@ static void putarray(zval* array, const upb_fielddef* f, upb_sink* sink,
   RepeatedField* intern = UNBOX(RepeatedField, array);
   HashTable *ht = PHP_PROTO_HASH_OF(intern->array);
   size = zend_hash_num_elements(ht);
-  // size = zend_hash_num_elements(PHP_PROTO_HASH_OF(intern->array));
   if (size == 0) return;
 
   upb_sink_startseq(sink, getsel(f, UPB_HANDLER_STARTSEQ), &subsink);
@@ -1615,11 +1614,101 @@ PHP_METHOD(Message, mergeFromJsonString) {
   }
 }
 
-PHP_METHOD(Message, discardUnknownFields) {
-  MessageHeader* msg = UNBOX(MessageHeader, getThis());
+// TODO(teboring): refactoring with putrawmsg
+static void discard_unknown_fields(MessageHeader* msg) {
+  upb_msg_field_iter it;
+
   stringsink* unknown = DEREF(message_data(msg), 0, stringsink*);
   if (unknown != NULL) {
     stringsink_uninit(unknown);
     DEREF(message_data(msg), 0, stringsink*) = NULL;
   }
+
+  // Recursively discard unknown fields of submessages.
+  Descriptor* desc = msg->descriptor;
+  TSRMLS_FETCH();
+  for (upb_msg_field_begin(&it, desc->msgdef);
+       !upb_msg_field_done(&it);
+       upb_msg_field_next(&it)) {
+    upb_fielddef* f = upb_msg_iter_field(&it);
+    uint32_t offset = desc->layout->fields[upb_fielddef_index(f)].offset;
+    bool containing_oneof = false;
+
+    if (upb_fielddef_containingoneof(f)) {
+      uint32_t oneof_case_offset =
+          desc->layout->fields[upb_fielddef_index(f)].case_offset;
+      // For a oneof, check that this field is actually present -- skip all the
+      // below if not.
+      if (DEREF(message_data(msg), oneof_case_offset, uint32_t) !=
+          upb_fielddef_number(f)) {
+        continue;
+      }
+      // Otherwise, fall through to the appropriate singular-field handler
+      // below.
+      containing_oneof = true;
+    }
+
+    if (is_map_field(f)) {
+      MapIter map_it;
+      int len, size;
+      const upb_fielddef* value_field;
+
+      value_field = map_field_value(f);
+      if (!upb_fielddef_issubmsg(value_field)) continue;
+
+      zval* map_php = CACHED_PTR_TO_ZVAL_PTR(
+          DEREF(message_data(msg), offset, CACHED_VALUE*));
+      if (map_php == NULL) continue;
+
+      Map* intern = UNBOX(Map, map_php);
+      for (map_begin(map_php, &map_it TSRMLS_CC);
+           !map_done(&map_it); map_next(&map_it)) {
+        upb_value value = map_iter_value(&map_it, &len);
+        void* memory = raw_value(upb_value_memory(&value), value_field);
+#if PHP_MAJOR_VERSION < 7
+        MessageHeader *submsg = UNBOX(MessageHeader, *(zval**)memory);
+#else
+        MessageHeader *submsg =
+            (MessageHeader*)((char*)(Z_OBJ_P((zval*)memory)) -
+                XtOffsetOf(MessageHeader, std));
+#endif
+        discard_unknown_fields(submsg);
+      }
+    } else if (upb_fielddef_isseq(f)) {
+      if (!upb_fielddef_issubmsg(f)) continue;
+
+      zval* array_php = CACHED_PTR_TO_ZVAL_PTR(
+          DEREF(message_data(msg), offset, CACHED_VALUE*));
+      if (array_php == NULL) continue;
+
+      int size, i;
+      RepeatedField* intern = UNBOX(RepeatedField, array_php);
+      HashTable *ht = PHP_PROTO_HASH_OF(intern->array);
+      size = zend_hash_num_elements(ht);
+      if (size == 0) continue;
+
+      for (i = 0; i < size; i++) {
+        void* memory = repeated_field_index_native(intern, i TSRMLS_CC);
+#if PHP_MAJOR_VERSION < 7
+        MessageHeader *submsg = UNBOX(MessageHeader, *(zval**)memory);
+#else
+        MessageHeader *submsg =
+            (MessageHeader*)((char*)(Z_OBJ_P((zval*)memory)) -
+                XtOffsetOf(MessageHeader, std));
+#endif
+        discard_unknown_fields(submsg);
+      }
+    } else if (upb_fielddef_issubmsg(f)) {
+      zval* submsg_php = CACHED_PTR_TO_ZVAL_PTR(
+          DEREF(message_data(msg), offset, CACHED_VALUE*));
+      if (Z_TYPE_P(submsg_php) == IS_NULL) continue;
+      MessageHeader* submsg = UNBOX(MessageHeader, submsg_php);
+      discard_unknown_fields(submsg);
+    }
+  }
+}
+
+PHP_METHOD(Message, discardUnknownFields) {
+  MessageHeader* msg = UNBOX(MessageHeader, getThis());
+  discard_unknown_fields(msg);
 }

php/ext/google/protobuf/message.c

@@ -29,7 +29,6 @@
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <php.h>
-#include <ext/date/php_date.h>
 #include <stdlib.h>
 
 #include "protobuf.h"
@@ -379,6 +378,7 @@ PHP_METHOD(Message, whichOneof) {
     PHP_PROTO_FAKE_SCOPE_BEGIN(LOWER_CLASS##_type);                            \
     zval* value = message_get_property_internal(getThis(), &member TSRMLS_CC); \
     PHP_PROTO_FAKE_SCOPE_END;                                                  \
+    zval_dtor(&member);                                                        \
     PHP_PROTO_RETVAL_ZVAL(value);                                              \
   }                                                                            \
   PHP_METHOD(UPPER_CLASS, set##UPPER_FIELD) {                                  \
@@ -390,6 +390,7 @@ PHP_METHOD(Message, whichOneof) {
     zval member;                                                               \
     PHP_PROTO_ZVAL_STRING(&member, LOWER_FIELD, 1);                            \
     message_set_property_internal(getThis(), &member, value TSRMLS_CC);        \
+    zval_dtor(&member);                                                        \
     PHP_PROTO_RETVAL_ZVAL(getThis());                                          \
   }
 
@@ -402,6 +403,7 @@ PHP_METHOD(Message, whichOneof) {
     message_get_oneof_property_internal(getThis(), &member,                    \
                                         return_value TSRMLS_CC);               \
     PHP_PROTO_FAKE_SCOPE_END;                                                  \
+    zval_dtor(&member);                                                        \
   }                                                                            \
   PHP_METHOD(UPPER_CLASS, set##UPPER_FIELD) {                                  \
     zval* value = NULL;                                                        \
@@ -412,6 +414,7 @@ PHP_METHOD(Message, whichOneof) {
     zval member;                                                               \
     PHP_PROTO_ZVAL_STRING(&member, LOWER_FIELD, 1);                            \
     message_set_property_internal(getThis(), &member, value TSRMLS_CC);        \
+    zval_dtor(&member);                                                        \
     PHP_PROTO_RETVAL_ZVAL(getThis());                                          \
   }
 
@@ -1120,38 +1123,60 @@ PHP_METHOD(Timestamp, fromDateTime) {
   zval* datetime;
   zval member;
 
+  PHP_PROTO_CE_DECLARE date_interface_ce;
+  if (php_proto_zend_lookup_class("\\DatetimeInterface", 18,
+                                  &date_interface_ce) == FAILURE) {
+    zend_error(E_ERROR, "Make sure date extension is enabled.");
+    return;
+  }
+
   if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "O", &datetime,
-                            php_date_get_date_ce()) == FAILURE) {
+                            PHP_PROTO_CE_UNREF(date_interface_ce)) == FAILURE) {
+    zend_error(E_USER_ERROR, "Expect DatetimeInterface.");
     return;
   }
 
-  php_date_obj* dateobj = UNBOX(php_date_obj, datetime);
-  if (!dateobj->time->sse_uptodate) {
-    timelib_update_ts(dateobj->time, NULL);
+  // Get timestamp from Datetime object.
+  zval retval;
+  zval function_name;
+  int64_t timestamp;
+
+#if PHP_MAJOR_VERSION < 7
+  INIT_ZVAL(retval);
+  INIT_ZVAL(function_name);
+#endif
+
+  PHP_PROTO_ZVAL_STRING(&function_name, "date_timestamp_get", 1);
+
+  if (call_user_function(EG(function_table), NULL, &function_name, &retval, 1,
+          ZVAL_PTR_TO_CACHED_PTR(datetime) TSRMLS_CC) == FAILURE) {
+    zend_error(E_ERROR, "Cannot get timestamp from DateTime.");
+    return;
   }
 
-  int64_t timestamp = dateobj->time->sse;
+  protobuf_convert_to_int64(&retval, &timestamp);
+
+  zval_dtor(&retval);
+  zval_dtor(&function_name);
 
   // Set seconds
   MessageHeader* self = UNBOX(MessageHeader, getThis());
   const upb_fielddef* field =
       upb_msgdef_ntofz(self->descriptor->msgdef, "seconds");
   void* storage = message_data(self);
   void* memory = slot_memory(self->descriptor->layout, storage, field);
-  *(int64_t*)memory = dateobj->time->sse;
+  *(int64_t*)memory = timestamp;
 
   // Set nanos
   field = upb_msgdef_ntofz(self->descriptor->msgdef, "nanos");
   storage = message_data(self);
   memory = slot_memory(self->descriptor->layout, storage, field);
   *(int32_t*)memory = 0;
+
+  RETURN_NULL();
 }
 
 PHP_METHOD(Timestamp, toDateTime) {
-  zval datetime;
-  php_date_instantiate(php_date_get_date_ce(), &datetime TSRMLS_CC);
-  php_date_obj* dateobj = UNBOX(php_date_obj, &datetime);
-
   // Get seconds
   MessageHeader* self = UNBOX(MessageHeader, getThis());
   const upb_fielddef* field =
@@ -1172,14 +1197,38 @@ PHP_METHOD(Timestamp, toDateTime) {
   strftime(formated_time, sizeof(formated_time), "%Y-%m-%dT%H:%M:%SUTC",
            utc_time);
 
-  if (!php_date_initialize(dateobj, formated_time, strlen(formated_time), NULL,
-                           NULL, 0 TSRMLS_CC)) {
-    zval_dtor(&datetime);
-    RETURN_NULL();
+  // Create Datetime object.
+  zval datetime;
+  zval formated_time_php;
+  zval function_name;
+  int64_t timestamp = 0;
+
+#if PHP_MAJOR_VERSION < 7
+  INIT_ZVAL(function_name);
+  INIT_ZVAL(formated_time_php);
+#endif
+
+  PHP_PROTO_ZVAL_STRING(&function_name, "date_create", 1);
+  PHP_PROTO_ZVAL_STRING(&formated_time_php, formated_time, 1);
+
+  CACHED_VALUE params[1] = {ZVAL_TO_CACHED_VALUE(formated_time_php)};
+
+  if (call_user_function(EG(function_table), NULL,
+                         &function_name, &datetime, 1,
+                         params TSRMLS_CC) == FAILURE) {
+    zend_error(E_ERROR, "Cannot create DateTime.");
+    return;
   }
 
+  zval_dtor(&formated_time_php);
+  zval_dtor(&function_name);
+
+#if PHP_MAJOR_VERSION < 7
   zval* datetime_ptr = &datetime;
   PHP_PROTO_RETVAL_ZVAL(datetime_ptr);
+#else
+  ZVAL_OBJ(return_value, Z_OBJ(datetime));
+#endif
 }
 
 // -----------------------------------------------------------------------------