Sync from Piper @436517178

PROTOBUF_SYNC_PIPER

CHANGES.txt

@@ -1,3 +1,18 @@
+Unreleased Changes
+
+C++
+  * Refactor generated message class layout
+  * Optimize tokenizer ParseInteger by removing division
+  * Reserve exactly the right amount of capacity in ExtensionSet::MergeFrom
+
+Java
+  * 6x speedup in ArrayEncoder.writeUInt32NotTag
+
+Python
+  * Added UnknownFieldSet(message) for pure Python. The old
+    message.UnknownFields() will be deprecated after UnknownFieldSet(message) is
+    added for cpp extension.
+
 2022-03-04 version 3.20.0 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
 
   Ruby

csharp/src/Google.Protobuf.Test.TestProtos/Google.Protobuf.Test.TestProtos.csproj

@@ -6,7 +6,7 @@
     and without the internal visibility from the test project (all of which have caused issues in the past).
   -->
   <PropertyGroup>
-    <TargetFrameworks>net45;netstandard1.1;netstandard2.0</TargetFrameworks>
+    <TargetFrameworks>net462;netstandard1.1;netstandard2.0</TargetFrameworks>
     <LangVersion>3.0</LangVersion>
     <AssemblyOriginatorKeyFile>../../keys/Google.Protobuf.snk</AssemblyOriginatorKeyFile>
     <SignAssembly>true</SignAssembly>

csharp/src/Google.Protobuf.Test/ExtensionSetTest.cs

@@ -1,4 +1,6 @@
-using Google.Protobuf.TestProtos.Proto2;
+using System;
+using System.Collections;
+using Google.Protobuf.TestProtos.Proto2;
 using NUnit.Framework;
 
 using static Google.Protobuf.TestProtos.Proto2.UnittestExtensions;
@@ -79,6 +81,63 @@ public void TryMergeFieldFrom_CodedInputStream()
             Assert.AreEqual("abcd", ExtensionSet.Get(ref extensionSet, OptionalStringExtension));
         }
 
+        [Test]
+        public void GetSingle()
+        {
+            var extensionValue = new TestAllTypes.Types.NestedMessage() { Bb = 42 };
+            var untypedExtension = new Extension<TestAllExtensions, object>(OptionalNestedMessageExtension.FieldNumber, codec: null);
+            var wrongTypedExtension = new Extension<TestAllExtensions, TestAllTypes>(OptionalNestedMessageExtension.FieldNumber, codec: null);
+
+            var message = new TestAllExtensions();
+
+            var value1 = message.GetExtension(untypedExtension);
+            Assert.IsNull(value1);
+
+            message.SetExtension(OptionalNestedMessageExtension, extensionValue);
+            var value2 = message.GetExtension(untypedExtension);
+            Assert.IsNotNull(value2);
+
+            var valueBytes = ((IMessage)value2).ToByteArray();
+            var parsedValue = TestProtos.Proto2.TestAllTypes.Types.NestedMessage.Parser.ParseFrom(valueBytes);
+            Assert.AreEqual(extensionValue, parsedValue);
+
+            var ex = Assert.Throws<InvalidOperationException>(() => message.GetExtension(wrongTypedExtension));
+
+            var expectedMessage = "The stored extension value has a type of 'Google.Protobuf.TestProtos.Proto2.TestAllTypes+Types+NestedMessage, Google.Protobuf.Test.TestProtos, Version=1.0.0.0, Culture=neutral, PublicKeyToken=a7d26565bac4d604'. " +
+                "This a different from the requested type of 'Google.Protobuf.TestProtos.Proto2.TestAllTypes, Google.Protobuf.Test.TestProtos, Version=1.0.0.0, Culture=neutral, PublicKeyToken=a7d26565bac4d604'.";
+            Assert.AreEqual(expectedMessage, ex.Message);
+        }
+
+        [Test]
+        public void GetRepeated()
+        {
+            var extensionValue = new TestAllTypes.Types.NestedMessage() { Bb = 42 };
+            var untypedExtension = new Extension<TestAllExtensions, IList>(RepeatedNestedMessageExtension.FieldNumber, codec: null);
+            var wrongTypedExtension = new RepeatedExtension<TestAllExtensions, TestAllTypes>(RepeatedNestedMessageExtension.FieldNumber, codec: null);
+
+            var message = new TestAllExtensions();
+
+            var value1 = message.GetExtension(untypedExtension);
+            Assert.IsNull(value1);
+
+            var repeatedField = message.GetOrInitializeExtension<TestAllTypes.Types.NestedMessage>(RepeatedNestedMessageExtension);
+            repeatedField.Add(extensionValue);
+
+            var value2 = message.GetExtension(untypedExtension);
+            Assert.IsNotNull(value2);
+            Assert.AreEqual(1, value2.Count);
+
+            var valueBytes = ((IMessage)value2[0]).ToByteArray();
+            var parsedValue = TestProtos.Proto2.TestAllTypes.Types.NestedMessage.Parser.ParseFrom(valueBytes);
+            Assert.AreEqual(extensionValue, parsedValue);
+
+            var ex = Assert.Throws<InvalidOperationException>(() => message.GetExtension(wrongTypedExtension));
+
+            var expectedMessage = "The stored extension value has a type of 'Google.Protobuf.TestProtos.Proto2.TestAllTypes+Types+NestedMessage, Google.Protobuf.Test.TestProtos, Version=1.0.0.0, Culture=neutral, PublicKeyToken=a7d26565bac4d604'. " +
+                "This a different from the requested type of 'Google.Protobuf.TestProtos.Proto2.TestAllTypes, Google.Protobuf.Test.TestProtos, Version=1.0.0.0, Culture=neutral, PublicKeyToken=a7d26565bac4d604'.";
+            Assert.AreEqual(expectedMessage, ex.Message);
+        }
+
         [Test]
         public void TestEquals()
         {

csharp/src/Google.Protobuf.Test/Google.Protobuf.Test.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>net451;netcoreapp3.1;net60</TargetFrameworks>
+    <TargetFrameworks>net462;netcoreapp3.1;net60</TargetFrameworks>
     <AssemblyOriginatorKeyFile>../../keys/Google.Protobuf.snk</AssemblyOriginatorKeyFile>
     <SignAssembly>true</SignAssembly>
     <IsPackable>False</IsPackable>

csharp/src/Google.Protobuf.Test/RefStructCompatibilityTest.cs

@@ -60,7 +60,7 @@ public void GeneratedCodeCompilesWithOldCsharpCompiler()
 
             var currentAssemblyDir = Path.GetDirectoryName(typeof(RefStructCompatibilityTest).GetTypeInfo().Assembly.Location);
             var testProtosProjectDir = Path.GetFullPath(Path.Combine(currentAssemblyDir, "..", "..", "..", "..", "Google.Protobuf.Test.TestProtos"));
-            var testProtosOutputDir = (currentAssemblyDir.Contains("bin/Debug/") || currentAssemblyDir.Contains("bin\\Debug\\")) ? "bin\\Debug\\net45" : "bin\\Release\\net45";
+            var testProtosOutputDir = (currentAssemblyDir.Contains("bin/Debug/") || currentAssemblyDir.Contains("bin\\Debug\\")) ? "bin\\Debug\\net462" : "bin\\Release\\net462";
 
             // If "ref struct" types are used in the generated code, compilation with an old compiler will fail with the following error:
             // "XYZ is obsolete: 'Types with embedded references are not supported in this version of your compiler.'"

csharp/src/Google.Protobuf/Extension.cs

@@ -77,7 +77,7 @@ public Extension(int fieldNumber, FieldCodec<TValue> codec) : base(fieldNumber)
             this.codec = codec;
         }
 
-        internal TValue DefaultValue => codec.DefaultValue;
+        internal TValue DefaultValue => codec != null ? codec.DefaultValue : default(TValue);
 
         internal override Type TargetType => typeof(TTarget);
 

csharp/src/Google.Protobuf/ExtensionSet.cs

@@ -34,6 +34,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Reflection;
 using System.Security;
 
 namespace Google.Protobuf
@@ -63,7 +64,39 @@ public static class ExtensionSet
             IExtensionValue value;
             if (TryGetValue(ref set, extension, out value))
             {
-                return ((ExtensionValue<TValue>)value).GetValue();
+                // The stored ExtensionValue can be a different type to what is being requested.
+                // This happens when the same extension proto is compiled in different assemblies.
+                // To allow consuming assemblies to still get the value when the TValue type is
+                // different, this get method:
+                // 1. Attempts to cast the value to the expected ExtensionValue<TValue>.
+                //    This is the usual case. It is used first because it avoids possibly boxing the value.
+                // 2. Fallback to get the value as object from IExtensionValue then casting.
+                //    This allows for someone to specify a TValue of object. They can then convert
+                //    the values to bytes and reparse using expected value.
+                // 3. If neither of these work, throw a user friendly error that the types aren't compatible.
+                if (value is ExtensionValue<TValue> extensionValue)
+                {
+                    return extensionValue.GetValue();
+                }
+                else if (value.GetValue() is TValue underlyingValue)
+                {
+                    return underlyingValue;
+                }
+                else
+                {
+                    var valueType = value.GetType().GetTypeInfo();
+                    if (valueType.IsGenericType && valueType.GetGenericTypeDefinition() == typeof(ExtensionValue<>))
+                    {
+                        var storedType = valueType.GenericTypeArguments[0];
+                        throw new InvalidOperationException(
+                            "The stored extension value has a type of '" + storedType.AssemblyQualifiedName + "'. " +
+                            "This a different from the requested type of '" + typeof(TValue).AssemblyQualifiedName + "'.");
+                    }
+                    else
+                    {
+                        throw new InvalidOperationException("Unexpected extension value type: " + valueType.AssemblyQualifiedName);
+                    }
+                }
             }
             else 
             {
@@ -79,7 +112,25 @@ public static class ExtensionSet
             IExtensionValue value;
             if (TryGetValue(ref set, extension, out value))
             {
-                return ((RepeatedExtensionValue<TValue>)value).GetValue();
+                if (value is RepeatedExtensionValue<TValue> extensionValue)
+                {
+                    return extensionValue.GetValue();
+                }
+                else
+                {
+                    var valueType = value.GetType().GetTypeInfo();
+                    if (valueType.IsGenericType && valueType.GetGenericTypeDefinition() == typeof(RepeatedExtensionValue<>))
+                    {
+                        var storedType = valueType.GenericTypeArguments[0];
+                        throw new InvalidOperationException(
+                            "The stored extension value has a type of '" + storedType.AssemblyQualifiedName + "'. " +
+                            "This a different from the requested type of '" + typeof(TValue).AssemblyQualifiedName + "'.");
+                    }
+                    else
+                    {
+                        throw new InvalidOperationException("Unexpected extension value type: " + valueType.AssemblyQualifiedName);
+                    }
+                }
             }
             else 
             {

csharp/src/Google.Protobuf/ExtensionValue.cs

@@ -44,6 +44,7 @@ internal interface IExtensionValue : IEquatable<IExtensionValue>, IDeepCloneable
         void WriteTo(ref WriteContext ctx);
         int CalculateSize();
         bool IsInitialized();
+        object GetValue();
     }
 
     internal sealed class ExtensionValue<T> : IExtensionValue
@@ -118,6 +119,8 @@ public void WriteTo(ref WriteContext ctx)
 
         public T GetValue() => field;
 
+        object IExtensionValue.GetValue() => field;
+
         public void SetValue(T value)
         {
             field = value;
@@ -201,6 +204,8 @@ public void WriteTo(ref WriteContext ctx)
 
         public RepeatedField<T> GetValue() => field;
 
+        object IExtensionValue.GetValue() => field;
+
         public bool IsInitialized()
         {
             for (int i = 0; i < field.Count; i++)

docs/options.md

@@ -304,3 +304,8 @@ with info about your project (name and website) so we can add an entry for you.
 1. Embedded Proto
    * Website: https://EmbeddedProto.com
    * Extension: 1141
+
+1. Protoc-gen-fieldmask
+   * Website: https://github.com/yeqown/protoc-gen-fieldmask
+   * Extension: 1142
+