Fix RestJson serialization of sparse null map values

generator/.DevConfigs/363f8b3b-9377-41b9-9ea0-63cc82b25cbeB.json

@@ -0,0 +1,9 @@
+{
+  "core": {
+    "changeLogMessages": [
+      "Fixed protocol test RestJsonSerializesSparseNullMapValues by adding proper null value handling in sparse map serialization"
+    ],
+    "type": "patch",
+    "updateMinimum": false
+  }
+}

generator/ProtocolTestsGenerator/smithy-dotnet-codegen/src/main/java/software/amazon/smithy/dotnet/codegen/customizations/ProtocolTestCustomizations.java

@@ -100,6 +100,7 @@ private ProtocolTestCustomizations() {
     );
     public static final List<String> VNextTests = Arrays.asList(
             //These are the tests that are failing in v4 after updating to 1.54.0 and artifacts 1.0.3004.0. Each one needs to be investigated.
-            "RestJsonSerializesSparseNullMapValues"
+            "RestJsonNullAndEmptyHeaders",
+            "NullAndEmptyHeaders"
     );
 }

generator/ServiceClientGeneratorLib/Generators/Marshallers/JsonRPCStructureMarshaller.tt

@@ -49,7 +49,12 @@ namespace <#=        this.Config.Namespace #>.Model.Internal.MarshallTransformat
 <#=new string(' ', level * 4)#>            {
 <#=new string(' ', level * 4)#>                context.Writer.WritePropertyName("<#=member.MarshallName#>");
 <#+
-			string memberProperty = variableName + "." + member.PropertyName + (member.IsNullable ? ".Value" : string.Empty);
+			string memberProperty = variableName + "." + member.PropertyName;
+			// Only append .Value for nullable types that aren't already complex types
+			if (member.IsNullable && !member.IsStructure && !member.IsList && !member.IsMap) 
+			{
+				memberProperty += ".Value";
+			}
 			if(member.IsStructure || member.IsList || member.IsMap)
 			{
 				this.ProcessStructure(level + 1, variableName + "." + member.PropertyName, member.Shape);
@@ -153,19 +158,7 @@ namespace <#=        this.Config.Namespace #>.Model.Internal.MarshallTransformat
 			}
 			else if(structure.IsMap)
 			{
-#>
-<#=new string(' ', level * 4)#>            context.Writer.WriteStartObject();
-<#=new string(' ', level * 4)#>            foreach (var <#=flatVariableName#>Kvp in <#=variableName#>)
-<#=new string(' ', level * 4)#>            {
-<#=new string(' ', level * 4)#>                context.Writer.WritePropertyName(<#=flatVariableName#>Kvp.Key);
-<#=new string(' ', level * 4)#>                var <#=flatVariableName#>Value = <#=flatVariableName#>Kvp.Value;
-
-<#+ 
-				ProcessStructure(level + 1, flatVariableName + "Value", structure.ValueShape);
-#>
-<#=new string(' ', level * 4)#>            }
-<#=new string(' ', level * 4)#>            context.Writer.WriteEndObject();
-<#+
+				ProcessMap(level, variableName, structure);
 			}
 			else if(structure.IsDocument)
 			{
@@ -207,25 +200,126 @@ namespace <#=        this.Config.Namespace #>.Model.Internal.MarshallTransformat
 
 			}
 	}
+
+	protected void ProcessMap(int level, string variableName, Shape structure)
+	{
+		string flatVariableName = variableName.Replace(".", "");
+#>
+<#=new string(' ', level * 4)#>            context.Writer.WriteStartObject();
+<#=new string(' ', level * 4)#>            foreach (var <#=flatVariableName#>Kvp in <#=variableName#>)
+<#=new string(' ', level * 4)#>            {
+<#=new string(' ', level * 4)#>                context.Writer.WritePropertyName(<#=flatVariableName#>Kvp.Key);
+<#=new string(' ', level * 4)#>                var <#=flatVariableName#>Value = <#=flatVariableName#>Kvp.Value;
+
+<#+             // Check for null values - only null checks for sparse maps as defined in customizations
+
+                bool isNullableMap = false;
+                if (this.Structure != null)
+                {
+                    var memberWithMap = this.Structure.Members.FirstOrDefault(m => m.PropertyName == variableName.Split('.').Last());
+                    isNullableMap = (memberWithMap != null && memberWithMap.UseNullable);
+                }
+
+                if (isNullableMap)
+                {
 #>
+<#=new string(' ', level * 4)#>                if (<#=flatVariableName#>Value == null)
+<#=new string(' ', level * 4)#>                {
+<#=new string(' ', level * 4)#>                    context.Writer.WriteNullValue();
+<#=new string(' ', level * 4)#>                }
+<#=new string(' ', level * 4)#>                else
+<#=new string(' ', level * 4)#>                {
 <#+
-	protected void DetermineCustomMarshallerJsonWriteMethod(Member member, string memberProperty, int level)
-	{
-		if (String.Equals(member.CustomMarshallerTransformation,"Amazon.Util.AWSSDKUtils.ConvertToUnixEpochMilliseconds", StringComparison.OrdinalIgnoreCase))
-		{
+                }
+
+                if(structure.ValueShape.IsBoolean)
+                {
+                    if(isNullableMap)
+                    {
 #>
-<#=new string(' ', level * 4)#>                context.Writer.WriteNumberValue(<#=member.CustomMarshallerTransformation#>(<#=memberProperty#>));
+<#=new string(' ', level * 4)#>                    context.Writer.WriteBooleanValue(<#=flatVariableName#>Value.Value);
 <#+
-		}
-		else
-		{
+                    }
+                    else
+                    {
 #>
-<#=new string(' ', level * 4)#>                context.Writer.WriteStringValue(<#=member.CustomMarshallerTransformation#>(<#=memberProperty#>));
+<#=new string(' ', level * 4)#>                    context.Writer.WriteBooleanValue(<#=flatVariableName#>Value);
 <#+
-		}	
-	}
+                    }
+                }
+                else if(structure.ValueShape.IsInt || structure.ValueShape.IsLong || structure.ValueShape.IsFloat || structure.ValueShape.IsDouble)
+                {
+                    if(isNullableMap)
+                    {
 #>
+<#=new string(' ', level * 4)#>                    context.Writer.WriteNumberValue(<#=flatVariableName#>Value.Value);
 <#+
+                    }
+                    else
+                    {
+#>
+<#=new string(' ', level * 4)#>                    context.Writer.WriteNumberValue(<#=flatVariableName#>Value);
+<#+
+                    }
+                }
+                else if(structure.ValueShape.IsTimeStamp)
+                {
+                    if(isNullableMap)
+                    {
+                        if (structure.ValueShape.data[Shape.TimestampFormatKey] != null && !string.Equals(structure.ValueShape.data["timestampFormat"].ToString(), "unixTimestamp"))
+                        {
+#>
+<#=new string(' ', level * 4)#>                    context.Writer.WriteStringValue(<#=flatVariableName#>Value.Value);
+<#+
+                        }
+                        else
+                        {
+#>
+<#=new string(' ', level * 4)#>                    context.Writer.WriteNumberValue(Convert.ToInt64(StringUtils.FromDateTimeToUnixTimestamp(<#=flatVariableName#>Value.Value)));
+<#+
+                        }
+                    }
+                    else
+                    {
+                        if (structure.ValueShape.data[Shape.TimestampFormatKey] != null && !string.Equals(structure.ValueShape.data["timestampFormat"].ToString(), "unixTimestamp"))
+                        {
+#>
+<#=new string(' ', level * 4)#>                    context.Writer.WriteStringValue(<#=flatVariableName#>Value);
+<#+
+                        }
+                        else
+                        {
+#>
+<#=new string(' ', level * 4)#>                    context.Writer.WriteNumberValue(Convert.ToInt64(StringUtils.FromDateTimeToUnixTimestamp(<#=flatVariableName#>Value)));
+<#+
+                        }
+                    }
+                }
+                else
+                {
+                    if(isNullableMap)
+                    {
+                            ProcessStructure(level + 2, flatVariableName + "Value", structure.ValueShape);
+                    }
+                    else
+                    {
+                            ProcessStructure(level + 1, flatVariableName + "Value", structure.ValueShape);
+                    }
+                }
+
+                // Close the else block for nullable maps
+                if (isNullableMap)
+                {
+#>
+<#=new string(' ', level * 4)#>                }
+<#+
+                }
+#>
+<#=new string(' ', level * 4)#>            }
+<#=new string(' ', level * 4)#>            context.Writer.WriteEndObject();
+<#+
+	}
+
 	/// https://smithy.io/2.0/aws/protocols/aws-restjson1-protocol.html#json-shape-serialization
 	/// timestamps in json protocols use unixtimestamp if none is specified (by default)
 	protected void DetermineNormalJsonWriteMethod(Shape shape, string memberProperty, int level)
@@ -256,6 +350,7 @@ namespace <#=        this.Config.Namespace #>.Model.Internal.MarshallTransformat
 		}
 		else if (shape.IsInt || shape.IsLong || shape.IsFloat || shape.IsDouble)
 		{
+			// Simple numeric value
 #>
 <#=new string(' ', level * 4)#>                context.Writer.WriteNumberValue(<#=memberProperty#>);
 <#+

generator/TestServiceModels/restjson-tests-client/rest-json-protocol-2019-12-16.normal.json

@@ -389,6 +389,17 @@
       "output":{"shape":"JsonMapsInputOutput"},
       "documentation":"<p>The example tests basic map serialization.</p>"
     },
+    "SparseJsonMaps":{
+      "name":"SparseJsonMaps",
+      "http":{
+        "method":"POST",
+        "requestUri":"/SparseJsonMaps",
+        "responseCode":200
+      },
+      "input":{"shape":"SparseJsonMapsInputOutput"},
+      "output":{"shape":"SparseJsonMapsInputOutput"},
+      "documentation":"<p>The example tests serialization of JSON map values in sparse maps.</p>"
+    },
     "JsonTimestamps":{
       "name":"JsonTimestamps",
       "http":{
@@ -1466,6 +1477,16 @@
         "denseSetMap":{"shape":"DenseSetMap"}
       }
     },
+    "SparseJsonMapsInputOutput":{
+      "type":"structure",
+      "members":{
+        "sparseStructMap":{"shape":"SparseStructMap"},
+        "sparseNumberMap":{"shape":"SparseNumberMap"},
+        "sparseBooleanMap":{"shape":"SparseBooleanMap"},
+        "sparseStringMap":{"shape":"SparseStringMap"},
+        "sparseSetMap":{"shape":"SparseSetMap"}
+      }
+    },
     "JsonTimestampsInputOutput":{
       "type":"structure",
       "members":{
@@ -1929,6 +1950,31 @@
         }
       },
       "union":true
+    },
+    "SparseBooleanMap":{
+      "type":"map",
+      "key":{"shape":"String"},
+      "value":{"shape":"Boolean"}
+    },
+    "SparseNumberMap":{
+      "type":"map",
+      "key":{"shape":"String"},
+      "value":{"shape":"Integer"}
+    },
+    "SparseSetMap":{
+      "type":"map",
+      "key":{"shape":"String"},
+      "value":{"shape":"StringList"}
+    },
+    "SparseStringMap":{
+      "type":"map",
+      "key":{"shape":"String"},
+      "value":{"shape":"String"}
+    },
+    "SparseStructMap":{
+      "type":"map",
+      "key":{"shape":"String"},
+      "value":{"shape":"GreetingStruct"}
     }
   },
   "documentation":"<p>A REST JSON service that sends JSON requests and responses.</p>"

sdk/test/ProtocolTests/Generated/RestJsonProtocol/dotnet-protocol-test-codegen/SparseJsonMaps.cs

@@ -86,12 +86,6 @@ public void RestJsonSparseJsonMapsRequest()
         /// <summary>
         /// Serializes JSON map values in sparse maps
         /// </summary>
-        /*
-        * This test either requires a breaking change and will be addressed
-        * in V4, or has a backlog item to be fixed in the future. Please
-        * refer to the VNextTests list to see which it is.
-        * */
-        [Ignore]
         [TestMethod]
         [TestCategory("ProtocolTest")]
         [TestCategory("RequestTest")]