Add support for preserve references on JSON

src/libraries/System.Text.Json/ref/System.Text.Json.cs

@@ -463,6 +463,7 @@ public sealed partial class JsonSerializerOptions
         public bool PropertyNameCaseInsensitive { get { throw null; } set { } }
         public System.Text.Json.JsonNamingPolicy? PropertyNamingPolicy { get { throw null; } set { } }
         public System.Text.Json.JsonCommentHandling ReadCommentHandling { get { throw null; } set { } }
+        public System.Text.Json.Serialization.ReferenceHandling ReferenceHandling { get { throw null; } set { } }
         public bool WriteIndented { get { throw null; } set { } }
         public System.Text.Json.Serialization.JsonConverter? GetConverter(System.Type typeToConvert) { throw null; }
     }
@@ -775,4 +776,10 @@ public sealed partial class JsonStringEnumConverter : System.Text.Json.Serializa
         public override bool CanConvert(System.Type typeToConvert) { throw null; }
         public override System.Text.Json.Serialization.JsonConverter CreateConverter(System.Type typeToConvert, System.Text.Json.JsonSerializerOptions options) { throw null; }
     }
+    public sealed partial class ReferenceHandling
+    {
+        internal ReferenceHandling() { }
+        public static System.Text.Json.Serialization.ReferenceHandling Default { get { throw null; } }
+        public static System.Text.Json.Serialization.ReferenceHandling Preserve { get { throw null; } }
+    }
 }

src/libraries/System.Text.Json/src/Resources/Strings.resx

@@ -421,7 +421,7 @@
     <value>Either the JSON value is not in a supported format, or is out of bounds for a UInt16.</value>
   </data>
   <data name="SerializerCycleDetected" xml:space="preserve">
-    <value>A possible object cycle was detected which is not supported. This can either be due to a cycle or if the object depth is larger than the maximum allowed depth of {0}.</value>
+    <value>A possible object cycle was detected. This can either be due to a cycle or if the object depth is larger than the maximum allowed depth of {0}. Consider using ReferenceHanlding.Preserve on JsonSerializerOptions to support cycles.</value>
   </data>
   <data name="EmptyStringToInitializeNumber" xml:space="preserve">
     <value>Expected a number, but instead got empty string.</value>
@@ -447,4 +447,42 @@
   <data name="NotNodeJsonElementParent" xml:space="preserve">
     <value>This JsonElement instance was not built from a JsonNode and is immutable.</value>
   </data>
-</root>
+  <data name="MetadataCannotParsePreservedObjectToImmutable" xml:space="preserve">
+    <value>Cannot parse a JSON object containing metadata properties like '$id' into an array or immutable collection type. Type '{0}'.</value>
+  </data>
+  <data name="MetadataDuplicateIdFound" xml:space="preserve">
+    <value>The value of the '$id' metadata property '{0}' conflicts with an existing identifier.</value>
+  </data>
+  <data name="MetadataIdIsNotFirstProperty" xml:space="preserve">
+    <value>The metadata property '$id' must be the first property in the JSON object.</value>
+  </data>
+  <data name="MetadataInvalidReferenceToValueType" xml:space="preserve">
+    <value>Invalid reference to value type '{0}'.</value>
+  </data>
+  <data name="MetadataInvalidTokenAfterValues" xml:space="preserve">
+    <value>The '$values' metadata property must be a JSON array. Current token type is '{0}'.</value>
+  </data>
+  <data name="MetadataPreservedArrayFailed" xml:space="preserve">
+    <value>Deserialization failed for one of these reasons:
+1. {0}
+2. {1}</value>
+  </data>
+  <data name="MetadataPreservedArrayInvalidProperty" xml:space="preserve">
+    <value>Invalid property '{0}' found within a JSON object that must only contain metadata properties and the nested JSON array to be preserved.</value>
+  </data>
+  <data name="MetadataPreservedArrayPropertyNotFound" xml:space="preserve">
+    <value>One or more metadata properties, such as '$id' and '$values', were not found within a JSON object that must only contain metadata properties and the nested JSON array to be preserved.</value>
+  </data>
+  <data name="MetadataReferenceCannotContainOtherProperties" xml:space="preserve">
+    <value>A JSON object that contains a '$ref' metadata property must not contain any other properties.</value>
+  </data>
+  <data name="MetadataReferenceNotFound" xml:space="preserve">
+    <value>Reference '{0}' not found.</value>
+  </data>
+  <data name="MetadataValueWasNotString" xml:space="preserve">
+    <value>The '$id' and '$ref' metadata properties must be JSON strings. Current token type is '{0}'.</value>
+  </data>
+  <data name="MetadataInvalidPropertyWithLeadingDollarSign" xml:space="preserve">
+    <value>Properties that start with '$' are not allowed on preserve mode, either escape the character or turn off preserve references by setting ReferenceHandling to ReferenceHandling.Default.</value>
+  </data>
+</root>

src/libraries/System.Text.Json/src/System.Text.Json.csproj

@@ -78,6 +78,7 @@
     <Compile Include="System\Text\Json\Serialization\Converters\JsonValueConverterUInt32.cs" />
     <Compile Include="System\Text\Json\Serialization\Converters\JsonValueConverterUInt64.cs" />
     <Compile Include="System\Text\Json\Serialization\Converters\JsonValueConverterUri.cs" />
+    <Compile Include="System\Text\Json\Serialization\DefaultReferenceResolver.cs" />
     <Compile Include="System\Text\Json\Serialization\ExtensionDataWriteStatus.cs" />
     <Compile Include="System\Text\Json\Serialization\ImmutableCollectionCreator.cs" />
     <Compile Include="System\Text\Json\Serialization\JsonAttribute.cs" />
@@ -94,6 +95,7 @@
     <Compile Include="System\Text\Json\Serialization\JsonExtensionDataAttribute.cs" />
     <Compile Include="System\Text\Json\Serialization\JsonIgnoreAttribute.cs" />
     <Compile Include="System\Text\Json\Serialization\JsonNamingPolicy.cs" />
+    <Compile Include="System\Text\Json\Serialization\JsonPreservableArrayReference.cs" />
     <Compile Include="System\Text\Json\Serialization\JsonPropertyInfo.cs" />
     <Compile Include="System\Text\Json\Serialization\JsonPropertyInfoCommon.cs" />
     <Compile Include="System\Text\Json\Serialization\JsonPropertyInfoNotNullable.cs" />
@@ -102,6 +104,7 @@
     <Compile Include="System\Text\Json\Serialization\JsonPropertyNameAttribute.cs" />
     <Compile Include="System\Text\Json\Serialization\JsonSerializer.Read.HandleArray.cs" />
     <Compile Include="System\Text\Json\Serialization\JsonSerializer.Read.HandleDictionary.cs" />
+    <Compile Include="System\Text\Json\Serialization\JsonSerializer.Read.HandleMetadata.cs" />
     <Compile Include="System\Text\Json\Serialization\JsonSerializer.Read.HandleObject.cs" />
     <Compile Include="System\Text\Json\Serialization\JsonSerializer.Read.HandlePropertyName.cs" />
     <Compile Include="System\Text\Json\Serialization\JsonSerializer.Read.HandleValue.cs" />
@@ -125,10 +128,13 @@
     <Compile Include="System\Text\Json\Serialization\JsonSerializerOptions.Converters.cs" />
     <Compile Include="System\Text\Json\Serialization\JsonStringEnumConverter.cs" />
     <Compile Include="System\Text\Json\Serialization\MemberAccessor.cs" />
+    <Compile Include="System\Text\Json\Serialization\MetadataPropertyName.cs" />
     <Compile Include="System\Text\Json\Serialization\PooledByteBufferWriter.cs" />
     <Compile Include="System\Text\Json\Serialization\PropertyRef.cs" />
     <Compile Include="System\Text\Json\Serialization\ReadStack.cs" />
     <Compile Include="System\Text\Json\Serialization\ReadStackFrame.cs" />
+    <Compile Include="System\Text\Json\Serialization\ReferenceEqualsEqualityComparer.cs" />
+    <Compile Include="System\Text\Json\Serialization\ReferenceHandling.cs" />
     <Compile Include="System\Text\Json\Serialization\ReflectionEmitMemberAccessor.cs" />
     <Compile Include="System\Text\Json\Serialization\ReflectionMemberAccessor.cs" />
     <Compile Include="System\Text\Json\Serialization\WriteStack.cs" />

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/DefaultReferenceResolver.cs

@@ -0,0 +1,92 @@
+// Licensed to the.NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+
+namespace System.Text.Json.Serialization
+{
+    /// <summary>
+    /// The default ReferenceResolver implementation to handle duplicate object references.
+    /// </summary>
+    /// <remarks>
+    /// It is currently a struct to save one unnecessary allcation while (de)serializing.
+    /// If we choose to expose the ReferenceResolver in a future, we may need to create an abstract class/interface and change this type to become a class that inherits from that abstract class/interface.
+    /// </remarks>
+    internal struct DefaultReferenceResolver
+    {
+        private uint _referenceCount;
+        private readonly Dictionary<string, object>? _referenceIdToObjectMap;
+        private readonly Dictionary<object, string>? _objectToReferenceIdMap;
+
+        public DefaultReferenceResolver(bool writing)
+        {
+            _referenceCount = default;
+
+            if (writing)
+            {
+                // Comparer used here to always do a Reference Equality comparison on serialization which is where we use the objects as the TKey in our dictionary.
+                _objectToReferenceIdMap = new Dictionary<object, string>(ReferenceEqualsEqualityComparer<object>.Comparer);
+                _referenceIdToObjectMap = null;
+            }
+            else
+            {
+                _referenceIdToObjectMap = new Dictionary<string, object>();
+                _objectToReferenceIdMap = null;
+            }
+        }
+
+
+        /// <summary>
+        /// Adds an entry to the bag of references using the specified id and value.
+        /// This method gets called when an $id metadata property from a JSON object is read.
+        /// </summary>
+        /// <param name="referenceId">The identifier of the respective JSON object or array.</param>
+        /// <param name="value">The value of the respective CLR reference type object that results from parsing the JSON object.</param>
+        public void AddReferenceOnDeserialize(string referenceId, object value)
+        {
+            if (!JsonHelpers.TryAdd(_referenceIdToObjectMap!, referenceId, value))
+            {
+                ThrowHelper.ThrowJsonException_MetadataDuplicateIdFound(referenceId);
+            }
+        }
+
+        /// <summary>
+        /// Gets the reference id of the specified value if exists; otherwise a new id is assigned.
+        /// This method gets called before a CLR object is written so we can decide whether to write $id and the rest of its properties or $ref and step into the next object.
+        /// The first $id value will be 1.
+        /// </summary>
+        /// <param name="value">The value of the CLR reference type object to get or add an id for.</param>
+        /// <param name="referenceId">The id realated to the object.</param>
+        /// <returns></returns>
+        public bool TryGetOrAddReferenceOnSerialize(object value, out string referenceId)
+        {
+            if (!_objectToReferenceIdMap!.TryGetValue(value, out referenceId!))
+            {
+                _referenceCount++;
+                referenceId = _referenceCount.ToString();
+                _objectToReferenceIdMap.Add(value, referenceId);
+
+                return false;
+            }
+
+            return true;
+        }
+
+        /// <summary>
+        /// Resolves the CLR reference type object related to the specified reference id.
+        /// This method gets called when $ref metadata property is read.
+        /// </summary>
+        /// <param name="referenceId">The id related to the returned object.</param>
+        /// <returns></returns>
+        public object ResolveReferenceOnDeserialize(string referenceId)
+        {
+            if (!_referenceIdToObjectMap!.TryGetValue(referenceId, out object? value))
+            {
+                ThrowHelper.ThrowJsonException_MetadataReferenceNotFound(referenceId);
+            }
+
+            return value;
+        }
+    }
+}

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonPreservableArrayReference.cs

@@ -0,0 +1,18 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+namespace System.Text.Json
+{
+    /// <summary>
+    /// JSON objects that contain metadata properties and the nested JSON array are wrapped into this class.
+    /// </summary>
+    /// <typeparam name="T">The original type of the enumerable.</typeparam>
+    internal class JsonPreservableArrayReference<T>
+    {
+        /// <summary>
+        ///  The actual enumerable instance being preserved is extracted when we finish processing the JSON object on HandleEndObject.
+        /// </summary>
+        public T Values { get; set; } = default!;
+    }
+}

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonPropertyInfo.cs

@@ -603,5 +603,7 @@ private void VerifyWrite(int originalDepth, Utf8JsonWriter writer)
                 ThrowHelper.ThrowJsonException_SerializationConverterWrite(ConverterBase);
             }
         }
+
+        public abstract Type GetJsonPreservableArrayReferenceType();
     }
 }

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonPropertyInfoCommon.cs

@@ -235,5 +235,10 @@ public override IDictionary CreateImmutableDictionaryInstance(ref ReadStack stat
 
             return collection;
         }
+
+        public override Type GetJsonPreservableArrayReferenceType()
+        {
+            return typeof(JsonPreservableArrayReference<TDeclaredProperty>);
+        }
     }
 }

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.HandleArray.cs

@@ -212,7 +212,7 @@ private static void HandleStartArray(JsonSerializerOptions options, ref ReadStac
             else if (state.Current.IsProcessingObject(ClassType.Dictionary) || (state.Current.IsProcessingProperty(ClassType.Dictionary) && !setPropertyDirectly))
             {
                 string? key = state.Current.KeyName;
-                Debug.Assert(!string.IsNullOrEmpty(key));
+                Debug.Assert(key != null);
 
                 if (state.Current.TempDictionaryValues != null)
                 {
@@ -288,7 +288,7 @@ private static void HandleStartArray(JsonSerializerOptions options, ref ReadStac
             else if (state.Current.IsProcessingDictionary())
             {
                 string? key = state.Current.KeyName;
-                Debug.Assert(!string.IsNullOrEmpty(key));
+                Debug.Assert(key != null);
 
                 if (state.Current.TempDictionaryValues != null)
                 {

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.HandleDictionary.cs

@@ -31,6 +31,8 @@ private static void HandleStartDictionary(JsonSerializerOptions options, ref Rea
 
                 JsonClassInfo classInfo = state.Current.JsonClassInfo;
 
+                Debug.Assert(state.Current.IsProcessingDictionary() || state.Current.IsProcessingObject(ClassType.Object) || state.Current.IsProcessingObject(ClassType.Enumerable));
+
                 if (state.Current.IsProcessingDictionary())
                 {
                     object? dictValue = ReadStackFrame.CreateDictionaryValue(ref state);
@@ -53,9 +55,15 @@ private static void HandleStartDictionary(JsonSerializerOptions options, ref Rea
 
                     state.Current.ReturnValue = classInfo.CreateObject();
                 }
-                else
+                else if (state.Current.IsProcessingObject(ClassType.Enumerable))
                 {
-                    ThrowHelper.ThrowJsonException_DeserializeUnableToConvertValue(classInfo.Type);
+                    // Array with metadata within the dictionary.
+                    HandleStartObjectInEnumerable(ref state, options, classInfo.Type);
+
+                    Debug.Assert(options.ReferenceHandling.ShouldReadPreservedReferences());
+                    Debug.Assert(state.Current.JsonClassInfo!.Type.GetGenericTypeDefinition() == typeof(JsonPreservableArrayReference<>));
+
+                    state.Current.ReturnValue = state.Current.JsonClassInfo.CreateObject!();
                 }
 
                 return;