Migrate Swift, Ivy, and pip detectors from Newtonsoft.Json to System.Text.Json

src/Microsoft.ComponentDetection.Detectors/ivy/IvyDetector.cs

@@ -7,14 +7,14 @@ namespace Microsoft.ComponentDetection.Detectors.Ivy;
 using System.Linq;
 using System.Reactive.Linq;
 using System.Reflection;
+using System.Text.Json.Nodes;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.ComponentDetection.Contracts;
 using Microsoft.ComponentDetection.Contracts.BcdeModels;
 using Microsoft.ComponentDetection.Contracts.Internal;
 using Microsoft.ComponentDetection.Contracts.TypedComponent;
 using Microsoft.Extensions.Logging;
-using Newtonsoft.Json.Linq;
 
 /// <summary>
 /// Detector for Maven components declared in ivy.xml files for Java projects that are built using Apache Ant
@@ -110,17 +110,17 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID
         }
     }
 
-    private static MavenComponent JsonGavToComponent(JToken gav)
+    private static MavenComponent JsonGavToComponent(JsonNode gav)
     {
         if (gav == null)
         {
             return null;
         }
 
         return new MavenComponent(
-            gav.Value<string>("g"),
-            gav.Value<string>("a"),
-            gav.Value<string>("v"));
+            gav["g"]?.GetValue<string>(),
+            gav["a"]?.GetValue<string>(),
+            gav["v"]?.GetValue<string>());
     }
 
     private async Task ProcessIvyAndIvySettingsFilesAsync(
@@ -222,14 +222,15 @@ private async Task<bool> RunAntToDetectDependenciesAsync(string workingDirectory
 
     private void RegisterUsagesFromFile(ISingleFileComponentRecorder singleFileComponentRecorder, string instructionsFile)
     {
-        var instructionsJson = JObject.Parse(File.ReadAllText(instructionsFile));
-        var instructionsList = (JContainer)instructionsJson["RegisterUsage"];
+        var instructionsJson = JsonNode.Parse(File.ReadAllText(instructionsFile));
+        var instructionsList = instructionsJson?["RegisterUsage"]?.AsArray();
+
         foreach (var dep in instructionsList)
         {
             var component = JsonGavToComponent(dep["gav"]);
-            var isDevDependency = dep.Value<bool>("DevelopmentDependency");
+            var isDevDependency = dep["DevelopmentDependency"]?.GetValue<bool>() ?? false;
             var parentComponent = JsonGavToComponent(dep["parent_gav"]);
-            var isResolved = dep.Value<bool>("resolved");
+            var isResolved = dep["resolved"]?.GetValue<bool>() ?? false;
             if (isResolved)
             {
                 singleFileComponentRecorder.RegisterUsage(

src/Microsoft.ComponentDetection.Detectors/pip/Contracts/PythonProject.cs

@@ -2,14 +2,17 @@
 namespace Microsoft.ComponentDetection.Detectors.Pip;
 
 using System.Collections.Generic;
+using System.Text.Json.Serialization;
 
 /// <summary>
 /// A project on pypi.
 /// </summary>
 public class PythonProject
 {
+    [JsonPropertyName("releases")]
     public SortedDictionary<string, IList<PythonProjectRelease>> Releases { get; set; }
 
 #nullable enable
+    [JsonPropertyName("info")]
     public PythonProjectInfo? Info { get; set; }
 }

src/Microsoft.ComponentDetection.Detectors/pip/Contracts/PythonProjectInfo.cs

@@ -3,14 +3,12 @@ namespace Microsoft.ComponentDetection.Detectors.Pip;
 
 using System.Collections.Generic;
 using System.Text.Json.Serialization;
-using Newtonsoft.Json;
 
 public class PythonProjectInfo
 {
     [JsonPropertyName("author")]
     public string Author { get; set; }
 
-    [JsonProperty("author_email")]
     [JsonPropertyName("author_email")]
     public string AuthorEmail { get; set; }
 
@@ -23,7 +21,6 @@ public class PythonProjectInfo
     [JsonPropertyName("maintainer")]
     public string Maintainer { get; set; }
 
-    [JsonProperty("maintainer_email")]
     [JsonPropertyName("maintainer_email")]
     public string MaintainerEmail { get; set; }
 

src/Microsoft.ComponentDetection.Detectors/pip/Contracts/PythonProjectRelease.cs

@@ -3,7 +3,6 @@ namespace Microsoft.ComponentDetection.Detectors.Pip;
 
 using System;
 using System.Text.Json.Serialization;
-using Newtonsoft.Json;
 
 /// <summary>
 /// A specific release of a project on pypy.
@@ -13,7 +12,6 @@ public class PythonProjectRelease
     [JsonPropertyName("packagetype")]
     public string PackageType { get; set; }
 
-    [JsonProperty("python_version")]
     [JsonPropertyName("python_version")]
     public string PythonVersion { get; set; }
 

src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs

@@ -8,13 +8,14 @@
 using System.Net.Http.Headers;
 using System.Reflection;
 using System.Runtime.CompilerServices;
+using System.Text.Encodings.Web;
+using System.Text.Json;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.ComponentDetection.Common.Telemetry.Records;
 using Microsoft.ComponentDetection.Contracts;
 using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Logging;
-using Newtonsoft.Json;
 using Polly;
 
 [assembly: InternalsVisibleTo("Microsoft.ComponentDetection.Detectors.Tests")]
@@ -49,6 +50,11 @@ public sealed class PyPiClient : IPyPiClient, IDisposable
 
     private static readonly ProductInfoHeaderValue CommentValue = new("(+https://github.com/microsoft/component-detection)");
 
+    private static readonly JsonSerializerOptions JsonSerializerOptions = new()
+    {
+        Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+    };
+
     // Keep telemetry on how the cache is being used for future refinements
     private readonly PypiCacheTelemetryRecord cacheTelemetry;
 
@@ -201,7 +207,7 @@ public async Task<PythonProject> GetProjectAsync(PipDependencySpecification spec
         }
 
         var response = await request.Content.ReadAsStringAsync();
-        var project = JsonConvert.DeserializeObject<PythonProject>(response);
+        var project = JsonSerializer.Deserialize<PythonProject>(response);
         var versions = new PythonProject
         {
             Info = project.Info,
@@ -226,7 +232,7 @@ public async Task<PythonProject> GetProjectAsync(PipDependencySpecification spec
                     ae,
                     "Component {ReleaseKey} : {ReleaseValue} could not be added to the sorted list of pip components for spec={SpecName}. Usually this happens with unexpected PyPi version formats (e.g. prerelease/dev versions).",
                     release.Key,
-                    JsonConvert.SerializeObject(release.Value),
+                    JsonSerializer.Serialize(release.Value, JsonSerializerOptions),
                     spec.Name);
                 continue;
             }

src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs

@@ -4,15 +4,21 @@ namespace Microsoft.ComponentDetection.Detectors.Pip;
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Text.Encodings.Web;
+using System.Text.Json;
 using System.Threading.Tasks;
 using Microsoft.ComponentDetection.Contracts;
 using Microsoft.ComponentDetection.Contracts.TypedComponent;
 using Microsoft.Extensions.Logging;
 using MoreLinq;
-using Newtonsoft.Json;
 
 public class PythonResolver : PythonResolverBase, IPythonResolver
 {
+    private static readonly JsonSerializerOptions JsonSerializerOptions = new()
+    {
+        Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+    };
+
     private readonly IPyPiClient pypiClient;
     private readonly ILogger<PythonResolver> logger;
     private readonly Dictionary<string, string> pythonEnvironmentVariables = [];
@@ -69,7 +75,7 @@ public async Task<IList<PipGraphNode>> ResolveRootsAsync(ISingleFileComponentRec
                     this.logger.LogWarning(
                         "Unable to resolve root dependency {PackageName} with version specifiers {PackageVersions} from pypi possibly due to computed version constraints. Skipping package.",
                         rootPackage.Name,
-                        JsonConvert.SerializeObject(rootPackage.DependencySpecifiers));
+                        JsonSerializer.Serialize(rootPackage.DependencySpecifiers, JsonSerializerOptions));
                     singleFileComponentRecorder.RegisterPackageParseFailure(rootPackage.Name);
                 }
             }
@@ -136,7 +142,7 @@ private async Task<IList<PipGraphNode>> ProcessQueueAsync(ISingleFileComponentRe
                             this.logger.LogWarning(
                                 "Unable to resolve non-root dependency {PackageName} with version specifiers {PackageVersions} from pypi possibly due to computed version constraints. Skipping package.",
                                 dependencyNode.Name,
-                                JsonConvert.SerializeObject(dependencyNode.DependencySpecifiers));
+                                JsonSerializer.Serialize(dependencyNode.DependencySpecifiers, JsonSerializerOptions));
                             singleFileComponentRecorder.RegisterPackageParseFailure(dependencyNode.Name);
                         }
                     }

src/Microsoft.ComponentDetection.Detectors/pip/PythonResolverBase.cs

@@ -3,14 +3,20 @@ namespace Microsoft.ComponentDetection.Detectors.Pip;
 
 using System.Collections.Generic;
 using System.Linq;
+using System.Text.Encodings.Web;
+using System.Text.Json;
 using System.Threading.Tasks;
 using Microsoft.ComponentDetection.Contracts.TypedComponent;
 using Microsoft.Extensions.Logging;
 using MoreLinq;
-using Newtonsoft.Json;
 
 public abstract class PythonResolverBase
 {
+    private static readonly JsonSerializerOptions JsonSerializerOptions = new()
+    {
+        Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+    };
+
     private readonly ILogger logger;
 
     internal PythonResolverBase(ILogger logger) => this.logger = logger;
@@ -93,8 +99,8 @@ public Dictionary<string, PipDependencySpecification> ResolveDependencySpecifica
                     "Duplicate package dependencies entry for component:{ComponentName} with dependency:{DependencyName}. Existing dependency specifiers: {ExistingSpecifiers}. New dependency specifiers: {NewSpecifiers}.",
                     component.Name,
                     d.Name,
-                    JsonConvert.SerializeObject(dependencies[d.Name].DependencySpecifiers),
-                    JsonConvert.SerializeObject(d.DependencySpecifiers));
+                    JsonSerializer.Serialize(dependencies[d.Name].DependencySpecifiers, JsonSerializerOptions),
+                    JsonSerializer.Serialize(d.DependencySpecifiers, JsonSerializerOptions));
                 dependencies[d.Name] = d;
             }
         });

src/Microsoft.ComponentDetection.Detectors/pip/SimplePythonResolver.cs

@@ -6,18 +6,24 @@ namespace Microsoft.ComponentDetection.Detectors.Pip;
 using System.IO;
 using System.IO.Compression;
 using System.Linq;
+using System.Text.Encodings.Web;
+using System.Text.Json;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using Microsoft.ComponentDetection.Contracts;
 using Microsoft.ComponentDetection.Contracts.TypedComponent;
 using Microsoft.Extensions.Logging;
 using MoreLinq;
-using Newtonsoft.Json;
 
 public class SimplePythonResolver : PythonResolverBase, ISimplePythonResolver
 {
     private static readonly Regex VersionRegex = new(@"-((\d+)((\.)\w+((\+|\.)\w*)*)*)(.tar|-)", RegexOptions.Compiled);
 
+    private static readonly JsonSerializerOptions JsonSerializerOptions = new()
+    {
+        Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+    };
+
     private readonly ISimplePyPiClient simplePypiClient;
     private readonly ILogger<SimplePythonResolver> logger;
 
@@ -141,7 +147,7 @@ await Parallel.ForEachAsync(initialPackages, async (rootPackage, ct) =>
                     this.logger.LogWarning(
                         "Unable to resolve root dependency {PackageName} with version specifiers {PackageVersions} from pypi possibly due to computed version constraints. Skipping package.",
                         rootPackage.Name,
-                        JsonConvert.SerializeObject(rootPackage.DependencySpecifiers));
+                        JsonSerializer.Serialize(rootPackage.DependencySpecifiers, JsonSerializerOptions));
                     singleFileComponentRecorder.RegisterPackageParseFailure(rootPackage.Name);
                 }
             }
@@ -215,7 +221,7 @@ private async Task<IList<PipGraphNode>> ProcessQueueAsync(ISingleFileComponentRe
                             this.logger.LogWarning(
                                 "Unable to resolve non-root dependency {PackageName} with version specifiers {PackageVersions} from pypi possibly due to computed version constraints. Skipping package.",
                                 dependencyNode.Name,
-                                JsonConvert.SerializeObject(dependencyNode.DependencySpecifiers));
+                                JsonSerializer.Serialize(dependencyNode.DependencySpecifiers, JsonSerializerOptions));
                             singleFileComponentRecorder.RegisterPackageParseFailure(dependencyNode.Name);
                         }
                     }
@@ -272,7 +278,7 @@ private SortedDictionary<string, IList<PythonProjectRelease>> ConvertSimplePypiP
                 this.logger.LogError(
                     ae,
                     "Release {Release} could not be added to the sorted list of pip components for spec={SpecName}. Usually this happens with unexpected PyPi version formats (e.g. prerelease/dev versions).",
-                    JsonConvert.SerializeObject(file),
+                    JsonSerializer.Serialize(file, JsonSerializerOptions),
                     spec.Name);
             }
         }

src/Microsoft.ComponentDetection.Detectors/swiftpm/Contracts/SwiftResolvedFile.cs

@@ -3,58 +3,48 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent;
 
 using System.Collections.Generic;
 using System.Text.Json.Serialization;
-using Newtonsoft.Json;
 
 /// <summary>
 /// Represents a Swift Package Manager component.
 /// </summary>
 public class SwiftResolvedFile
 {
-    [JsonProperty("pins")]
     [JsonPropertyName("pins")]
     public IList<SwiftDependency> Pins { get; set; }
 
-    [JsonProperty("version")]
     [JsonPropertyName("version")]
     public int Version { get; set; }
 
     public class SwiftDependency
     {
         // The name of the package
-        [JsonProperty("identity")]
         [JsonPropertyName("identity")]
         public string Identity { get; set; }
 
         // How the package is imported. Example: "remoteSourceControl"
         // This is not an enum because the Swift contract does not specify the possible values.
-        [JsonProperty("kind")]
         [JsonPropertyName("kind")]
         public string Kind { get; set; }
 
         // The unique path to the repository where the package is located. Example: Git repo URL.
-        [JsonProperty("location")]
         [JsonPropertyName("location")]
         public string Location { get; set; }
 
         // Data about the package version and commit hash.
-        [JsonProperty("state")]
         [JsonPropertyName("state")]
         public SwiftState State { get; set; }
 
         public class SwiftState
         {
             // The commit hash of the package.
-            [JsonProperty("revision")]
             [JsonPropertyName("revision")]
             public string Revision { get; set; }
 
             // The version of the package. Might be missing.
-            [JsonProperty("version")]
             [JsonPropertyName("version")]
             public string Version { get; set; }
 
             // The branch of the package. Might be missing.
-            [JsonProperty("branch")]
             [JsonPropertyName("branch")]
             public string Branch { get; set; }
         }

src/Microsoft.ComponentDetection.Detectors/swiftpm/SwiftResolvedComponentDetector.cs

@@ -4,13 +4,13 @@ namespace Microsoft.ComponentDetection.Detectors.Swift;
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Text.Json;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.ComponentDetection.Contracts;
 using Microsoft.ComponentDetection.Contracts.Internal;
 using Microsoft.ComponentDetection.Contracts.TypedComponent;
 using Microsoft.Extensions.Logging;
-using Newtonsoft.Json;
 
 /// <summary>
 /// Detects Swift Package Manager components.
@@ -109,6 +109,6 @@ private SwiftResolvedFile ReadAndParseResolvedFile(Stream stream)
             resolvedFile = reader.ReadToEnd();
         }
 
-        return JsonConvert.DeserializeObject<SwiftResolvedFile>(resolvedFile);
+        return JsonSerializer.Deserialize<SwiftResolvedFile>(resolvedFile);
     }
 }

test/Microsoft.ComponentDetection.Detectors.Tests/IvyDetectorTests.cs

@@ -47,7 +47,7 @@ public async Task AntAvailableHappyPathAsync()
                                    "{ \"gav\": { \"g\": \"d0g\", \"a\": \"d0a\", \"v\": \"0.0.0\"}, \"DevelopmentDependency\": false, \"resolved\": false},\n" +
                                    "{ \"gav\": { \"g\": \"d1g\", \"a\": \"d1a\", \"v\": \"1.1.1\"}, \"DevelopmentDependency\": true, \"resolved\": true},\n" +
                                    "{ \"gav\": { \"g\": \"d2g\", \"a\": \"d2a\", \"v\": \"2.2.2\"}, \"DevelopmentDependency\": false, \"resolved\": true},\n" +
-                                   "{ \"gav\": { \"g\": \"d3g\", \"a\": \"d3a\", \"v\": \"3.3.3\"}, \"DevelopmentDependency\": false, \"resolved\": true, \"parent_gav\": { \"g\": \"d2g\", \"a\": \"d2a\", \"v\": \"2.2.2\"}},\n" +
+                                   "{ \"gav\": { \"g\": \"d3g\", \"a\": \"d3a\", \"v\": \"3.3.3\"}, \"DevelopmentDependency\": false, \"resolved\": true, \"parent_gav\": { \"g\": \"d2g\", \"a\": \"d2a\", \"v\": \"2.2.2\"}}\n" +
                                    "]}";
 
         this.IvyHappyPath(content: registerUsageContent);
@@ -93,7 +93,7 @@ public async Task IvyDependencyGraphAsync()
                                    "{ \"gav\": { \"g\": \"d0g\", \"a\": \"d0a\", \"v\": \"0.0.0\"}, \"DevelopmentDependency\": false, \"resolved\": false},\n" +
                                    "{ \"gav\": { \"g\": \"d1g\", \"a\": \"d1a\", \"v\": \"1.1.1\"}, \"DevelopmentDependency\": true, \"resolved\": true},\n" +
                                    "{ \"gav\": { \"g\": \"d2g\", \"a\": \"d2a\", \"v\": \"2.2.2\"}, \"DevelopmentDependency\": false, \"resolved\": true},\n" +
-                                   "{ \"gav\": { \"g\": \"d3g\", \"a\": \"d3a\", \"v\": \"3.3.3\"}, \"DevelopmentDependency\": false, \"resolved\": true, \"parent_gav\": { \"g\": \"d2g\", \"a\": \"d2a\", \"v\": \"2.2.2\"}},\n" +
+                                   "{ \"gav\": { \"g\": \"d3g\", \"a\": \"d3a\", \"v\": \"3.3.3\"}, \"DevelopmentDependency\": false, \"resolved\": true, \"parent_gav\": { \"g\": \"d2g\", \"a\": \"d2a\", \"v\": \"2.2.2\"}}\n" +
                                    "]}";
 
         var d1Id = "d1g d1a 1.1.1 - Maven";