From 0545fe4620156d13d324fe1143ed47013b8168d3 Mon Sep 17 00:00:00 2001 From: Jamie Magee Date: Fri, 19 Aug 2022 11:21:58 -0700 Subject: [PATCH 01/11] WIP --- Directory.Packages.props | 1 + .../Microsoft.ComponentDetection.Detectors.csproj | 3 ++- .../poetry/PoetryComponentDetector.cs | 11 ++++++----- .../rust/Contracts/CargoLock.cs | 3 ++- .../rust/RustCrateDetector.cs | 1 - 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 4ce7a50ef..bc2857909 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -40,6 +40,7 @@ + diff --git a/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj b/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj index 26c167bc5..d58c12aa9 100644 --- a/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj +++ b/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj @@ -18,6 +18,7 @@ + @@ -34,4 +35,4 @@ - \ No newline at end of file + diff --git a/src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs index ef743bce0..42fd82e83 100644 --- a/src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs @@ -7,10 +7,12 @@ using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.Poetry.Contracts; -using Nett; namespace Microsoft.ComponentDetection.Detectors.Poetry { + using System.IO; + using Tomlyn; + [Export(typeof(IComponentDetector))] public class PoetryComponentDetector : FileComponentDetector, IExperimentalDetector { @@ -24,13 +26,14 @@ public class PoetryComponentDetector : FileComponentDetector, IExperimentalDetec public override IEnumerable Categories => new List { "Python" }; - protected override Task OnFileFound(ProcessRequest processRequest, IDictionary detectorArgs) + protected override async Task OnFileFound(ProcessRequest processRequest, IDictionary detectorArgs) { var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; var poetryLockFile = processRequest.ComponentStream; this.Logger.LogVerbose("Found Poetry lockfile: " + poetryLockFile); - var poetryLock = StreamTomlSerializer.Deserialize(poetryLockFile.Stream, TomlSettings.Create()).Get(); + var reader = new StreamReader(poetryLockFile.Stream); + var poetryLock = Toml.ToModel(await reader.ReadToEndAsync()); poetryLock.package.ToList().ForEach(package => { var isDevelopmentDependency = package.category != "main"; @@ -46,8 +49,6 @@ protected override Task OnFileFound(ProcessRequest processRequest, IDictionary Date: Tue, 23 Aug 2022 14:47:15 -0700 Subject: [PATCH 02/11] Updated some Nett reference -> Toml --- .../rust/RustCrateDetector.cs | 2 +- .../rust/RustCrateUtilities.cs | 20 +++++++++++-------- .../rust/RustCrateV2Detector.cs | 2 +- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs index 12506a106..48264fea2 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs @@ -43,7 +43,7 @@ protected override Task OnFileFound(ProcessRequest processRequest, IDictionary { RustCrateUtilities.CargoTomlSearchPattern }, (name, directoryName) => false, recursivelyScanDirectories: false); - var cargoDependencyData = RustCrateUtilities.ExtractRootDependencyAndWorkspaceSpecifications(cargoTomlComponentStream, singleFileComponentRecorder); + var cargoDependencyData = RustCrateUtilities.ExtractRootDependencyAndWorkspaceSpecificationsAsync(cargoTomlComponentStream, singleFileComponentRecorder); // If workspaces have been defined in the root cargo.toml file, scan for specified cargo.toml manifests var numWorkspaceComponentStreams = 0; diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateUtilities.cs b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateUtilities.cs index f211052ef..7d46f8b3b 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateUtilities.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateUtilities.cs @@ -8,8 +8,10 @@ using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.Rust.Contracts; -using Nett; +using Tomlyn; using Semver; +using System.Threading.Tasks; +using Tomlyn.Model; namespace Microsoft.ComponentDetection.Detectors.Rust { @@ -42,7 +44,7 @@ public class RustCrateUtilities /// A CargoDependencyData containing populated lists of CargoWorkspaces that will be included from search, CargoWorkspaceExclusions that will be excluded from search, /// a list of non-development dependencies, and a list of development dependencies. /// - public static CargoDependencyData ExtractRootDependencyAndWorkspaceSpecifications(IEnumerable cargoTomlComponentStream, ISingleFileComponentRecorder singleFileComponentRecorder) + public static async Task ExtractRootDependencyAndWorkspaceSpecificationsAsync(IEnumerable cargoTomlComponentStream, ISingleFileComponentRecorder singleFileComponentRecorder) { var cargoDependencyData = new CargoDependencyData(); @@ -51,23 +53,25 @@ public static CargoDependencyData ExtractRootDependencyAndWorkspaceSpecification // We break at the end of this loop foreach (var cargoTomlFile in cargoTomlComponentStream) { - var cargoToml = StreamTomlSerializer.Deserialize(cargoTomlFile.Stream, TomlSettings.Create()); + var reader = new StreamReader(cargoTomlFile.Stream); + var cargoToml = Toml.ToModel(await reader.ReadToEndAsync()); + //var cargoToml = StreamTomlSerializer.Deserialize(cargoTomlFile.Stream, TomlSettings.Create()); singleFileComponentRecorder.AddAdditionalRelatedFile(cargoTomlFile.Location); // Extract the workspaces present, if any if (cargoToml.ContainsKey(WorkspaceKey)) { - var workspaces = cargoToml.Get(WorkspaceKey); + var workspaces = cargoToml["WorkspaceKey"] as TomlTable; - var workspaceMembers = workspaces.ContainsKey(WorkspaceMemberKey) ? workspaces[WorkspaceMemberKey] : null; - var workspaceExclusions = workspaces.ContainsKey(WorkspaceExcludeKey) ? workspaces[WorkspaceExcludeKey] : null; + var workspaceMembers = workspaces.ContainsKey(WorkspaceMemberKey) ? workspaces[WorkspaceMemberKey] as TomlObject : null; + var workspaceExclusions = workspaces.ContainsKey(WorkspaceExcludeKey) ? workspaces[WorkspaceExcludeKey] as TomlObject : null; if (workspaceMembers != null) { if (workspaceMembers.TomlType != TomlObjectType.Array) { - throw new InvalidRustTomlFileException($"In accompanying Cargo.toml file expected {WorkspaceMemberKey} within {WorkspaceKey} to be of type Array, but found {workspaceMembers.TomlType}"); + throw new InvalidRustTomlFileException($"In accompanying Cargo.toml file expected {WorkspaceMemberKey} within {WorkspaceKey} to be of type Array, but found {workspaceMembers.GetType}"); } // TomlObject arrays do not natively implement a HashSet get, so add from a list @@ -78,7 +82,7 @@ public static CargoDependencyData ExtractRootDependencyAndWorkspaceSpecification { if (workspaceExclusions.TomlType != TomlObjectType.Array) { - throw new InvalidRustTomlFileException($"In accompanying Cargo.toml file expected {WorkspaceExcludeKey} within {WorkspaceKey} to be of type Array, but found {workspaceExclusions.TomlType}"); + throw new InvalidRustTomlFileException($"In accompanying Cargo.toml file expected {WorkspaceExcludeKey} within {WorkspaceKey} to be of type Array, but found {workspaceExclusions.GetType}"); } cargoDependencyData.CargoWorkspaceExclusions.UnionWith(workspaceExclusions.Get>()); diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateV2Detector.cs b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateV2Detector.cs index bcaae2295..925d07d51 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateV2Detector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateV2Detector.cs @@ -44,7 +44,7 @@ protected override Task OnFileFound(ProcessRequest processRequest, IDictionary { RustCrateUtilities.CargoTomlSearchPattern }, (name, directoryName) => false, recursivelyScanDirectories: false); - var cargoDependencyData = RustCrateUtilities.ExtractRootDependencyAndWorkspaceSpecifications(cargoTomlComponentStream, singleFileComponentRecorder); + var cargoDependencyData = RustCrateUtilities.ExtractRootDependencyAndWorkspaceSpecificationsAsync(cargoTomlComponentStream, singleFileComponentRecorder); // If workspaces have been defined in the root cargo.toml file, scan for specified cargo.toml manifests var numWorkspaceComponentStreams = 0; From 07dc91b6c510c5f404063be66ee9d9529d8ef6db Mon Sep 17 00:00:00 2001 From: Omotola Akeredolu Date: Fri, 26 Aug 2022 10:08:02 -0700 Subject: [PATCH 03/11] More changes for Nett to Tomlyn --- Directory.Packages.props | 1 - ...rosoft.ComponentDetection.Detectors.csproj | 3 +- .../poetry/Contracts/PoetryLock.cs | 5 +- .../poetry/Contracts/PoetryPackage.cs | 7 +- .../poetry/PoetryComponentDetector.cs | 9 +- .../rust/Contracts/CargoLock.cs | 5 +- .../rust/RustCrateDetector.cs | 9 +- .../rust/RustCrateUtilities.cs | 100 ++++++++++-------- .../rust/RustCrateV2Detector.cs | 8 +- ...delProjectCentricComponentDetectorTests.cs | 4 +- 10 files changed, 85 insertions(+), 66 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index bc2857909..1b82140de 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -23,7 +23,6 @@ - diff --git a/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj b/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj index d58c12aa9..6b0313e14 100644 --- a/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj +++ b/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj @@ -1,9 +1,8 @@ - + - diff --git a/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryLock.cs b/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryLock.cs index cb8bfd9da..0783a0184 100644 --- a/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryLock.cs +++ b/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryLock.cs @@ -1,4 +1,5 @@ -using System.Diagnostics.CodeAnalysis; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; namespace Microsoft.ComponentDetection.Detectors.Poetry.Contracts { @@ -6,6 +7,6 @@ namespace Microsoft.ComponentDetection.Detectors.Poetry.Contracts public class PoetryLock { [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")] - public PoetryPackage[] package { get; set; } + public List package { get; set; } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryPackage.cs b/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryPackage.cs index 984e367f4..001c4e27c 100644 --- a/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryPackage.cs +++ b/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryPackage.cs @@ -1,4 +1,5 @@ -using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.CodeAnalysis; +using Tomlyn.Model; namespace Microsoft.ComponentDetection.Detectors.Poetry.Contracts { @@ -15,5 +16,9 @@ public class PoetryPackage [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")] public PoetrySource source { get; set; } + + public TomlTable dependencies { get; set; } + + public TomlTable extras { get; set; } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs index 42fd82e83..be5d6c552 100644 --- a/src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs @@ -26,14 +26,18 @@ public class PoetryComponentDetector : FileComponentDetector, IExperimentalDetec public override IEnumerable Categories => new List { "Python" }; - protected override async Task OnFileFound(ProcessRequest processRequest, IDictionary detectorArgs) + protected override Task OnFileFound(ProcessRequest processRequest, IDictionary detectorArgs) { var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; var poetryLockFile = processRequest.ComponentStream; this.Logger.LogVerbose("Found Poetry lockfile: " + poetryLockFile); var reader = new StreamReader(poetryLockFile.Stream); - var poetryLock = Toml.ToModel(await reader.ReadToEndAsync()); + var options = new TomlModelOptions + { + IgnoreMissingProperties = true + }; + var poetryLock = Toml.ToModel(reader.ReadToEnd(), options: options); poetryLock.package.ToList().ForEach(package => { var isDevelopmentDependency = package.category != "main"; @@ -49,6 +53,7 @@ protected override async Task OnFileFound(ProcessRequest processRequest, IDictio singleFileComponentRecorder.RegisterUsage(component, isDevelopmentDependency: isDevelopmentDependency); } }); + return Task.CompletedTask; } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/Contracts/CargoLock.cs b/src/Microsoft.ComponentDetection.Detectors/rust/Contracts/CargoLock.cs index 06f3c4934..e9bf1a050 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/Contracts/CargoLock.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/Contracts/CargoLock.cs @@ -1,14 +1,15 @@ -using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.CodeAnalysis; namespace Microsoft.ComponentDetection.Detectors.Rust.Contracts { + using System.Collections.Generic; using Tomlyn.Model; // Represents Cargo.Lock file structure. public class CargoLock { [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")] - public CargoPackage[] package { get; set; } + public List package { get; set; } [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")] public TomlTable metadata { get; set; } diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs index 48264fea2..23adc61ff 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Composition; using System.IO; @@ -8,6 +8,7 @@ using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.Rust.Contracts; +using Tomlyn; namespace Microsoft.ComponentDetection.Detectors.Rust { @@ -31,8 +32,8 @@ protected override Task OnFileFound(ProcessRequest processRequest, IDictionary(); - + var reader = new StreamReader(cargoLockFile.Stream); + var cargoLock= Toml.ToModel(reader.ReadToEnd()); // This makes sure we're only trying to parse Cargo.lock v1 formats if (cargoLock.metadata == null) { @@ -43,7 +44,7 @@ protected override Task OnFileFound(ProcessRequest processRequest, IDictionary { RustCrateUtilities.CargoTomlSearchPattern }, (name, directoryName) => false, recursivelyScanDirectories: false); - var cargoDependencyData = RustCrateUtilities.ExtractRootDependencyAndWorkspaceSpecificationsAsync(cargoTomlComponentStream, singleFileComponentRecorder); + var cargoDependencyData = RustCrateUtilities.ExtractRootDependencyAndWorkspaceSpecifications(cargoTomlComponentStream, singleFileComponentRecorder); // If workspaces have been defined in the root cargo.toml file, scan for specified cargo.toml manifests var numWorkspaceComponentStreams = 0; diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateUtilities.cs b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateUtilities.cs index 7d46f8b3b..d9ec3cc10 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateUtilities.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateUtilities.cs @@ -44,7 +44,7 @@ public class RustCrateUtilities /// A CargoDependencyData containing populated lists of CargoWorkspaces that will be included from search, CargoWorkspaceExclusions that will be excluded from search, /// a list of non-development dependencies, and a list of development dependencies. /// - public static async Task ExtractRootDependencyAndWorkspaceSpecificationsAsync(IEnumerable cargoTomlComponentStream, ISingleFileComponentRecorder singleFileComponentRecorder) + public static CargoDependencyData ExtractRootDependencyAndWorkspaceSpecifications(IEnumerable cargoTomlComponentStream, ISingleFileComponentRecorder singleFileComponentRecorder) { var cargoDependencyData = new CargoDependencyData(); @@ -54,38 +54,36 @@ public static async Task ExtractRootDependencyAndWorkspaceS foreach (var cargoTomlFile in cargoTomlComponentStream) { var reader = new StreamReader(cargoTomlFile.Stream); - var cargoToml = Toml.ToModel(await reader.ReadToEndAsync()); - //var cargoToml = StreamTomlSerializer.Deserialize(cargoTomlFile.Stream, TomlSettings.Create()); + var cargoToml = Toml.ToModel(reader.ReadToEnd()); singleFileComponentRecorder.AddAdditionalRelatedFile(cargoTomlFile.Location); // Extract the workspaces present, if any - if (cargoToml.ContainsKey(WorkspaceKey)) + if (cargoToml.TryGetValue(WorkspaceKey, out var value) && value is TomlTable workspaces) { - var workspaces = cargoToml["WorkspaceKey"] as TomlTable; - - var workspaceMembers = workspaces.ContainsKey(WorkspaceMemberKey) ? workspaces[WorkspaceMemberKey] as TomlObject : null; - var workspaceExclusions = workspaces.ContainsKey(WorkspaceExcludeKey) ? workspaces[WorkspaceExcludeKey] as TomlObject : null; - - if (workspaceMembers != null) + if (workspaces.TryGetValue(WorkspaceMemberKey, out var workspaceMembers)) { - if (workspaceMembers.TomlType != TomlObjectType.Array) + if (workspaceMembers is TomlArray workspaceMembersArray) { - throw new InvalidRustTomlFileException($"In accompanying Cargo.toml file expected {WorkspaceMemberKey} within {WorkspaceKey} to be of type Array, but found {workspaceMembers.GetType}"); + // TomlObject arrays do not natively implement a HashSet get, so add from a list + cargoDependencyData.CargoWorkspaces.UnionWith(workspaceMembersArray.Select(i => i.ToString())); + } + else + { + throw new InvalidRustTomlFileException($"In accompanying Cargo.toml file expected {WorkspaceMemberKey} within {WorkspaceKey} to be of type Array, but found {workspaceMembers?.GetType()}"); } - - // TomlObject arrays do not natively implement a HashSet get, so add from a list - cargoDependencyData.CargoWorkspaces.UnionWith(workspaceMembers.Get>()); } - if (workspaceExclusions != null) + if (workspaces.TryGetValue(WorkspaceExcludeKey, out var workspaceExclusions)) { - if (workspaceExclusions.TomlType != TomlObjectType.Array) + if (workspaceExclusions is TomlArray workspaceExclusionsArray) { - throw new InvalidRustTomlFileException($"In accompanying Cargo.toml file expected {WorkspaceExcludeKey} within {WorkspaceKey} to be of type Array, but found {workspaceExclusions.GetType}"); + cargoDependencyData.CargoWorkspaceExclusions.UnionWith(workspaceExclusionsArray.Select(i => i.ToString())); + } + else + { + throw new InvalidRustTomlFileException($"In accompanying Cargo.toml file expected {WorkspaceExcludeKey} within {WorkspaceKey} to be of type Array, but found {workspaceExclusions?.GetType()}"); } - - cargoDependencyData.CargoWorkspaceExclusions.UnionWith(workspaceExclusions.Get>()); } } @@ -110,7 +108,8 @@ public static void ExtractDependencySpecifications(IEnumerable // This method is only used in non root toml extraction, so the whole list should be iterated foreach (var cargoTomlFile in cargoTomlComponentStreams) { - var cargoToml = StreamTomlSerializer.Deserialize(cargoTomlFile.Stream, TomlSettings.Create()); + var reader = new StreamReader(cargoTomlFile.Stream); + var cargoToml = Toml.ToModel(reader.ReadToEnd()); singleFileComponentRecorder.AddAdditionalRelatedFile(cargoTomlFile.Location); @@ -302,34 +301,41 @@ public static DependencySpecification GenerateDependencySpecifications(TomlTable { foreach (var dependency in dependencies.Keys) { - string versionSpecifier; - if (dependencies[dependency].TomlType == TomlObjectType.String) - { - versionSpecifier = dependencies.Get(dependency); - } - else if (dependencies.Get(dependency).ContainsKey("version") && dependencies.Get(dependency).Get("version") != "0.0.0") - { - // We have a valid version that doesn't indicate 'internal' like 0.0.0 does. - versionSpecifier = dependencies.Get(dependency).Get("version"); - } - else if (dependencies.Get(dependency).ContainsKey("path")) - { - // If this is a workspace dependency specification that specifies a component by path reference, skip adding it directly here. - // Example: kubos-app = { path = "../../apis/app-api/rust" } - continue; - } - else + string versionSpecifier = string.Empty; + + + if (dependencies.TryGetValue(dependency, out var value)) { - return null; + if (value is string valueAsString) + { + versionSpecifier = valueAsString; + } + + else if ((value is TomlTable versionTable) && versionTable.TryGetValue("version", out var versionValue) && versionValue is string versionValueAsSring && (versionValueAsSring != "0.0.0")) + { + // We have a valid version that doesn't indicate 'internal' like 0.0.0 does. + versionSpecifier = versionValueAsSring; + } + else if ((value is TomlTable pathTable) && pathTable.TryGetValue("path", out var pathValue)) + { + // If this is a workspace dependency specification that specifies a component by path reference, skip adding it directly here. + // Example: kubos-app = { path = "../../apis/app-api/rust" } + continue; + } + else + { + return null; + } } // If the dependency is renamed, use the actual name of the package: // https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#renaming-dependencies-in-cargotoml string dependencyName; - if (dependencies[dependency].TomlType == TomlObjectType.Table && - dependencies.Get(dependency).ContainsKey("package")) + + + if (dependencies.TryGetValue(dependency, out var dependencyValue) && dependencyValue is TomlTable tomlTable && tomlTable.TryGetValue("package", out var packageValue) && packageValue is string packageValueAsString) { - dependencyName = dependencies.Get(dependency).Get("package"); + dependencyName = packageValueAsString; } else { @@ -462,16 +468,17 @@ private static IEnumerable GetDependencies(TomlTable cargoToml, IEnum { if (cargoToml.ContainsKey(tomlDependencyKey)) { - dependencies.Add(cargoToml.Get(tomlDependencyKey)); + var newDependencyKey = cargoToml[tomlDependencyKey] as TomlTable; + dependencies.Add(newDependencyKey); } } if (cargoToml.ContainsKey(targetKey)) { - var configs = cargoToml.Get(targetKey); + var configs = cargoToml[targetKey] as TomlTable; foreach (var config in configs) { - var properties = configs.Get(config.Key); + var properties = configs[config.Key] as TomlTable; foreach (var propertyKey in properties.Keys) { var isRelevantKey = tomlDependencyKeys.Any(dependencyKey => @@ -479,7 +486,8 @@ private static IEnumerable GetDependencies(TomlTable cargoToml, IEnum if (isRelevantKey) { - dependencies.Add(properties.Get(propertyKey)); + var newDependencyKey = properties[propertyKey] as TomlTable; + dependencies.Add(newDependencyKey); } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateV2Detector.cs b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateV2Detector.cs index 925d07d51..f44e8c855 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateV2Detector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateV2Detector.cs @@ -8,7 +8,7 @@ using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.Rust.Contracts; -using Nett; +using Tomlyn; namespace Microsoft.ComponentDetection.Detectors.Rust { @@ -32,8 +32,8 @@ protected override Task OnFileFound(ProcessRequest processRequest, IDictionary(); - + var reader = new StreamReader(cargoLockFile.Stream); + var cargoLock = Toml.ToModel(reader.ReadToEnd()); // This makes sure we're only trying to parse Cargo.lock v2 formats if (cargoLock.metadata != null) { @@ -44,7 +44,7 @@ protected override Task OnFileFound(ProcessRequest processRequest, IDictionary { RustCrateUtilities.CargoTomlSearchPattern }, (name, directoryName) => false, recursivelyScanDirectories: false); - var cargoDependencyData = RustCrateUtilities.ExtractRootDependencyAndWorkspaceSpecificationsAsync(cargoTomlComponentStream, singleFileComponentRecorder); + var cargoDependencyData = RustCrateUtilities.ExtractRootDependencyAndWorkspaceSpecifications(cargoTomlComponentStream, singleFileComponentRecorder); // If workspaces have been defined in the root cargo.toml file, scan for specified cargo.toml manifests var numWorkspaceComponentStreams = 0; diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/NuGetProjectModelProjectCentricComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/NuGetProjectModelProjectCentricComponentDetectorTests.cs index 1e3e19f83..db0961c67 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/NuGetProjectModelProjectCentricComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/NuGetProjectModelProjectCentricComponentDetectorTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Runtime.InteropServices; @@ -166,7 +166,7 @@ public async Task ScanDirectoryAsync_DependencyGraph_2_2_additional_Verification { "DotNet.Glob", "MinVer", - "Nett", + "Tomlyn", "Newtonsoft.Json", "NuGet.ProjectModel", "NuGet.Versioning", From 12eec8b386d5a1f7ce75070a6024d636bd2a94e6 Mon Sep 17 00:00:00 2001 From: Omotola Akeredolu Date: Fri, 26 Aug 2022 10:40:31 -0700 Subject: [PATCH 04/11] Changing ref back to Nett since project_assets_2_2 is only used in tests --- .../NuGetProjectModelProjectCentricComponentDetectorTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/NuGetProjectModelProjectCentricComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/NuGetProjectModelProjectCentricComponentDetectorTests.cs index db0961c67..6bc0c4be3 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/NuGetProjectModelProjectCentricComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/NuGetProjectModelProjectCentricComponentDetectorTests.cs @@ -166,7 +166,7 @@ public async Task ScanDirectoryAsync_DependencyGraph_2_2_additional_Verification { "DotNet.Glob", "MinVer", - "Tomlyn", + "Nett", "Newtonsoft.Json", "NuGet.ProjectModel", "NuGet.Versioning", From 26bcd0fc3e36da2796138bbef98bc532d3434de8 Mon Sep 17 00:00:00 2001 From: Omotola Akeredolu Date: Fri, 26 Aug 2022 12:39:07 -0700 Subject: [PATCH 05/11] Updated Data attributes and removed unnecessary comments --- .../poetry/Contracts/PoetryLock.cs | 2 +- .../poetry/Contracts/PoetryPackage.cs | 24 +++++++++++-------- .../poetry/PoetryComponentDetector.cs | 10 ++++---- .../rust/Contracts/CargoLock.cs | 10 ++++---- .../rust/RustCrateDetector.cs | 4 ++-- .../rust/RustCrateUtilities.cs | 5 ++-- .../rust/RustCrateV2Detector.cs | 2 +- 7 files changed, 31 insertions(+), 26 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryLock.cs b/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryLock.cs index 0783a0184..20eb37bb5 100644 --- a/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryLock.cs +++ b/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryLock.cs @@ -7,6 +7,6 @@ namespace Microsoft.ComponentDetection.Detectors.Poetry.Contracts public class PoetryLock { [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")] - public List package { get; set; } + public List Package { get; set; } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryPackage.cs b/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryPackage.cs index 001c4e27c..cdb0bd874 100644 --- a/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryPackage.cs +++ b/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryPackage.cs @@ -1,24 +1,28 @@ using System.Diagnostics.CodeAnalysis; +using System.Runtime.Serialization; using Tomlyn.Model; namespace Microsoft.ComponentDetection.Detectors.Poetry.Contracts { + [DataContract] public class PoetryPackage { - [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")] - public string category { get; set; } + [DataMember(Name = "category")] + public string Category { get; set; } - [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")] - public string name { get; set; } + [DataMember(Name = "name")] + public string Name { get; set; } - [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")] - public string version { get; set; } + [DataMember(Name = "version")] + public string Version { get; set; } - [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")] - public PoetrySource source { get; set; } + [DataMember(Name = "source")] + public PoetrySource Source { get; set; } - public TomlTable dependencies { get; set; } + [DataMember(Name = "dependencies")] + public TomlTable Dependencies { get; set; } - public TomlTable extras { get; set; } + [DataMember(Name = "extras")] + public TomlTable Extras { get; set; } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs index be5d6c552..7127a214f 100644 --- a/src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs @@ -38,18 +38,18 @@ protected override Task OnFileFound(ProcessRequest processRequest, IDictionary(reader.ReadToEnd(), options: options); - poetryLock.package.ToList().ForEach(package => + poetryLock.Package.ToList().ForEach(package => { - var isDevelopmentDependency = package.category != "main"; + var isDevelopmentDependency = package.Category != "main"; - if (package.source != null && package.source.type == "git") + if (package.Source != null && package.Source.type == "git") { - var component = new DetectedComponent(new GitComponent(new Uri(package.source.url), package.source.resolved_reference)); + var component = new DetectedComponent(new GitComponent(new Uri(package.Source.url), package.Source.resolved_reference)); singleFileComponentRecorder.RegisterUsage(component, isDevelopmentDependency: isDevelopmentDependency); } else { - var component = new DetectedComponent(new PipComponent(package.name, package.version)); + var component = new DetectedComponent(new PipComponent(package.Name, package.Version)); singleFileComponentRecorder.RegisterUsage(component, isDevelopmentDependency: isDevelopmentDependency); } }); diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/Contracts/CargoLock.cs b/src/Microsoft.ComponentDetection.Detectors/rust/Contracts/CargoLock.cs index e9bf1a050..2b2efcc4b 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/Contracts/CargoLock.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/Contracts/CargoLock.cs @@ -3,15 +3,17 @@ namespace Microsoft.ComponentDetection.Detectors.Rust.Contracts { using System.Collections.Generic; + using System.Runtime.Serialization; using Tomlyn.Model; // Represents Cargo.Lock file structure. + [DataContract] public class CargoLock { - [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")] - public List package { get; set; } + [DataMember(Name = "package")] + public List Package { get; set; } - [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")] - public TomlTable metadata { get; set; } + [DataMember(Name = "metadata")] + public TomlTable Metadata { get; set; } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs index 23adc61ff..2156aa0db 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs @@ -35,7 +35,7 @@ protected override Task OnFileFound(ProcessRequest processRequest, IDictionary(reader.ReadToEnd()); // This makes sure we're only trying to parse Cargo.lock v1 formats - if (cargoLock.metadata == null) + if (cargoLock.Metadata == null) { this.Logger.LogInfo($"Cargo.lock file at {cargoLockFile.Location} contains no metadata section so we're parsing it as the v2 format. The v1 detector will not process it."); return Task.CompletedTask; @@ -79,7 +79,7 @@ protected override Task OnFileFound(ProcessRequest processRequest, IDictionary i.ToString())); } else @@ -352,7 +351,7 @@ public static DependencySpecification GenerateDependencySpecifications(TomlTable public static IEnumerable ConvertCargoLockV2PackagesToV1(CargoLock cargoLock) { var packageMap = new Dictionary>(); - cargoLock.package.ToList().ForEach(package => + cargoLock.Package.ToList().ForEach(package => { if (!packageMap.TryGetValue(package.name, out var packageList)) { @@ -364,7 +363,7 @@ public static IEnumerable ConvertCargoLockV2PackagesToV1(CargoLock } }); - return cargoLock.package.Select(package => + return cargoLock.Package.Select(package => { if (package.dependencies == null) { diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateV2Detector.cs b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateV2Detector.cs index f44e8c855..7223bc4ca 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateV2Detector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateV2Detector.cs @@ -35,7 +35,7 @@ protected override Task OnFileFound(ProcessRequest processRequest, IDictionary(reader.ReadToEnd()); // This makes sure we're only trying to parse Cargo.lock v2 formats - if (cargoLock.metadata != null) + if (cargoLock.Metadata != null) { this.Logger.LogInfo($"Cargo.lock file at {cargoLockFile.Location} contains a metadata section so we're parsing it as the v1 format. The v2 detector will no process it."); return Task.CompletedTask; From 212ad1c845b1586a9667f46f196fc414e495ca85 Mon Sep 17 00:00:00 2001 From: Omotola Akeredolu Date: Fri, 26 Aug 2022 12:54:35 -0700 Subject: [PATCH 06/11] Updated Data attributes in PoetryLock file --- .../poetry/Contracts/PoetryLock.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryLock.cs b/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryLock.cs index 20eb37bb5..c73c42bee 100644 --- a/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryLock.cs +++ b/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryLock.cs @@ -1,12 +1,14 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Runtime.Serialization; namespace Microsoft.ComponentDetection.Detectors.Poetry.Contracts { // Represents Poetry.Lock file structure. + [DataContract] public class PoetryLock { - [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")] + [DataMember(Name = "Package")] public List Package { get; set; } } } From df8cc9128642b96f0ee9b039389e5b3bb776ec15 Mon Sep 17 00:00:00 2001 From: Omotola Akeredolu Date: Fri, 26 Aug 2022 13:21:57 -0700 Subject: [PATCH 07/11] Made property type more specific --- .../poetry/Contracts/PoetryPackage.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryPackage.cs b/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryPackage.cs index cdb0bd874..2cb38e07b 100644 --- a/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryPackage.cs +++ b/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryPackage.cs @@ -1,6 +1,5 @@ -using System.Diagnostics.CodeAnalysis; +using System.Collections.Generic; using System.Runtime.Serialization; -using Tomlyn.Model; namespace Microsoft.ComponentDetection.Detectors.Poetry.Contracts { @@ -20,9 +19,9 @@ public class PoetryPackage public PoetrySource Source { get; set; } [DataMember(Name = "dependencies")] - public TomlTable Dependencies { get; set; } + public Dictionary Dependencies { get; set; } [DataMember(Name = "extras")] - public TomlTable Extras { get; set; } + public Dictionary Extras { get; set; } } } From 60a852ae16f252adc2e1e469c64db9638268f514 Mon Sep 17 00:00:00 2001 From: Omotola Akeredolu Date: Thu, 1 Sep 2022 09:55:58 -0700 Subject: [PATCH 08/11] Formatting fixes --- .../poetry/PoetryComponentDetector.cs | 4 ++-- .../rust/RustCrateDetector.cs | 3 ++- .../rust/RustCrateUtilities.cs | 9 ++------- .../rust/RustCrateV2Detector.cs | 1 + 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs index 7127a214f..441198014 100644 --- a/src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs @@ -35,7 +35,7 @@ protected override Task OnFileFound(ProcessRequest processRequest, IDictionary(reader.ReadToEnd(), options: options); poetryLock.Package.ToList().ForEach(package => @@ -53,7 +53,7 @@ protected override Task OnFileFound(ProcessRequest processRequest, IDictionary(reader.ReadToEnd()); + var cargoLock = Toml.ToModel(reader.ReadToEnd()); + // This makes sure we're only trying to parse Cargo.lock v1 formats if (cargoLock.Metadata == null) { diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateUtilities.cs b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateUtilities.cs index 39656c0a3..7a6059d7a 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateUtilities.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateUtilities.cs @@ -3,14 +3,14 @@ using System.IO; using System.Linq; using System.Text.RegularExpressions; +using System.Threading.Tasks; using DotNet.Globbing; using Microsoft.ComponentDetection.Common.Telemetry.Records; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.Rust.Contracts; -using Tomlyn; using Semver; -using System.Threading.Tasks; +using Tomlyn; using Tomlyn.Model; namespace Microsoft.ComponentDetection.Detectors.Rust @@ -299,15 +299,12 @@ public static DependencySpecification GenerateDependencySpecifications(TomlTable foreach (var dependency in dependencies.Keys) { string versionSpecifier = string.Empty; - - if (dependencies.TryGetValue(dependency, out var value)) { if (value is string valueAsString) { versionSpecifier = valueAsString; } - else if ((value is TomlTable versionTable) && versionTable.TryGetValue("version", out var versionValue) && versionValue is string versionValueAsSring && (versionValueAsSring != "0.0.0")) { // We have a valid version that doesn't indicate 'internal' like 0.0.0 does. @@ -328,8 +325,6 @@ public static DependencySpecification GenerateDependencySpecifications(TomlTable // If the dependency is renamed, use the actual name of the package: // https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#renaming-dependencies-in-cargotoml string dependencyName; - - if (dependencies.TryGetValue(dependency, out var dependencyValue) && dependencyValue is TomlTable tomlTable && tomlTable.TryGetValue("package", out var packageValue) && packageValue is string packageValueAsString) { dependencyName = packageValueAsString; diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateV2Detector.cs b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateV2Detector.cs index 7223bc4ca..263948390 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateV2Detector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateV2Detector.cs @@ -34,6 +34,7 @@ protected override Task OnFileFound(ProcessRequest processRequest, IDictionary(reader.ReadToEnd()); + // This makes sure we're only trying to parse Cargo.lock v2 formats if (cargoLock.Metadata != null) { From 06223fcc7be9c4636135b4897b1ebc69f5a8c154 Mon Sep 17 00:00:00 2001 From: Omotola Akeredolu Date: Tue, 6 Sep 2022 19:36:12 -0700 Subject: [PATCH 09/11] Made updates to add or ignore rproperties used in Toml Deserialization --- .../poetry/Contracts/PoetryLock.cs | 3 +++ .../rust/Contracts/CargoLock.cs | 2 +- .../rust/RustCrateDetector.cs | 6 +++++- .../rust/RustCrateV2Detector.cs | 7 +++++-- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryLock.cs b/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryLock.cs index c73c42bee..87dce51c5 100644 --- a/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryLock.cs +++ b/src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryLock.cs @@ -10,5 +10,8 @@ public class PoetryLock { [DataMember(Name = "Package")] public List Package { get; set; } + + [DataMember(Name = "metadata")] + public Dictionary Metadata { get; set; } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/Contracts/CargoLock.cs b/src/Microsoft.ComponentDetection.Detectors/rust/Contracts/CargoLock.cs index 2b2efcc4b..9b714d029 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/Contracts/CargoLock.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/Contracts/CargoLock.cs @@ -14,6 +14,6 @@ public class CargoLock public List Package { get; set; } [DataMember(Name = "metadata")] - public TomlTable Metadata { get; set; } + public Dictionary Metadata { get; set; } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs index a101e9bac..7b1bd4cb5 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs @@ -33,7 +33,11 @@ protected override Task OnFileFound(ProcessRequest processRequest, IDictionary(reader.ReadToEnd()); + var options = new TomlModelOptions + { + IgnoreMissingProperties = true, + }; + var cargoLock = Toml.ToModel(reader.ReadToEnd(), options: options); // This makes sure we're only trying to parse Cargo.lock v1 formats if (cargoLock.Metadata == null) diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateV2Detector.cs b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateV2Detector.cs index 263948390..dd529ea44 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateV2Detector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateV2Detector.cs @@ -29,11 +29,14 @@ protected override Task OnFileFound(ProcessRequest processRequest, IDictionary(reader.ReadToEnd()); + var cargoLock = Toml.ToModel(reader.ReadToEnd(), options: options); // This makes sure we're only trying to parse Cargo.lock v2 formats if (cargoLock.Metadata != null) From f442df9a9be825bd3c39eebd8bd9f8685ca8c3fb Mon Sep 17 00:00:00 2001 From: Omotola Akeredolu Date: Wed, 7 Sep 2022 11:02:50 -0700 Subject: [PATCH 10/11] Added documentation for running verification tests --- docs/running-verification-tests.md | 40 ++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 docs/running-verification-tests.md diff --git a/docs/running-verification-tests.md b/docs/running-verification-tests.md new file mode 100644 index 000000000..93739e24e --- /dev/null +++ b/docs/running-verification-tests.md @@ -0,0 +1,40 @@ +# Running Verification Tests On Your Local Machine +Verification tests are used to confirm that no detectors are lost when changes are made to the project. The tests are run on every PR build. They work by comparing the detection results from the main branch to detection results from the new changes on your PR. You can follow the steps below to run them locally. + +## Step 1 : Run Detection on the main branch + +- Checkout the main branch in your local repo +- Create a folder to store detection results (e.g C:\old-output-folder) +- Use command line to run detection on the main branch with the code below: + +```dotnet run scan --Verbosity Verbose --SourceDirectory {path to your local repo} --Output {path to the output folder you created}``` + +For Example: + +```dotnet run scan --Verbosity Verbose --SourceDirectory C:\componentdetection --Output C:\old-output-folder``` + + +## Step 2 : Run Detection on your new branch + +- Checkout the branch with the new changes you are trying to merge +- Create a folder to store detection results. This folder should be seperate from the one you used in Step 1 (e.g C:\new-output-folder) +- Use command line to run detection on the main branch with the code below: + +```dotnet run scan --Verbosity Verbose --SourceDirectory {path to your local repo} --Output {path to the output folder you created}``` + +For Example: + +```dotnet run scan --Verbosity Verbose --SourceDirectory C:\componentdetection --Output C:\new-output-folder``` + +## Step 3 : Update variables in the test + +- Open the Microsoft.ComponentDetection.VerificationTests project in VS Studio +- Navigate to `GatherResources()` in `ComponentDetectionIntegrationTests.cs` +- Update the following variables: + - `oldGithubArtifactsDir` : This should be the output folder you created in Step 1 + - `newGithubArtifactsDir` : This should be the output folder you created in Step 2 + - `allowedTimeDriftRatioString`: This should be ".75" + + +## Step 4: Run The tests + Run or Debug the tests in test explorer like you would any other test. \ No newline at end of file From 218cfa20a87122accb348b3ece774e0d0b90f497 Mon Sep 17 00:00:00 2001 From: Omotola Akeredolu Date: Wed, 7 Sep 2022 12:02:37 -0700 Subject: [PATCH 11/11] Small formatting and documentation fix --- docs/running-verification-tests.md | 6 ++++-- .../poetry/PoetryComponentDetector.cs | 5 ++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/running-verification-tests.md b/docs/running-verification-tests.md index 93739e24e..4898a9e77 100644 --- a/docs/running-verification-tests.md +++ b/docs/running-verification-tests.md @@ -14,7 +14,7 @@ For Example: ```dotnet run scan --Verbosity Verbose --SourceDirectory C:\componentdetection --Output C:\old-output-folder``` -## Step 2 : Run Detection on your new branch +## Step 2 : Run Detection on your new branch - Checkout the branch with the new changes you are trying to merge - Create a folder to store detection results. This folder should be seperate from the one you used in Step 1 (e.g C:\new-output-folder) @@ -37,4 +37,6 @@ For Example: ## Step 4: Run The tests - Run or Debug the tests in test explorer like you would any other test. \ No newline at end of file +You can run the tests in two ways: +- Run or Debug the tests in test explorer. +- Use the command line to navigate to the Microsoft.ComponentDetection.VerificationTests folder, and run `dotnet test`. \ No newline at end of file diff --git a/src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs index 441198014..ecd3543e7 100644 --- a/src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs @@ -3,6 +3,8 @@ using System.Composition; using System.Linq; using System.Threading.Tasks; +using System.IO; +using Tomlyn; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; @@ -10,9 +12,6 @@ namespace Microsoft.ComponentDetection.Detectors.Poetry { - using System.IO; - using Tomlyn; - [Export(typeof(IComponentDetector))] public class PoetryComponentDetector : FileComponentDetector, IExperimentalDetector {