From 0b4d5408a72c4f1dc82ef4a0ccf3bc14a8a2e07b Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Mon, 24 Jan 2022 19:12:08 +0000 Subject: [PATCH 01/10] Initial implementation of VcpkgDetector and VcpkgComponent --- .../DetectorClass.cs | 3 + .../TypedComponent/ComponentType.cs | 3 + .../TypedComponent/VcpkgComponent.cs | 63 ++++++++++ .../vcpkg/Contracts/Package.cs | 19 +++ .../vcpkg/Contracts/VcpkgSBOM.cs | 14 +++ .../vcpkg/VcpkgComponentDetector.cs | 101 +++++++++++++++ .../vcpkg/nlohmann-json/vcpkg.spdx.json | 115 ++++++++++++++++++ .../Resources/vcpkg/tinyxml2/vcpkg.spdx.json | 115 ++++++++++++++++++ 8 files changed, 433 insertions(+) create mode 100644 src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs create mode 100644 src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Package.cs create mode 100644 src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/VcpkgSBOM.cs create mode 100644 src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs create mode 100644 test/Microsoft.ComponentDetection.Detectors.Tests/Resources/vcpkg/nlohmann-json/vcpkg.spdx.json create mode 100644 test/Microsoft.ComponentDetection.Detectors.Tests/Resources/vcpkg/tinyxml2/vcpkg.spdx.json diff --git a/src/Microsoft.ComponentDetection.Contracts/DetectorClass.cs b/src/Microsoft.ComponentDetection.Contracts/DetectorClass.cs index db8898b3f..443edaa9d 100644 --- a/src/Microsoft.ComponentDetection.Contracts/DetectorClass.cs +++ b/src/Microsoft.ComponentDetection.Contracts/DetectorClass.cs @@ -35,5 +35,8 @@ public enum DetectorClass /// Indicates a detector applies to Conda packages. Conda, + + /// Indicates a detector applies to Vcpkg packages. + Vcpkg, } } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ComponentType.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ComponentType.cs index c441c129c..72398c3e6 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ComponentType.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ComponentType.cs @@ -44,5 +44,8 @@ public enum ComponentType : byte [EnumMember] Conda = 13, + + [EnumMember] + Vcpkg = 14, } } \ No newline at end of file diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs new file mode 100644 index 000000000..d5749b8b7 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs @@ -0,0 +1,63 @@ +using PackageUrl; + +namespace Microsoft.ComponentDetection.Contracts.TypedComponent +{ + public class VcpkgComponent : TypedComponent + { + private VcpkgComponent() + { + /* Reserved for deserialization */ + } + + public VcpkgComponent(string spdxid, string name, string version, string triplet = null, string portVersion = null, string description = null, string downloadLocation = null) + { + SPDXID = ValidateRequiredInput(spdxid, nameof(SPDXID), nameof(ComponentType.Vcpkg)); + Name = ValidateRequiredInput(name, nameof(Name), nameof(ComponentType.Vcpkg)); + Version = version; + PortVersion = portVersion; + Triplet = triplet; + Description = description; + DownloadLocation = downloadLocation; + } + + public string SPDXID { get; set; } + + public string Name { get; set; } + + public string DownloadLocation { get; set; } + + public string Triplet { get; set; } + + public string Version { get; set; } + + public string Description { get; set; } + + public string PortVersion { get; set; } + + public override ComponentType Type => ComponentType.Vcpkg; + + public override string Id + { + get + { + if (PortVersion != null) + return $"{Name} {Version}#{PortVersion} - {Type}"; + else + return $"{Name} {Version} - {Type}"; + } + } + + public override PackageURL PackageUrl + { + get + { + if (PortVersion != null) + return new PackageURL($"pkg:vcpkg/{Name}@{Version}?port_version={PortVersion}"); + else if (Version != null) + return new PackageURL($"pkg:vcpkg/{Name}@{Version}"); + else + return new PackageURL($"pkg:vcpkg/{Name}"); + } + } + } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Package.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Package.cs new file mode 100644 index 000000000..055e17b36 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Package.cs @@ -0,0 +1,19 @@ +namespace Microsoft.ComponentDetection.Detectors.Vcpkg.Contracts +{ + public class Package + { + public string SPDXID { get; set; } + + public string VersionInfo { get; set; } + + public string DownloadLocation { get; set; } + + public string Filename { get; set; } + + public string Homepage { get; set; } + + public string Description { get; set; } + + public string Name { get; set; } + } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/VcpkgSBOM.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/VcpkgSBOM.cs new file mode 100644 index 000000000..0fd9de256 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/VcpkgSBOM.cs @@ -0,0 +1,14 @@ +namespace Microsoft.ComponentDetection.Detectors.Vcpkg.Contracts +{ + /// + /// Take from https://github.com/anchore/syft/tree/main/schema/json. + /// Match version to tag used i.e. https://github.com/anchore/syft/blob/v0.16.1/internal/constants.go#L9 + /// Can convert JSON Schema to C# using quicktype.io. + /// + public class VcpkgSBOM + { + public Package[] Packages { get; set; } + + public string Name { get; set; } + } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs new file mode 100644 index 000000000..0c6dcf066 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs @@ -0,0 +1,101 @@ +using System; +using System.Collections.Generic; +using System.Composition; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Microsoft.ComponentDetection.Common; +using Microsoft.ComponentDetection.Common.Telemetry.Records; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.Internal; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.ComponentDetection.Detectors.Vcpkg.Contracts; +using Newtonsoft.Json; + +namespace Microsoft.ComponentDetection.Detectors.Vcpkg +{ + [Export(typeof(IComponentDetector))] + public class VcpkgComponentDetector : FileComponentDetector, IDefaultOffComponentDetector + { + [Import] + public ICommandLineInvocationService CommandLineInvocationService { get; set; } + + [Import] + public IEnvironmentVariableService EnvVarService { get; set; } + + public override string Id { get; } = "Vcpkg"; + + public override IEnumerable Categories => new[] { Enum.GetName(typeof(DetectorClass), DetectorClass.Vcpkg) }; + + public override IList SearchPatterns { get; } = new List { "vcpkg.spdx.json" }; + + public override IEnumerable SupportedComponentTypes { get; } = new[] { ComponentType.Go }; + + public override int Version => 1; + + private HashSet projectRoots = new HashSet(); + + protected override async Task OnFileFound(ProcessRequest processRequest, IDictionary detectorArgs) + { + var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; + var file = processRequest.ComponentStream; + + Logger.LogWarning($"vcpkg detector found {file}"); + + var projectRootDirectory = Directory.GetParent(file.Location); + if (projectRoots.Any(path => projectRootDirectory.FullName.StartsWith(path))) + { + return; + } + + ParseSpdxFile(singleFileComponentRecorder, file); + } + + private void ParseSpdxFile( + ISingleFileComponentRecorder singleFileComponentRecorder, + IComponentStream file) + { + using var reader = new StreamReader(file.Stream); + var sbom = JsonConvert.DeserializeObject(reader.ReadToEnd()); + + foreach (var item in sbom.Packages) + { + if (item.Name == null || item.Name.Length == 0) { continue; } + try + { + Logger.LogWarning($"parsed package {item.Name}"); + if (item.SPDXID == "SPDXRef-port") + { + var split = item.VersionInfo.Split('#'); + var component = new VcpkgComponent(item.SPDXID, item.Name, split[0], portVersion: split.Length >= 2 ? split[1] : "0", downloadLocation: item.DownloadLocation); + singleFileComponentRecorder.RegisterUsage(new DetectedComponent(component)); + } + else if (item.SPDXID == "SPDXRef-binary") + { + var split = item.Name.Split(':'); + var component = new VcpkgComponent(item.SPDXID, item.Name, item.VersionInfo, triplet: split[1], downloadLocation: item.DownloadLocation); + singleFileComponentRecorder.RegisterUsage(new DetectedComponent(component)); + } + else if (item.SPDXID.StartsWith("SPDXRef-resource-")) + { + var dl = item.DownloadLocation; + var split = dl.Split("#"); + var subpath = split.Length > 1 ? split[1] : null; + dl = split.Length > 1 ? split[0] : dl; + split = dl.Split("@"); + var version = split.Length > 1 ? split[1] : null; + dl = split.Length > 1 ? split[0] : dl; + + var component = new VcpkgComponent(item.SPDXID, item.Name, version, downloadLocation: dl); + singleFileComponentRecorder.RegisterUsage(new DetectedComponent(component)); + } + } + catch (Exception) + { + Logger.LogWarning($"failed while handling {item.Name}"); + } + } + } + } +} diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Resources/vcpkg/nlohmann-json/vcpkg.spdx.json b/test/Microsoft.ComponentDetection.Detectors.Tests/Resources/vcpkg/nlohmann-json/vcpkg.spdx.json new file mode 100644 index 000000000..29bae26a3 --- /dev/null +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/Resources/vcpkg/nlohmann-json/vcpkg.spdx.json @@ -0,0 +1,115 @@ +{ + "$schema": "https://raw.githubusercontent.com/spdx/spdx-spec/v2.2.1/schemas/spdx-schema.json", + "spdxVersion": "SPDX-2.2", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "documentNamespace": "https://spdx.org/spdxdocs/nlohmann-json-x64-linux-3.10.4-78c7f190-b402-44d1-a364-b9ac86392b84", + "name": "nlohmann-json:x64-linux@3.10.4 69dcfc6886529ad2d210f71f132d743672a7e65d2c39f53456f17fc5fc08b278", + "creationInfo": { + "creators": [ + "Tool: vcpkg-unknownhash" + ], + "created": "2022-01-18T21:12:48Z" + }, + "relationships": [ + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "GENERATES", + "relatedSpdxElement": "SPDXRef-binary" + }, + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-file-0" + }, + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-file-1" + }, + { + "spdxElementId": "SPDXRef-binary", + "relationshipType": "GENERATED_FROM", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-0", + "relationshipType": "CONTAINED_BY", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-1", + "relationshipType": "CONTAINED_BY", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-1", + "relationshipType": "DEPENDENCY_MANIFEST_OF", + "relatedSpdxElement": "SPDXRef-port" + } + ], + "packages": [ + { + "name": "nlohmann-json", + "SPDXID": "SPDXRef-port", + "versionInfo": "3.10.4", + "downloadLocation": "git+https://github.com/Microsoft/vcpkg#ports/nlohmann-json", + "homepage": "https://github.com/nlohmann/json", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "copyrightText": "NOASSERTION", + "description": "JSON for Modern C++", + "comment": "This is the port (recipe) consumed by vcpkg." + }, + { + "name": "nlohmann-json:x64-linux", + "SPDXID": "SPDXRef-binary", + "versionInfo": "69dcfc6886529ad2d210f71f132d743672a7e65d2c39f53456f17fc5fc08b278", + "downloadLocation": "NONE", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "copyrightText": "NOASSERTION", + "comment": "This is a binary package built by vcpkg." + }, + { + "SPDXID": "SPDXRef-resource-1", + "name": "nlohmann/json", + "downloadLocation": "git+https://github.com/nlohmann/json@v3.10.4", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "copyrightText": "NOASSERTION", + "checksums": [ + { + "algorithm": "SHA512", + "checksumValue": "f78592db6218165cbc74c10bcba40366f1bfea84405b7ee25fe97a056d5b7a15aeeb956d93296673928dcbd6e26ffcfb152f885b4a44d5d55751396ccf090835" + } + ] + } + ], + "files": [ + { + "fileName": "./portfile.cmake", + "SPDXID": "SPDXRef-file-0", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "49b4f9e11cdd0ef697a2750187fa5fa42e7e833dbf411d299ca4d1e9f147773a" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION" + }, + { + "fileName": "./vcpkg.json", + "SPDXID": "SPDXRef-file-1", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "b97c75815135eb812a88ee92d50170928d4c69517bfb332e181070cbce2c081b" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION" + } + ] +} \ No newline at end of file diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Resources/vcpkg/tinyxml2/vcpkg.spdx.json b/test/Microsoft.ComponentDetection.Detectors.Tests/Resources/vcpkg/tinyxml2/vcpkg.spdx.json new file mode 100644 index 000000000..91d7433c3 --- /dev/null +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/Resources/vcpkg/tinyxml2/vcpkg.spdx.json @@ -0,0 +1,115 @@ +{ + "$schema": "https://raw.githubusercontent.com/spdx/spdx-spec/v2.2.1/schemas/spdx-schema.json", + "spdxVersion": "SPDX-2.2", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "documentNamespace": "https://spdx.org/spdxdocs/tinyxml2-x64-linux-9.0.0-c99e4f03-5275-458b-8a69-b5f8dfa45f18", + "name": "tinyxml2:x64-linux@9.0.0 5c7679507def92c5c71df44aec08a90a5c749f7f805b3f0e8e70f5e8a5b1b8d0", + "creationInfo": { + "creators": [ + "Tool: vcpkg-unknownhash" + ], + "created": "2022-01-14T00:28:41Z" + }, + "relationships": [ + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "GENERATES", + "relatedSpdxElement": "SPDXRef-binary" + }, + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-file-0" + }, + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-file-1" + }, + { + "spdxElementId": "SPDXRef-binary", + "relationshipType": "GENERATED_FROM", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-0", + "relationshipType": "CONTAINED_BY", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-1", + "relationshipType": "CONTAINED_BY", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-1", + "relationshipType": "DEPENDENCY_MANIFEST_OF", + "relatedSpdxElement": "SPDXRef-port" + } + ], + "packages": [ + { + "name": "tinyxml2", + "SPDXID": "SPDXRef-port", + "versionInfo": "9.0.0", + "downloadLocation": "git+https://github.com/Microsoft/vcpkg#ports/tinyxml2", + "homepage": "https://github.com/leethomason/tinyxml2", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "copyrightText": "NOASSERTION", + "description": "A simple, small, efficient, C++ XML parser", + "comment": "This is the port (recipe) consumed by vcpkg." + }, + { + "name": "tinyxml2:x64-linux", + "SPDXID": "SPDXRef-binary", + "versionInfo": "5c7679507def92c5c71df44aec08a90a5c749f7f805b3f0e8e70f5e8a5b1b8d0", + "downloadLocation": "NONE", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "copyrightText": "NOASSERTION", + "comment": "This is a binary package built by vcpkg." + }, + { + "SPDXID": "SPDXRef-resource-1", + "name": "leethomason/tinyxml2", + "downloadLocation": "git+https://github.com/leethomason/tinyxml2@9.0.0", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "copyrightText": "NOASSERTION", + "checksums": [ + { + "algorithm": "SHA512", + "checksumValue": "9c5ce8131984690df302ca3e32314573b137180ed522c92fd631692979c942372a28f697fdb3d5e56bcf2d3dc596262b724d088153f3e1d721c9536f2a883367" + } + ] + } + ], + "files": [ + { + "fileName": "./portfile.cmake", + "SPDXID": "SPDXRef-file-0", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "dad309c46aea9ccd9a3779723970963b2792d6a7fade26f95437be8ed18ccd60" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION" + }, + { + "fileName": "./vcpkg.json", + "SPDXID": "SPDXRef-file-1", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "4f8ddfeb9d3faa3ecf03f22a07678a060e353cde5e2b46be1cbe519993aab841" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION" + } + ] +} From da1f2837de8b65824bf1d31d4fc0a79a2e44f1b1 Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Thu, 27 Jan 2022 17:25:45 +0000 Subject: [PATCH 02/10] Fix warnings --- .../TypedComponent/VcpkgComponent.cs | 10 ++++++++++ .../vcpkg/VcpkgComponentDetector.cs | 12 ++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs index d5749b8b7..41037f922 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs @@ -41,9 +41,13 @@ public override string Id get { if (PortVersion != null) + { return $"{Name} {Version}#{PortVersion} - {Type}"; + } else + { return $"{Name} {Version} - {Type}"; + } } } @@ -52,11 +56,17 @@ public override PackageURL PackageUrl get { if (PortVersion != null) + { return new PackageURL($"pkg:vcpkg/{Name}@{Version}?port_version={PortVersion}"); + } else if (Version != null) + { return new PackageURL($"pkg:vcpkg/{Name}@{Version}"); + } else + { return new PackageURL($"pkg:vcpkg/{Name}"); + } } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs index 0c6dcf066..8674a99e1 100644 --- a/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs @@ -49,19 +49,23 @@ protected override async Task OnFileFound(ProcessRequest processRequest, IDictio return; } - ParseSpdxFile(singleFileComponentRecorder, file); + await ParseSpdxFile(singleFileComponentRecorder, file); } - private void ParseSpdxFile( + private async Task ParseSpdxFile( ISingleFileComponentRecorder singleFileComponentRecorder, IComponentStream file) { using var reader = new StreamReader(file.Stream); - var sbom = JsonConvert.DeserializeObject(reader.ReadToEnd()); + var sbom = JsonConvert.DeserializeObject(await reader.ReadToEndAsync()); foreach (var item in sbom.Packages) { - if (item.Name == null || item.Name.Length == 0) { continue; } + if (item.Name == null || item.Name.Length == 0) + { + continue; + } + try { Logger.LogWarning($"parsed package {item.Name}"); From 0e48dbe2ff5b99484ef07e8e8524179cedb3fa01 Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Mon, 24 Jan 2022 19:12:08 +0000 Subject: [PATCH 03/10] Initial implementation of VcpkgDetector and VcpkgComponent --- .../DetectorClass.cs | 3 + .../TypedComponent/ComponentType.cs | 3 + .../TypedComponent/VcpkgComponent.cs | 63 ++++++++++ .../vcpkg/Contracts/Package.cs | 19 +++ .../vcpkg/Contracts/VcpkgSBOM.cs | 14 +++ .../vcpkg/VcpkgComponentDetector.cs | 101 +++++++++++++++ .../vcpkg/nlohmann-json/vcpkg.spdx.json | 115 ++++++++++++++++++ .../Resources/vcpkg/tinyxml2/vcpkg.spdx.json | 115 ++++++++++++++++++ 8 files changed, 433 insertions(+) create mode 100644 src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs create mode 100644 src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Package.cs create mode 100644 src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/VcpkgSBOM.cs create mode 100644 src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs create mode 100644 test/Microsoft.ComponentDetection.Detectors.Tests/Resources/vcpkg/nlohmann-json/vcpkg.spdx.json create mode 100644 test/Microsoft.ComponentDetection.Detectors.Tests/Resources/vcpkg/tinyxml2/vcpkg.spdx.json diff --git a/src/Microsoft.ComponentDetection.Contracts/DetectorClass.cs b/src/Microsoft.ComponentDetection.Contracts/DetectorClass.cs index db8898b3f..443edaa9d 100644 --- a/src/Microsoft.ComponentDetection.Contracts/DetectorClass.cs +++ b/src/Microsoft.ComponentDetection.Contracts/DetectorClass.cs @@ -35,5 +35,8 @@ public enum DetectorClass /// Indicates a detector applies to Conda packages. Conda, + + /// Indicates a detector applies to Vcpkg packages. + Vcpkg, } } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ComponentType.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ComponentType.cs index c441c129c..72398c3e6 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ComponentType.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ComponentType.cs @@ -44,5 +44,8 @@ public enum ComponentType : byte [EnumMember] Conda = 13, + + [EnumMember] + Vcpkg = 14, } } \ No newline at end of file diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs new file mode 100644 index 000000000..d5749b8b7 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs @@ -0,0 +1,63 @@ +using PackageUrl; + +namespace Microsoft.ComponentDetection.Contracts.TypedComponent +{ + public class VcpkgComponent : TypedComponent + { + private VcpkgComponent() + { + /* Reserved for deserialization */ + } + + public VcpkgComponent(string spdxid, string name, string version, string triplet = null, string portVersion = null, string description = null, string downloadLocation = null) + { + SPDXID = ValidateRequiredInput(spdxid, nameof(SPDXID), nameof(ComponentType.Vcpkg)); + Name = ValidateRequiredInput(name, nameof(Name), nameof(ComponentType.Vcpkg)); + Version = version; + PortVersion = portVersion; + Triplet = triplet; + Description = description; + DownloadLocation = downloadLocation; + } + + public string SPDXID { get; set; } + + public string Name { get; set; } + + public string DownloadLocation { get; set; } + + public string Triplet { get; set; } + + public string Version { get; set; } + + public string Description { get; set; } + + public string PortVersion { get; set; } + + public override ComponentType Type => ComponentType.Vcpkg; + + public override string Id + { + get + { + if (PortVersion != null) + return $"{Name} {Version}#{PortVersion} - {Type}"; + else + return $"{Name} {Version} - {Type}"; + } + } + + public override PackageURL PackageUrl + { + get + { + if (PortVersion != null) + return new PackageURL($"pkg:vcpkg/{Name}@{Version}?port_version={PortVersion}"); + else if (Version != null) + return new PackageURL($"pkg:vcpkg/{Name}@{Version}"); + else + return new PackageURL($"pkg:vcpkg/{Name}"); + } + } + } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Package.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Package.cs new file mode 100644 index 000000000..055e17b36 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Package.cs @@ -0,0 +1,19 @@ +namespace Microsoft.ComponentDetection.Detectors.Vcpkg.Contracts +{ + public class Package + { + public string SPDXID { get; set; } + + public string VersionInfo { get; set; } + + public string DownloadLocation { get; set; } + + public string Filename { get; set; } + + public string Homepage { get; set; } + + public string Description { get; set; } + + public string Name { get; set; } + } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/VcpkgSBOM.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/VcpkgSBOM.cs new file mode 100644 index 000000000..0fd9de256 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/VcpkgSBOM.cs @@ -0,0 +1,14 @@ +namespace Microsoft.ComponentDetection.Detectors.Vcpkg.Contracts +{ + /// + /// Take from https://github.com/anchore/syft/tree/main/schema/json. + /// Match version to tag used i.e. https://github.com/anchore/syft/blob/v0.16.1/internal/constants.go#L9 + /// Can convert JSON Schema to C# using quicktype.io. + /// + public class VcpkgSBOM + { + public Package[] Packages { get; set; } + + public string Name { get; set; } + } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs new file mode 100644 index 000000000..0c6dcf066 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs @@ -0,0 +1,101 @@ +using System; +using System.Collections.Generic; +using System.Composition; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Microsoft.ComponentDetection.Common; +using Microsoft.ComponentDetection.Common.Telemetry.Records; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.Internal; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.ComponentDetection.Detectors.Vcpkg.Contracts; +using Newtonsoft.Json; + +namespace Microsoft.ComponentDetection.Detectors.Vcpkg +{ + [Export(typeof(IComponentDetector))] + public class VcpkgComponentDetector : FileComponentDetector, IDefaultOffComponentDetector + { + [Import] + public ICommandLineInvocationService CommandLineInvocationService { get; set; } + + [Import] + public IEnvironmentVariableService EnvVarService { get; set; } + + public override string Id { get; } = "Vcpkg"; + + public override IEnumerable Categories => new[] { Enum.GetName(typeof(DetectorClass), DetectorClass.Vcpkg) }; + + public override IList SearchPatterns { get; } = new List { "vcpkg.spdx.json" }; + + public override IEnumerable SupportedComponentTypes { get; } = new[] { ComponentType.Go }; + + public override int Version => 1; + + private HashSet projectRoots = new HashSet(); + + protected override async Task OnFileFound(ProcessRequest processRequest, IDictionary detectorArgs) + { + var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; + var file = processRequest.ComponentStream; + + Logger.LogWarning($"vcpkg detector found {file}"); + + var projectRootDirectory = Directory.GetParent(file.Location); + if (projectRoots.Any(path => projectRootDirectory.FullName.StartsWith(path))) + { + return; + } + + ParseSpdxFile(singleFileComponentRecorder, file); + } + + private void ParseSpdxFile( + ISingleFileComponentRecorder singleFileComponentRecorder, + IComponentStream file) + { + using var reader = new StreamReader(file.Stream); + var sbom = JsonConvert.DeserializeObject(reader.ReadToEnd()); + + foreach (var item in sbom.Packages) + { + if (item.Name == null || item.Name.Length == 0) { continue; } + try + { + Logger.LogWarning($"parsed package {item.Name}"); + if (item.SPDXID == "SPDXRef-port") + { + var split = item.VersionInfo.Split('#'); + var component = new VcpkgComponent(item.SPDXID, item.Name, split[0], portVersion: split.Length >= 2 ? split[1] : "0", downloadLocation: item.DownloadLocation); + singleFileComponentRecorder.RegisterUsage(new DetectedComponent(component)); + } + else if (item.SPDXID == "SPDXRef-binary") + { + var split = item.Name.Split(':'); + var component = new VcpkgComponent(item.SPDXID, item.Name, item.VersionInfo, triplet: split[1], downloadLocation: item.DownloadLocation); + singleFileComponentRecorder.RegisterUsage(new DetectedComponent(component)); + } + else if (item.SPDXID.StartsWith("SPDXRef-resource-")) + { + var dl = item.DownloadLocation; + var split = dl.Split("#"); + var subpath = split.Length > 1 ? split[1] : null; + dl = split.Length > 1 ? split[0] : dl; + split = dl.Split("@"); + var version = split.Length > 1 ? split[1] : null; + dl = split.Length > 1 ? split[0] : dl; + + var component = new VcpkgComponent(item.SPDXID, item.Name, version, downloadLocation: dl); + singleFileComponentRecorder.RegisterUsage(new DetectedComponent(component)); + } + } + catch (Exception) + { + Logger.LogWarning($"failed while handling {item.Name}"); + } + } + } + } +} diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Resources/vcpkg/nlohmann-json/vcpkg.spdx.json b/test/Microsoft.ComponentDetection.Detectors.Tests/Resources/vcpkg/nlohmann-json/vcpkg.spdx.json new file mode 100644 index 000000000..29bae26a3 --- /dev/null +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/Resources/vcpkg/nlohmann-json/vcpkg.spdx.json @@ -0,0 +1,115 @@ +{ + "$schema": "https://raw.githubusercontent.com/spdx/spdx-spec/v2.2.1/schemas/spdx-schema.json", + "spdxVersion": "SPDX-2.2", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "documentNamespace": "https://spdx.org/spdxdocs/nlohmann-json-x64-linux-3.10.4-78c7f190-b402-44d1-a364-b9ac86392b84", + "name": "nlohmann-json:x64-linux@3.10.4 69dcfc6886529ad2d210f71f132d743672a7e65d2c39f53456f17fc5fc08b278", + "creationInfo": { + "creators": [ + "Tool: vcpkg-unknownhash" + ], + "created": "2022-01-18T21:12:48Z" + }, + "relationships": [ + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "GENERATES", + "relatedSpdxElement": "SPDXRef-binary" + }, + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-file-0" + }, + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-file-1" + }, + { + "spdxElementId": "SPDXRef-binary", + "relationshipType": "GENERATED_FROM", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-0", + "relationshipType": "CONTAINED_BY", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-1", + "relationshipType": "CONTAINED_BY", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-1", + "relationshipType": "DEPENDENCY_MANIFEST_OF", + "relatedSpdxElement": "SPDXRef-port" + } + ], + "packages": [ + { + "name": "nlohmann-json", + "SPDXID": "SPDXRef-port", + "versionInfo": "3.10.4", + "downloadLocation": "git+https://github.com/Microsoft/vcpkg#ports/nlohmann-json", + "homepage": "https://github.com/nlohmann/json", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "copyrightText": "NOASSERTION", + "description": "JSON for Modern C++", + "comment": "This is the port (recipe) consumed by vcpkg." + }, + { + "name": "nlohmann-json:x64-linux", + "SPDXID": "SPDXRef-binary", + "versionInfo": "69dcfc6886529ad2d210f71f132d743672a7e65d2c39f53456f17fc5fc08b278", + "downloadLocation": "NONE", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "copyrightText": "NOASSERTION", + "comment": "This is a binary package built by vcpkg." + }, + { + "SPDXID": "SPDXRef-resource-1", + "name": "nlohmann/json", + "downloadLocation": "git+https://github.com/nlohmann/json@v3.10.4", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "copyrightText": "NOASSERTION", + "checksums": [ + { + "algorithm": "SHA512", + "checksumValue": "f78592db6218165cbc74c10bcba40366f1bfea84405b7ee25fe97a056d5b7a15aeeb956d93296673928dcbd6e26ffcfb152f885b4a44d5d55751396ccf090835" + } + ] + } + ], + "files": [ + { + "fileName": "./portfile.cmake", + "SPDXID": "SPDXRef-file-0", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "49b4f9e11cdd0ef697a2750187fa5fa42e7e833dbf411d299ca4d1e9f147773a" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION" + }, + { + "fileName": "./vcpkg.json", + "SPDXID": "SPDXRef-file-1", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "b97c75815135eb812a88ee92d50170928d4c69517bfb332e181070cbce2c081b" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION" + } + ] +} \ No newline at end of file diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Resources/vcpkg/tinyxml2/vcpkg.spdx.json b/test/Microsoft.ComponentDetection.Detectors.Tests/Resources/vcpkg/tinyxml2/vcpkg.spdx.json new file mode 100644 index 000000000..91d7433c3 --- /dev/null +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/Resources/vcpkg/tinyxml2/vcpkg.spdx.json @@ -0,0 +1,115 @@ +{ + "$schema": "https://raw.githubusercontent.com/spdx/spdx-spec/v2.2.1/schemas/spdx-schema.json", + "spdxVersion": "SPDX-2.2", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "documentNamespace": "https://spdx.org/spdxdocs/tinyxml2-x64-linux-9.0.0-c99e4f03-5275-458b-8a69-b5f8dfa45f18", + "name": "tinyxml2:x64-linux@9.0.0 5c7679507def92c5c71df44aec08a90a5c749f7f805b3f0e8e70f5e8a5b1b8d0", + "creationInfo": { + "creators": [ + "Tool: vcpkg-unknownhash" + ], + "created": "2022-01-14T00:28:41Z" + }, + "relationships": [ + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "GENERATES", + "relatedSpdxElement": "SPDXRef-binary" + }, + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-file-0" + }, + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-file-1" + }, + { + "spdxElementId": "SPDXRef-binary", + "relationshipType": "GENERATED_FROM", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-0", + "relationshipType": "CONTAINED_BY", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-1", + "relationshipType": "CONTAINED_BY", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-1", + "relationshipType": "DEPENDENCY_MANIFEST_OF", + "relatedSpdxElement": "SPDXRef-port" + } + ], + "packages": [ + { + "name": "tinyxml2", + "SPDXID": "SPDXRef-port", + "versionInfo": "9.0.0", + "downloadLocation": "git+https://github.com/Microsoft/vcpkg#ports/tinyxml2", + "homepage": "https://github.com/leethomason/tinyxml2", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "copyrightText": "NOASSERTION", + "description": "A simple, small, efficient, C++ XML parser", + "comment": "This is the port (recipe) consumed by vcpkg." + }, + { + "name": "tinyxml2:x64-linux", + "SPDXID": "SPDXRef-binary", + "versionInfo": "5c7679507def92c5c71df44aec08a90a5c749f7f805b3f0e8e70f5e8a5b1b8d0", + "downloadLocation": "NONE", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "copyrightText": "NOASSERTION", + "comment": "This is a binary package built by vcpkg." + }, + { + "SPDXID": "SPDXRef-resource-1", + "name": "leethomason/tinyxml2", + "downloadLocation": "git+https://github.com/leethomason/tinyxml2@9.0.0", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "copyrightText": "NOASSERTION", + "checksums": [ + { + "algorithm": "SHA512", + "checksumValue": "9c5ce8131984690df302ca3e32314573b137180ed522c92fd631692979c942372a28f697fdb3d5e56bcf2d3dc596262b724d088153f3e1d721c9536f2a883367" + } + ] + } + ], + "files": [ + { + "fileName": "./portfile.cmake", + "SPDXID": "SPDXRef-file-0", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "dad309c46aea9ccd9a3779723970963b2792d6a7fade26f95437be8ed18ccd60" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION" + }, + { + "fileName": "./vcpkg.json", + "SPDXID": "SPDXRef-file-1", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "4f8ddfeb9d3faa3ecf03f22a07678a060e353cde5e2b46be1cbe519993aab841" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION" + } + ] +} From b7703827d316129f6b6a33da5189f4fc8247ea98 Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Thu, 27 Jan 2022 17:25:45 +0000 Subject: [PATCH 04/10] Fix warnings --- .../TypedComponent/VcpkgComponent.cs | 10 ++++++++++ .../vcpkg/VcpkgComponentDetector.cs | 12 ++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs index d5749b8b7..41037f922 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs @@ -41,9 +41,13 @@ public override string Id get { if (PortVersion != null) + { return $"{Name} {Version}#{PortVersion} - {Type}"; + } else + { return $"{Name} {Version} - {Type}"; + } } } @@ -52,11 +56,17 @@ public override PackageURL PackageUrl get { if (PortVersion != null) + { return new PackageURL($"pkg:vcpkg/{Name}@{Version}?port_version={PortVersion}"); + } else if (Version != null) + { return new PackageURL($"pkg:vcpkg/{Name}@{Version}"); + } else + { return new PackageURL($"pkg:vcpkg/{Name}"); + } } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs index 0c6dcf066..8674a99e1 100644 --- a/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs @@ -49,19 +49,23 @@ protected override async Task OnFileFound(ProcessRequest processRequest, IDictio return; } - ParseSpdxFile(singleFileComponentRecorder, file); + await ParseSpdxFile(singleFileComponentRecorder, file); } - private void ParseSpdxFile( + private async Task ParseSpdxFile( ISingleFileComponentRecorder singleFileComponentRecorder, IComponentStream file) { using var reader = new StreamReader(file.Stream); - var sbom = JsonConvert.DeserializeObject(reader.ReadToEnd()); + var sbom = JsonConvert.DeserializeObject(await reader.ReadToEndAsync()); foreach (var item in sbom.Packages) { - if (item.Name == null || item.Name.Length == 0) { continue; } + if (item.Name == null || item.Name.Length == 0) + { + continue; + } + try { Logger.LogWarning($"parsed package {item.Name}"); From 243a8f4221b66ef76d2e64952bff49125938cb12 Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Wed, 9 Mar 2022 16:12:51 -0800 Subject: [PATCH 05/10] Update src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs Co-authored-by: Greg Villicana <58237075+grvillic@users.noreply.github.com> --- .../vcpkg/VcpkgComponentDetector.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs index 8674a99e1..51648e7fd 100644 --- a/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs @@ -30,7 +30,7 @@ public class VcpkgComponentDetector : FileComponentDetector, IDefaultOffComponen public override IList SearchPatterns { get; } = new List { "vcpkg.spdx.json" }; - public override IEnumerable SupportedComponentTypes { get; } = new[] { ComponentType.Go }; + public override IEnumerable SupportedComponentTypes { get; } = new[] { ComponentType.Vcpkg }; public override int Version => 1; From dbd4ad215d6fa0dfdb78cf78a1d2d9219377cd6b Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Fri, 11 Mar 2022 00:04:00 +0000 Subject: [PATCH 06/10] Address PR comments. Add parsing for Annotations. --- .../vcpkg/Contracts/Annotation.cs | 13 +++++++++++++ .../vcpkg/Contracts/Package.cs | 2 ++ .../vcpkg/Contracts/VcpkgSBOM.cs | 4 +--- .../vcpkg/VcpkgComponentDetector.cs | 2 +- .../Resources/vcpkg/tinyxml2/vcpkg.spdx.json | 12 ++++++++++-- 5 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Annotation.cs diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Annotation.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Annotation.cs new file mode 100644 index 000000000..f9e62f5e8 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Annotation.cs @@ -0,0 +1,13 @@ +namespace Microsoft.ComponentDetection.Detectors.Vcpkg.Contracts +{ + public class Annotation + { + public string Date { get; set; } + + public string Comment { get; set; } + + public string Type { get; set; } + + public string Annotator { get; set; } + } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Package.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Package.cs index 055e17b36..fd4b7a6e8 100644 --- a/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Package.cs +++ b/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Package.cs @@ -15,5 +15,7 @@ public class Package public string Description { get; set; } public string Name { get; set; } + + public Annotation[] Annotations { get; set; } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/VcpkgSBOM.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/VcpkgSBOM.cs index 0fd9de256..77c53be03 100644 --- a/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/VcpkgSBOM.cs +++ b/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/VcpkgSBOM.cs @@ -1,9 +1,7 @@ namespace Microsoft.ComponentDetection.Detectors.Vcpkg.Contracts { /// - /// Take from https://github.com/anchore/syft/tree/main/schema/json. - /// Match version to tag used i.e. https://github.com/anchore/syft/blob/v0.16.1/internal/constants.go#L9 - /// Can convert JSON Schema to C# using quicktype.io. + /// Matches a subset of https://raw.githubusercontent.com/spdx/spdx-spec/v2.2.1/schemas/spdx-schema.json. /// public class VcpkgSBOM { diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs index 51648e7fd..79a8f8e19 100644 --- a/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs @@ -61,7 +61,7 @@ private async Task ParseSpdxFile( foreach (var item in sbom.Packages) { - if (item.Name == null || item.Name.Length == 0) + if (string.IsNullOrEmpty(item.Name)) { continue; } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Resources/vcpkg/tinyxml2/vcpkg.spdx.json b/test/Microsoft.ComponentDetection.Detectors.Tests/Resources/vcpkg/tinyxml2/vcpkg.spdx.json index 91d7433c3..fc97fee5c 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/Resources/vcpkg/tinyxml2/vcpkg.spdx.json +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/Resources/vcpkg/tinyxml2/vcpkg.spdx.json @@ -59,7 +59,15 @@ "licenseDeclared": "NOASSERTION", "copyrightText": "NOASSERTION", "description": "A simple, small, efficient, C++ XML parser", - "comment": "This is the port (recipe) consumed by vcpkg." + "comment": "This is the port (recipe) consumed by vcpkg.", + "annotations": [ + { + "annotationDate": "2010-01-29T18:30:22Z", + "annotationType": "OTHER", + "annotator": "Tool: vcpkg - 2020-01-01", + "comment": "vcpkgPackageType:port" + } + ] }, { "name": "tinyxml2:x64-linux", @@ -112,4 +120,4 @@ "copyrightText": "NOASSERTION" } ] -} +} \ No newline at end of file From 0d7a37b26eb888adb36f9d5c831c1d58b29c7f25 Mon Sep 17 00:00:00 2001 From: Greg Villicana Date: Tue, 22 Mar 2022 19:11:31 -0700 Subject: [PATCH 07/10] Use DateTime property for annotation object --- .../vcpkg/Contracts/Annotation.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Annotation.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Annotation.cs index f9e62f5e8..0a11ca4cd 100644 --- a/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Annotation.cs +++ b/src/Microsoft.ComponentDetection.Detectors/vcpkg/Contracts/Annotation.cs @@ -1,8 +1,10 @@ +using System; + namespace Microsoft.ComponentDetection.Detectors.Vcpkg.Contracts { public class Annotation { - public string Date { get; set; } + public DateTime Date { get; set; } public string Comment { get; set; } From 40e62a5c3de06433b61dcfe3542ace89db114814 Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Wed, 30 Mar 2022 21:03:18 +0000 Subject: [PATCH 08/10] Add tests for VcpkgComponentDetector --- .../vcpkg/VcpkgComponentDetector.cs | 21 ++- .../VcpkgComponentDetectorTests.cs | 174 ++++++++++++++++++ .../vcpkg/nlohmann-json/vcpkg.spdx.json | 0 .../resources}/vcpkg/tinyxml2/vcpkg.spdx.json | 0 4 files changed, 189 insertions(+), 6 deletions(-) create mode 100644 test/Microsoft.ComponentDetection.Detectors.Tests/VcpkgComponentDetectorTests.cs rename test/{Microsoft.ComponentDetection.Detectors.Tests/Resources => Microsoft.ComponentDetection.VerificationTests/resources}/vcpkg/nlohmann-json/vcpkg.spdx.json (100%) rename test/{Microsoft.ComponentDetection.Detectors.Tests/Resources => Microsoft.ComponentDetection.VerificationTests/resources}/vcpkg/tinyxml2/vcpkg.spdx.json (100%) diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs index 79a8f8e19..e53cc899b 100644 --- a/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs @@ -57,17 +57,26 @@ private async Task ParseSpdxFile( IComponentStream file) { using var reader = new StreamReader(file.Stream); - var sbom = JsonConvert.DeserializeObject(await reader.ReadToEndAsync()); + VcpkgSBOM sbom; + try + { + sbom = JsonConvert.DeserializeObject(await reader.ReadToEndAsync()); + } + catch (Exception) + { + return; + } + if (sbom == null || sbom.Packages == null) { return; } foreach (var item in sbom.Packages) { - if (string.IsNullOrEmpty(item.Name)) - { - continue; - } - try { + if (string.IsNullOrEmpty(item.Name)) + { + continue; + } + Logger.LogWarning($"parsed package {item.Name}"); if (item.SPDXID == "SPDXRef-port") { diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/VcpkgComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/VcpkgComponentDetectorTests.cs new file mode 100644 index 000000000..9e28ccaa6 --- /dev/null +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/VcpkgComponentDetectorTests.cs @@ -0,0 +1,174 @@ +using Microsoft.ComponentDetection.Common.DependencyGraph; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.TestsUtilities; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading.Tasks; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.ComponentDetection.Detectors.Vcpkg; +using System.Reflection; + +namespace Microsoft.ComponentDetection.Detectors.Tests +{ + [TestClass] + [TestCategory("Governance/All")] + [TestCategory("Governance/ComponentDetection")] + public class VcpkgComponentDetectorTests + { + private DetectorTestUtility detectorTestUtility; + + [TestInitialize] + public void TestInitialize() + { + var componentRecorder = new ComponentRecorder(enableManualTrackingOfExplicitReferences: false); + detectorTestUtility = DetectorTestUtilityCreator.Create() + .WithScanRequest(new ScanRequest(new DirectoryInfo(Path.GetTempPath()), null, null, new Dictionary(), null, componentRecorder)); + } + + [TestMethod] + public async Task TestNlohmann() + { + var spdxFile = @"{ + ""SPDXID"": ""SPDXRef - DOCUMENT"", + ""documentNamespace"": + ""https://spdx.org/spdxdocs/nlohmann-json-x64-linux-3.10.4-78c7f190-b402-44d1-a364-b9ac86392b84"", + ""name"": ""nlohmann-json:x64-linux@3.10.4 69dcfc6886529ad2d210f71f132d743672a7e65d2c39f53456f17fc5fc08b278"", + ""packages"": [ + { + ""name"": ""nlohmann-json"", + ""SPDXID"": ""SPDXRef-port"", + ""versionInfo"": ""3.10.4"", + ""downloadLocation"": ""git+https://github.com/Microsoft/vcpkg#ports/nlohmann-json"", + ""homepage"": ""https://github.com/nlohmann/json"", + ""licenseConcluded"": ""NOASSERTION"", + ""licenseDeclared"": ""NOASSERTION"", + ""copyrightText"": ""NOASSERTION"", + ""description"": ""JSON for Modern C++"", + ""comment"": ""This is the port (recipe) consumed by vcpkg."" + } + ] +}"; + var (scanResult, componentRecorder) = await detectorTestUtility + .WithFile("vcpkg.spdx.json", spdxFile) + .ExecuteDetector(); + + Assert.AreEqual(ProcessingResultCode.Success, scanResult.ResultCode); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + var components = detectedComponents.ToList(); + var sbomComponent = (VcpkgComponent)components.FirstOrDefault()?.Component; + + if (sbomComponent is null) + { + throw new AssertFailedException($"{nameof(sbomComponent)} is null"); + } + + Assert.AreEqual(1, components.Count()); + Assert.AreEqual("nlohmann-json", sbomComponent.Name); + Assert.AreEqual("3.10.4", sbomComponent.Version); + Assert.AreEqual("0", sbomComponent.PortVersion); + Assert.AreEqual("SPDXRef-port", sbomComponent.SPDXID); + Assert.AreEqual("git+https://github.com/Microsoft/vcpkg#ports/nlohmann-json", sbomComponent.DownloadLocation); + Assert.AreEqual("pkg:vcpkg/nlohmann-json@3.10.4?port_version=0", sbomComponent.PackageUrl.ToString()); + } + + [TestMethod] + public async Task TestTinyxmlAndResource() + { + var spdxFile = @"{ + ""SPDXID"": ""SPDXRef - DOCUMENT"", + ""documentNamespace"": + ""https://spdx.org/spdxdocs/tinyxml2-x64-linux-9.0.0-c99e4f03-5275-458b-8a69-b5f8dfa45f18"", + ""name"": ""tinyxml2:x64-linux@9.0.0 5c7679507def92c5c71df44aec08a90a5c749f7f805b3f0e8e70f5e8a5b1b8d0"", + ""packages"": [ + { + ""name"": ""tinyxml2:x64-linux"", + ""SPDXID"": ""SPDXRef-binary"", + ""versionInfo"": ""5c7679507def92c5c71df44aec08a90a5c749f7f805b3f0e8e70f5e8a5b1b8d0"", + ""downloadLocation"": ""NONE"", + ""licenseConcluded"": ""NOASSERTION"", + ""licenseDeclared"": ""NOASSERTION"", + ""copyrightText"": ""NOASSERTION"", + ""comment"": ""This is a binary package built by vcpkg."" + }, + { + ""SPDXID"": ""SPDXRef-resource-1"", + ""name"": ""leethomason/tinyxml2"", + ""downloadLocation"": ""git+https://github.com/leethomason/tinyxml2@9.0.0"", + ""licenseConcluded"": ""NOASSERTION"", + ""licenseDeclared"": ""NOASSERTION"", + ""copyrightText"": ""NOASSERTION"", + ""checksums"": [ + { + ""algorithm"": ""SHA512"", + ""checksumValue"": ""9c5ce8131984690df302ca3e32314573b137180ed522c92fd631692979c942372a28f697fdb3d5e56bcf2d3dc596262b724d088153f3e1d721c9536f2a883367"" + } + ] + } + ] +}"; + var (scanResult, componentRecorder) = await detectorTestUtility + .WithFile("vcpkg.spdx.json", spdxFile) + .ExecuteDetector(); + + Assert.AreEqual(ProcessingResultCode.Success, scanResult.ResultCode); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + var components = detectedComponents.ToList(); + + Assert.AreEqual(2, components.Count()); + { + var sbomComponent = (VcpkgComponent)components[0].Component; + Assert.AreEqual("tinyxml2:x64-linux", sbomComponent.Name); + Assert.AreEqual("5c7679507def92c5c71df44aec08a90a5c749f7f805b3f0e8e70f5e8a5b1b8d0", sbomComponent.Version); + Assert.AreEqual("SPDXRef-binary", sbomComponent.SPDXID); + Assert.AreEqual("NONE", sbomComponent.DownloadLocation); + } + + { + var sbomComponent = (VcpkgComponent)components[1].Component; + Assert.AreEqual("leethomason/tinyxml2", sbomComponent.Name); + Assert.AreEqual("9.0.0", sbomComponent.Version); + Assert.AreEqual("SPDXRef-resource-1", sbomComponent.SPDXID); + Assert.AreEqual("git+https://github.com/leethomason/tinyxml2", sbomComponent.DownloadLocation); + } + } + + [TestMethod] + public async Task TestBlankJson() + { + var spdxFile = "{}"; + + var (scanResult, componentRecorder) = await detectorTestUtility + .WithFile("vcpkg.spdx.json", spdxFile) + .ExecuteDetector(); + + Assert.AreEqual(ProcessingResultCode.Success, scanResult.ResultCode); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + var components = detectedComponents.ToList(); + Assert.IsFalse(components.Any()); + } + + [TestMethod] + public async Task TestInvalidFile() + { + var spdxFile = "invalidspdxfile"; + + var (scanResult, componentRecorder) = await detectorTestUtility + .WithFile("vcpkg.spdx.json", spdxFile) + .ExecuteDetector(); + + Assert.AreEqual(ProcessingResultCode.Success, scanResult.ResultCode); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + var components = detectedComponents.ToList(); + Assert.IsFalse(components.Any()); + } + } +} diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Resources/vcpkg/nlohmann-json/vcpkg.spdx.json b/test/Microsoft.ComponentDetection.VerificationTests/resources/vcpkg/nlohmann-json/vcpkg.spdx.json similarity index 100% rename from test/Microsoft.ComponentDetection.Detectors.Tests/Resources/vcpkg/nlohmann-json/vcpkg.spdx.json rename to test/Microsoft.ComponentDetection.VerificationTests/resources/vcpkg/nlohmann-json/vcpkg.spdx.json diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Resources/vcpkg/tinyxml2/vcpkg.spdx.json b/test/Microsoft.ComponentDetection.VerificationTests/resources/vcpkg/tinyxml2/vcpkg.spdx.json similarity index 100% rename from test/Microsoft.ComponentDetection.Detectors.Tests/Resources/vcpkg/tinyxml2/vcpkg.spdx.json rename to test/Microsoft.ComponentDetection.VerificationTests/resources/vcpkg/tinyxml2/vcpkg.spdx.json From 037d2c60cc151a6af0e2e298d43350b51478f274 Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Wed, 30 Mar 2022 21:10:16 +0000 Subject: [PATCH 09/10] Satisfy format detector --- .../vcpkg/VcpkgComponentDetector.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs index e53cc899b..b7e5da1b9 100644 --- a/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs @@ -66,7 +66,11 @@ private async Task ParseSpdxFile( { return; } - if (sbom == null || sbom.Packages == null) { return; } + + if (sbom == null || sbom.Packages == null) + { + return; + } foreach (var item in sbom.Packages) { From a2cc24a6028894e6ea409ad84cfba3f8b8836bc0 Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Wed, 30 Mar 2022 16:55:41 -0700 Subject: [PATCH 10/10] Update src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs Co-authored-by: Greg Villicana <58237075+grvillic@users.noreply.github.com> --- .../vcpkg/VcpkgComponentDetector.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs index b7e5da1b9..60a8e99c6 100644 --- a/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs @@ -67,7 +67,7 @@ private async Task ParseSpdxFile( return; } - if (sbom == null || sbom.Packages == null) + if (sbom?.Packages == null) { return; }