From 04fd4ee21769611ad923ae482cbe898f4fdc6ae3 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Thu, 27 Feb 2025 14:26:59 -0800 Subject: [PATCH 1/7] Add DotNetComponent The DotNetComponent type is used to report usage of the .NET SDK version This is important as the .NET SDK will add redistributable content to the project, so using an older version may mean your application contains unserviced copies of the host or runtime. The TargetFramework and project type is reported as well since these influence what content the SDK may reference. --- .../CommandLineInvocationService.cs | 2 +- .../TypedComponent/ComponentType.cs | 3 + .../TypedComponent/DotNetComponent.cs | 64 ++++ .../dotnet/DotNetComponentDetector.cs | 186 ++++++++++ .../yarn/YarnLockComponentDetector.cs | 2 +- .../Experiments/DotNetDetectorExperiment.cs | 22 ++ .../Extensions/ServiceCollectionExtensions.cs | 5 + .../DotNetComponentDetectorTests.cs | 348 ++++++++++++++++++ ....ComponentDetection.Detectors.Tests.csproj | 1 + 9 files changed, 631 insertions(+), 2 deletions(-) create mode 100644 src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs create mode 100644 src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs create mode 100644 src/Microsoft.ComponentDetection.Orchestrator/Experiments/DotNetDetectorExperiment.cs create mode 100644 test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs diff --git a/src/Microsoft.ComponentDetection.Common/CommandLineInvocationService.cs b/src/Microsoft.ComponentDetection.Common/CommandLineInvocationService.cs index e391212c5..f82c1eede 100644 --- a/src/Microsoft.ComponentDetection.Common/CommandLineInvocationService.cs +++ b/src/Microsoft.ComponentDetection.Common/CommandLineInvocationService.cs @@ -60,7 +60,7 @@ public async Task ExecuteCommandAsync( CancellationToken cancellationToken = default, params string[] parameters) { - var isCommandLocatable = await this.CanCommandBeLocatedAsync(command, additionalCandidateCommands); + var isCommandLocatable = await this.CanCommandBeLocatedAsync(command, additionalCandidateCommands, workingDirectory, parameters); if (!isCommandLocatable) { throw new InvalidOperationException( diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ComponentType.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ComponentType.cs index 6e2becf9d..31b245d59 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ComponentType.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ComponentType.cs @@ -59,4 +59,7 @@ public enum ComponentType : byte [EnumMember] Swift = 18, + + [EnumMember] + DotNet = 19, } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs new file mode 100644 index 000000000..65e21c349 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs @@ -0,0 +1,64 @@ +namespace Microsoft.ComponentDetection.Contracts.TypedComponent; + +using System.Text; + +#nullable enable + +using PackageUrl; + +public class DotNetComponent : TypedComponent +{ + private DotNetComponent() + { + /* Reserved for deserialization */ + } + + public DotNetComponent(string? sdkVersion, string? targetFramework = null, string? projectType = null) + { + this.SdkVersion = sdkVersion; + this.TargetFramework = targetFramework; + this.ProjectType = projectType; // application, library, or null + } + + /// + /// SDK Version detected, could be null if no global.json exists and no dotnet is on the path. + /// + public string? SdkVersion { get; set; } + + /// + /// Target framework for this instance. Null in the case of global.json. + /// + public string? TargetFramework { get; set; } + + /// + /// Project type: application, library. Null in the case of global.json or if no project output could be discovered. + /// + public string? ProjectType { get; set; } + + public override ComponentType Type => ComponentType.DotNet; + + /// + /// Provides an id like `dotnet {SdkVersion} - {TargetFramework} - {ProjectType}` where targetFramework and projectType are only present if not null. + /// + public override string Id + { + get + { + var builder = new StringBuilder($"dotnet {this.SdkVersion ?? "unknown"}"); + if (this.TargetFramework is not null) + { + builder.Append($" - {this.TargetFramework}"); + + if (this.ProjectType is not null) + { + builder.Append($" - {this.ProjectType}"); + } + } + + return builder.ToString(); + } + } + + // TODO - do we need to add a type to prul https://github.com/package-url/purl-spec/blob/main/PURL-TYPES.rst + public override PackageURL PackageUrl => new PackageURL("generic", null, "dotnet-sdk", this.SdkVersion ?? "unknown", null, null); +} diff --git a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs new file mode 100644 index 000000000..c7cd71364 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs @@ -0,0 +1,186 @@ +namespace Microsoft.ComponentDetection.Detectors.DotNet; + +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection.PortableExecutable; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using global::NuGet.ProjectModel; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.Internal; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; + +public class DotNetComponentDetector : FileComponentDetector, IExperimentalDetector +{ + private const string GlobalJsonFileName = "global.json"; + private readonly ICommandLineInvocationService commandLineInvocationService; + private readonly IDirectoryUtilityService directoryUtilityService; + private readonly IFileUtilityService fileUtilityService; + private readonly IPathUtilityService pathUtilityService; + private readonly LockFileFormat lockFileFormat = new(); + private readonly Dictionary sdkVersionCache = []; + private string? sourceDirectory; + private string? sourceFileRootDirectory; + + public DotNetComponentDetector( + IComponentStreamEnumerableFactory componentStreamEnumerableFactory, + ICommandLineInvocationService commandLineInvocationService, + IDirectoryUtilityService directoryUtilityService, + IFileUtilityService fileUtilityService, + IPathUtilityService pathUtilityService, + IObservableDirectoryWalkerFactory walkerFactory, + ILogger logger) + { + this.commandLineInvocationService = commandLineInvocationService; + this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; + this.directoryUtilityService = directoryUtilityService; + this.fileUtilityService = fileUtilityService; + this.pathUtilityService = pathUtilityService; + this.Scanner = walkerFactory; + this.Logger = logger; + } + + public override string Id => "DotNet"; + + public override IList SearchPatterns { get; } = [LockFileFormat.AssetsFileName]; + + public override IEnumerable SupportedComponentTypes => [ComponentType.DotNet]; + + public override int Version { get; } = 1; + + public override IEnumerable Categories => ["DotNet"]; + + private async Task RunDotNetVersionAsync(string workingDirectoryPath, CancellationToken cancellationToken) + { + var workingDirectory = new DirectoryInfo(workingDirectoryPath); + + var process = await this.commandLineInvocationService.ExecuteCommandAsync("dotnet", ["dotnet.exe"], workingDirectory, cancellationToken, "--version").ConfigureAwait(false); + return process.ExitCode == 0 ? process.StdOut.Trim() : null; + } + + public override Task ExecuteDetectorAsync(ScanRequest request, CancellationToken cancellationToken = default) + { + this.sourceDirectory = this.pathUtilityService.NormalizePath(request.SourceDirectory.FullName); + this.sourceFileRootDirectory = this.pathUtilityService.NormalizePath(request.SourceFileRoot?.FullName); + + return base.ExecuteDetectorAsync(request, cancellationToken); + } + + protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) + { + var lockFile = this.lockFileFormat.Read(processRequest.ComponentStream.Stream, processRequest.ComponentStream.Location); + + var projectPath = lockFile.PackageSpec.RestoreMetadata.ProjectPath; + var projectDirectory = this.pathUtilityService.GetParentDirectory(projectPath); + var sdkVersion = await this.GetSdkVersionAsync(projectDirectory, cancellationToken); + + var projectName = lockFile.PackageSpec.RestoreMetadata.ProjectName; + var projectOutputPath = lockFile.PackageSpec.RestoreMetadata.OutputPath; + var targetType = this.GetProjectType(projectOutputPath, projectName, cancellationToken); + + var componentReporter = this.ComponentRecorder.CreateSingleFileComponentRecorder(projectPath); + foreach (var target in lockFile.Targets) + { + var targetFramework = target.TargetFramework?.GetShortFolderName(); + + componentReporter.RegisterUsage(new DetectedComponent(new DotNetComponent(sdkVersion, targetFramework, targetType))); + } + } + + private string? GetProjectType(string projectOutputPath, string projectName, CancellationToken cancellationToken) + { + if (this.directoryUtilityService.Exists(projectOutputPath)) + { + var namePattern = (projectName ?? "*") + ".dll"; + + // look for the compiled output, first as dll then as exe. + var candidates = this.directoryUtilityService.EnumerateFiles(projectOutputPath, namePattern, SearchOption.AllDirectories) + .Concat(this.directoryUtilityService.EnumerateFiles(projectOutputPath, namePattern, SearchOption.AllDirectories)); + foreach (var candidate in candidates) + { + if (this.IsApplication(candidate)) + { + return "application"; + } + else + { + return "library"; + } + } + } + + return null; + } + + private bool IsApplication(string assemblyPath) + { + try + { + using var peReader = new PEReader(this.fileUtilityService.MakeFileStream(assemblyPath)); + + // despite the name `IsExe` this is actually based of the CoffHeader Characteristics + return peReader.PEHeaders.IsExe; + } + catch (Exception) + { + return false; + } + } + + /// + /// Recursively get the sdk version from the project directory or parent directories. + /// + /// Directory to start the search. + /// Cancellation token to halt the search. + /// Sdk version found, or null if no version can be detected. + private async Task GetSdkVersionAsync(string projectDirectory, CancellationToken cancellationToken) + { + // normalize since we need to use as a key + projectDirectory = this.pathUtilityService.NormalizePath(projectDirectory); + if (this.sdkVersionCache.TryGetValue(projectDirectory, out var sdkVersion)) + { + return sdkVersion; + } + + var parentDirectory = this.pathUtilityService.GetParentDirectory(projectDirectory); + var globalJsonPath = Path.Combine(projectDirectory, GlobalJsonFileName); + + if (this.fileUtilityService.Exists(globalJsonPath)) + { + var globalJson = await JsonDocument.ParseAsync(this.fileUtilityService.MakeFileStream(globalJsonPath), cancellationToken: cancellationToken); + if (globalJson.RootElement.TryGetProperty("sdk", out var sdk)) + { + if (sdk.TryGetProperty("version", out var version)) + { + sdkVersion = version.GetString(); + var globalJsonComponent = new DetectedComponent(new DotNetComponent(sdkVersion)); + var recorder = this.ComponentRecorder.CreateSingleFileComponentRecorder(globalJsonPath); + recorder.RegisterUsage(globalJsonComponent, isExplicitReferencedDependency: true); + } + } + } + else if (projectDirectory.Equals(this.sourceDirectory, StringComparison.OrdinalIgnoreCase) || + projectDirectory.Equals(this.sourceFileRootDirectory, StringComparison.OrdinalIgnoreCase) || + parentDirectory is null || + projectDirectory.Equals(parentDirectory, StringComparison.OrdinalIgnoreCase)) + { + // if we are at the source directory, source file root, or have reached a root directory, run `dotnet --version` + // this could fail if dotnet is not on the path, or if the global.json is malformed + sdkVersion = await this.RunDotNetVersionAsync(projectDirectory, cancellationToken); + } + else + { + // recurse up the directory tree + sdkVersion = await this.GetSdkVersionAsync(parentDirectory, cancellationToken); + } + + this.sdkVersionCache[projectDirectory] = sdkVersion; + + return sdkVersion; + } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs index 30407ff45..c2110e2a1 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs @@ -7,7 +7,7 @@ namespace Microsoft.ComponentDetection.Detectors.Yarn; using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; -using DotNet.Globbing; +using global::DotNet.Globbing; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DotNetDetectorExperiment.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DotNetDetectorExperiment.cs new file mode 100644 index 000000000..ab0c33de3 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DotNetDetectorExperiment.cs @@ -0,0 +1,22 @@ +namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Configs; + +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Detectors.DotNet; + +/// +/// Validating the . +/// +public class DotNetDetectorExperiment : IExperimentConfiguration +{ + /// + public string Name => "DotNetDetector"; + + /// + public bool IsInControlGroup(IComponentDetector componentDetector) => false; + + /// + public bool IsInExperimentGroup(IComponentDetector componentDetector) => componentDetector is DotNetComponentDetector; + + /// + public bool ShouldRecord(IComponentDetector componentDetector, int numComponents) => true; +} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs index c0faf5dbb..306199b56 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs @@ -6,6 +6,7 @@ namespace Microsoft.ComponentDetection.Orchestrator.Extensions; using Microsoft.ComponentDetection.Detectors.CocoaPods; using Microsoft.ComponentDetection.Detectors.Conan; using Microsoft.ComponentDetection.Detectors.Dockerfile; +using Microsoft.ComponentDetection.Detectors.DotNet; using Microsoft.ComponentDetection.Detectors.Go; using Microsoft.ComponentDetection.Detectors.Gradle; using Microsoft.ComponentDetection.Detectors.Ivy; @@ -65,6 +66,7 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); // Detectors // CocoaPods @@ -79,6 +81,9 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s // Dockerfile services.AddSingleton(); + // DotNet + services.AddSingleton(); + // Go services.AddSingleton(); services.AddSingleton(); diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs new file mode 100644 index 000000000..28e1e589a --- /dev/null +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs @@ -0,0 +1,348 @@ +namespace Microsoft.ComponentDetection.Detectors.Tests; + +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reactive.Linq; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using global::NuGet.Frameworks; +using global::NuGet.ProjectModel; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.ComponentDetection.Detectors.DotNet; +using Microsoft.ComponentDetection.TestsUtilities; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +[TestClass] +[TestCategory("Governance/All")] +[TestCategory("Governance/ComponentDetection")] +public class DotNetComponentDetectorTests : BaseDetectorTest +{ + private static readonly string RootDir = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "C:" : @"\"; + + private readonly Mock> mockLogger = new(); + + // uses ExecuteCommandAsync + private readonly Mock mockCommandLineInvocationService = new(); + private readonly CommandLineExecutionResult commandLineExecutionResult = new(); + + // uses Exists, EnumerateFiles + private readonly Mock mockDirectoryUtilityService = new(); + + // uses Exists, MakeFileStream + private readonly Mock mockFileUtilityService = new(); + private readonly Dictionary> files = []; + + // uses GetParentDirectory, NormalizePath + private readonly Mock mockPathUtilityService = new(); + + /// + /// Initializes a new instance of the class. + /// + public DotNetComponentDetectorTests() + { + this.DetectorTestUtility.AddServiceMock(this.mockLogger) + .AddServiceMock(this.mockCommandLineInvocationService) + .AddServiceMock(this.mockDirectoryUtilityService) + .AddServiceMock(this.mockFileUtilityService) + .AddServiceMock(this.mockPathUtilityService); + + this.mockFileUtilityService.Setup(x => x.Exists(It.IsAny())).Returns((string p) => this.FileExists(p)); + this.mockFileUtilityService.Setup(x => x.MakeFileStream(It.IsAny())).Returns((string p) => this.OpenFile(p)); + this.mockDirectoryUtilityService.Setup(x => x.Exists(It.IsAny())).Returns((string p) => this.DirectoryExists(p)); + + // ignore pattern and search option since we don't really need them for tests + this.mockDirectoryUtilityService.Setup(x => x.EnumerateFiles(It.IsAny(), It.IsAny(), It.IsAny())).Returns((string d, string p, SearchOption s) => this.EnumerateFilesRecursive(d)); + + this.mockPathUtilityService.Setup(x => x.NormalizePath(It.IsAny())).Returns((string p) => p); // don't do normalization + this.mockPathUtilityService.Setup(x => x.GetParentDirectory(It.IsAny())).Returns((string p) => Path.GetDirectoryName(p)); + + this.mockCommandLineInvocationService.Setup(x => x.ExecuteCommandAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())).Returns(Task.FromResult(this.commandLineExecutionResult)); + } + + private bool FileExists(string path) + { + var fileName = Path.GetFileName(path); + var directory = Path.GetDirectoryName(path); + + return this.files.TryGetValue(directory, out var fileNames) && + fileNames.TryGetValue(fileName, out _); + } + + private Stream OpenFile(string path) + { + var fileName = Path.GetFileName(path); + var directory = Path.GetDirectoryName(path); + + return this.files.TryGetValue(directory, out var fileNames) && + fileNames.TryGetValue(fileName, out var stream) ? stream : null; + } + + private bool DirectoryExists(string directory) => this.files.ContainsKey(directory); + + private IEnumerable EnumerateFilesRecursive(string directory) + { + if (this.files.TryGetValue(directory, out var fileNames)) + { + foreach (var fileName in fileNames.Keys) + { + var filePath = Path.Combine(directory, fileName); + + if (fileName.EndsWith(Path.DirectorySeparatorChar)) + { + foreach (var subFile in this.EnumerateFilesRecursive(filePath.TrimEnd(Path.DirectorySeparatorChar))) + { + yield return subFile; + } + } + else + { + yield return filePath; + } + } + } + } + + private void AddFile(string path, Stream content) + { + var fileName = Path.GetFileName(path); + var directory = Path.GetDirectoryName(path); + this.AddDirectory(directory); + this.files[directory][fileName] = content; + } + + private void AddDirectory(string path, string subDirectory = null) + { + if (path is null) + { + return; + } + + if (subDirectory is not null) + { + // use a trailing slash to indicate a sub directory in the files collection + subDirectory += Path.DirectorySeparatorChar; + } + + if (this.files.TryGetValue(path, out var directoryFiles)) + { + if (subDirectory is not null) + { + directoryFiles.Add(subDirectory, null); + } + } + else + { + this.files.Add(path, subDirectory is null ? [] : new() { { subDirectory, null } }); + this.AddDirectory(Path.GetDirectoryName(path), Path.GetFileName(path)); + } + } + + private void SetCommandResult(int exitCode, string stdOut = null, string stdErr = null) + { + this.commandLineExecutionResult.ExitCode = exitCode; + this.commandLineExecutionResult.StdOut = stdOut; + this.commandLineExecutionResult.StdErr = stdErr; + } + + [TestCleanup] + public void ClearMocks() + { + this.files.Clear(); + this.SetCommandResult(-1); + } + + private static string ProjectAssets(string projectName, string outputPath, string projectPath, params string[] targetFrameworks) + { + LockFileFormat format = new(); + LockFile lockFile = new(); + using var textWriter = new StringWriter(); + + lockFile.Targets = targetFrameworks.Select(tfm => new LockFileTarget() { TargetFramework = NuGetFramework.Parse(tfm) }).ToList(); + lockFile.PackageSpec = new() + { + RestoreMetadata = new() + { + ProjectName = projectName, + OutputPath = outputPath, + ProjectPath = projectPath, + }, + }; + + format.Write(textWriter, lockFile); + return textWriter.ToString(); + } + + private static Stream GlobalJson(string sdkVersion) + { + var stream = new MemoryStream(); + using (var writer = new Utf8JsonWriter(stream, new() { Indented = true })) + { + writer.WriteStartObject(); + writer.WritePropertyName("sdk"); + writer.WriteStartObject(); + writer.WriteString("version", sdkVersion); + writer.WriteEndObject(); + writer.WriteEndObject(); + } + + stream.Position = 0; + return stream; + } + + [TestMethod] + public async Task TestDotNetDetectorWithNoFiles_ReturnsSuccessfullyAsync() + { + var (scanResult, componentRecorder) = await this.DetectorTestUtility.ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().BeEmpty(); + } + + [TestMethod] + public async Task TestDotNetDetectorGlobalJson_ReturnsSDKVersion() + { + var projectPath = Path.Combine(RootDir, "path", "to", "project"); + var projectAssets = ProjectAssets("projectName", "does-not-exist", projectPath, "net8.0"); + var globalJson = GlobalJson("42.0.800"); + this.AddFile(projectPath, null); + this.AddFile(Path.Combine(RootDir, "path", "global.json"), globalJson); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("project.assets.json", projectAssets) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().HaveCount(2); + + var discoveredComponents = detectedComponents.ToArray(); + discoveredComponents.Where(component => component.Component.Id == "dotnet 42.0.800").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "dotnet 42.0.800 - net8.0").Should().ContainSingle(); + } + + [TestMethod] + public async Task TestDotNetDetectorNoGlobalJson_ReturnsDotNetVersion() + { + var projectPath = Path.Combine(RootDir, "path", "to", "project"); + var projectAssets = ProjectAssets("projectName", "does-not-exist", projectPath, "net8.0"); + this.AddFile(projectPath, null); + this.SetCommandResult(0, "86.75.309"); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("project.assets.json", projectAssets) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().ContainSingle(); + + var discoveredComponents = detectedComponents.ToArray(); + discoveredComponents.Where(component => component.Component.Id == "dotnet 86.75.309 - net8.0").Should().ContainSingle(); + } + + [TestMethod] + public async Task TestDotNetDetectorNoGlobalJsonNoDotnet_ReturnsUnknownVersion() + { + var projectPath = Path.Combine(RootDir, "path", "to", "project"); + var projectAssets = ProjectAssets("projectName", "does-not-exist", projectPath, "net8.0"); + this.AddFile(projectPath, null); + this.SetCommandResult(-1); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("project.assets.json", projectAssets) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().ContainSingle(); + + var discoveredComponents = detectedComponents.ToArray(); + discoveredComponents.Where(component => component.Component.Id == "dotnet unknown - net8.0").Should().ContainSingle(); + } + + [TestMethod] + public async Task TestDotNetDetectorMultipleTargetFrameworks() + { + var projectPath = Path.Combine(RootDir, "path", "to", "project"); + var projectAssets = ProjectAssets("projectName", "does-not-exist", projectPath, "net8.0", "net6.0", "netstandard2.0"); + this.AddFile(projectPath, null); + this.SetCommandResult(0, "1.2.3"); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("project.assets.json", projectAssets) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().HaveCount(3); + + var discoveredComponents = detectedComponents.ToArray(); + discoveredComponents.Where(component => component.Component.Id == "dotnet 1.2.3 - net8.0").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "dotnet 1.2.3 - net6.0").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "dotnet 1.2.3 - netstandard2.0").Should().ContainSingle(); + } + + [TestMethod] + public async Task TestDotNetDetectorMultipleProjectsWithDifferentOutputTypeAndSdkVersion() + { + // dotnet on the path with be version 1.2.3 + this.SetCommandResult(0, "1.2.3"); + + // dotnet from global.json will be 4.5.6 + var globalJson = GlobalJson("4.5.6"); + this.AddFile(Path.Combine(RootDir, "path", "global.json"), globalJson); + + // set up a library project - under global.json + var libraryProjectName = "library"; + var libraryProjectPath = Path.Combine(RootDir, "path", "to", "project", $"{libraryProjectName}.csproj"); + this.AddFile(libraryProjectPath, null); + var libraryOutputPath = Path.Combine(Path.GetDirectoryName(libraryProjectPath), "obj"); + var libraryAssetsPath = Path.Combine(libraryOutputPath, "project.assets.json"); + var libraryAssets = ProjectAssets("library", libraryOutputPath, libraryProjectPath, "net8.0", "net6.0", "netstandard2.0"); + var libraryAssemblyStream = File.OpenRead(typeof(DotNetComponent).Assembly.Location); + this.AddFile(Path.Combine(libraryOutputPath, "Release", "net8.0", "library.dll"), libraryAssemblyStream); + this.AddFile(Path.Combine(libraryOutputPath, "Release", "net6.0", "library.dll"), libraryAssemblyStream); + this.AddFile(Path.Combine(libraryOutputPath, "Release", "netstandard2.0", "library.dll"), libraryAssemblyStream); + + // set up an application - not under global.json + var applicationProjectName = "application"; + var applicationProjectPath = Path.Combine(RootDir, "anotherPath", "to", "project", $"{applicationProjectName}.csproj"); + this.AddFile(applicationProjectPath, null); + var applicationOutputPath = Path.Combine(Path.GetDirectoryName(applicationProjectPath), "obj"); + var applicationAssetsPath = Path.Combine(applicationOutputPath, "project.assets.json"); + var applicationAssets = ProjectAssets("application", applicationOutputPath, applicationProjectPath, "net8.0", "net4.8"); + var applicationAssemblyStream = File.OpenRead(Assembly.GetEntryAssembly().Location); + this.AddFile(Path.Combine(applicationOutputPath, "Release", "net8.0", "application.dll"), applicationAssemblyStream); + this.AddFile(Path.Combine(applicationOutputPath, "Release", "net4.8", "application.exe"), applicationAssemblyStream); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile(libraryAssetsPath, libraryAssets) + .WithFile(applicationAssetsPath, applicationAssets) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().HaveCount(6); + + var discoveredComponents = detectedComponents.ToArray(); + discoveredComponents.Where(component => component.Component.Id == "dotnet 4.5.6").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "dotnet 4.5.6 - net8.0 - library").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "dotnet 4.5.6 - net6.0 - library").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "dotnet 4.5.6 - netstandard2.0 - library").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "dotnet 1.2.3 - net8.0 - application").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "dotnet 1.2.3 - net48 - application").Should().ContainSingle(); + } +} diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Microsoft.ComponentDetection.Detectors.Tests.csproj b/test/Microsoft.ComponentDetection.Detectors.Tests/Microsoft.ComponentDetection.Detectors.Tests.csproj index b6cc9fc79..b219414bc 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/Microsoft.ComponentDetection.Detectors.Tests.csproj +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/Microsoft.ComponentDetection.Detectors.Tests.csproj @@ -8,6 +8,7 @@ + From c16984c4f06fba6017cc4613bfc0f2898be14c24 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Thu, 27 Feb 2025 15:17:59 -0800 Subject: [PATCH 2/7] Fix parent directory identification on linux. --- .../dotnet/DotNetComponentDetector.cs | 2 +- .../DotNetComponentDetectorTests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs index c7cd71364..5277a1791 100644 --- a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs @@ -166,7 +166,7 @@ private bool IsApplication(string assemblyPath) } else if (projectDirectory.Equals(this.sourceDirectory, StringComparison.OrdinalIgnoreCase) || projectDirectory.Equals(this.sourceFileRootDirectory, StringComparison.OrdinalIgnoreCase) || - parentDirectory is null || + string.IsNullOrEmpty(parentDirectory) || projectDirectory.Equals(parentDirectory, StringComparison.OrdinalIgnoreCase)) { // if we are at the source directory, source file root, or have reached a root directory, run `dotnet --version` diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs index 28e1e589a..23950bc1e 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs @@ -120,7 +120,7 @@ private void AddFile(string path, Stream content) private void AddDirectory(string path, string subDirectory = null) { - if (path is null) + if (string.IsNullOrEmpty(path)) { return; } From a62326844cc2baa1170f77f3659dd601034ba261 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Mon, 3 Mar 2025 14:16:15 -0800 Subject: [PATCH 3/7] Address feedback --- .../TypedComponent/DotNetComponent.cs | 49 ++++-------- .../dotnet/DotNetComponentDetector.cs | 55 ++++++++++--- .../DotNetComponentDetectorTests.cs | 78 ++++++++++++++----- 3 files changed, 120 insertions(+), 62 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs index 65e21c349..d7d64975d 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs @@ -1,10 +1,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; -using System.Text; - -#nullable enable - -using PackageUrl; +using System; public class DotNetComponent : TypedComponent { @@ -13,52 +9,37 @@ private DotNetComponent() /* Reserved for deserialization */ } - public DotNetComponent(string? sdkVersion, string? targetFramework = null, string? projectType = null) + public DotNetComponent(string sdkVersion, string targetFramework = null, string projectType = null) { - this.SdkVersion = sdkVersion; - this.TargetFramework = targetFramework; - this.ProjectType = projectType; // application, library, or null + if (string.IsNullOrWhiteSpace(sdkVersion) && string.IsNullOrWhiteSpace(targetFramework)) + { + throw new ArgumentNullException(nameof(sdkVersion), $"Either {nameof(sdkVersion)} or {nameof(targetFramework)} of component type {nameof(DotNetComponent)} must be specified."); + } + + this.SdkVersion = string.IsNullOrWhiteSpace(sdkVersion) ? "unknown" : sdkVersion; + this.TargetFramework = string.IsNullOrWhiteSpace(targetFramework) ? "unknown" : targetFramework; + this.ProjectType = string.IsNullOrWhiteSpace(projectType) ? "unknown" : projectType; } /// /// SDK Version detected, could be null if no global.json exists and no dotnet is on the path. /// - public string? SdkVersion { get; set; } + public string SdkVersion { get; set; } /// /// Target framework for this instance. Null in the case of global.json. /// - public string? TargetFramework { get; set; } + public string TargetFramework { get; set; } /// /// Project type: application, library. Null in the case of global.json or if no project output could be discovered. /// - public string? ProjectType { get; set; } + public string ProjectType { get; set; } public override ComponentType Type => ComponentType.DotNet; /// - /// Provides an id like `dotnet {SdkVersion} - {TargetFramework} - {ProjectType}` where targetFramework and projectType are only present if not null. + /// Provides an id like `{SdkVersion} - {TargetFramework} - {ProjectType} - dotnet` where unspecified values are represented as 'unknown'. /// - public override string Id - { - get - { - var builder = new StringBuilder($"dotnet {this.SdkVersion ?? "unknown"}"); - if (this.TargetFramework is not null) - { - builder.Append($" - {this.TargetFramework}"); - - if (this.ProjectType is not null) - { - builder.Append($" - {this.ProjectType}"); - } - } - - return builder.ToString(); - } - } - - // TODO - do we need to add a type to prul https://github.com/package-url/purl-spec/blob/main/PURL-TYPES.rst - public override PackageURL PackageUrl => new PackageURL("generic", null, "dotnet-sdk", this.SdkVersion ?? "unknown", null, null); + public override string Id => $"{this.SdkVersion} {this.TargetFramework} {this.ProjectType} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs index 5277a1791..fd053c7c8 100644 --- a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs @@ -2,6 +2,7 @@ namespace Microsoft.ComponentDetection.Detectors.DotNet; #nullable enable using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; @@ -23,7 +24,7 @@ public class DotNetComponentDetector : FileComponentDetector, IExperimentalDetec private readonly IFileUtilityService fileUtilityService; private readonly IPathUtilityService pathUtilityService; private readonly LockFileFormat lockFileFormat = new(); - private readonly Dictionary sdkVersionCache = []; + private readonly ConcurrentDictionary sdkVersionCache = []; private string? sourceDirectory; private string? sourceFileRootDirectory; @@ -60,7 +61,15 @@ public DotNetComponentDetector( var workingDirectory = new DirectoryInfo(workingDirectoryPath); var process = await this.commandLineInvocationService.ExecuteCommandAsync("dotnet", ["dotnet.exe"], workingDirectory, cancellationToken, "--version").ConfigureAwait(false); - return process.ExitCode == 0 ? process.StdOut.Trim() : null; + + if (process.ExitCode != 0) + { + // debug only - it could be that dotnet is not actually on the path and specified directly by the build scripts. + this.Logger.LogDebug("Failed to invoke 'dotnet --version'. Return: {Return} StdErr: {StdErr} StdOut: {StdOut}.", process.ExitCode, process.StdErr, process.StdOut); + return null; + } + + return process.StdOut.Trim(); } public override Task ExecuteDetectorAsync(ScanRequest request, CancellationToken cancellationToken = default) @@ -76,11 +85,24 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID var lockFile = this.lockFileFormat.Read(processRequest.ComponentStream.Stream, processRequest.ComponentStream.Location); var projectPath = lockFile.PackageSpec.RestoreMetadata.ProjectPath; + + if (!this.fileUtilityService.Exists(projectPath)) + { + // Could be the assets file was not actually from this build + this.Logger.LogWarning("Project path {ProjectPath} specified by {ProjectAssetsPath} does not exist.", projectPath, processRequest.ComponentStream.Location); + } + var projectDirectory = this.pathUtilityService.GetParentDirectory(projectPath); var sdkVersion = await this.GetSdkVersionAsync(projectDirectory, cancellationToken); var projectName = lockFile.PackageSpec.RestoreMetadata.ProjectName; var projectOutputPath = lockFile.PackageSpec.RestoreMetadata.OutputPath; + + if (!this.directoryUtilityService.Exists(projectOutputPath)) + { + this.Logger.LogWarning("Project output path {ProjectOutputPath} specified by {ProjectAssetsPath} does not exist.", projectOutputPath, processRequest.ComponentStream.Location); + } + var targetType = this.GetProjectType(projectOutputPath, projectName, cancellationToken); var componentReporter = this.ComponentRecorder.CreateSingleFileComponentRecorder(projectPath); @@ -126,8 +148,9 @@ private bool IsApplication(string assemblyPath) // despite the name `IsExe` this is actually based of the CoffHeader Characteristics return peReader.PEHeaders.IsExe; } - catch (Exception) + catch (Exception e) { + this.Logger.LogWarning("Failed to open output assembly {AssemblyPath} error {Message}.", assemblyPath, e.Message); return false; } } @@ -140,6 +163,12 @@ private bool IsApplication(string assemblyPath) /// Sdk version found, or null if no version can be detected. private async Task GetSdkVersionAsync(string projectDirectory, CancellationToken cancellationToken) { + if (string.IsNullOrWhiteSpace(projectDirectory)) + { + // not expected + return null; + } + // normalize since we need to use as a key projectDirectory = this.pathUtilityService.NormalizePath(projectDirectory); if (this.sdkVersionCache.TryGetValue(projectDirectory, out var sdkVersion)) @@ -152,17 +181,23 @@ private bool IsApplication(string assemblyPath) if (this.fileUtilityService.Exists(globalJsonPath)) { - var globalJson = await JsonDocument.ParseAsync(this.fileUtilityService.MakeFileStream(globalJsonPath), cancellationToken: cancellationToken); - if (globalJson.RootElement.TryGetProperty("sdk", out var sdk)) + sdkVersion = await this.RunDotNetVersionAsync(projectDirectory, cancellationToken); + + if (string.IsNullOrWhiteSpace(sdkVersion)) { - if (sdk.TryGetProperty("version", out var version)) + var globalJson = await JsonDocument.ParseAsync(this.fileUtilityService.MakeFileStream(globalJsonPath), cancellationToken: cancellationToken); + if (globalJson.RootElement.TryGetProperty("sdk", out var sdk)) { - sdkVersion = version.GetString(); - var globalJsonComponent = new DetectedComponent(new DotNetComponent(sdkVersion)); - var recorder = this.ComponentRecorder.CreateSingleFileComponentRecorder(globalJsonPath); - recorder.RegisterUsage(globalJsonComponent, isExplicitReferencedDependency: true); + if (sdk.TryGetProperty("version", out var version)) + { + sdkVersion = version.GetString(); + } } } + + var globalJsonComponent = new DetectedComponent(new DotNetComponent(sdkVersion)); + var recorder = this.ComponentRecorder.CreateSingleFileComponentRecorder(globalJsonPath); + recorder.RegisterUsage(globalJsonComponent, isExplicitReferencedDependency: true); } else if (projectDirectory.Equals(this.sourceDirectory, StringComparison.OrdinalIgnoreCase) || projectDirectory.Equals(this.sourceFileRootDirectory, StringComparison.OrdinalIgnoreCase) || diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs index 23950bc1e..39c4462ef 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs @@ -1,5 +1,6 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -43,6 +44,8 @@ public class DotNetComponentDetectorTests : BaseDetectorTest mockPathUtilityService = new(); + private Func commandLineCallback; + /// /// Initializes a new instance of the class. /// @@ -64,7 +67,8 @@ public DotNetComponentDetectorTests() this.mockPathUtilityService.Setup(x => x.NormalizePath(It.IsAny())).Returns((string p) => p); // don't do normalization this.mockPathUtilityService.Setup(x => x.GetParentDirectory(It.IsAny())).Returns((string p) => Path.GetDirectoryName(p)); - this.mockCommandLineInvocationService.Setup(x => x.ExecuteCommandAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())).Returns(Task.FromResult(this.commandLineExecutionResult)); + this.mockCommandLineInvocationService.Setup(x => x.ExecuteCommandAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((string c, IEnumerable ac, DirectoryInfo d, CancellationToken ct, string[] args) => Task.FromResult(this.CommandResult(c, d))); } private bool FileExists(string path) @@ -147,11 +151,20 @@ private void AddDirectory(string path, string subDirectory = null) private void SetCommandResult(int exitCode, string stdOut = null, string stdErr = null) { + this.commandLineCallback = null; this.commandLineExecutionResult.ExitCode = exitCode; this.commandLineExecutionResult.StdOut = stdOut; this.commandLineExecutionResult.StdErr = stdErr; } + private void SetCommandResult(Func callback) + { + this.commandLineCallback = callback; + } + + private CommandLineExecutionResult CommandResult(string command, DirectoryInfo directory) => + (this.commandLineCallback != null) ? this.commandLineCallback(command, directory) : this.commandLineExecutionResult; + [TestCleanup] public void ClearMocks() { @@ -214,6 +227,31 @@ public async Task TestDotNetDetectorGlobalJson_ReturnsSDKVersion() var globalJson = GlobalJson("42.0.800"); this.AddFile(projectPath, null); this.AddFile(Path.Combine(RootDir, "path", "global.json"), globalJson); + this.SetCommandResult(-1); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("project.assets.json", projectAssets) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().HaveCount(2); + + var discoveredComponents = detectedComponents.ToArray(); + discoveredComponents.Where(component => component.Component.Id == "42.0.800 unknown unknown - DotNet").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "42.0.800 net8.0 unknown - DotNet").Should().ContainSingle(); + } + + [TestMethod] + public async Task TestDotNetDetectorGlobalJsonRollForward_ReturnsSDKVersion() + { + var projectPath = Path.Combine(RootDir, "path", "to", "project"); + var projectAssets = ProjectAssets("projectName", "does-not-exist", projectPath, "net8.0"); + var globalJson = GlobalJson("8.0.100"); + this.AddFile(projectPath, null); + this.AddFile(Path.Combine(RootDir, "path", "global.json"), globalJson); + this.SetCommandResult(0, "8.0.808"); var (scanResult, componentRecorder) = await this.DetectorTestUtility .WithFile("project.assets.json", projectAssets) @@ -225,8 +263,8 @@ public async Task TestDotNetDetectorGlobalJson_ReturnsSDKVersion() detectedComponents.Should().HaveCount(2); var discoveredComponents = detectedComponents.ToArray(); - discoveredComponents.Where(component => component.Component.Id == "dotnet 42.0.800").Should().ContainSingle(); - discoveredComponents.Where(component => component.Component.Id == "dotnet 42.0.800 - net8.0").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "8.0.808 unknown unknown - DotNet").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "8.0.808 net8.0 unknown - DotNet").Should().ContainSingle(); } [TestMethod] @@ -247,7 +285,7 @@ public async Task TestDotNetDetectorNoGlobalJson_ReturnsDotNetVersion() detectedComponents.Should().ContainSingle(); var discoveredComponents = detectedComponents.ToArray(); - discoveredComponents.Where(component => component.Component.Id == "dotnet 86.75.309 - net8.0").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "86.75.309 net8.0 unknown - DotNet").Should().ContainSingle(); } [TestMethod] @@ -268,7 +306,7 @@ public async Task TestDotNetDetectorNoGlobalJsonNoDotnet_ReturnsUnknownVersion() detectedComponents.Should().ContainSingle(); var discoveredComponents = detectedComponents.ToArray(); - discoveredComponents.Where(component => component.Component.Id == "dotnet unknown - net8.0").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "unknown net8.0 unknown - DotNet").Should().ContainSingle(); } [TestMethod] @@ -289,20 +327,24 @@ public async Task TestDotNetDetectorMultipleTargetFrameworks() detectedComponents.Should().HaveCount(3); var discoveredComponents = detectedComponents.ToArray(); - discoveredComponents.Where(component => component.Component.Id == "dotnet 1.2.3 - net8.0").Should().ContainSingle(); - discoveredComponents.Where(component => component.Component.Id == "dotnet 1.2.3 - net6.0").Should().ContainSingle(); - discoveredComponents.Where(component => component.Component.Id == "dotnet 1.2.3 - netstandard2.0").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "1.2.3 net8.0 unknown - DotNet").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "1.2.3 net6.0 unknown - DotNet").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "1.2.3 netstandard2.0 unknown - DotNet").Should().ContainSingle(); } [TestMethod] public async Task TestDotNetDetectorMultipleProjectsWithDifferentOutputTypeAndSdkVersion() { - // dotnet on the path with be version 1.2.3 - this.SetCommandResult(0, "1.2.3"); - // dotnet from global.json will be 4.5.6 var globalJson = GlobalJson("4.5.6"); - this.AddFile(Path.Combine(RootDir, "path", "global.json"), globalJson); + var globalJsonDir = Path.Combine(RootDir, "path"); + this.AddFile(Path.Combine(globalJsonDir, "global.json"), globalJson); + + this.SetCommandResult((c, d) => new CommandLineExecutionResult() + { + ExitCode = 0, + StdOut = d.FullName == globalJsonDir ? "4.5.6" : "1.2.3", + }); // set up a library project - under global.json var libraryProjectName = "library"; @@ -338,11 +380,11 @@ public async Task TestDotNetDetectorMultipleProjectsWithDifferentOutputTypeAndSd detectedComponents.Should().HaveCount(6); var discoveredComponents = detectedComponents.ToArray(); - discoveredComponents.Where(component => component.Component.Id == "dotnet 4.5.6").Should().ContainSingle(); - discoveredComponents.Where(component => component.Component.Id == "dotnet 4.5.6 - net8.0 - library").Should().ContainSingle(); - discoveredComponents.Where(component => component.Component.Id == "dotnet 4.5.6 - net6.0 - library").Should().ContainSingle(); - discoveredComponents.Where(component => component.Component.Id == "dotnet 4.5.6 - netstandard2.0 - library").Should().ContainSingle(); - discoveredComponents.Where(component => component.Component.Id == "dotnet 1.2.3 - net8.0 - application").Should().ContainSingle(); - discoveredComponents.Where(component => component.Component.Id == "dotnet 1.2.3 - net48 - application").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "4.5.6 unknown unknown - DotNet").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "4.5.6 net8.0 library - DotNet").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "4.5.6 net6.0 library - DotNet").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "4.5.6 netstandard2.0 library - DotNet").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "1.2.3 net8.0 application - DotNet").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "1.2.3 net48 application - DotNet").Should().ContainSingle(); } } From 679fefafd9ec8b6fc82aa9ff0bf5541eb2a208bb Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Mon, 3 Mar 2025 15:52:06 -0800 Subject: [PATCH 4/7] Additional tests --- .../dotnet/DotNetComponentDetector.cs | 26 +++---- .../DotNetComponentDetectorTests.cs | 71 +++++++++++++++++++ 2 files changed, 80 insertions(+), 17 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs index fd053c7c8..68caefbda 100644 --- a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs @@ -74,8 +74,8 @@ public DotNetComponentDetector( public override Task ExecuteDetectorAsync(ScanRequest request, CancellationToken cancellationToken = default) { - this.sourceDirectory = this.pathUtilityService.NormalizePath(request.SourceDirectory.FullName); - this.sourceFileRootDirectory = this.pathUtilityService.NormalizePath(request.SourceFileRoot?.FullName); + this.sourceDirectory = this.pathUtilityService.NormalizePath(request.SourceDirectory.FullName.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)); + this.sourceFileRootDirectory = this.pathUtilityService.NormalizePath(request.SourceFileRoot?.FullName.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)); return base.ExecuteDetectorAsync(request, cancellationToken); } @@ -125,13 +125,13 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID .Concat(this.directoryUtilityService.EnumerateFiles(projectOutputPath, namePattern, SearchOption.AllDirectories)); foreach (var candidate in candidates) { - if (this.IsApplication(candidate)) + try { - return "application"; + return this.IsApplication(candidate) ? "application" : "library"; } - else + catch (Exception e) { - return "library"; + this.Logger.LogWarning("Failed to open output assembly {AssemblyPath} error {Message}.", candidate, e.Message); } } } @@ -141,18 +141,10 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID private bool IsApplication(string assemblyPath) { - try - { - using var peReader = new PEReader(this.fileUtilityService.MakeFileStream(assemblyPath)); + using var peReader = new PEReader(this.fileUtilityService.MakeFileStream(assemblyPath)); - // despite the name `IsExe` this is actually based of the CoffHeader Characteristics - return peReader.PEHeaders.IsExe; - } - catch (Exception e) - { - this.Logger.LogWarning("Failed to open output assembly {AssemblyPath} error {Message}.", assemblyPath, e.Message); - return false; - } + // despite the name `IsExe` this is actually based of the CoffHeader Characteristics + return peReader.PEHeaders.IsExe; } /// diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs index 39c4462ef..f2ffe7f20 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs @@ -387,4 +387,75 @@ public async Task TestDotNetDetectorMultipleProjectsWithDifferentOutputTypeAndSd discoveredComponents.Where(component => component.Component.Id == "1.2.3 net8.0 application - DotNet").Should().ContainSingle(); discoveredComponents.Where(component => component.Component.Id == "1.2.3 net48 application - DotNet").Should().ContainSingle(); } + + [TestMethod] + public async Task TestDotNetDetectorInvalidOutputAssembly() + { + // dotnet from global.json will be 4.5.6 + var globalJson = GlobalJson("4.5.6"); + var globalJsonDir = Path.Combine(RootDir, "path"); + this.AddFile(Path.Combine(globalJsonDir, "global.json"), globalJson); + this.SetCommandResult(1, "4.5.6"); + + // set up a library project - under global.json + var libraryProjectName = "library"; + var libraryProjectPath = Path.Combine(RootDir, "path", "to", "project", $"{libraryProjectName}.csproj"); + this.AddFile(libraryProjectPath, null); + var libraryOutputPath = Path.Combine(Path.GetDirectoryName(libraryProjectPath), "obj"); + var libraryAssetsPath = Path.Combine(libraryOutputPath, "project.assets.json"); + var libraryAssets = ProjectAssets("library", libraryOutputPath, libraryProjectPath, "net8.0", "net6.0", "netstandard2.0"); + + // empty 8KB stream + var libraryAssemblyStream = new MemoryStream() { Position = 8 * 1024 }; + this.AddFile(Path.Combine(libraryOutputPath, "Release", "net8.0", "library.dll"), libraryAssemblyStream); + this.AddFile(Path.Combine(libraryOutputPath, "Release", "net6.0", "library.dll"), libraryAssemblyStream); + this.AddFile(Path.Combine(libraryOutputPath, "Release", "netstandard2.0", "library.dll"), libraryAssemblyStream); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile(libraryAssetsPath, libraryAssets) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().HaveCount(4); + + var discoveredComponents = detectedComponents.ToArray(); + discoveredComponents.Where(component => component.Component.Id == "4.5.6 unknown unknown - DotNet").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "4.5.6 net8.0 unknown - DotNet").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "4.5.6 net6.0 unknown - DotNet").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "4.5.6 netstandard2.0 unknown - DotNet").Should().ContainSingle(); + } + + [TestMethod] + public async Task TestDotNetDetectorNoGlobalJsonSourceRoot() + { + // DetectorTestUtility runs under Path.GetTempPath() + var scanRoot = Path.GetTempPath().TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + + var projectPath = Path.Combine(scanRoot, "path", "to", "project"); + var projectAssets = ProjectAssets("projectName", "does-not-exist", projectPath, "net8.0"); + this.AddFile(projectPath, null); + this.SetCommandResult((c, d) => + { + d.FullName.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar).Should().BeEquivalentTo(scanRoot); + return new CommandLineExecutionResult() + { + ExitCode = 0, + StdOut = "0.0.0", + }; + }); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("project.assets.json", projectAssets) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().ContainSingle(); + + var discoveredComponents = detectedComponents.ToArray(); + discoveredComponents.Where(component => component.Component.Id == "0.0.0 net8.0 unknown - DotNet").Should().ContainSingle(); + } } From fa5c6c486fda6c7206c74a399a518b874a177122 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Mon, 3 Mar 2025 16:07:25 -0800 Subject: [PATCH 5/7] Trim trailing directory separator --- .../dotnet/DotNetComponentDetector.cs | 13 ++++++++----- .../DotNetComponentDetectorTests.cs | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs index 68caefbda..ea683c3b0 100644 --- a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs @@ -56,6 +56,8 @@ public DotNetComponentDetector( public override IEnumerable Categories => ["DotNet"]; + private string? NormalizeDirectory(string? path) => this.pathUtilityService.NormalizePath(path)?.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + private async Task RunDotNetVersionAsync(string workingDirectoryPath, CancellationToken cancellationToken) { var workingDirectory = new DirectoryInfo(workingDirectoryPath); @@ -74,8 +76,8 @@ public DotNetComponentDetector( public override Task ExecuteDetectorAsync(ScanRequest request, CancellationToken cancellationToken = default) { - this.sourceDirectory = this.pathUtilityService.NormalizePath(request.SourceDirectory.FullName.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)); - this.sourceFileRootDirectory = this.pathUtilityService.NormalizePath(request.SourceFileRoot?.FullName.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)); + this.sourceDirectory = this.NormalizeDirectory(request.SourceDirectory.FullName); + this.sourceFileRootDirectory = this.NormalizeDirectory(request.SourceFileRoot?.FullName); return base.ExecuteDetectorAsync(request, cancellationToken); } @@ -153,16 +155,17 @@ private bool IsApplication(string assemblyPath) /// Directory to start the search. /// Cancellation token to halt the search. /// Sdk version found, or null if no version can be detected. - private async Task GetSdkVersionAsync(string projectDirectory, CancellationToken cancellationToken) + private async Task GetSdkVersionAsync(string? projectDirectory, CancellationToken cancellationToken) { + // normalize since we need to use as a key + projectDirectory = this.NormalizeDirectory(projectDirectory); + if (string.IsNullOrWhiteSpace(projectDirectory)) { // not expected return null; } - // normalize since we need to use as a key - projectDirectory = this.pathUtilityService.NormalizePath(projectDirectory); if (this.sdkVersionCache.TryGetValue(projectDirectory, out var sdkVersion)) { return sdkVersion; diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs index f2ffe7f20..dc783400b 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs @@ -343,7 +343,7 @@ public async Task TestDotNetDetectorMultipleProjectsWithDifferentOutputTypeAndSd this.SetCommandResult((c, d) => new CommandLineExecutionResult() { ExitCode = 0, - StdOut = d.FullName == globalJsonDir ? "4.5.6" : "1.2.3", + StdOut = d.FullName.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) == globalJsonDir ? "4.5.6" : "1.2.3", }); // set up a library project - under global.json From 342e415720befd9de7c91562f333e591132dcb00 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Mon, 3 Mar 2025 20:34:22 -0800 Subject: [PATCH 6/7] Fix path trimming --- .../dotnet/DotNetComponentDetector.cs | 2 +- .../DotNetComponentDetectorTests.cs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs index ea683c3b0..dd0d59fc5 100644 --- a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs @@ -56,7 +56,7 @@ public DotNetComponentDetector( public override IEnumerable Categories => ["DotNet"]; - private string? NormalizeDirectory(string? path) => this.pathUtilityService.NormalizePath(path)?.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + private string? NormalizeDirectory(string? path) => string.IsNullOrEmpty(path) ? path : Path.TrimEndingDirectorySeparator(this.pathUtilityService.NormalizePath(path)); private async Task RunDotNetVersionAsync(string workingDirectoryPath, CancellationToken cancellationToken) { diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs index dc783400b..6877fa15a 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs @@ -26,7 +26,7 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; [TestCategory("Governance/ComponentDetection")] public class DotNetComponentDetectorTests : BaseDetectorTest { - private static readonly string RootDir = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "C:" : @"\"; + private static readonly string RootDir = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "C:" : @"/"; private readonly Mock> mockLogger = new(); @@ -101,7 +101,7 @@ private IEnumerable EnumerateFilesRecursive(string directory) if (fileName.EndsWith(Path.DirectorySeparatorChar)) { - foreach (var subFile in this.EnumerateFilesRecursive(filePath.TrimEnd(Path.DirectorySeparatorChar))) + foreach (var subFile in this.EnumerateFilesRecursive(Path.TrimEndingDirectorySeparator(filePath))) { yield return subFile; } @@ -343,7 +343,7 @@ public async Task TestDotNetDetectorMultipleProjectsWithDifferentOutputTypeAndSd this.SetCommandResult((c, d) => new CommandLineExecutionResult() { ExitCode = 0, - StdOut = d.FullName.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) == globalJsonDir ? "4.5.6" : "1.2.3", + StdOut = d.FullName == globalJsonDir ? "4.5.6" : "1.2.3", }); // set up a library project - under global.json @@ -431,14 +431,14 @@ public async Task TestDotNetDetectorInvalidOutputAssembly() public async Task TestDotNetDetectorNoGlobalJsonSourceRoot() { // DetectorTestUtility runs under Path.GetTempPath() - var scanRoot = Path.GetTempPath().TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + var scanRoot = Path.TrimEndingDirectorySeparator(Path.GetTempPath()); var projectPath = Path.Combine(scanRoot, "path", "to", "project"); var projectAssets = ProjectAssets("projectName", "does-not-exist", projectPath, "net8.0"); this.AddFile(projectPath, null); this.SetCommandResult((c, d) => { - d.FullName.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar).Should().BeEquivalentTo(scanRoot); + d.FullName.Should().BeEquivalentTo(scanRoot); return new CommandLineExecutionResult() { ExitCode = 0, From 596ff655052276fcccd0fb32d2824e6cb5917a38 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Tue, 4 Mar 2025 16:09:44 -0800 Subject: [PATCH 7/7] Add doc for dotnet detector --- docs/detectors/dotnet.md | 36 +++++++++++++++++++ .../TypedComponent/DotNetComponent.cs | 8 +++-- 2 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 docs/detectors/dotnet.md diff --git a/docs/detectors/dotnet.md b/docs/detectors/dotnet.md new file mode 100644 index 000000000..fbeccef5f --- /dev/null +++ b/docs/detectors/dotnet.md @@ -0,0 +1,36 @@ +# DotNet SDK Detection + +## Requirements + +DotNet SDK Detection depends on `project.assets.json` files in the project output / intermediates. + +## Detection Strategy + +The `project.assets.json` will be produced when a project is restored and built. From this we can locate +the project file that was built. From the project file location we probe for the .NET SDK version used. +We look up the directory path to find a `global.json`[1] then will run `dotnet --version` in that +directory to determine which version of the .NET SDK it will select. If no `global.json` is found, then +probing will stop when the detector encounters `SourceDirectory`, `SourceFileRoot`, or the root of the drive +and proceed to run `dotnet --version` in that directory. Repositories control the version of the .NET SDK +used to build their project by either preinstalling on their build machine or container, or acquiring during +the build pipeline. The .NET SDK version used is important as this version selects redistributable content +that becomes part of the application (the dotnet host, runtime for self-contained or AOT apps, build tools +which generate source, etc). + +In addition to recording the SDK version used, the detector will report the framework versions targeted by +the project as `TargetFramework` values as well as the type of project `application` or `library`. These +are important because applications may be built to target old framework versions which may be out of support +and have unreported vulnerabilities. `TargetFramework` is determined from the `project.assets.json` while +the type of the project is determined by locating the project's output assembly in a subdirectory of the +output path and reading the PE COFF header's characteristics for `IMAGE_FILE_EXECUTABLE_IMAGE`[2]. + +[1]: https://learn.microsoft.com/en-us/dotnet/core/tools/global-json +[2]: https://learn.microsoft.com/en-us/windows/win32/debug/pe-format#characteristics + +## Known Limitations + +If the `dotnet` executable is not on the path the detector may fail to locate the version used to build the +project. The detector will fallback to parsing the `global.json` in this case if it is present. +Detection of the output type is done by locating the output assembly under the output path specified in +`project.assets.json`. Some build systems may place project intermediates in a different location. In this +case the project type will be reported as `unknown`. \ No newline at end of file diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs index d7d64975d..edafc3d61 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs @@ -4,6 +4,8 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class DotNetComponent : TypedComponent { + private const string UnknownValue = "unknown"; + private DotNetComponent() { /* Reserved for deserialization */ @@ -16,9 +18,9 @@ public DotNetComponent(string sdkVersion, string targetFramework = null, string throw new ArgumentNullException(nameof(sdkVersion), $"Either {nameof(sdkVersion)} or {nameof(targetFramework)} of component type {nameof(DotNetComponent)} must be specified."); } - this.SdkVersion = string.IsNullOrWhiteSpace(sdkVersion) ? "unknown" : sdkVersion; - this.TargetFramework = string.IsNullOrWhiteSpace(targetFramework) ? "unknown" : targetFramework; - this.ProjectType = string.IsNullOrWhiteSpace(projectType) ? "unknown" : projectType; + this.SdkVersion = string.IsNullOrWhiteSpace(sdkVersion) ? UnknownValue : sdkVersion; + this.TargetFramework = string.IsNullOrWhiteSpace(targetFramework) ? UnknownValue : targetFramework; + this.ProjectType = string.IsNullOrWhiteSpace(projectType) ? UnknownValue : projectType; } ///