From 5d645a236e71442a8304542c1422c821dd6d3acf Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Wed, 12 Mar 2025 16:22:10 -0700 Subject: [PATCH 1/2] Fix dotnet detector for exes / newer version not on path We weren't handling EXEs correctly - simply missed this case. When a global.json mentions a version that doesn't exist on the path, the `dotnet --version` command will fail, and the `commandLineInvocationService` will throw InvalidOperationException since it cannot distinguish this failure from a missing command. Catch this exception and treat it like a failure so we fallback to parsing `global.json`. --- .../dotnet/DotNetComponentDetector.cs | 25 +++-- .../DotNetComponentDetectorTests.cs | 97 ++++++++++++++++++- 2 files changed, 110 insertions(+), 12 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs index dd0d59fc5..74f2e1507 100644 --- a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs @@ -62,16 +62,25 @@ public DotNetComponentDetector( { var workingDirectory = new DirectoryInfo(workingDirectoryPath); - var process = await this.commandLineInvocationService.ExecuteCommandAsync("dotnet", ["dotnet.exe"], workingDirectory, cancellationToken, "--version").ConfigureAwait(false); + try + { + var process = await this.commandLineInvocationService.ExecuteCommandAsync("dotnet", ["dotnet.exe"], workingDirectory, cancellationToken, "--version").ConfigureAwait(false); + + if (process.ExitCode != 0) + { + // debug only - it could be that dotnet is not actually on the path and specified directly by the build scripts. + this.Logger.LogDebug("Failed to invoke 'dotnet --version'. Return: {Return} StdErr: {StdErr} StdOut: {StdOut}.", process.ExitCode, process.StdErr, process.StdOut); + return null; + } - if (process.ExitCode != 0) + return process.StdOut.Trim(); + } + catch (InvalidOperationException ioe) { // debug only - it could be that dotnet is not actually on the path and specified directly by the build scripts. - this.Logger.LogDebug("Failed to invoke 'dotnet --version'. Return: {Return} StdErr: {StdErr} StdOut: {StdOut}.", process.ExitCode, process.StdErr, process.StdOut); + this.Logger.LogDebug("Failed to invoke 'dotnet --version'. {Message}", ioe.Message); return null; } - - return process.StdOut.Trim(); } public override Task ExecuteDetectorAsync(ScanRequest request, CancellationToken cancellationToken = default) @@ -120,11 +129,11 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID { if (this.directoryUtilityService.Exists(projectOutputPath)) { - var namePattern = (projectName ?? "*") + ".dll"; + var namePattern = projectName ?? "*"; // look for the compiled output, first as dll then as exe. - var candidates = this.directoryUtilityService.EnumerateFiles(projectOutputPath, namePattern, SearchOption.AllDirectories) - .Concat(this.directoryUtilityService.EnumerateFiles(projectOutputPath, namePattern, SearchOption.AllDirectories)); + var candidates = this.directoryUtilityService.EnumerateFiles(projectOutputPath, namePattern + ".dll", SearchOption.AllDirectories) + .Concat(this.directoryUtilityService.EnumerateFiles(projectOutputPath, namePattern + ".exe", SearchOption.AllDirectories)); foreach (var candidate in candidates) { try diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs index 6877fa15a..b99ca258d 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs @@ -8,6 +8,7 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; using System.Reflection; using System.Runtime.InteropServices; using System.Text.Json; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using FluentAssertions; @@ -62,7 +63,7 @@ public DotNetComponentDetectorTests() this.mockDirectoryUtilityService.Setup(x => x.Exists(It.IsAny())).Returns((string p) => this.DirectoryExists(p)); // ignore pattern and search option since we don't really need them for tests - this.mockDirectoryUtilityService.Setup(x => x.EnumerateFiles(It.IsAny(), It.IsAny(), It.IsAny())).Returns((string d, string p, SearchOption s) => this.EnumerateFilesRecursive(d)); + this.mockDirectoryUtilityService.Setup(x => x.EnumerateFiles(It.IsAny(), It.IsAny(), It.IsAny())).Returns((string d, string p, SearchOption s) => this.EnumerateFilesRecursive(d, p)); this.mockPathUtilityService.Setup(x => x.NormalizePath(It.IsAny())).Returns((string p) => p); // don't do normalization this.mockPathUtilityService.Setup(x => x.GetParentDirectory(It.IsAny())).Returns((string p) => Path.GetDirectoryName(p)); @@ -91,24 +92,30 @@ private Stream OpenFile(string path) private bool DirectoryExists(string directory) => this.files.ContainsKey(directory); - private IEnumerable EnumerateFilesRecursive(string directory) + private IEnumerable EnumerateFilesRecursive(string directory, string pattern) { if (this.files.TryGetValue(directory, out var fileNames)) { + // a basic approximation of globbing + var patternRegex = new Regex(pattern.Replace(".", "\\.").Replace("*", ".*")); + foreach (var fileName in fileNames.Keys) { var filePath = Path.Combine(directory, fileName); if (fileName.EndsWith(Path.DirectorySeparatorChar)) { - foreach (var subFile in this.EnumerateFilesRecursive(Path.TrimEndingDirectorySeparator(filePath))) + foreach (var subFile in this.EnumerateFilesRecursive(Path.TrimEndingDirectorySeparator(filePath), pattern)) { yield return subFile; } } else { - yield return filePath; + if (patternRegex.IsMatch(fileName)) + { + yield return filePath; + } } } } @@ -267,6 +274,54 @@ public async Task TestDotNetDetectorGlobalJsonRollForward_ReturnsSDKVersion() discoveredComponents.Where(component => component.Component.Id == "8.0.808 net8.0 unknown - DotNet").Should().ContainSingle(); } + [TestMethod] + public async Task TestDotNetDetectorGlobalJsonDotNetVersionFails_ReturnsSDKVersion() + { + var projectPath = Path.Combine(RootDir, "path", "to", "project"); + var projectAssets = ProjectAssets("projectName", "does-not-exist", projectPath, "net8.0"); + var globalJson = GlobalJson("8.0.100"); + this.AddFile(projectPath, null); + this.AddFile(Path.Combine(RootDir, "path", "global.json"), globalJson); + this.SetCommandResult(-1); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("project.assets.json", projectAssets) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().HaveCount(2); + + var discoveredComponents = detectedComponents.ToArray(); + discoveredComponents.Where(component => component.Component.Id == "8.0.100 unknown unknown - DotNet").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "8.0.100 net8.0 unknown - DotNet").Should().ContainSingle(); + } + + [TestMethod] + public async Task TestDotNetDetectorGlobalJsonDotNetVersionThrows_ReturnsSDKVersion() + { + var projectPath = Path.Combine(RootDir, "path", "to", "project"); + var projectAssets = ProjectAssets("projectName", "does-not-exist", projectPath, "net8.0"); + var globalJson = GlobalJson("8.0.100"); + this.AddFile(projectPath, null); + this.AddFile(Path.Combine(RootDir, "path", "global.json"), globalJson); + this.SetCommandResult((c, d) => throw new InvalidOperationException()); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("project.assets.json", projectAssets) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().HaveCount(2); + + var discoveredComponents = detectedComponents.ToArray(); + discoveredComponents.Where(component => component.Component.Id == "8.0.100 unknown unknown - DotNet").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "8.0.100 net8.0 unknown - DotNet").Should().ContainSingle(); + } + [TestMethod] public async Task TestDotNetDetectorNoGlobalJson_ReturnsDotNetVersion() { @@ -388,6 +443,40 @@ public async Task TestDotNetDetectorMultipleProjectsWithDifferentOutputTypeAndSd discoveredComponents.Where(component => component.Component.Id == "1.2.3 net48 application - DotNet").Should().ContainSingle(); } + [TestMethod] + public async Task TestDotNetDetectorExe() + { + // dotnet from global.json will be 4.5.6 + var globalJson = GlobalJson("4.5.6"); + var globalJsonDir = Path.Combine(RootDir, "path"); + this.AddFile(Path.Combine(globalJsonDir, "global.json"), globalJson); + + this.SetCommandResult(0, "4.5.6"); + + // set up an application - not under global.json + var applicationProjectName = "application"; + var applicationProjectPath = Path.Combine(RootDir, "path", "to", "project", $"{applicationProjectName}.csproj"); + this.AddFile(applicationProjectPath, null); + var applicationOutputPath = Path.Combine(Path.GetDirectoryName(applicationProjectPath), "obj"); + var applicationAssetsPath = Path.Combine(applicationOutputPath, "project.assets.json"); + var applicationAssets = ProjectAssets("application", applicationOutputPath, applicationProjectPath, "net4.8"); + var applicationAssemblyStream = File.OpenRead(Assembly.GetEntryAssembly().Location); + this.AddFile(Path.Combine(applicationOutputPath, "Release", "net4.8", "application.exe"), applicationAssemblyStream); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile(applicationAssetsPath, applicationAssets) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().HaveCount(2); + + var discoveredComponents = detectedComponents.ToArray(); + discoveredComponents.Where(component => component.Component.Id == "4.5.6 unknown unknown - DotNet").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "4.5.6 net48 application - DotNet").Should().ContainSingle(); + } + [TestMethod] public async Task TestDotNetDetectorInvalidOutputAssembly() { From ebfba5dc6dc5242085e136a4f631b22582ad0ba7 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Wed, 12 Mar 2025 22:53:17 -0700 Subject: [PATCH 2/2] Support moved project.assets.json files We've seen cases where the project.assets.json files refer to paths with a different root than the current scan. One example is building inside a container then running on the filesystem mounted outside the container. Address this by comparing the relative path to the root, and finding if the same relative path is present in the assets file's output path. --- .../dotnet/DotNetComponentDetector.cs | 55 ++++++++++++++++++- .../DotNetComponentDetectorTests.cs | 48 ++++++++++++++++ 2 files changed, 102 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs index 74f2e1507..81be72d75 100644 --- a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs @@ -58,6 +58,44 @@ public DotNetComponentDetector( private string? NormalizeDirectory(string? path) => string.IsNullOrEmpty(path) ? path : Path.TrimEndingDirectorySeparator(this.pathUtilityService.NormalizePath(path)); + /// + /// Given a path relative to sourceDirectory, and the same path in another filesystem, + /// determine what path could be replaced with root. + /// + /// Some path under root, including the root path. + /// Path to the same file as but in a different root. + /// Portion of that corresponds to root, or null if it can not be rebased. + private string? GetRootRebasePath(string rootBasedPath, string? rebasePath) + { + if (string.IsNullOrEmpty(rebasePath) || string.IsNullOrEmpty(this.sourceDirectory) || string.IsNullOrEmpty(rootBasedPath)) + { + return null; + } + + // sourceDirectory is normalized, normalize others + rootBasedPath = this.pathUtilityService.NormalizePath(rootBasedPath); + rebasePath = this.pathUtilityService.NormalizePath(rebasePath); + + // nothing to do if the paths are the same + if (rebasePath.Equals(rootBasedPath, StringComparison.Ordinal)) + { + return null; + } + + // find the relative path under root. + var rootRelativePath = this.pathUtilityService.NormalizePath(Path.GetRelativePath(this.sourceDirectory!, rootBasedPath)); + + // if the rebase path has the same relative portion, then we have a replacement. + if (rebasePath.EndsWith(rootRelativePath)) + { + return rebasePath[..^rootRelativePath.Length]; + } + + // The path didn't have a common relative path, it might have been copied from a completely different location since it was built. + // We cannot rebase the paths. + return null; + } + private async Task RunDotNetVersionAsync(string workingDirectoryPath, CancellationToken cancellationToken) { var workingDirectory = new DirectoryInfo(workingDirectoryPath); @@ -95,7 +133,20 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID { var lockFile = this.lockFileFormat.Read(processRequest.ComponentStream.Stream, processRequest.ComponentStream.Location); + var projectAssetsDirectory = this.pathUtilityService.GetParentDirectory(processRequest.ComponentStream.Location); var projectPath = lockFile.PackageSpec.RestoreMetadata.ProjectPath; + var projectOutputPath = lockFile.PackageSpec.RestoreMetadata.OutputPath; + + // The output path should match the location that the assets file, if it doesn't we could be analyzing paths + // on a different filesystem root than they were created. + // Attempt to rebase paths based on the difference between this file's location and the output path. + var rebasePath = this.GetRootRebasePath(projectAssetsDirectory, projectOutputPath); + + if (rebasePath is not null) + { + projectPath = Path.Combine(this.sourceDirectory!, Path.GetRelativePath(rebasePath, projectPath)); + projectOutputPath = Path.Combine(this.sourceDirectory!, Path.GetRelativePath(rebasePath, projectOutputPath)); + } if (!this.fileUtilityService.Exists(projectPath)) { @@ -107,11 +158,13 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID var sdkVersion = await this.GetSdkVersionAsync(projectDirectory, cancellationToken); var projectName = lockFile.PackageSpec.RestoreMetadata.ProjectName; - var projectOutputPath = lockFile.PackageSpec.RestoreMetadata.OutputPath; if (!this.directoryUtilityService.Exists(projectOutputPath)) { this.Logger.LogWarning("Project output path {ProjectOutputPath} specified by {ProjectAssetsPath} does not exist.", projectOutputPath, processRequest.ComponentStream.Location); + + // default to use the location of the assets file. + projectOutputPath = projectAssetsDirectory; } var targetType = this.GetProjectType(projectOutputPath, projectName, cancellationToken); diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs index b99ca258d..bd0d98bb7 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs @@ -547,4 +547,52 @@ public async Task TestDotNetDetectorNoGlobalJsonSourceRoot() var discoveredComponents = detectedComponents.ToArray(); discoveredComponents.Where(component => component.Component.Id == "0.0.0 net8.0 unknown - DotNet").Should().ContainSingle(); } + + [TestMethod] + public async Task TestDotNetDetectorRebasePaths() + { + // DetectorTestUtility runs under Path.GetTempPath() + var scanRoot = Path.TrimEndingDirectorySeparator(Path.GetTempPath()); + + // dotnet from global.json will be 4.5.6 + var globalJson = GlobalJson("4.5.6"); + var globalJsonDir = Path.Combine(scanRoot, "path"); + this.AddFile(Path.Combine(globalJsonDir, "global.json"), globalJson); + + // make sure we find global.json and read it + this.SetCommandResult(-1); + + // set up a library project - under global.json + var libraryProjectName = "library"; + + var libraryProjectPath = Path.Combine(scanRoot, "path", "to", "project", $"{libraryProjectName}.csproj"); + var libraryBuildProjectPath = Path.Combine(RootDir, "path", "to", "project", $"{libraryProjectName}.csproj"); + this.AddFile(libraryProjectPath, null); + + var libraryOutputPath = Path.Combine(Path.GetDirectoryName(libraryProjectPath), "obj"); + var libraryBuildOutputPath = Path.Combine(Path.GetDirectoryName(libraryBuildProjectPath), "obj"); + var libraryAssetsPath = Path.Combine(libraryOutputPath, "project.assets.json"); + + // use "build" paths to simulate an Assets file that has a different root. Here the build assets have RootDir, but the scanned filesystem has scanRoot. + var libraryAssets = ProjectAssets("library", libraryBuildOutputPath, libraryBuildProjectPath, "net8.0", "net6.0", "netstandard2.0"); + var libraryAssemblyStream = File.OpenRead(typeof(DotNetComponent).Assembly.Location); + this.AddFile(Path.Combine(libraryOutputPath, "Release", "net8.0", "library.dll"), libraryAssemblyStream); + this.AddFile(Path.Combine(libraryOutputPath, "Release", "net6.0", "library.dll"), libraryAssemblyStream); + this.AddFile(Path.Combine(libraryOutputPath, "Release", "netstandard2.0", "library.dll"), libraryAssemblyStream); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile(libraryAssetsPath, libraryAssets) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().HaveCount(4); + + var discoveredComponents = detectedComponents.ToArray(); + discoveredComponents.Where(component => component.Component.Id == "4.5.6 unknown unknown - DotNet").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "4.5.6 net8.0 library - DotNet").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "4.5.6 net6.0 library - DotNet").Should().ContainSingle(); + discoveredComponents.Where(component => component.Component.Id == "4.5.6 netstandard2.0 library - DotNet").Should().ContainSingle(); + } }