From e9776da0c695da3bce69d6cebec86c6bd40c315b Mon Sep 17 00:00:00 2001 From: Greg Villicana Date: Sun, 9 Apr 2023 18:06:15 -0700 Subject: [PATCH] Accept custom individual component file paths --- .../DependencyGraph/DependencyGraph.cs | 5 +- .../DetectedComponent.cs | 3 +- .../IComponentRecorder.cs | 5 +- .../DefaultGraphTranslationService.cs | 12 ++- .../DefaultGraphTranslationServiceTests.cs | 100 ++++++++++++++++++ 5 files changed, 121 insertions(+), 4 deletions(-) create mode 100644 test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DefaultGraphTranslationServiceTests.cs diff --git a/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs b/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs index 7e7cdf484..edfcd4932 100644 --- a/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs +++ b/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.Immutable; @@ -87,6 +87,9 @@ public ICollection GetExplicitReferencedDependencyIds(string componentId return explicitReferencedDependencyIds; } + /// + /// Any file added here will be reported as a location on ALL components found in current graph. + /// public void AddAdditionalRelatedFile(string additionalRelatedFile) { this.AdditionalRelatedFiles.AddOrUpdate(additionalRelatedFile, 0, (notUsed, notUsed2) => 0); diff --git a/src/Microsoft.ComponentDetection.Contracts/DetectedComponent.cs b/src/Microsoft.ComponentDetection.Contracts/DetectedComponent.cs index 0d42a69d0..9bee98b7e 100644 --- a/src/Microsoft.ComponentDetection.Contracts/DetectedComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/DetectedComponent.cs @@ -60,7 +60,8 @@ public DetectedComponent(TypedComponent.TypedComponent component, IComponentDete private string DebuggerDisplay => $"{this.Component.DebuggerDisplay}"; - /// Adds a filepath to the FilePaths hashset for this detected component. + /// Adds a filepath to the FilePaths hashset for this detected component. + /// Note: Dependency Graph automatically captures the location where a component is found, no need to call it at all inside package manager detectors. /// The file path to add to the hashset. public void AddComponentFilePath(string filePath) { diff --git a/src/Microsoft.ComponentDetection.Contracts/IComponentRecorder.cs b/src/Microsoft.ComponentDetection.Contracts/IComponentRecorder.cs index 1ce79372b..be4baf715 100644 --- a/src/Microsoft.ComponentDetection.Contracts/IComponentRecorder.cs +++ b/src/Microsoft.ComponentDetection.Contracts/IComponentRecorder.cs @@ -1,4 +1,4 @@ -namespace Microsoft.ComponentDetection.Contracts; +namespace Microsoft.ComponentDetection.Contracts; using System.Collections.Generic; using Microsoft.ComponentDetection.Contracts.BcdeModels; @@ -45,6 +45,9 @@ void RegisterUsage( DetectedComponent GetComponent(string componentId); + /// + /// Any file added here will be reported as a location on ALL components found in current graph. + /// void AddAdditionalRelatedFile(string relatedFilePath); IReadOnlyDictionary GetDetectedComponents(); diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs index 168b8a23e..b45e90157 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs @@ -12,6 +12,7 @@ namespace Microsoft.ComponentDetection.Orchestrator.Services.GraphTranslation; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Orchestrator.ArgumentSets; using Microsoft.Extensions.Logging; +using Newtonsoft.Json; public class DefaultGraphTranslationService : IGraphTranslationService { @@ -70,7 +71,9 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn // to look like a pipeline. foreach (var component in detectedComponents) { - // Reinitialize properties that might still be getting populated in ways we don't want to support, because the data is authoritatively stored in the graph. + // clone custom locations and make them relative to root. + var declaredRawFilePaths = component.FilePaths ?? new HashSet(); + var componentCustomLocations = JsonConvert.DeserializeObject>(JsonConvert.SerializeObject(declaredRawFilePaths)); component.FilePaths?.Clear(); // Information about each component is relative to all of the graphs it is present in, so we take all graphs containing a given component and apply the graph data. @@ -87,8 +90,15 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn // Return in a format that allows us to add the additional files for the components var locations = dependencyGraph.GetAdditionalRelatedFiles(); + + // graph authoritatively stores the location of the component locations.Add(location); + foreach (var customLocation in componentCustomLocations) + { + locations.Add(customLocation); + } + var relativePaths = this.MakeFilePathsRelative(this.logger, rootDirectory, locations); foreach (var additionalRelatedFile in relativePaths ?? Enumerable.Empty()) diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DefaultGraphTranslationServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DefaultGraphTranslationServiceTests.cs new file mode 100644 index 000000000..63f49f2a0 --- /dev/null +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DefaultGraphTranslationServiceTests.cs @@ -0,0 +1,100 @@ +namespace Microsoft.ComponentDetection.Orchestrator.Tests.Services; + +using System.Collections.Generic; +using System.IO; +using System.Linq; +using FluentAssertions; +using Microsoft.ComponentDetection.Common.DependencyGraph; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.ComponentDetection.Orchestrator.ArgumentSets; +using Microsoft.ComponentDetection.Orchestrator.Services; +using Microsoft.ComponentDetection.Orchestrator.Services.GraphTranslation; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +[TestClass] +[TestCategory("Governance/All")] +[TestCategory("Governance/ComponentDetection")] +public class DefaultGraphTranslationServiceTests +{ + private readonly DefaultGraphTranslationService serviceUnderTest; + private readonly ContainerDetails sampleContainerDetails; + private readonly ComponentRecorder componentRecorder; + private readonly Mock componentDetectorMock; + private readonly DirectoryInfo sourceDirectory; + + public DefaultGraphTranslationServiceTests() + { + this.serviceUnderTest = new DefaultGraphTranslationService(new Mock>().Object); + this.componentRecorder = new ComponentRecorder(new Mock().Object); + + this.sampleContainerDetails = new ContainerDetails { Id = 1 }; + this.componentDetectorMock = new Mock(); + this.componentDetectorMock.SetupGet(x => x.Id).Returns("Detector1"); + this.componentDetectorMock.SetupGet(x => x.Version).Returns(1); + this.sourceDirectory = new DirectoryInfo(Path.Combine(Path.GetTempPath(), Path.GetRandomFileName())); + this.sourceDirectory.Create(); + } + + [TestMethod] + public void GenerateScanResultFromResult_WithCustomLocations() + { + var detectedFilePath = "/some/file/path"; + var npmCustomPath = "/custom/path.js"; + var nugetCustomPath = "/custom/path2.csproj"; + var relatedFilePath = "/generic/relevant/path"; + + var singleFileComponentRecorder = this.componentRecorder.CreateSingleFileComponentRecorder(Path.Join(this.sourceDirectory.FullName, detectedFilePath)); + var processingResult = new DetectorProcessingResult + { + ResultCode = ProcessingResultCode.Success, + ContainersDetailsMap = new Dictionary + { + { + this.sampleContainerDetails.Id, this.sampleContainerDetails + }, + }, + ComponentRecorders = new[] { (this.componentDetectorMock.Object, this.componentRecorder) }, + }; + + var expectedNpmComponent = new NpmComponent("npm-component", "1.2.3"); + var expectedNugetComponent = new NuGetComponent("nugetComponent", "4.5.6"); + var detectedNpmComponent = new DetectedComponent(expectedNpmComponent); + var detectedNugetComponent = new DetectedComponent(expectedNugetComponent); + + // Any Related File will be reported for ALL components found in this graph + singleFileComponentRecorder.AddAdditionalRelatedFile(Path.Join(this.sourceDirectory.FullName, relatedFilePath)); + + // Registering components in same manifest with different custom paths + detectedNpmComponent.AddComponentFilePath(Path.Join(this.sourceDirectory.FullName, npmCustomPath)); + detectedNugetComponent.AddComponentFilePath(Path.Join(this.sourceDirectory.FullName, nugetCustomPath)); + + singleFileComponentRecorder.RegisterUsage(detectedNpmComponent, isDevelopmentDependency: false); + singleFileComponentRecorder.RegisterUsage(detectedNugetComponent, isDevelopmentDependency: true); + + var args = new BcdeArguments + { + SourceDirectory = this.sourceDirectory, + }; + + var result = this.serviceUnderTest.GenerateScanResultFromProcessingResult(processingResult, args); + result.Should().NotBeNull(); + result.ComponentsFound.Should().HaveCount(2); + result.ResultCode.Should().Be(ProcessingResultCode.Success); + + var resultNpmComponent = result.ComponentsFound.Single(c => c.Component.Type == ComponentType.Npm); + var resultNugetComponent = result.ComponentsFound.Single(c => c.Component.Type == ComponentType.NuGet); + + resultNpmComponent.LocationsFoundAt.Should().BeEquivalentTo(new[] { npmCustomPath, detectedFilePath, relatedFilePath }); + resultNugetComponent.LocationsFoundAt.Should().BeEquivalentTo(new[] { nugetCustomPath, detectedFilePath, relatedFilePath }); + + var actualNpmComponent = resultNpmComponent.Component as NpmComponent; + var actualNugetComponent = resultNugetComponent.Component as NuGetComponent; + + actualNpmComponent.Should().BeEquivalentTo(expectedNpmComponent); + actualNugetComponent.Should().BeEquivalentTo(expectedNugetComponent); + } +}