diff --git a/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs b/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs index 268eb6564..8b35d226e 100644 --- a/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs +++ b/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.Immutable; @@ -62,6 +62,20 @@ public IEnumerable GetDetectedComponents() return detectedComponents; } + public IEnumerable GetSkippedComponents() + { + if (this.singleFileRecorders == null) + { + return Enumerable.Empty(); + } + + return this.singleFileRecorders + .Select(x => x.GetSkippedComponents().Keys) + .SelectMany(x => x) + .Distinct() + .ToImmutableList(); + } + public ISingleFileComponentRecorder CreateSingleFileComponentRecorder(string location) { if (string.IsNullOrWhiteSpace(location)) @@ -96,6 +110,11 @@ public sealed class SingleFileComponentRecorder : ISingleFileComponentRecorder private readonly ConcurrentDictionary detectedComponentsInternal = new ConcurrentDictionary(); + /// + /// Dictionary of components which had an error during parsing and a dummy data value that only allocates 1 byte. + /// + private readonly ConcurrentDictionary skippedComponentsInternal = new ConcurrentDictionary(); + private readonly ComponentRecorder recorder; private readonly object registerUsageLock = new object(); @@ -130,6 +149,11 @@ public IReadOnlyDictionary GetDetectedComponents() return this.detectedComponentsInternal; } + public IReadOnlyDictionary GetSkippedComponents() + { + return this.skippedComponentsInternal; + } + public void RegisterUsage( DetectedComponent detectedComponent, bool isExplicitReferencedDependency = false, @@ -173,6 +197,16 @@ public void RegisterUsage( } } + public void RegisterPackageParseFailure(string skippedComponent) + { + if (skippedComponent == null) + { + throw new ArgumentNullException(paramName: nameof(skippedComponent)); + } + + _ = this.skippedComponentsInternal[skippedComponent] = default; + } + public void AddAdditionalRelatedFile(string relatedFilePath) { this.DependencyGraph.AddAdditionalRelatedFile(relatedFilePath); diff --git a/src/Microsoft.ComponentDetection.Contracts/IComponentRecorder.cs b/src/Microsoft.ComponentDetection.Contracts/IComponentRecorder.cs index ad009fda4..e9f346b18 100644 --- a/src/Microsoft.ComponentDetection.Contracts/IComponentRecorder.cs +++ b/src/Microsoft.ComponentDetection.Contracts/IComponentRecorder.cs @@ -8,6 +8,8 @@ public interface IComponentRecorder IEnumerable GetDetectedComponents(); + IEnumerable GetSkippedComponents(); + ISingleFileComponentRecorder CreateSingleFileComponentRecorder(string location); IReadOnlyDictionary GetDependencyGraphsByLocation(); @@ -35,6 +37,12 @@ void RegisterUsage( bool? isDevelopmentDependency = null, DependencyScope? dependencyScope = null); + /// + /// Register that a package was unable to be processed. + /// + /// Component version identifier. + void RegisterPackageParseFailure(string skippedComponent); + DetectedComponent GetComponent(string componentId); void AddAdditionalRelatedFile(string relatedFilePath); diff --git a/src/Microsoft.ComponentDetection.Detectors/cocoapods/PodComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/cocoapods/PodComponentDetector.cs index 20b5e9923..bc79d666a 100644 --- a/src/Microsoft.ComponentDetection.Detectors/cocoapods/PodComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/cocoapods/PodComponentDetector.cs @@ -237,6 +237,7 @@ private void ProcessPodfileLock( else { this.Logger.LogWarning($"Missing podspec declaration. podspec={dependency.Podspec}, version={dependency.PodVersion}"); + singleFileComponentRecorder.RegisterPackageParseFailure($"{dependency.Podspec} - {dependency.PodVersion}"); } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs index de5fe59a2..708d5cade 100644 --- a/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs @@ -170,7 +170,9 @@ private void ParseGoModFile( } else { - this.Logger.LogWarning($"Line could not be parsed for component [{line.Trim()}]"); + var lineTrim = line.Trim(); + this.Logger.LogWarning($"Line could not be parsed for component [{lineTrim}]"); + singleFileComponentRecorder.RegisterPackageParseFailure(lineTrim); } } } @@ -209,7 +211,9 @@ private void ParseGoSumFile( } else { - this.Logger.LogWarning($"Line could not be parsed for component [{line.Trim()}]"); + var lineTrim = line.Trim(); + this.Logger.LogWarning($"Line could not be parsed for component [{lineTrim}]"); + singleFileComponentRecorder.RegisterPackageParseFailure(lineTrim); } } } @@ -270,6 +274,7 @@ private void PopulateDependencyGraph(string goGraphOutput, ISingleFileComponentR else { this.Logger.LogWarning($"Failed to parse components from relationship string {relationship}"); + componentRecorder.RegisterPackageParseFailure(relationship); } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/ivy/IvyDetector.cs b/src/Microsoft.ComponentDetection.Detectors/ivy/IvyDetector.cs index 3ab12f09a..6a173f9d5 100644 --- a/src/Microsoft.ComponentDetection.Detectors/ivy/IvyDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/ivy/IvyDetector.cs @@ -237,6 +237,7 @@ private void RegisterUsagesFromFile(ISingleFileComponentRecorder singleFileCompo else { this.Logger.LogWarning($"Dependency \"{component.Id}\" could not be resolved by Ivy, and so has not been recorded by Component Detection."); + singleFileComponentRecorder.RegisterPackageParseFailure(component.Id); } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/linux/LinuxContainerDetector.cs b/src/Microsoft.ComponentDetection.Detectors/linux/LinuxContainerDetector.cs index d734edc26..e174abe65 100644 --- a/src/Microsoft.ComponentDetection.Detectors/linux/LinuxContainerDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/linux/LinuxContainerDetector.cs @@ -153,6 +153,9 @@ await this.dockerService.TryPullImageAsync(image, cancellationToken))) StackTrace = e.StackTrace, ImageId = image, }; + + var singleFileComponentRecorder = componentRecorder.CreateSingleFileComponentRecorder(image); + singleFileComponentRecorder.RegisterPackageParseFailure(image); } }); @@ -196,6 +199,9 @@ await this.dockerService.TryPullImageAsync(image, cancellationToken))) StackTrace = e.StackTrace, ImageId = kvp.Value.ImageId, }; + + var singleFileComponentRecorder = componentRecorder.CreateSingleFileComponentRecorder(kvp.Value.ImageId); + singleFileComponentRecorder.RegisterPackageParseFailure(kvp.Key); } return EmptyImageScanningResult(); diff --git a/src/Microsoft.ComponentDetection.Detectors/maven/MavenCommandService.cs b/src/Microsoft.ComponentDetection.Detectors/maven/MavenCommandService.cs index 91d116d10..ab28e6654 100644 --- a/src/Microsoft.ComponentDetection.Detectors/maven/MavenCommandService.cs +++ b/src/Microsoft.ComponentDetection.Detectors/maven/MavenCommandService.cs @@ -43,6 +43,7 @@ public async Task GenerateDependenciesFileAsync(ProcessRequest processRequest) { this.logger.LogVerbose($"Mvn execution failed for pom file: {pomFile.Location}"); this.logger.LogError(string.IsNullOrEmpty(result.StdErr) ? result.StdOut : result.StdErr); + processRequest.SingleFileComponentRecorder.RegisterPackageParseFailure(pomFile.Location); } } diff --git a/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentDetector.cs index 8ca1c9192..57c877376 100644 --- a/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentDetector.cs @@ -80,6 +80,7 @@ protected virtual bool ProcessIndividualPackageJTokens(string filePath, ISingleF if (!SemanticVersion.TryParse(version, out _)) { this.Logger.LogWarning($"Unable to parse version \"{version}\" for package \"{name}\" found at path \"{filePath}\". This may indicate an invalid npm package component and it will not be registered."); + singleFileComponentRecorder.RegisterPackageParseFailure($"{name} - {version}"); return false; } diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetComponentDetector.cs index 4ad4d78eb..460ad2372 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetComponentDetector.cs @@ -126,6 +126,7 @@ private async Task ProcessFileAsync(ProcessRequest processRequest) if (!NuGetVersion.TryParse(version, out var parsedVer)) { this.Logger.LogInfo($"Version '{version}' from {stream.Location} could not be parsed as a NuGet version"); + singleFileComponentRecorder.RegisterPackageParseFailure(stream.Location); return; } @@ -140,6 +141,7 @@ private async Task ProcessFileAsync(ProcessRequest processRequest) { // If something went wrong, just ignore the component this.Logger.LogFailedReadingFile(stream.Location, e); + singleFileComponentRecorder.RegisterPackageParseFailure(stream.Location); } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/IPythonResolver.cs b/src/Microsoft.ComponentDetection.Detectors/pip/IPythonResolver.cs index c45b2e057..e3a6e5fdc 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/IPythonResolver.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/IPythonResolver.cs @@ -1,13 +1,15 @@ namespace Microsoft.ComponentDetection.Detectors.Pip; using System.Collections.Generic; using System.Threading.Tasks; +using Microsoft.ComponentDetection.Contracts; public interface IPythonResolver { /// /// Resolves the root Python packages from the initial list of packages. /// + /// The component recorder for file that is been processed. /// The initial list of packages. /// The root packages, with dependencies associated as children. - Task> ResolveRootsAsync(IList initialPackages); + Task> ResolveRootsAsync(ISingleFileComponentRecorder singleFileComponentRecorder, IList initialPackages); } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PipComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PipComponentDetector.cs index 216c9aebd..e760f788c 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/PipComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/PipComponentDetector.cs @@ -66,7 +66,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID .Where(x => !x.PackageIsUnsafe()) .ToList(); - var roots = await this.pythonResolver.ResolveRootsAsync(listedPackage); + var roots = await this.pythonResolver.ResolveRootsAsync(singleFileComponentRecorder, listedPackage); RecordComponents( singleFileComponentRecorder, diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs index 756b27b65..ea4d4c16e 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs @@ -20,9 +20,10 @@ public PythonResolver(IPyPiClient pypiClient, ILogger logger) /// /// Resolves the root Python packages from the initial list of packages. /// + /// The component recorder for file that is been processed. /// The initial list of packages. /// The root packages, with dependencies associated as children. - public async Task> ResolveRootsAsync(IList initialPackages) + public async Task> ResolveRootsAsync(ISingleFileComponentRecorder singleFileComponentRecorder, IList initialPackages) { var state = new PythonResolverState(); @@ -53,15 +54,16 @@ public async Task> ResolveRootsAsync(IList(); + return await this.ProcessQueueAsync(singleFileComponentRecorder, state) ?? new List(); } - private async Task> ProcessQueueAsync(PythonResolverState state) + private async Task> ProcessQueueAsync(ISingleFileComponentRecorder singleFileComponentRecorder, PythonResolverState state) { while (state.ProcessingQueue.Count > 0) { @@ -109,7 +111,7 @@ private async Task> ProcessQueueAsync(PythonResolverState st } else { - this.logger.LogWarning($"Dependency Package {dependencyNode.Name} not found in Pypi. Skipping package"); + singleFileComponentRecorder.RegisterPackageParseFailure(dependencyNode.Name); } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/ruby/RubyComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/ruby/RubyComponentDetector.cs index caf80edf3..110a56aea 100644 --- a/src/Microsoft.ComponentDetection.Detectors/ruby/RubyComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/ruby/RubyComponentDetector.cs @@ -117,13 +117,13 @@ private void ParseGemLockFile(ISingleFileComponentRecorder singleFileComponentRe switch (heading) { case "GIT": - this.ParseSection(SectionType.GIT, sublines, components, dependencies, file); + this.ParseSection(singleFileComponentRecorder, SectionType.GIT, sublines, components, dependencies, file); break; case "GEM": - this.ParseSection(SectionType.GEM, sublines, components, dependencies, file); + this.ParseSection(singleFileComponentRecorder, SectionType.GEM, sublines, components, dependencies, file); break; case "PATH": - this.ParseSection(SectionType.PATH, sublines, components, dependencies, file); + this.ParseSection(singleFileComponentRecorder, SectionType.PATH, sublines, components, dependencies, file); break; case "BUNDLED WITH": var line = sublines[0].Trim(); @@ -143,7 +143,7 @@ private void ParseGemLockFile(ISingleFileComponentRecorder singleFileComponentRe { // Throw this line away. Is this malformed? We were expecting a header this.Logger.LogVerbose(lines[0]); - this.Logger.LogVerbose("Appears to be malformed/is not expected here. Expected heading."); + this.Logger.LogVerbose("Appears to be malformed/is not expected here. Expected heading."); lines.RemoveAt(0); } } @@ -171,7 +171,7 @@ private void ParseGemLockFile(ISingleFileComponentRecorder singleFileComponentRe } } - private void ParseSection(SectionType sectionType, List lines, Dictionary components, Dictionary> dependencies, IComponentStream file) + private void ParseSection(ISingleFileComponentRecorder singleFileComponentRecorder, SectionType sectionType, List lines, Dictionary components, Dictionary> dependencies, IComponentStream file) { string name, remote, revision; name = remote = revision = string.Empty; @@ -221,6 +221,7 @@ private void ParseSection(SectionType sectionType, List lines, Dictionar if (this.IsVersionRelative(version)) { this.Logger.LogWarning($"Found component with invalid version, name = {name} and version = {version}"); + singleFileComponentRecorder.RegisterPackageParseFailure($"{name} - {version}"); wasParentDependencyExcluded = true; continue; } diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs index bb67ffb2f..f16c43104 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs @@ -241,6 +241,7 @@ private void ProcessDependency( record.Dependencies = dependency; this.Logger.LogFailedReadingFile(cargoLockFile.Location, e); + singleFileComponentRecorder.RegisterPackageParseFailure(record.PackageInfo); } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs index 0b31bffc2..da6fbde2b 100644 --- a/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs @@ -46,7 +46,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; var file = processRequest.ComponentStream; - this.Logger.LogInfo($"vcpkg detector found {file}"); + this.Logger.LogVerbose($"vcpkg detector found {file}"); var projectRootDirectory = Directory.GetParent(file.Location); if (this.projectRoots.Any(path => projectRootDirectory.FullName.StartsWith(path))) @@ -116,6 +116,7 @@ private async Task ParseSpdxFileAsync( catch (Exception) { this.Logger.LogWarning($"failed while handling {item.Name}"); + singleFileComponentRecorder.RegisterPackageParseFailure(item.Name); } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/IYarnLockFileFactory.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/IYarnLockFileFactory.cs index f2dc248ec..593dc7fd0 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/IYarnLockFileFactory.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/IYarnLockFileFactory.cs @@ -6,5 +6,5 @@ namespace Microsoft.ComponentDetection.Detectors.Yarn; public interface IYarnLockFileFactory { - Task ParseYarnLockFileAsync(Stream file, ILogger logger); + Task ParseYarnLockFileAsync(ISingleFileComponentRecorder singleFileComponentRecorder, Stream file, ILogger logger); } diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/IYarnLockParser.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/IYarnLockParser.cs index 088bcca6b..6336058ef 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/IYarnLockParser.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/IYarnLockParser.cs @@ -6,5 +6,5 @@ public interface IYarnLockParser { bool CanParse(YarnLockVersion yarnLockVersion); - YarnLockFile Parse(IYarnBlockFile fileLines, ILogger logger); + YarnLockFile Parse(ISingleFileComponentRecorder singleFileComponentRecorder, IYarnBlockFile fileLines, ILogger logger); } diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnLockParser.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnLockParser.cs index 177a298cb..a0358a63e 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnLockParser.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnLockParser.cs @@ -30,7 +30,7 @@ public bool CanParse(YarnLockVersion yarnLockVersion) return SupportedVersions.Contains(yarnLockVersion); } - public YarnLockFile Parse(IYarnBlockFile fileLines, ILogger logger) + public YarnLockFile Parse(ISingleFileComponentRecorder singleFileComponentRecorder, IYarnBlockFile fileLines, ILogger logger) { if (fileLines == null) { @@ -70,6 +70,7 @@ public YarnLockFile Parse(IYarnBlockFile fileLines, ILogger logger) if (!block.Values.TryGetValue(VersionString, out var version)) { logger.LogWarning($"Failed to read a version for {yarnEntry.Name}. The entry will be skipped."); + singleFileComponentRecorder.RegisterPackageParseFailure(yarnEntry.Name); continue; } diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs index c1223acc2..b269e0c2b 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs @@ -57,7 +57,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID try { - var parsed = await this.yarnLockFileFactory.ParseYarnLockFileAsync(file.Stream, this.Logger); + var parsed = await this.yarnLockFileFactory.ParseYarnLockFileAsync(singleFileComponentRecorder, file.Stream, this.Logger); this.DetectComponents(parsed, file.Location, singleFileComponentRecorder); } catch (Exception ex) @@ -89,7 +89,7 @@ private void DetectComponents(YarnLockFile file, string location, ISingleFileCom } } - if (yarnPackages.Count == 0 || !this.TryReadPeerPackageJsonRequestsAsYarnEntries(location, yarnPackages, out var yarnRoots)) + if (yarnPackages.Count == 0 || !this.TryReadPeerPackageJsonRequestsAsYarnEntries(singleFileComponentRecorder, location, yarnPackages, out var yarnRoots)) { return; } @@ -178,11 +178,12 @@ private void ParseTreeWithAssignedRoot(YarnEntry root, Dictionary + /// The component recorder for file that is been processed. /// The file location of the yarn.lock file. /// All the yarn entries that we know about. /// The output yarnRoots that we care about using as starting points. /// False if no package.json file was found at location, otherwise it returns true. - private bool TryReadPeerPackageJsonRequestsAsYarnEntries(string location, Dictionary yarnEntries, out List yarnRoots) + private bool TryReadPeerPackageJsonRequestsAsYarnEntries(ISingleFileComponentRecorder singleFileComponentRecorder, string location, Dictionary yarnEntries, out List yarnRoots) { yarnRoots = new List(); @@ -221,6 +222,7 @@ private bool TryReadPeerPackageJsonRequestsAsYarnEntries(string location, Dictio if (!yarnEntries.ContainsKey(entryKey)) { this.Logger.LogWarning($"A package was requested in the package.json file that was a peer of {location} but was not contained in the lockfile. {name} - {version.Key}"); + singleFileComponentRecorder.RegisterPackageParseFailure($"{name} - {version.Key}"); continue; } diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockFileFactory.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockFileFactory.cs index 52ac0402b..0d19a4ab7 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockFileFactory.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockFileFactory.cs @@ -11,7 +11,7 @@ public class YarnLockFileFactory : IYarnLockFileFactory public YarnLockFileFactory(IEnumerable parsers) => this.parsers = parsers; - public async Task ParseYarnLockFileAsync(Stream file, ILogger logger) + public async Task ParseYarnLockFileAsync(ISingleFileComponentRecorder singleFileComponentRecorder, Stream file, ILogger logger) { var blockFile = await YarnBlockFile.CreateBlockFileAsync(file); @@ -19,7 +19,7 @@ public async Task ParseYarnLockFileAsync(Stream file, ILogger logg { if (parser.CanParse(blockFile.YarnLockVersion)) { - return parser.Parse(blockFile, logger); + return parser.Parse(singleFileComponentRecorder, blockFile, logger); } } diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs index 8aa12be0d..86aa96293 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs @@ -119,6 +119,33 @@ public async Task ProcessDetectorsAsync(IDetectionArgu var totalElapsedTime = stopwatch.Elapsed.TotalSeconds; this.LogTabularOutput(this.Logger, providerElapsedTime, totalElapsedTime); + // If there are components which are skipped due to connection or parsing + // errors, log them by detector. + var parseWarningShown = false; + foreach (var (_, recorder, detector) in results) + { + var skippedComponents = recorder.GetSkippedComponents(); + if (!skippedComponents.Any()) + { + continue; + } + + if (!parseWarningShown) + { + this.Logger.LogCreateLoggingGroup(); + this.Logger.LogWarning($"Some components or files were not detected due to parsing failures or connectivity issues."); + this.Logger.LogWarning($"Please review the logs above for more detailed information."); + parseWarningShown = true; + } + + this.Logger.LogCreateLoggingGroup(); + this.Logger.LogWarning($"Components skipped for {detector.Id} detector:"); + foreach (var component in skippedComponents) + { + this.Logger.LogWarning($"- {component}"); + } + } + this.Logger.LogCreateLoggingGroup(); this.Logger.LogInfo($"Detection time: {totalElapsedTime} seconds."); diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/PipComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/PipComponentDetectorTests.cs index f09c97e0f..8b591fe3b 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/PipComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/PipComponentDetectorTests.cs @@ -83,8 +83,8 @@ public async Task TestPipDetector_SetupPyAndRequirementsTxtAsync() new PipGraphNode(new PipComponent("f", "1.1")), }; - this.pythonResolver.Setup(x => x.ResolveRootsAsync(It.Is>(p => p.Any(d => d.Name == "b")))).ReturnsAsync(setupPyRoots); - this.pythonResolver.Setup(x => x.ResolveRootsAsync(It.Is>(p => p.Any(d => d.Name == "d")))).ReturnsAsync(requirementsTxtRoots); + this.pythonResolver.Setup(x => x.ResolveRootsAsync(It.IsAny(), It.Is>(p => p.Any(d => d.Name == "b")))).ReturnsAsync(setupPyRoots); + this.pythonResolver.Setup(x => x.ResolveRootsAsync(It.IsAny(), It.Is>(p => p.Any(d => d.Name == "d")))).ReturnsAsync(requirementsTxtRoots); var (result, componentRecorder) = await this.DetectorTestUtility .WithFile("setup.py", string.Empty) @@ -139,8 +139,8 @@ public async Task TestPipDetector_ComponentsDedupedAcrossFilesAsync() new PipGraphNode(new PipComponent("g", "1.2")), }; - this.pythonResolver.Setup(x => x.ResolveRootsAsync(It.Is>(p => p.Any(d => d.Name == "h")))).ReturnsAsync(requirementsTxtRoots); - this.pythonResolver.Setup(x => x.ResolveRootsAsync(It.Is>(p => p.Any(d => d.Name == "g")))).ReturnsAsync(requirementsTxtRoots2); + this.pythonResolver.Setup(x => x.ResolveRootsAsync(It.IsAny(), It.Is>(p => p.Any(d => d.Name == "h")))).ReturnsAsync(requirementsTxtRoots); + this.pythonResolver.Setup(x => x.ResolveRootsAsync(It.IsAny(), It.Is>(p => p.Any(d => d.Name == "g")))).ReturnsAsync(requirementsTxtRoots2); var (result, componentRecorder) = await this.DetectorTestUtility .WithFile("requirements.txt", string.Empty) @@ -188,11 +188,11 @@ public async Task TestPipDetector_ComponentRecorderAsync() blue.Children.Add(dog); this.pythonResolver.Setup(x => - x.ResolveRootsAsync(It.Is>(p => p.Any(d => d.Name == "a")))) + x.ResolveRootsAsync(It.IsAny(), It.Is>(p => p.Any(d => d.Name == "a")))) .ReturnsAsync(new List { rootA, rootB, }); this.pythonResolver.Setup(x => - x.ResolveRootsAsync(It.Is>(p => p.Any(d => d.Name == "c")))) + x.ResolveRootsAsync(It.IsAny(), It.Is>(p => p.Any(d => d.Name == "c")))) .ReturnsAsync(new List { rootC, rootD, rootE, }); var (result, componentRecorder) = await this.DetectorTestUtility diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/PipResolverTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/PipResolverTests.cs index 17d925c3f..756da8f2b 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/PipResolverTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/PipResolverTests.cs @@ -16,12 +16,14 @@ public class PipResolverTests { private Mock loggerMock; private Mock pyPiClient; + private Mock recorderMock; [TestInitialize] public void TestInitialize() { this.loggerMock = new Mock(); this.pyPiClient = new Mock(); + this.recorderMock = new Mock(); } [TestMethod] @@ -49,7 +51,7 @@ public async Task TestPipResolverSimpleGraphAsync() var resolver = new PythonResolver(this.pyPiClient.Object, this.loggerMock.Object); - var resolveResult = await resolver.ResolveRootsAsync(dependencies); + var resolveResult = await resolver.ResolveRootsAsync(this.recorderMock.Object, dependencies); Assert.IsNotNull(resolveResult); @@ -92,7 +94,7 @@ public async Task TestPipResolverNonExistantRootAsync() var resolver = new PythonResolver(this.pyPiClient.Object, this.loggerMock.Object); - var resolveResult = await resolver.ResolveRootsAsync(dependencies); + var resolveResult = await resolver.ResolveRootsAsync(this.recorderMock.Object, dependencies); Assert.IsNotNull(resolveResult); @@ -132,7 +134,7 @@ public async Task TestPipResolverNonExistantLeafAsync() var resolver = new PythonResolver(this.pyPiClient.Object, this.loggerMock.Object); - var resolveResult = await resolver.ResolveRootsAsync(dependencies); + var resolveResult = await resolver.ResolveRootsAsync(this.recorderMock.Object, dependencies); Assert.IsNotNull(resolveResult); @@ -175,7 +177,7 @@ public async Task TestPipResolverBacktrackAsync() var resolver = new PythonResolver(this.pyPiClient.Object, this.loggerMock.Object); - var resolveResult = await resolver.ResolveRootsAsync(dependencies); + var resolveResult = await resolver.ResolveRootsAsync(this.recorderMock.Object, dependencies); Assert.IsNotNull(resolveResult); diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/YarnLockDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/YarnLockDetectorTests.cs index 5f5d52b34..ed80ca71d 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/YarnLockDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/YarnLockDetectorTests.cs @@ -1,4 +1,4 @@ -namespace Microsoft.ComponentDetection.Detectors.Tests; +namespace Microsoft.ComponentDetection.Detectors.Tests; using System; using System.Collections.Generic; using System.IO; @@ -33,8 +33,12 @@ public YarnLockDetectorTests() this.yarnLockParser = new YarnLockParser(loggerMock.Object); this.yarnLockFileFactory = new YarnLockFileFactory(new[] { this.yarnLockParser }); var yarnLockFileFactoryMock = new Mock(); - yarnLockFileFactoryMock.Setup(x => x.ParseYarnLockFileAsync(It.IsAny(), It.IsAny())) - .Returns((Stream stream, ILogger logger) => this.yarnLockFileFactory.ParseYarnLockFileAsync(stream, logger)); + + var recorderMock = new Mock(); + + yarnLockFileFactoryMock.Setup(x => x.ParseYarnLockFileAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((ISingleFileComponentRecorder recorder, Stream stream, ILogger logger) => this.yarnLockFileFactory.ParseYarnLockFileAsync(recorder, stream, logger)); + this.DetectorTestUtility.AddServiceMock(yarnLockFileFactoryMock); } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/YarnParserTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/YarnParserTests.cs index d06eb5c30..8bdaa515f 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/YarnParserTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/YarnParserTests.cs @@ -1,4 +1,4 @@ -namespace Microsoft.ComponentDetection.Detectors.Tests; +namespace Microsoft.ComponentDetection.Detectors.Tests; using System; using System.Collections.Generic; using System.Linq; @@ -14,15 +14,20 @@ public class YarnParserTests { private readonly Mock loggerMock; + private readonly Mock recorderMock; - public YarnParserTests() => this.loggerMock = new Mock(); + public YarnParserTests() + { + this.loggerMock = new Mock(); + this.recorderMock = new Mock(); + } [TestMethod] public void YarnLockParserWithNullBlockFile_Fails() { var parser = new YarnLockParser(this.loggerMock.Object); - void Action() => parser.Parse(null, this.loggerMock.Object); + void Action() => parser.Parse(this.recorderMock.Object, null, this.loggerMock.Object); Assert.ThrowsException(Action); } @@ -65,7 +70,7 @@ public void YarnLockParser_ParsesEmptyFile() blockFile.Setup(x => x.YarnLockVersion).Returns(yarnLockFileVersion); blockFile.Setup(x => x.GetEnumerator()).Returns(blocks.GetEnumerator()); - var file = parser.Parse(blockFile.Object, this.loggerMock.Object); + var file = parser.Parse(this.recorderMock.Object, blockFile.Object, this.loggerMock.Object); Assert.AreEqual(YarnLockVersion.V1, file.LockVersion); Assert.AreEqual(0, file.Entries.Count()); @@ -95,7 +100,7 @@ public void YarnLockParser_ParsesBlocks() blockFile.Setup(x => x.YarnLockVersion).Returns(yarnLockFileVersion); blockFile.Setup(x => x.GetEnumerator()).Returns(blocks.GetEnumerator()); - var file = parser.Parse(blockFile.Object, this.loggerMock.Object); + var file = parser.Parse(this.recorderMock.Object, blockFile.Object, this.loggerMock.Object); Assert.AreEqual(YarnLockVersion.V1, file.LockVersion); Assert.AreEqual(3, file.Entries.Count()); @@ -131,7 +136,7 @@ public void YarnLockParser_ParsesNoVersionInTitleBlock() blockFile.Setup(x => x.YarnLockVersion).Returns(yarnLockFileVersion); blockFile.Setup(x => x.GetEnumerator()).Returns(blocks.GetEnumerator()); - var file = parser.Parse(blockFile.Object, this.loggerMock.Object); + var file = parser.Parse(this.recorderMock.Object, blockFile.Object, this.loggerMock.Object); Assert.AreEqual(YarnLockVersion.V1, file.LockVersion); Assert.AreEqual(2, file.Entries.Count());