Add output log of skipped / failed components

src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Collections.Immutable;
@@ -62,6 +62,20 @@ public IEnumerable<DetectedComponent> GetDetectedComponents()
         return detectedComponents;
     }
 
+    public IEnumerable<string> GetSkippedComponents()
+    {
+        if (this.singleFileRecorders == null)
+        {
+            return Enumerable.Empty<string>();
+        }
+
+        return this.singleFileRecorders
+            .Select(x => x.GetSkippedComponents().Keys)
+            .SelectMany(x => x)
+            .Distinct()
+            .ToImmutableList();
+    }
+
     public ISingleFileComponentRecorder CreateSingleFileComponentRecorder(string location)
     {
         if (string.IsNullOrWhiteSpace(location))
@@ -96,6 +110,11 @@ public sealed class SingleFileComponentRecorder : ISingleFileComponentRecorder
 
         private readonly ConcurrentDictionary<string, DetectedComponent> detectedComponentsInternal = new ConcurrentDictionary<string, DetectedComponent>();
 
+        /// <summary>
+        /// Dictionary of components which had an error during parsing and a dummy data value that only allocates 1 byte.
+        /// </summary>
+        private readonly ConcurrentDictionary<string, byte> skippedComponentsInternal = new ConcurrentDictionary<string, byte>();
+
         private readonly ComponentRecorder recorder;
 
         private readonly object registerUsageLock = new object();
@@ -130,6 +149,11 @@ public IReadOnlyDictionary<string, DetectedComponent> GetDetectedComponents()
             return this.detectedComponentsInternal;
         }
 
+        public IReadOnlyDictionary<string, byte> GetSkippedComponents()
+        {
+            return this.skippedComponentsInternal;
+        }
+
         public void RegisterUsage(
             DetectedComponent detectedComponent,
             bool isExplicitReferencedDependency = false,
@@ -173,6 +197,16 @@ public void RegisterUsage(
             }
         }
 
+        public void RegisterPackageParseFailure(string skippedComponent)
+        {
+            if (skippedComponent == null)
+            {
+                throw new ArgumentNullException(paramName: nameof(skippedComponent));
+            }
+
+            _ = this.skippedComponentsInternal[skippedComponent] = default;
+        }
+
         public void AddAdditionalRelatedFile(string relatedFilePath)
         {
             this.DependencyGraph.AddAdditionalRelatedFile(relatedFilePath);

src/Microsoft.ComponentDetection.Contracts/IComponentRecorder.cs

@@ -8,6 +8,8 @@ public interface IComponentRecorder
 
     IEnumerable<DetectedComponent> GetDetectedComponents();
 
+    IEnumerable<string> GetSkippedComponents();
+
     ISingleFileComponentRecorder CreateSingleFileComponentRecorder(string location);
 
     IReadOnlyDictionary<string, IDependencyGraph> GetDependencyGraphsByLocation();
@@ -35,6 +37,12 @@ void RegisterUsage(
         bool? isDevelopmentDependency = null,
         DependencyScope? dependencyScope = null);
 
+    /// <summary>
+    /// Register that a package was unable to be processed.
+    /// </summary>
+    /// <param name="skippedComponent">Component version identifier.</param>
+    void RegisterPackageParseFailure(string skippedComponent);
+
     DetectedComponent GetComponent(string componentId);
 
     void AddAdditionalRelatedFile(string relatedFilePath);

src/Microsoft.ComponentDetection.Detectors/cocoapods/PodComponentDetector.cs

@@ -237,6 +237,7 @@ private void ProcessPodfileLock(
                 else
                 {
                     this.Logger.LogWarning($"Missing podspec declaration. podspec={dependency.Podspec}, version={dependency.PodVersion}");
+                    singleFileComponentRecorder.RegisterPackageParseFailure($"{dependency.Podspec} - {dependency.PodVersion}");
                 }
             }
         }

src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs

@@ -170,7 +170,9 @@ private void ParseGoModFile(
             }
             else
             {
-                this.Logger.LogWarning($"Line could not be parsed for component [{line.Trim()}]");
+                var lineTrim = line.Trim();
+                this.Logger.LogWarning($"Line could not be parsed for component [{lineTrim}]");
+                singleFileComponentRecorder.RegisterPackageParseFailure(lineTrim);
             }
         }
     }
@@ -209,7 +211,9 @@ private void ParseGoSumFile(
             }
             else
             {
-                this.Logger.LogWarning($"Line could not be parsed for component [{line.Trim()}]");
+                var lineTrim = line.Trim();
+                this.Logger.LogWarning($"Line could not be parsed for component [{lineTrim}]");
+                singleFileComponentRecorder.RegisterPackageParseFailure(lineTrim);
             }
         }
     }
@@ -270,6 +274,7 @@ private void PopulateDependencyGraph(string goGraphOutput, ISingleFileComponentR
             else
             {
                 this.Logger.LogWarning($"Failed to parse components from relationship string {relationship}");
+                componentRecorder.RegisterPackageParseFailure(relationship);
             }
         }
     }

src/Microsoft.ComponentDetection.Detectors/ivy/IvyDetector.cs

@@ -237,6 +237,7 @@ private void RegisterUsagesFromFile(ISingleFileComponentRecorder singleFileCompo
             else
             {
                 this.Logger.LogWarning($"Dependency \"{component.Id}\" could not be resolved by Ivy, and so has not been recorded by Component Detection.");
+                singleFileComponentRecorder.RegisterPackageParseFailure(component.Id);
             }
         }
     }

src/Microsoft.ComponentDetection.Detectors/linux/LinuxContainerDetector.cs

@@ -153,6 +153,9 @@ await this.dockerService.TryPullImageAsync(image, cancellationToken)))
                         StackTrace = e.StackTrace,
                         ImageId = image,
                     };
+
+                    var singleFileComponentRecorder = componentRecorder.CreateSingleFileComponentRecorder(image);
+                    singleFileComponentRecorder.RegisterPackageParseFailure(image);
                 }
             });
 
@@ -196,6 +199,9 @@ await this.dockerService.TryPullImageAsync(image, cancellationToken)))
                     StackTrace = e.StackTrace,
                     ImageId = kvp.Value.ImageId,
                 };
+
+                var singleFileComponentRecorder = componentRecorder.CreateSingleFileComponentRecorder(kvp.Value.ImageId);
+                singleFileComponentRecorder.RegisterPackageParseFailure(kvp.Key);
             }
 
             return EmptyImageScanningResult();

src/Microsoft.ComponentDetection.Detectors/maven/MavenCommandService.cs

@@ -43,6 +43,7 @@ public async Task GenerateDependenciesFileAsync(ProcessRequest processRequest)
         {
             this.logger.LogVerbose($"Mvn execution failed for pom file: {pomFile.Location}");
             this.logger.LogError(string.IsNullOrEmpty(result.StdErr) ? result.StdOut : result.StdErr);
+            processRequest.SingleFileComponentRecorder.RegisterPackageParseFailure(pomFile.Location);
         }
     }
 

src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentDetector.cs

@@ -80,6 +80,7 @@ protected virtual bool ProcessIndividualPackageJTokens(string filePath, ISingleF
         if (!SemanticVersion.TryParse(version, out _))
         {
             this.Logger.LogWarning($"Unable to parse version \"{version}\" for package \"{name}\" found at path \"{filePath}\". This may indicate an invalid npm package component and it will not be registered.");
+            singleFileComponentRecorder.RegisterPackageParseFailure($"{name} - {version}");
             return false;
         }
 

src/Microsoft.ComponentDetection.Detectors/nuget/NuGetComponentDetector.cs

@@ -126,6 +126,7 @@ private async Task ProcessFileAsync(ProcessRequest processRequest)
             if (!NuGetVersion.TryParse(version, out var parsedVer))
             {
                 this.Logger.LogInfo($"Version '{version}' from {stream.Location} could not be parsed as a NuGet version");
+                singleFileComponentRecorder.RegisterPackageParseFailure(stream.Location);
 
                 return;
             }
@@ -140,6 +141,7 @@ private async Task ProcessFileAsync(ProcessRequest processRequest)
         {
             // If something went wrong, just ignore the component
             this.Logger.LogFailedReadingFile(stream.Location, e);
+            singleFileComponentRecorder.RegisterPackageParseFailure(stream.Location);
         }
     }
 

src/Microsoft.ComponentDetection.Detectors/pip/IPythonResolver.cs

@@ -1,13 +1,15 @@
 namespace Microsoft.ComponentDetection.Detectors.Pip;
 using System.Collections.Generic;
 using System.Threading.Tasks;
+using Microsoft.ComponentDetection.Contracts;
 
 public interface IPythonResolver
 {
     /// <summary>
     /// Resolves the root Python packages from the initial list of packages.
     /// </summary>
+    /// <param name="singleFileComponentRecorder">The component recorder for file that is been processed.</param>
     /// <param name="initialPackages">The initial list of packages.</param>
     /// <returns>The root packages, with dependencies associated as children.</returns>
-    Task<IList<PipGraphNode>> ResolveRootsAsync(IList<PipDependencySpecification> initialPackages);
+    Task<IList<PipGraphNode>> ResolveRootsAsync(ISingleFileComponentRecorder singleFileComponentRecorder, IList<PipDependencySpecification> initialPackages);
 }

src/Microsoft.ComponentDetection.Detectors/pip/PipComponentDetector.cs

@@ -66,7 +66,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID
                 .Where(x => !x.PackageIsUnsafe())
                 .ToList();
 
-            var roots = await this.pythonResolver.ResolveRootsAsync(listedPackage);
+            var roots = await this.pythonResolver.ResolveRootsAsync(singleFileComponentRecorder, listedPackage);
 
             RecordComponents(
                 singleFileComponentRecorder,

src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs

@@ -20,9 +20,10 @@ public PythonResolver(IPyPiClient pypiClient, ILogger logger)
     /// <summary>
     /// Resolves the root Python packages from the initial list of packages.
     /// </summary>
+    /// <param name="singleFileComponentRecorder">The component recorder for file that is been processed.</param>
     /// <param name="initialPackages">The initial list of packages.</param>
     /// <returns>The root packages, with dependencies associated as children.</returns>
-    public async Task<IList<PipGraphNode>> ResolveRootsAsync(IList<PipDependencySpecification> initialPackages)
+    public async Task<IList<PipGraphNode>> ResolveRootsAsync(ISingleFileComponentRecorder singleFileComponentRecorder, IList<PipDependencySpecification> initialPackages)
     {
         var state = new PythonResolverState();
 
@@ -53,15 +54,16 @@ public async Task<IList<PipGraphNode>> ResolveRootsAsync(IList<PipDependencySpec
                 else
                 {
                     this.logger.LogWarning($"Root dependency {rootPackage.Name} not found on pypi. Skipping package.");
+                    singleFileComponentRecorder.RegisterPackageParseFailure(rootPackage.Name);
                 }
             }
         }
 
         // Now queue packages for processing
-        return await this.ProcessQueueAsync(state) ?? new List<PipGraphNode>();
+        return await this.ProcessQueueAsync(singleFileComponentRecorder, state) ?? new List<PipGraphNode>();
     }
 
-    private async Task<IList<PipGraphNode>> ProcessQueueAsync(PythonResolverState state)
+    private async Task<IList<PipGraphNode>> ProcessQueueAsync(ISingleFileComponentRecorder singleFileComponentRecorder, PythonResolverState state)
     {
         while (state.ProcessingQueue.Count > 0)
         {
@@ -109,7 +111,7 @@ private async Task<IList<PipGraphNode>> ProcessQueueAsync(PythonResolverState st
                     }
                     else
                     {
-                        this.logger.LogWarning($"Dependency Package {dependencyNode.Name} not found in Pypi. Skipping package");
+                        singleFileComponentRecorder.RegisterPackageParseFailure(dependencyNode.Name);
                     }
                 }
             }

src/Microsoft.ComponentDetection.Detectors/ruby/RubyComponentDetector.cs

@@ -117,13 +117,13 @@ private void ParseGemLockFile(ISingleFileComponentRecorder singleFileComponentRe
                 switch (heading)
                 {
                     case "GIT":
-                        this.ParseSection(SectionType.GIT, sublines, components, dependencies, file);
+                        this.ParseSection(singleFileComponentRecorder, SectionType.GIT, sublines, components, dependencies, file);
                         break;
                     case "GEM":
-                        this.ParseSection(SectionType.GEM, sublines, components, dependencies, file);
+                        this.ParseSection(singleFileComponentRecorder, SectionType.GEM, sublines, components, dependencies, file);
                         break;
                     case "PATH":
-                        this.ParseSection(SectionType.PATH, sublines, components, dependencies, file);
+                        this.ParseSection(singleFileComponentRecorder, SectionType.PATH, sublines, components, dependencies, file);
                         break;
                     case "BUNDLED WITH":
                         var line = sublines[0].Trim();
@@ -143,7 +143,7 @@ private void ParseGemLockFile(ISingleFileComponentRecorder singleFileComponentRe
             {
                 // Throw this line away. Is this malformed? We were expecting a header
                 this.Logger.LogVerbose(lines[0]);
-                this.Logger.LogVerbose("Appears to be malformed/is not expected here.  Expected heading.");
+                this.Logger.LogVerbose("Appears to be malformed/is not expected here. Expected heading.");
                 lines.RemoveAt(0);
             }
         }
@@ -171,7 +171,7 @@ private void ParseGemLockFile(ISingleFileComponentRecorder singleFileComponentRe
         }
     }
 
-    private void ParseSection(SectionType sectionType, List<string> lines, Dictionary<string, DetectedComponent> components, Dictionary<string, List<Dependency>> dependencies, IComponentStream file)
+    private void ParseSection(ISingleFileComponentRecorder singleFileComponentRecorder, SectionType sectionType, List<string> lines, Dictionary<string, DetectedComponent> components, Dictionary<string, List<Dependency>> dependencies, IComponentStream file)
     {
         string name, remote, revision;
         name = remote = revision = string.Empty;
@@ -221,6 +221,7 @@ private void ParseSection(SectionType sectionType, List<string> lines, Dictionar
                         if (this.IsVersionRelative(version))
                         {
                             this.Logger.LogWarning($"Found component with invalid version, name = {name} and version = {version}");
+                            singleFileComponentRecorder.RegisterPackageParseFailure($"{name} - {version}");
                             wasParentDependencyExcluded = true;
                             continue;
                         }

src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs

@@ -241,6 +241,7 @@ private void ProcessDependency(
             record.Dependencies = dependency;
 
             this.Logger.LogFailedReadingFile(cargoLockFile.Location, e);
+            singleFileComponentRecorder.RegisterPackageParseFailure(record.PackageInfo);
         }
     }
 }

src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs

@@ -46,7 +46,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID
         var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder;
         var file = processRequest.ComponentStream;
 
-        this.Logger.LogInfo($"vcpkg detector found {file}");
+        this.Logger.LogVerbose($"vcpkg detector found {file}");
 
         var projectRootDirectory = Directory.GetParent(file.Location);
         if (this.projectRoots.Any(path => projectRootDirectory.FullName.StartsWith(path)))
@@ -116,6 +116,7 @@ private async Task ParseSpdxFileAsync(
             catch (Exception)
             {
                 this.Logger.LogWarning($"failed while handling {item.Name}");
+                singleFileComponentRecorder.RegisterPackageParseFailure(item.Name);
             }
         }
     }

src/Microsoft.ComponentDetection.Detectors/yarn/IYarnLockFileFactory.cs

@@ -6,5 +6,5 @@ namespace Microsoft.ComponentDetection.Detectors.Yarn;
 
 public interface IYarnLockFileFactory
 {
-    Task<YarnLockFile> ParseYarnLockFileAsync(Stream file, ILogger logger);
+    Task<YarnLockFile> ParseYarnLockFileAsync(ISingleFileComponentRecorder singleFileComponentRecorder, Stream file, ILogger logger);
 }

src/Microsoft.ComponentDetection.Detectors/yarn/IYarnLockParser.cs

@@ -6,5 +6,5 @@ public interface IYarnLockParser
 {
     bool CanParse(YarnLockVersion yarnLockVersion);
 
-    YarnLockFile Parse(IYarnBlockFile fileLines, ILogger logger);
+    YarnLockFile Parse(ISingleFileComponentRecorder singleFileComponentRecorder, IYarnBlockFile fileLines, ILogger logger);
 }

src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnLockParser.cs

@@ -30,7 +30,7 @@ public bool CanParse(YarnLockVersion yarnLockVersion)
         return SupportedVersions.Contains(yarnLockVersion);
     }
 
-    public YarnLockFile Parse(IYarnBlockFile fileLines, ILogger logger)
+    public YarnLockFile Parse(ISingleFileComponentRecorder singleFileComponentRecorder, IYarnBlockFile fileLines, ILogger logger)
     {
         if (fileLines == null)
         {
@@ -70,6 +70,7 @@ public YarnLockFile Parse(IYarnBlockFile fileLines, ILogger logger)
             if (!block.Values.TryGetValue(VersionString, out var version))
             {
                 logger.LogWarning($"Failed to read a version for {yarnEntry.Name}. The entry will be skipped.");
+                singleFileComponentRecorder.RegisterPackageParseFailure(yarnEntry.Name);
                 continue;
             }
 

src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs

@@ -57,7 +57,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID
 
         try
         {
-            var parsed = await this.yarnLockFileFactory.ParseYarnLockFileAsync(file.Stream, this.Logger);
+            var parsed = await this.yarnLockFileFactory.ParseYarnLockFileAsync(singleFileComponentRecorder, file.Stream, this.Logger);
             this.DetectComponents(parsed, file.Location, singleFileComponentRecorder);
         }
         catch (Exception ex)
@@ -89,7 +89,7 @@ private void DetectComponents(YarnLockFile file, string location, ISingleFileCom
             }
         }
 
-        if (yarnPackages.Count == 0 || !this.TryReadPeerPackageJsonRequestsAsYarnEntries(location, yarnPackages, out var yarnRoots))
+        if (yarnPackages.Count == 0 || !this.TryReadPeerPackageJsonRequestsAsYarnEntries(singleFileComponentRecorder, location, yarnPackages, out var yarnRoots))
         {
             return;
         }
@@ -178,11 +178,12 @@ private void ParseTreeWithAssignedRoot(YarnEntry root, Dictionary<string, YarnEn
     /// This function reads those from the package.json so that they can later be used as the starting points
     /// in traversing the dependency graph.
     /// </summary>
+    /// <param name="singleFileComponentRecorder">The component recorder for file that is been processed.</param>
     /// <param name="location">The file location of the yarn.lock file.</param>
     /// <param name="yarnEntries">All the yarn entries that we know about.</param>
     /// <param name="yarnRoots">The output yarnRoots that we care about using as starting points.</param>
     /// <returns>False if no package.json file was found at location, otherwise it returns true. </returns>
-    private bool TryReadPeerPackageJsonRequestsAsYarnEntries(string location, Dictionary<string, YarnEntry> yarnEntries, out List<YarnEntry> yarnRoots)
+    private bool TryReadPeerPackageJsonRequestsAsYarnEntries(ISingleFileComponentRecorder singleFileComponentRecorder, string location, Dictionary<string, YarnEntry> yarnEntries, out List<YarnEntry> yarnRoots)
     {
         yarnRoots = new List<YarnEntry>();
 
@@ -221,6 +222,7 @@ private bool TryReadPeerPackageJsonRequestsAsYarnEntries(string location, Dictio
                 if (!yarnEntries.ContainsKey(entryKey))
                 {
                     this.Logger.LogWarning($"A package was requested in the package.json file that was a peer of {location} but was not contained in the lockfile. {name} - {version.Key}");
+                    singleFileComponentRecorder.RegisterPackageParseFailure($"{name} - {version.Key}");
                     continue;
                 }