From e5ae1c5f4e64591e0e669597076ec0f76e4a2365 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C2=8DAmitla=20Vannikumar?= Date: Wed, 31 Aug 2022 13:47:45 -0700 Subject: [PATCH 01/12] Fis SA1202 --- .editorconfig | 3 - .../CommandLineInvocationService.cs | 22 +- .../DependencyGraph/ComponentRecorder.cs | 16 +- .../DependencyGraph/DependencyGraph.cs | 74 ++-- .../FastDirectoryWalkerFactory.cs | 32 +- .../FileWritingService.cs | 5 +- .../Logger.cs | 20 +- .../PathUtilityService.cs | 26 +- .../TabularStringFormat.cs | 8 +- .../Records/BaseDetectionTelemetryRecord.cs | 12 +- .../NuGetProjectAssetsTelemetryRecord.cs | 12 +- .../DetectedComponent.cs | 8 +- .../FileComponentDetector.cs | 32 +- .../TypedComponent/CargoComponent.cs | 10 +- .../TypedComponent/CondaComponent.cs | 12 +- .../TypedComponent/DockerImageComponent.cs | 10 +- .../DockerReferenceComponent.cs | 10 +- .../TypedComponent/GitComponent.cs | 12 +- .../TypedComponent/GoComponent.cs | 12 +- .../TypedComponent/LinuxComponent.cs | 12 +- .../TypedComponent/MavenComponent.cs | 12 +- .../TypedComponent/NpmComponent.cs | 12 +- .../TypedComponent/NugetComponent.cs | 12 +- .../TypedComponent/OtherComponent.cs | 10 +- .../TypedComponent/PipComponent.cs | 10 +- .../TypedComponent/PodComponent.cs | 12 +- .../TypedComponent/RubyGemsComponent.cs | 10 +- .../TypedComponent/SpdxComponent.cs | 10 +- .../TypedComponent/VcpkgComponent.cs | 10 +- .../cocoapods/PodComponentDetector.cs | 38 +- .../ivy/IvyDetector.cs | 28 +- .../linux/LinuxContainerDetector.cs | 64 ++-- .../maven/MavenStyleDependencyGraphParser.cs | 20 +- .../npm/NpmComponentDetector.cs | 32 +- .../npm/NpmComponentDetectorWithRoots.cs | 108 +++--- .../pip/IPyPiClient.cs | 130 +++---- .../pip/PipDependencySpecification.cs | 14 +- .../pip/PythonVersionUtilities.cs | 98 ++--- .../rust/RustCrateUtilities.cs | 346 +++++++++--------- .../rust/SemVer/Comparator.cs | 42 +-- .../yarn/Parsers/YarnBlockFile.cs | 12 +- .../yarn/Parsers/YarnLockParser.cs | 12 +- .../Services/BcdeScanExecutionService.cs | 38 +- .../Services/DetectorProcessingService.cs | 198 +++++----- .../LoggerTests.cs | 34 +- .../IvyDetectorTests.cs | 14 +- .../MvnCliDetectorTests.cs | 14 +- .../nuget/NuGetPackagesConfigDetectorTests.cs | 4 +- .../DetectorProcessingServiceTests.cs | 144 ++++---- .../DetectorRestrictionServiceTests.cs | 28 +- .../DetectorTestUtility.cs | 20 +- 51 files changed, 941 insertions(+), 943 deletions(-) diff --git a/.editorconfig b/.editorconfig index 2cc2d88c2..631e60a7f 100644 --- a/.editorconfig +++ b/.editorconfig @@ -479,9 +479,6 @@ dotnet_diagnostic.SA1200.severity = suggestion # A field should not follow a property dotnet_diagnostic.SA1201.severity = suggestion -# https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1202.md -# Constant fields should appear before non-constant fields -dotnet_diagnostic.SA1202.severity = suggestion # https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1203.md # 'public' members should come before 'private' members diff --git a/src/Microsoft.ComponentDetection.Common/CommandLineInvocationService.cs b/src/Microsoft.ComponentDetection.Common/CommandLineInvocationService.cs index b937f3aa5..9702b5709 100644 --- a/src/Microsoft.ComponentDetection.Common/CommandLineInvocationService.cs +++ b/src/Microsoft.ComponentDetection.Common/CommandLineInvocationService.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.ComponentModel; @@ -88,6 +88,16 @@ public bool IsCommandLineExecution() return true; } + public async Task CanCommandBeLocated(string command, IEnumerable additionalCandidateCommands = null, params string[] parameters) + { + return await this.CanCommandBeLocated(command, additionalCandidateCommands, workingDirectory: null, parameters); + } + + public async Task ExecuteCommand(string command, IEnumerable additionalCandidateCommands = null, params string[] parameters) + { + return await this.ExecuteCommand(command, additionalCandidateCommands, workingDirectory: null, parameters); + } + private static Task RunProcessAsync(string fileName, string parameters, DirectoryInfo workingDirectory = null) { var tcs = new TaskCompletionSource(); @@ -144,15 +154,5 @@ private static Task RunProcessAsync(string fileName, return tcs.Task; } - - public async Task CanCommandBeLocated(string command, IEnumerable additionalCandidateCommands = null, params string[] parameters) - { - return await this.CanCommandBeLocated(command, additionalCandidateCommands, workingDirectory: null, parameters); - } - - public async Task ExecuteCommand(string command, IEnumerable additionalCandidateCommands = null, params string[] parameters) - { - return await this.ExecuteCommand(command, additionalCandidateCommands, workingDirectory: null, parameters); - } } } diff --git a/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs b/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs index 5346f9487..fb7a0aa14 100644 --- a/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs +++ b/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.Immutable; @@ -79,27 +79,27 @@ public ISingleFileComponentRecorder CreateSingleFileComponentRecorder(string loc return matching; } - internal DependencyGraph GetDependencyGraphForLocation(string location) - { - return this.singleFileRecorders.Single(x => x.ManifestFileLocation == location).DependencyGraph; - } - public IReadOnlyDictionary GetDependencyGraphsByLocation() { return this.singleFileRecorders.Where(x => x.DependencyGraph.HasComponents()) .ToImmutableDictionary(x => x.ManifestFileLocation, x => x.DependencyGraph as IDependencyGraph); } + internal DependencyGraph GetDependencyGraphForLocation(string location) + { + return this.singleFileRecorders.Single(x => x.ManifestFileLocation == location).DependencyGraph; + } + public class SingleFileComponentRecorder : ISingleFileComponentRecorder { private readonly ILogger log; public string ManifestFileLocation { get; } - internal DependencyGraph DependencyGraph { get; } - IDependencyGraph ISingleFileComponentRecorder.DependencyGraph => this.DependencyGraph; + internal DependencyGraph DependencyGraph { get; } + private readonly ConcurrentDictionary detectedComponentsInternal = new ConcurrentDictionary(); private readonly ComponentRecorder recorder; diff --git a/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs b/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs index 0d4175a36..41090f354 100644 --- a/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs +++ b/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.Immutable; @@ -119,6 +119,42 @@ public IEnumerable GetAllExplicitlyReferencedComponents() .Select(componentRefNode => componentRefNode.Id); } + IEnumerable IDependencyGraph.GetDependenciesForComponent(string componentId) + { + return this.GetDependenciesForComponent(componentId).ToImmutableList(); + } + + IEnumerable IDependencyGraph.GetComponents() + { + return this.componentNodes.Keys.ToImmutableList(); + } + + bool IDependencyGraph.IsComponentExplicitlyReferenced(string componentId) + { + return this.IsExplicitReferencedDependency(this.componentNodes[componentId]); + } + + internal class ComponentRefNode + { + internal bool IsExplicitReferencedDependency { get; set; } + + internal string Id { get; set; } + + internal ISet DependencyIds { get; private set; } + + internal ISet DependedOnByIds { get; private set; } + + internal bool? IsDevelopmentDependency { get; set; } + + internal DependencyScope? DependencyScope { get; set; } + + internal ComponentRefNode() + { + this.DependencyIds = new HashSet(); + this.DependedOnByIds = new HashSet(); + } + } + private void GetExplicitReferencedDependencies(ComponentRefNode component, IList explicitReferencedDependencyIds, ISet visited) { if (this.IsExplicitReferencedDependency(component)) @@ -158,41 +194,5 @@ private void AddDependency(string componentId, string parentComponentId) parentComponentRefNode.DependencyIds.Add(componentId); this.componentNodes[componentId].DependedOnByIds.Add(parentComponentId); } - - IEnumerable IDependencyGraph.GetDependenciesForComponent(string componentId) - { - return this.GetDependenciesForComponent(componentId).ToImmutableList(); - } - - IEnumerable IDependencyGraph.GetComponents() - { - return this.componentNodes.Keys.ToImmutableList(); - } - - bool IDependencyGraph.IsComponentExplicitlyReferenced(string componentId) - { - return this.IsExplicitReferencedDependency(this.componentNodes[componentId]); - } - - internal class ComponentRefNode - { - internal bool IsExplicitReferencedDependency { get; set; } - - internal string Id { get; set; } - - internal ISet DependencyIds { get; private set; } - - internal ISet DependedOnByIds { get; private set; } - - internal bool? IsDevelopmentDependency { get; set; } - - internal DependencyScope? DependencyScope { get; set; } - - internal ComponentRefNode() - { - this.DependencyIds = new HashSet(); - this.DependedOnByIds = new HashSet(); - } - } } } diff --git a/src/Microsoft.ComponentDetection.Common/FastDirectoryWalkerFactory.cs b/src/Microsoft.ComponentDetection.Common/FastDirectoryWalkerFactory.cs index cbf93bbc9..aefcb97b0 100644 --- a/src/Microsoft.ComponentDetection.Common/FastDirectoryWalkerFactory.cs +++ b/src/Microsoft.ComponentDetection.Common/FastDirectoryWalkerFactory.cs @@ -196,22 +196,6 @@ public IObservable GetDirectoryScanner(DirectoryInfo root, Concu }); } - private FileSystemInfo Transform(ref FileSystemEntry entry) - { - return entry.ToFileSystemInfo(); - } - - private IObservable CreateDirectoryWalker(DirectoryInfo di, ExcludeDirectoryPredicate directoryExclusionPredicate, int minimumConnectionCount, IEnumerable filePatterns) - { - return this.GetDirectoryScanner(di, new ConcurrentDictionary(), directoryExclusionPredicate, filePatterns, true).Replay() // Returns a replay subject which will republish anything found to new subscribers. - .AutoConnect(minimumConnectionCount); // Specifies that this connectable observable should start when minimumConnectionCount subscribe. - } - - private bool MatchesAnyPattern(FileInfo fi, params string[] searchPatterns) - { - return searchPatterns != null && searchPatterns.Any(sp => this.PathUtilityService.MatchesPattern(sp, fi.Name)); - } - /// /// Initialized an observable file enumerator. /// @@ -290,5 +274,21 @@ public void Reset() { this.pendingScans.Clear(); } + + private FileSystemInfo Transform(ref FileSystemEntry entry) + { + return entry.ToFileSystemInfo(); + } + + private IObservable CreateDirectoryWalker(DirectoryInfo di, ExcludeDirectoryPredicate directoryExclusionPredicate, int minimumConnectionCount, IEnumerable filePatterns) + { + return this.GetDirectoryScanner(di, new ConcurrentDictionary(), directoryExclusionPredicate, filePatterns, true).Replay() // Returns a replay subject which will republish anything found to new subscribers. + .AutoConnect(minimumConnectionCount); // Specifies that this connectable observable should start when minimumConnectionCount subscribe. + } + + private bool MatchesAnyPattern(FileInfo fi, params string[] searchPatterns) + { + return searchPatterns != null && searchPatterns.Any(sp => this.PathUtilityService.MatchesPattern(sp, fi.Name)); + } } } diff --git a/src/Microsoft.ComponentDetection.Common/FileWritingService.cs b/src/Microsoft.ComponentDetection.Common/FileWritingService.cs index fc44ad6e6..b83d10193 100644 --- a/src/Microsoft.ComponentDetection.Common/FileWritingService.cs +++ b/src/Microsoft.ComponentDetection.Common/FileWritingService.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Composition; using System.IO; using Microsoft.ComponentDetection.Common.Exceptions; @@ -10,9 +10,10 @@ namespace Microsoft.ComponentDetection.Common [Shared] public class FileWritingService : IFileWritingService { + public const string TimestampFormatString = "yyyyMMddHHmmss"; + private object lockObject = new object(); private string timestamp = DateTime.Now.ToString(TimestampFormatString); - public const string TimestampFormatString = "yyyyMMddHHmmss"; public string BasePath { get; private set; } diff --git a/src/Microsoft.ComponentDetection.Common/Logger.cs b/src/Microsoft.ComponentDetection.Common/Logger.cs index dfed734c5..721276805 100644 --- a/src/Microsoft.ComponentDetection.Common/Logger.cs +++ b/src/Microsoft.ComponentDetection.Common/Logger.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Composition; using System.Runtime.CompilerServices; using Microsoft.ComponentDetection.Common.Telemetry.Records; @@ -68,15 +68,6 @@ public void LogError(string message) this.LogInternal("ERROR", message, VerbosityMode.Quiet); } - private void LogInternal(string prefix, string message, VerbosityMode verbosity = VerbosityMode.Normal) - { - var formattedPrefix = string.IsNullOrWhiteSpace(prefix) ? string.Empty : $"[{prefix}] "; - var text = $"{formattedPrefix}{message} {NewLine}"; - - this.PrintToConsole(text, verbosity); - this.AppendToFile(text); - } - public void LogFailedReadingFile(string filePath, Exception e) { this.PrintToConsole(NewLine, VerbosityMode.Verbose); @@ -150,5 +141,14 @@ private void PrintToConsole(string text, VerbosityMode minVerbosity) this.ConsoleWriter.Write(text); } } + + private void LogInternal(string prefix, string message, VerbosityMode verbosity = VerbosityMode.Normal) + { + var formattedPrefix = string.IsNullOrWhiteSpace(prefix) ? string.Empty : $"[{prefix}] "; + var text = $"{formattedPrefix}{message} {NewLine}"; + + this.PrintToConsole(text, verbosity); + this.AppendToFile(text); + } } } diff --git a/src/Microsoft.ComponentDetection.Common/PathUtilityService.cs b/src/Microsoft.ComponentDetection.Common/PathUtilityService.cs index 4e40bb320..cda65a524 100644 --- a/src/Microsoft.ComponentDetection.Common/PathUtilityService.cs +++ b/src/Microsoft.ComponentDetection.Common/PathUtilityService.cs @@ -17,19 +17,6 @@ namespace Microsoft.ComponentDetection.Common [Shared] public class PathUtilityService : IPathUtilityService { - [DllImport("kernel32.dll", EntryPoint = "CreateFileW", CharSet = CharSet.Unicode, SetLastError = true)] - private static extern SafeFileHandle CreateFile( - [In] string lpFileName, - [In] uint dwDesiredAccess, - [In] uint dwShareMode, - [In] IntPtr lpSecurityAttributes, - [In] uint dwCreationDisposition, - [In] uint dwFlagsAndAttributes, - [In] IntPtr hTemplateFile); - - [DllImport("kernel32.dll", EntryPoint = "GetFinalPathNameByHandleW", CharSet = CharSet.Unicode, SetLastError = true)] - private static extern int GetFinalPathNameByHandle([In] IntPtr hFile, [Out] StringBuilder lpszFilePath, [In] int cchFilePath, [In] int dwFlags); - /// /// This call can be made on a linux system to get the absolute path of a file. It will resolve nested layers. /// Note: You may pass IntPtr.Zero to the output parameter. You MUST then free the IntPtr that RealPathLinux returns @@ -242,6 +229,19 @@ public string ResolvePhysicalPathLinux(string path) } } + [DllImport("kernel32.dll", EntryPoint = "CreateFileW", CharSet = CharSet.Unicode, SetLastError = true)] + private static extern SafeFileHandle CreateFile( + [In] string lpFileName, + [In] uint dwDesiredAccess, + [In] uint dwShareMode, + [In] IntPtr lpSecurityAttributes, + [In] uint dwCreationDisposition, + [In] uint dwFlagsAndAttributes, + [In] IntPtr hTemplateFile); + + [DllImport("kernel32.dll", EntryPoint = "GetFinalPathNameByHandleW", CharSet = CharSet.Unicode, SetLastError = true)] + private static extern int GetFinalPathNameByHandle([In] IntPtr hFile, [Out] StringBuilder lpszFilePath, [In] int cchFilePath, [In] int dwFlags); + private bool CheckIfRunningOnWindowsContainer() { if (IsLinux) diff --git a/src/Microsoft.ComponentDetection.Common/TabularStringFormat.cs b/src/Microsoft.ComponentDetection.Common/TabularStringFormat.cs index 493f12bc3..c5b12a8c2 100644 --- a/src/Microsoft.ComponentDetection.Common/TabularStringFormat.cs +++ b/src/Microsoft.ComponentDetection.Common/TabularStringFormat.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Text; @@ -7,15 +7,15 @@ namespace Microsoft.ComponentDetection.Common { public class TabularStringFormat { + public const char DefaultVerticalLineChar = '|'; + public const char DefaultHorizontalLineChar = '_'; + private IList columns; private int totalWidth; private char horizontalLineChar; private char verticalLineChar; private string tableTitle; - public const char DefaultVerticalLineChar = '|'; - public const char DefaultHorizontalLineChar = '_'; - public TabularStringFormat(IList columns, char horizontalLineChar = DefaultHorizontalLineChar, char verticalLineChar = DefaultVerticalLineChar, string tableTitle = null) { this.columns = columns; diff --git a/src/Microsoft.ComponentDetection.Common/Telemetry/Records/BaseDetectionTelemetryRecord.cs b/src/Microsoft.ComponentDetection.Common/Telemetry/Records/BaseDetectionTelemetryRecord.cs index f7cb52677..a0f87f4e2 100644 --- a/src/Microsoft.ComponentDetection.Common/Telemetry/Records/BaseDetectionTelemetryRecord.cs +++ b/src/Microsoft.ComponentDetection.Common/Telemetry/Records/BaseDetectionTelemetryRecord.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Diagnostics; using Microsoft.ComponentDetection.Common.Telemetry.Attributes; @@ -27,6 +27,11 @@ public void StopExecutionTimer() } } + public void Dispose() + { + this.Dispose(true); + } + private bool disposedValue = false; protected virtual void Dispose(bool disposing) @@ -42,10 +47,5 @@ protected virtual void Dispose(bool disposing) this.disposedValue = true; } } - - public void Dispose() - { - this.Dispose(true); - } } } diff --git a/src/Microsoft.ComponentDetection.Common/Telemetry/Records/NuGetProjectAssetsTelemetryRecord.cs b/src/Microsoft.ComponentDetection.Common/Telemetry/Records/NuGetProjectAssetsTelemetryRecord.cs index 3e1f7dc64..a516a013c 100644 --- a/src/Microsoft.ComponentDetection.Common/Telemetry/Records/NuGetProjectAssetsTelemetryRecord.cs +++ b/src/Microsoft.ComponentDetection.Common/Telemetry/Records/NuGetProjectAssetsTelemetryRecord.cs @@ -1,4 +1,4 @@ -using System; +using System; namespace Microsoft.ComponentDetection.Common.Telemetry.Records { @@ -14,6 +14,11 @@ public class NuGetProjectAssetsTelemetryRecord : IDetectionTelemetryRecord, IDis private bool disposedValue = false; + public void Dispose() + { + this.Dispose(true); + } + protected virtual void Dispose(bool disposing) { if (!this.disposedValue) @@ -26,10 +31,5 @@ protected virtual void Dispose(bool disposing) this.disposedValue = true; } } - - public void Dispose() - { - this.Dispose(true); - } } } diff --git a/src/Microsoft.ComponentDetection.Contracts/DetectedComponent.cs b/src/Microsoft.ComponentDetection.Contracts/DetectedComponent.cs index a3e8aa0f6..4571498b9 100644 --- a/src/Microsoft.ComponentDetection.Contracts/DetectedComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/DetectedComponent.cs @@ -8,8 +8,6 @@ namespace Microsoft.ComponentDetection.Contracts [DebuggerDisplay("{DebuggerDisplay,nq}")] public class DetectedComponent { - private readonly object hashLock = new object(); - /// Creates a new DetectedComponent. /// The typed component instance to base this detection on. /// The detector that detected this component. @@ -32,8 +30,6 @@ public DetectedComponent(TypedComponent.TypedComponent component, IComponentDete } } - private string DebuggerDisplay => $"{this.Component.DebuggerDisplay}"; - /// /// Gets or sets the detector that detected this component. /// @@ -70,5 +66,9 @@ public void AddComponentFilePath(string filePath) this.FilePaths.Add(filePath); } } + + private string DebuggerDisplay => $"{this.Component.DebuggerDisplay}"; + + private readonly object hashLock = new object(); } } diff --git a/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs b/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs index fb7dffa3b..5e3ce989a 100644 --- a/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Composition; using System.IO; @@ -34,17 +34,6 @@ public abstract class FileComponentDetector : IComponentDetector /// Gets the categories this detector is considered a member of. Used by the DetectorCategories arg to include detectors. public abstract IEnumerable Categories { get; } - /// - /// Gets the folder names that will be skipped by the Component Detector. - /// - protected virtual IList SkippedFolders => new List { }; - - /// - /// Gets or sets the active scan request -- only populated after a ScanDirectoryAsync is invoked. If ScanDirectoryAsync is overridden, - /// the overrider should ensure this property is populated. - /// - protected ScanRequest CurrentScanRequest { get; set; } - /// Gets the supported component types. public abstract IEnumerable SupportedComponentTypes { get; } @@ -54,12 +43,8 @@ public abstract class FileComponentDetector : IComponentDetector [Import] public IObservableDirectoryWalkerFactory Scanner { get; set; } - protected IObservable ComponentStreams { get; private set; } - public bool NeedsAutomaticRootDependencyCalculation { get; protected set; } - protected Dictionary Telemetry { get; set; } = new Dictionary(); - /// public async virtual Task ExecuteDetectorAsync(ScanRequest request) { @@ -78,6 +63,8 @@ private Task ScanDirectoryAsync(ScanRequest reques return this.ProcessAsync(filteredObservable, request.DetectorArgs); } + protected Dictionary Telemetry { get; set; } = new Dictionary(); + /// /// Gets the file streams for the Detector's declared as an . /// @@ -110,6 +97,8 @@ private async Task ProcessAsync(IObservable ComponentStreams { get; private set; } + protected virtual Task> OnPrepareDetection(IObservable processRequests, IDictionary detectorArgs) { return Task.FromResult(processRequests); @@ -121,5 +110,16 @@ protected virtual Task OnDetectionFinished() { return Task.CompletedTask; } + + /// + /// Gets the folder names that will be skipped by the Component Detector. + /// + protected virtual IList SkippedFolders => new List { }; + + /// + /// Gets or sets the active scan request -- only populated after a ScanDirectoryAsync is invoked. If ScanDirectoryAsync is overridden, + /// the overrider should ensure this property is populated. + /// + protected ScanRequest CurrentScanRequest { get; set; } } } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs index ca196e6b6..e69b51e3d 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs @@ -4,11 +4,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent { public class CargoComponent : TypedComponent { - private CargoComponent() - { - // reserved for deserialization - } - public CargoComponent(string name, string version) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Cargo)); @@ -24,5 +19,10 @@ public CargoComponent(string name, string version) public override string Id => $"{this.Name} {this.Version} - {this.Type}"; public override PackageURL PackageUrl => new PackageURL("cargo", string.Empty, this.Name, this.Version, null, string.Empty); + + private CargoComponent() + { + // reserved for deserialization + } } } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CondaComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CondaComponent.cs index 55637e208..5710dc4bd 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CondaComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CondaComponent.cs @@ -1,12 +1,7 @@ -namespace Microsoft.ComponentDetection.Contracts.TypedComponent +namespace Microsoft.ComponentDetection.Contracts.TypedComponent { public class CondaComponent : TypedComponent { - private CondaComponent() - { - /* Reserved for deserialization */ - } - public CondaComponent(string name, string version, string build, string channel, string subdir, string @namespace, string url, string md5) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Conda)); @@ -38,5 +33,10 @@ public CondaComponent(string name, string version, string build, string channel, public override ComponentType Type => ComponentType.Conda; public override string Id => $"{this.Name} {this.Version} {this.Build} {this.Channel} {this.Subdir} {this.Namespace} {this.Url} {this.MD5} - {this.Type}"; + + private CondaComponent() + { + /* Reserved for deserialization */ + } } } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs index c104665ff..5b513d099 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs @@ -2,11 +2,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent { public class DockerImageComponent : TypedComponent { - private DockerImageComponent() - { - /* Reserved for deserialization */ - } - public DockerImageComponent(string hash, string name = null, string tag = null) { this.Digest = this.ValidateRequiredInput(hash, nameof(this.Digest), nameof(ComponentType.DockerImage)); @@ -23,5 +18,10 @@ public DockerImageComponent(string hash, string name = null, string tag = null) public override ComponentType Type => ComponentType.DockerImage; public override string Id => $"{this.Name} {this.Tag} {this.Digest}"; + + private DockerImageComponent() + { + /* Reserved for deserialization */ + } } } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerReferenceComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerReferenceComponent.cs index 5ad27a1ed..d08b95da7 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerReferenceComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerReferenceComponent.cs @@ -2,11 +2,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent { public class DockerReferenceComponent : TypedComponent { - private DockerReferenceComponent() - { - /* Reserved for deserialization */ - } - public DockerReferenceComponent(string hash, string repository = null, string tag = null) { this.Digest = this.ValidateRequiredInput(hash, nameof(this.Digest), nameof(ComponentType.DockerReference)); @@ -37,5 +32,10 @@ public DockerReference FullReference } public override string Id => $"{this.Repository} {this.Tag} {this.Digest}"; + + private DockerReferenceComponent() + { + /* Reserved for deserialization */ + } } } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GitComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GitComponent.cs index fd1fb3bfd..fbe0db638 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GitComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GitComponent.cs @@ -1,14 +1,9 @@ -using System; +using System; namespace Microsoft.ComponentDetection.Contracts.TypedComponent { public class GitComponent : TypedComponent { - private GitComponent() - { - /* Reserved for deserialization */ - } - public GitComponent(Uri repositoryUrl, string commitHash) { this.RepositoryUrl = this.ValidateRequiredInput(repositoryUrl, nameof(this.RepositoryUrl), nameof(ComponentType.Git)); @@ -30,5 +25,10 @@ public GitComponent(Uri repositoryUrl, string commitHash, string tag) public override ComponentType Type => ComponentType.Git; public override string Id => $"{this.RepositoryUrl} : {this.CommitHash} - {this.Type}"; + + private GitComponent() + { + /* Reserved for deserialization */ + } } } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs index ad6cca046..7b682b084 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs @@ -1,15 +1,10 @@ -using System; +using System; using PackageUrl; namespace Microsoft.ComponentDetection.Contracts.TypedComponent { public class GoComponent : TypedComponent, IEquatable { - private GoComponent() - { - /* Reserved for deserialization */ - } - public GoComponent(string name, string version) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Go)); @@ -58,5 +53,10 @@ public override int GetHashCode() // Commit should be used in place of version when available // https://github.com/package-url/purl-spec/blame/180c46d266c45aa2bd81a2038af3f78e87bb4a25/README.rst#L610 public override PackageURL PackageUrl => new PackageURL("golang", null, this.Name, string.IsNullOrWhiteSpace(this.Hash) ? this.Version : this.Hash, null, null); + + private GoComponent() + { + /* Reserved for deserialization */ + } } } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs index 30f18aaf0..412237f18 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs @@ -1,14 +1,9 @@ -using PackageUrl; +using PackageUrl; namespace Microsoft.ComponentDetection.Contracts.TypedComponent { public class LinuxComponent : TypedComponent { - private LinuxComponent() - { - /* Reserved for deserialization */ - } - public LinuxComponent(string distribution, string release, string name, string version) { this.Distribution = this.ValidateRequiredInput(distribution, nameof(this.Distribution), nameof(ComponentType.Linux)); @@ -53,6 +48,11 @@ public override PackageURL PackageUrl } } + private LinuxComponent() + { + /* Reserved for deserialization */ + } + private bool IsUbuntu() { return this.Distribution.ToLowerInvariant() == "ubuntu"; diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/MavenComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/MavenComponent.cs index a644b052f..8beb3c980 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/MavenComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/MavenComponent.cs @@ -1,14 +1,9 @@ -using PackageUrl; +using PackageUrl; namespace Microsoft.ComponentDetection.Contracts.TypedComponent { public class MavenComponent : TypedComponent { - private MavenComponent() - { - /* Reserved for deserialization */ - } - public MavenComponent(string groupId, string artifactId, string version) { this.GroupId = this.ValidateRequiredInput(groupId, nameof(this.GroupId), nameof(ComponentType.Maven)); @@ -27,5 +22,10 @@ public MavenComponent(string groupId, string artifactId, string version) public override string Id => $"{this.GroupId} {this.ArtifactId} {this.Version} - {this.Type}"; public override PackageURL PackageUrl => new PackageURL("maven", this.GroupId, this.ArtifactId, this.Version, null, null); + + private MavenComponent() + { + /* Reserved for deserialization */ + } } } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs index c656caebc..59841b80d 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs @@ -1,15 +1,10 @@ -using Microsoft.ComponentDetection.Contracts.Internal; +using Microsoft.ComponentDetection.Contracts.Internal; using PackageUrl; namespace Microsoft.ComponentDetection.Contracts.TypedComponent { public class NpmComponent : TypedComponent { - private NpmComponent() - { - /* Reserved for deserialization */ - } - public NpmComponent(string name, string version, string hash = null, NpmAuthor author = null) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Npm)); @@ -31,5 +26,10 @@ public NpmComponent(string name, string version, string hash = null, NpmAuthor a public override string Id => $"{this.Name} {this.Version} - {this.Type}"; public override PackageURL PackageUrl => new PackageURL("npm", null, this.Name, this.Version, null, null); + + private NpmComponent() + { + /* Reserved for deserialization */ + } } } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs index 683d95d92..9c12e62d8 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs @@ -1,14 +1,9 @@ -using PackageUrl; +using PackageUrl; namespace Microsoft.ComponentDetection.Contracts.TypedComponent { public class NuGetComponent : TypedComponent { - private NuGetComponent() - { - /* Reserved for deserialization */ - } - public NuGetComponent(string name, string version, string[] authors = null) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.NuGet)); @@ -27,5 +22,10 @@ public NuGetComponent(string name, string version, string[] authors = null) public override string Id => $"{this.Name} {this.Version} - {this.Type}"; public override PackageURL PackageUrl => new PackageURL("nuget", null, this.Name, this.Version, null, null); + + private NuGetComponent() + { + /* Reserved for deserialization */ + } } } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs index 584cd34b7..034dc6c07 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs @@ -4,11 +4,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent { public class OtherComponent : TypedComponent { - private OtherComponent() - { - /* Reserved for deserialization */ - } - public OtherComponent(string name, string version, Uri downloadUrl, string hash) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Other)); @@ -28,5 +23,10 @@ public OtherComponent(string name, string version, Uri downloadUrl, string hash) public override ComponentType Type => ComponentType.Other; public override string Id => $"{this.Name} {this.Version} {this.DownloadUrl} - {this.Type}"; + + private OtherComponent() + { + /* Reserved for deserialization */ + } } } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs index e83365e2e..cc2970180 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs @@ -4,11 +4,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent { public class PipComponent : TypedComponent { - private PipComponent() - { - /* Reserved for deserialization */ - } - public PipComponent(string name, string version) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Pip)); @@ -24,5 +19,10 @@ public PipComponent(string name, string version) public override string Id => $"{this.Name} {this.Version} - {this.Type}".ToLowerInvariant(); public override PackageURL PackageUrl => new PackageURL("pypi", null, this.Name, this.Version, null, null); + + private PipComponent() + { + /* Reserved for deserialization */ + } } } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs index 63c99a860..37b28c60f 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs @@ -1,15 +1,10 @@ -using System.Collections.Generic; +using System.Collections.Generic; using PackageUrl; namespace Microsoft.ComponentDetection.Contracts.TypedComponent { public class PodComponent : TypedComponent { - private PodComponent() - { - /* Reserved for deserialization */ - } - public PodComponent(string name, string version, string specRepo = "") { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Pod)); @@ -40,5 +35,10 @@ public override PackageURL PackageUrl return new PackageURL("cocoapods", null, this.Name, this.Version, qualifiers, null); } } + + private PodComponent() + { + /* Reserved for deserialization */ + } } } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs index 130e4acdb..e347386de 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs @@ -4,11 +4,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent { public class RubyGemsComponent : TypedComponent { - private RubyGemsComponent() - { - /* Reserved for deserialization */ - } - public RubyGemsComponent(string name, string version, string source = "") { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.RubyGems)); @@ -27,5 +22,10 @@ public RubyGemsComponent(string name, string version, string source = "") public override string Id => $"{this.Name} {this.Version} - {this.Type}"; public override PackageURL PackageUrl => new PackageURL("gem", null, this.Name, this.Version, null, null); + + private RubyGemsComponent() + { + /* Reserved for deserialization */ + } } } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs index 83fc86048..0a44736bb 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs @@ -4,11 +4,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent { public class SpdxComponent : TypedComponent { - private SpdxComponent() - { - /* Reserved for deserialization */ - } - public SpdxComponent(string spdxVersion, Uri documentNamespace, string name, string checksum, string rootElementId, string path) { this.SpdxVersion = this.ValidateRequiredInput(spdxVersion, nameof(this.SpdxVersion), nameof(ComponentType.Spdx)); @@ -34,5 +29,10 @@ public SpdxComponent(string spdxVersion, Uri documentNamespace, string name, str public string Path { get; } public override string Id => $"{this.Name}-{this.SpdxVersion}-{this.Checksum}"; + + private SpdxComponent() + { + /* Reserved for deserialization */ + } } } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs index 3765b6632..cec0f9eb1 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs @@ -4,11 +4,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent { public class VcpkgComponent : TypedComponent { - private VcpkgComponent() - { - /* Reserved for deserialization */ - } - public VcpkgComponent(string spdxid, string name, string version, string triplet = null, string portVersion = null, string description = null, string downloadLocation = null) { int.TryParse(portVersion, out var port); @@ -71,5 +66,10 @@ public override PackageURL PackageUrl } } } + + private VcpkgComponent() + { + /* Reserved for deserialization */ + } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/cocoapods/PodComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/cocoapods/PodComponentDetector.cs index 6a318594e..f12651f4a 100644 --- a/src/Microsoft.ComponentDetection.Detectors/cocoapods/PodComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/cocoapods/PodComponentDetector.cs @@ -154,6 +154,25 @@ public string GetSpecRepositoryOfSpec(string specName) } } + protected override async Task OnFileFound(ProcessRequest processRequest, IDictionary detectorArgs) + { + var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; + var file = processRequest.ComponentStream; + + this.Logger.LogVerbose($"Found {file.Pattern}: {file.Location}"); + + try + { + var podfileLock = await ParsePodfileLock(file); + + this.ProcessPodfileLock(singleFileComponentRecorder, podfileLock); + } + catch (Exception e) + { + this.Logger.LogFailedReadingFile(file.Location, e); + } + } + private static async Task ParsePodfileLock(IComponentStream file) { var fileContent = await new StreamReader(file.Stream).ReadToEndAsync(); @@ -210,25 +229,6 @@ private static string NormalizePodfileGitUri(string gitOption) return gitOption; } - protected override async Task OnFileFound(ProcessRequest processRequest, IDictionary detectorArgs) - { - var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; - var file = processRequest.ComponentStream; - - this.Logger.LogVerbose($"Found {file.Pattern}: {file.Location}"); - - try - { - var podfileLock = await ParsePodfileLock(file); - - this.ProcessPodfileLock(singleFileComponentRecorder, podfileLock); - } - catch (Exception e) - { - this.Logger.LogFailedReadingFile(file.Location, e); - } - } - private void ProcessPodfileLock( ISingleFileComponentRecorder singleFileComponentRecorder, PodfileLock podfileLock) diff --git a/src/Microsoft.ComponentDetection.Detectors/ivy/IvyDetector.cs b/src/Microsoft.ComponentDetection.Detectors/ivy/IvyDetector.cs index 9f4fb3a4c..8c35f1253 100644 --- a/src/Microsoft.ComponentDetection.Detectors/ivy/IvyDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/ivy/IvyDetector.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Composition; using System.IO; @@ -58,19 +58,6 @@ public class IvyDetector : FileComponentDetector, IExperimentalDetector [Import] public ICommandLineInvocationService CommandLineInvocationService { get; set; } - private static MavenComponent JsonGavToComponent(JToken gav) - { - if (gav == null) - { - return null; - } - - return new MavenComponent( - gav.Value("g"), - gav.Value("a"), - gav.Value("v")); - } - protected override async Task> OnPrepareDetection(IObservable processRequests, IDictionary detectorArgs) { if (await this.IsAntLocallyAvailableAsync()) @@ -108,6 +95,19 @@ protected override async Task OnFileFound(ProcessRequest processRequest, IDictio } } + private static MavenComponent JsonGavToComponent(JToken gav) + { + if (gav == null) + { + return null; + } + + return new MavenComponent( + gav.Value("g"), + gav.Value("a"), + gav.Value("v")); + } + private async Task ProcessIvyAndIvySettingsFilesAsync( ISingleFileComponentRecorder singleFileComponentRecorder, string ivyXmlFile, diff --git a/src/Microsoft.ComponentDetection.Detectors/linux/LinuxContainerDetector.cs b/src/Microsoft.ComponentDetection.Detectors/linux/LinuxContainerDetector.cs index dd62986a0..86fb7d402 100644 --- a/src/Microsoft.ComponentDetection.Detectors/linux/LinuxContainerDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/linux/LinuxContainerDetector.cs @@ -38,38 +38,6 @@ public class LinuxContainerDetector : IComponentDetector public bool NeedsAutomaticRootDependencyCalculation => false; - /// - /// Extracts and returns the timeout defined by the user, or a default value if one is not provided. - /// - /// The arguments provided by the user. - /// - private static TimeSpan GetTimeout(IDictionary detectorArgs) - { - if (detectorArgs == null || !detectorArgs.TryGetValue("Linux.ScanningTimeoutSec", out var timeout)) - { - return TimeSpan.FromMinutes(10); - } - - return double.TryParse(timeout, out var parsedTimeout) ? TimeSpan.FromSeconds(parsedTimeout) : TimeSpan.FromMinutes(10); - } - - private static IndividualDetectorScanResult EmptySuccessfulScan() - { - return new IndividualDetectorScanResult - { - ResultCode = ProcessingResultCode.Success, - }; - } - - private static ImageScanningResult EmptyImageScanningResult() - { - return new ImageScanningResult - { - ContainerDetails = null, - Components = Enumerable.Empty(), - }; - } - public async Task ExecuteDetectorAsync(ScanRequest request) { var imagesToProcess = request.ImagesToScan?.Where(image => !string.IsNullOrWhiteSpace(image)) @@ -111,6 +79,38 @@ public async Task ExecuteDetectorAsync(ScanRequest }; } + /// + /// Extracts and returns the timeout defined by the user, or a default value if one is not provided. + /// + /// The arguments provided by the user. + /// + private static TimeSpan GetTimeout(IDictionary detectorArgs) + { + if (detectorArgs == null || !detectorArgs.TryGetValue("Linux.ScanningTimeoutSec", out var timeout)) + { + return TimeSpan.FromMinutes(10); + } + + return double.TryParse(timeout, out var parsedTimeout) ? TimeSpan.FromSeconds(parsedTimeout) : TimeSpan.FromMinutes(10); + } + + private static IndividualDetectorScanResult EmptySuccessfulScan() + { + return new IndividualDetectorScanResult + { + ResultCode = ProcessingResultCode.Success, + }; + } + + private static ImageScanningResult EmptyImageScanningResult() + { + return new ImageScanningResult + { + ContainerDetails = null, + Components = Enumerable.Empty(), + }; + } + private async Task> ProcessImagesAsync( IEnumerable imagesToProcess, IComponentRecorder componentRecorder, diff --git a/src/Microsoft.ComponentDetection.Detectors/maven/MavenStyleDependencyGraphParser.cs b/src/Microsoft.ComponentDetection.Detectors/maven/MavenStyleDependencyGraphParser.cs index b0365c358..fa4887004 100644 --- a/src/Microsoft.ComponentDetection.Detectors/maven/MavenStyleDependencyGraphParser.cs +++ b/src/Microsoft.ComponentDetection.Detectors/maven/MavenStyleDependencyGraphParser.cs @@ -19,16 +19,6 @@ public class MavenStyleDependencyGraphParser private DetectedComponent topLevelComponent = null; - private void StartDependencyCategory(string categoryName) - { - if (this.DependencyCategory != null) - { - throw new InvalidOperationException("Current category must be finished before starting new category."); - } - - this.DependencyCategory = new GraphNode(categoryName); - } - public GraphNode Parse(string[] lines) { foreach (var line in lines) @@ -77,6 +67,16 @@ public void Parse(string[] lines, ISingleFileComponentRecorder singleFileCompone } } + private void StartDependencyCategory(string categoryName) + { + if (this.DependencyCategory != null) + { + throw new InvalidOperationException("Current category must be finished before starting new category."); + } + + this.DependencyCategory = new GraphNode(categoryName); + } + private void TrackDependency(int position, string versionedComponent) { while (this.stack.Count > 0 && this.stack.Peek().ParseLevel >= position) diff --git a/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentDetector.cs index 9017e7e4d..33468580d 100644 --- a/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentDetector.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Composition; using System.IO; @@ -56,21 +56,6 @@ await this.SafeProcessAllPackageJTokens(filePath, contents, (token) => }); } - private async Task SafeProcessAllPackageJTokens(string sourceFilePath, string contents, JTokenProcessingDelegate jtokenProcessor) - { - try - { - await this.ProcessAllPackageJTokensAsync(contents, jtokenProcessor); - } - catch (Exception e) - { - // If something went wrong, just ignore the component - this.Logger.LogBuildWarning($"Could not parse Jtokens from file {sourceFilePath}."); - this.Logger.LogFailedReadingFile(sourceFilePath, e); - return; - } - } - protected virtual Task ProcessAllPackageJTokensAsync(string contents, JTokenProcessingDelegate jtokenProcessor) { var o = JToken.Parse(contents); @@ -96,6 +81,21 @@ protected virtual bool ProcessIndividualPackageJTokens(string filePath, ISingleF return true; } + private async Task SafeProcessAllPackageJTokens(string sourceFilePath, string contents, JTokenProcessingDelegate jtokenProcessor) + { + try + { + await this.ProcessAllPackageJTokensAsync(contents, jtokenProcessor); + } + catch (Exception e) + { + // If something went wrong, just ignore the component + this.Logger.LogBuildWarning($"Could not parse Jtokens from file {sourceFilePath}."); + this.Logger.LogFailedReadingFile(sourceFilePath, e); + return; + } + } + private NpmAuthor GetAuthor(JToken authorToken, string packageName, string filePath) { var authorString = authorToken?.ToString(); diff --git a/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentDetectorWithRoots.cs b/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentDetectorWithRoots.cs index 173e13212..9c4606d33 100644 --- a/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentDetectorWithRoots.cs +++ b/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentDetectorWithRoots.cs @@ -109,6 +109,60 @@ await this.SafeProcessAllPackageJTokens(file, (token) => }); } + protected Task ProcessAllPackageJTokensAsync(IComponentStream componentStream, JTokenProcessingDelegate jtokenProcessor) + { + try + { + if (!componentStream.Stream.CanRead) + { + componentStream.Stream.ReadByte(); + } + } + catch (Exception ex) + { + this.Logger.LogBuildWarning($"Could not read {componentStream.Location} file."); + this.Logger.LogFailedReadingFile(componentStream.Location, ex); + return Task.CompletedTask; + } + + using var file = new StreamReader(componentStream.Stream); + using var reader = new JsonTextReader(file); + + var o = JToken.ReadFrom(reader); + jtokenProcessor(o); + return Task.CompletedTask; + } + + protected void ProcessIndividualPackageJTokens(ISingleFileComponentRecorder singleFileComponentRecorder, JToken packageLockJToken, IEnumerable packageJsonComponentStream, bool skipValidation = false) + { + var dependencies = packageLockJToken["dependencies"]; + var topLevelDependencies = new Queue<(JProperty, TypedComponent)>(); + + var dependencyLookup = dependencies.Children().ToDictionary(dependency => dependency.Name); + + foreach (var stream in packageJsonComponentStream) + { + using var file = new StreamReader(stream.Stream); + using var reader = new JsonTextReader(file); + + var packageJsonToken = JToken.ReadFrom(reader); + var enqueued = this.TryEnqueueFirstLevelDependencies(topLevelDependencies, packageJsonToken["dependencies"], dependencyLookup, skipValidation: skipValidation); + enqueued = enqueued && this.TryEnqueueFirstLevelDependencies(topLevelDependencies, packageJsonToken["devDependencies"], dependencyLookup, skipValidation: skipValidation); + if (!enqueued) + { + // This represents a mismatch between lock file and package.json, break out and do not register anything for these files + throw new InvalidOperationException(string.Format("InvalidPackageJson -- There was a mismatch between the components in the package.json and the lock file at '{0}'", singleFileComponentRecorder.ManifestFileLocation)); + } + } + + if (!packageJsonComponentStream.Any()) + { + throw new InvalidOperationException(string.Format("InvalidPackageJson -- There must be a package.json file at '{0}' for components to be registered", singleFileComponentRecorder.ManifestFileLocation)); + } + + this.TraverseRequirementAndDependencyTree(topLevelDependencies, dependencyLookup, singleFileComponentRecorder); + } + private IObservable RemoveNodeModuleNestedFiles(IObservable componentStreams) { var directoryItemFacades = new List(); @@ -198,60 +252,6 @@ private async Task SafeProcessAllPackageJTokens(IComponentStream componentStream } } - protected Task ProcessAllPackageJTokensAsync(IComponentStream componentStream, JTokenProcessingDelegate jtokenProcessor) - { - try - { - if (!componentStream.Stream.CanRead) - { - componentStream.Stream.ReadByte(); - } - } - catch (Exception ex) - { - this.Logger.LogBuildWarning($"Could not read {componentStream.Location} file."); - this.Logger.LogFailedReadingFile(componentStream.Location, ex); - return Task.CompletedTask; - } - - using var file = new StreamReader(componentStream.Stream); - using var reader = new JsonTextReader(file); - - var o = JToken.ReadFrom(reader); - jtokenProcessor(o); - return Task.CompletedTask; - } - - protected void ProcessIndividualPackageJTokens(ISingleFileComponentRecorder singleFileComponentRecorder, JToken packageLockJToken, IEnumerable packageJsonComponentStream, bool skipValidation = false) - { - var dependencies = packageLockJToken["dependencies"]; - var topLevelDependencies = new Queue<(JProperty, TypedComponent)>(); - - var dependencyLookup = dependencies.Children().ToDictionary(dependency => dependency.Name); - - foreach (var stream in packageJsonComponentStream) - { - using var file = new StreamReader(stream.Stream); - using var reader = new JsonTextReader(file); - - var packageJsonToken = JToken.ReadFrom(reader); - var enqueued = this.TryEnqueueFirstLevelDependencies(topLevelDependencies, packageJsonToken["dependencies"], dependencyLookup, skipValidation: skipValidation); - enqueued = enqueued && this.TryEnqueueFirstLevelDependencies(topLevelDependencies, packageJsonToken["devDependencies"], dependencyLookup, skipValidation: skipValidation); - if (!enqueued) - { - // This represents a mismatch between lock file and package.json, break out and do not register anything for these files - throw new InvalidOperationException(string.Format("InvalidPackageJson -- There was a mismatch between the components in the package.json and the lock file at '{0}'", singleFileComponentRecorder.ManifestFileLocation)); - } - } - - if (!packageJsonComponentStream.Any()) - { - throw new InvalidOperationException(string.Format("InvalidPackageJson -- There must be a package.json file at '{0}' for components to be registered", singleFileComponentRecorder.ManifestFileLocation)); - } - - this.TraverseRequirementAndDependencyTree(topLevelDependencies, dependencyLookup, singleFileComponentRecorder); - } - private void TraverseRequirementAndDependencyTree(IEnumerable<(JProperty, TypedComponent)> topLevelDependencies, IDictionary dependencyLookup, ISingleFileComponentRecorder singleFileComponentRecorder) { // iterate through everything for a top level dependency with a single explicitReferencedDependency value diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs b/src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs index 41e12acb5..f91c96db9 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs @@ -34,10 +34,10 @@ public class PyPiClient : IPyPiClient [Import] public IEnvironmentVariableService EnvironmentVariableService { get; set; } - private static HttpClientHandler httpClientHandler = new HttpClientHandler() { CheckCertificateRevocationList = true }; - internal static HttpClient HttpClient = new HttpClient(httpClientHandler); + private static HttpClientHandler httpClientHandler = new HttpClientHandler() { CheckCertificateRevocationList = true }; + // time to wait before retrying a failed call to pypi.org private static readonly TimeSpan RETRYDELAY = TimeSpan.FromSeconds(1); @@ -61,69 +61,6 @@ public class PyPiClient : IPyPiClient // Keep telemetry on how the cache is being used for future refinements private PypiCacheTelemetryRecord cacheTelemetry; - public PyPiClient() - { - this.cacheTelemetry = new PypiCacheTelemetryRecord() - { - NumCacheHits = 0, - FinalCacheSize = 0, - }; - } - - ~PyPiClient() - { - this.cacheTelemetry.FinalCacheSize = this.cachedResponses.Count; - this.cacheTelemetry.Dispose(); - } - - /// - /// Returns a cached response if it exists, otherwise returns the response from PyPi REST call. - /// The response from PyPi is automatically added to the cache. - /// - /// The REST Uri to call. - /// The cached response or a new result from PyPi. - private async Task GetAndCachePyPiResponse(string uri) - { - if (!this.checkedMaxEntriesVariable) - { - this.InitializeNonDefaultMemoryCache(); - } - - if (this.cachedResponses.TryGetValue(uri, out HttpResponseMessage result)) - { - this.cacheTelemetry.NumCacheHits++; - this.Logger.LogVerbose("Retrieved cached Python data from " + uri); - return result; - } - - this.Logger.LogInfo("Getting Python data from " + uri); - var response = await HttpClient.GetAsync(uri); - - // The `first - wins` response accepted into the cache. This might be different from the input if another caller wins the race. - return await this.cachedResponses.GetOrCreateAsync(uri, cacheEntry => - { - cacheEntry.SlidingExpiration = TimeSpan.FromSeconds(CACHEINTERVALSECONDS); // This entry will expire after CACHEINTERVALSECONDS seconds from last use - cacheEntry.Size = 1; // Specify a size of 1 so a set number of entries can always be in the cache - return Task.FromResult(response); - }); - } - - /// - /// On the initial caching attempt, see if the user specified an override for - /// PyPiMaxCacheEntries and recreate the cache if needed. - /// - private void InitializeNonDefaultMemoryCache() - { - var maxEntriesVariable = this.EnvironmentVariableService.GetEnvironmentVariable("PyPiMaxCacheEntries"); - if (!string.IsNullOrEmpty(maxEntriesVariable) && long.TryParse(maxEntriesVariable, out var maxEntries)) - { - this.Logger.LogInfo($"Setting IPyPiClient max cache entries to {maxEntries}"); - this.cachedResponses = new MemoryCache(new MemoryCacheOptions { SizeLimit = maxEntries }); - } - - this.checkedMaxEntriesVariable = true; - } - public async Task> FetchPackageDependencies(string name, string version, PythonProjectRelease release) { var dependencies = new List(); @@ -255,5 +192,68 @@ public async Task>> GetRele return versions; } + + public PyPiClient() + { + this.cacheTelemetry = new PypiCacheTelemetryRecord() + { + NumCacheHits = 0, + FinalCacheSize = 0, + }; + } + + ~PyPiClient() + { + this.cacheTelemetry.FinalCacheSize = this.cachedResponses.Count; + this.cacheTelemetry.Dispose(); + } + + /// + /// Returns a cached response if it exists, otherwise returns the response from PyPi REST call. + /// The response from PyPi is automatically added to the cache. + /// + /// The REST Uri to call. + /// The cached response or a new result from PyPi. + private async Task GetAndCachePyPiResponse(string uri) + { + if (!this.checkedMaxEntriesVariable) + { + this.InitializeNonDefaultMemoryCache(); + } + + if (this.cachedResponses.TryGetValue(uri, out HttpResponseMessage result)) + { + this.cacheTelemetry.NumCacheHits++; + this.Logger.LogVerbose("Retrieved cached Python data from " + uri); + return result; + } + + this.Logger.LogInfo("Getting Python data from " + uri); + var response = await HttpClient.GetAsync(uri); + + // The `first - wins` response accepted into the cache. This might be different from the input if another caller wins the race. + return await this.cachedResponses.GetOrCreateAsync(uri, cacheEntry => + { + cacheEntry.SlidingExpiration = TimeSpan.FromSeconds(CACHEINTERVALSECONDS); // This entry will expire after CACHEINTERVALSECONDS seconds from last use + cacheEntry.Size = 1; // Specify a size of 1 so a set number of entries can always be in the cache + return Task.FromResult(response); + }); + } + + /// + /// On the initial caching attempt, see if the user specified an override for + /// PyPiMaxCacheEntries and recreate the cache if needed. + /// + private void InitializeNonDefaultMemoryCache() + { + var maxEntriesVariable = this.EnvironmentVariableService.GetEnvironmentVariable("PyPiMaxCacheEntries"); + if (!string.IsNullOrEmpty(maxEntriesVariable) && long.TryParse(maxEntriesVariable, out var maxEntries)) + { + this.Logger.LogInfo($"Setting IPyPiClient max cache entries to {maxEntries}"); + this.cachedResponses = new MemoryCache(new MemoryCacheOptions { SizeLimit = maxEntries }); + } + + this.checkedMaxEntriesVariable = true; + } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PipDependencySpecification.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PipDependencySpecification.cs index 84efab5de..63c0c297c 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/PipDependencySpecification.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/PipDependencySpecification.cs @@ -11,8 +11,6 @@ namespace Microsoft.ComponentDetection.Detectors.Pip [DebuggerDisplay("{DebuggerDisplay,nq}")] public class PipDependencySpecification { - private string DebuggerDisplay => $"{this.Name} ({string.Join(';', this.DependencySpecifiers)})"; - /// /// Gets or sets the package (ex: pyyaml). /// @@ -23,6 +21,13 @@ public class PipDependencySpecification /// public IList DependencySpecifiers { get; set; } = new List(); + private string DebuggerDisplay => $"{this.Name} ({string.Join(';', this.DependencySpecifiers)})"; + + // Extracts name and version from a Requires-Dist string that is found in a metadata file + public static readonly Regex RequiresDistRegex = new Regex( + @"Requires-Dist:\s*(?:(.*?)\s*\((.*?)\)|([^\s;]*))", + RegexOptions.Compiled); + /// /// These are packages that we don't want to evaluate in our graph as they are generally python builtins. /// @@ -47,11 +52,6 @@ public class PipDependencySpecification @"((?=<)|(?=>)|(?=>=)|(?=<=)|(?===)|(?=!=)|(?=~=)|(?====))(.*)", RegexOptions.Compiled); - // Extracts name and version from a Requires-Dist string that is found in a metadata file - public static readonly Regex RequiresDistRegex = new Regex( - @"Requires-Dist:\s*(?:(.*?)\s*\((.*?)\)|([^\s;]*))", - RegexOptions.Compiled); - /// /// Whether or not the package is safe to resolve based on the packagesToIgnore. /// diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PythonVersionUtilities.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PythonVersionUtilities.cs index f7daf33c5..b4b9b3655 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/PythonVersionUtilities.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/PythonVersionUtilities.cs @@ -26,55 +26,6 @@ public static bool VersionValidForSpec(string version, IList specs) return true; } - private static bool VersionValidForSpec(string version, string spec) - { - var opChars = new char[] { '=', '<', '>', '~', '!' }; - var specArray = spec.ToCharArray(); - - var i = 0; - while (i < spec.Length && i < 3 && opChars.Contains(specArray[i])) - { - i++; - } - - var op = spec.Substring(0, i); - - var targetVer = new PythonVersion(version); - var specVer = new PythonVersion(spec.Substring(i)); - - if (!targetVer.Valid) - { - throw new ArgumentException($"{version} is not a valid python version"); - } - - if (!specVer.Valid) - { - throw new ArgumentException($"The version specification {spec.Substring(i)} is not a valid python version"); - } - - switch (op) - { - case "==": - return targetVer.CompareTo(specVer) == 0; - case "===": - return targetVer.CompareTo(specVer) == 0; - case "<": - return specVer > targetVer; - case ">": - return targetVer > specVer; - case "<=": - return specVer >= targetVer; - case ">=": - return targetVer >= specVer; - case "!=": - return targetVer.CompareTo(specVer) != 0; - case "~=": - return CheckEquality(version, spec.Substring(i), true); - default: - return false; - } - } - // Todo, remove this code once * parsing is handled in the python version class public static bool CheckEquality(string version, string specVer, bool fuzzy = false) { @@ -136,5 +87,54 @@ public static bool CheckEquality(string version, string specVer, bool fuzzy = fa } } } + + private static bool VersionValidForSpec(string version, string spec) + { + var opChars = new char[] { '=', '<', '>', '~', '!' }; + var specArray = spec.ToCharArray(); + + var i = 0; + while (i < spec.Length && i < 3 && opChars.Contains(specArray[i])) + { + i++; + } + + var op = spec.Substring(0, i); + + var targetVer = new PythonVersion(version); + var specVer = new PythonVersion(spec.Substring(i)); + + if (!targetVer.Valid) + { + throw new ArgumentException($"{version} is not a valid python version"); + } + + if (!specVer.Valid) + { + throw new ArgumentException($"The version specification {spec.Substring(i)} is not a valid python version"); + } + + switch (op) + { + case "==": + return targetVer.CompareTo(specVer) == 0; + case "===": + return targetVer.CompareTo(specVer) == 0; + case "<": + return specVer > targetVer; + case ">": + return targetVer > specVer; + case "<=": + return specVer >= targetVer; + case ">=": + return targetVer >= specVer; + case "!=": + return targetVer.CompareTo(specVer) != 0; + case "~=": + return CheckEquality(version, spec.Substring(i), true); + default: + return false; + } + } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateUtilities.cs b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateUtilities.cs index 38495e58a..6228ddfcf 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateUtilities.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateUtilities.cs @@ -15,9 +15,6 @@ namespace Microsoft.ComponentDetection.Detectors.Rust { public class RustCrateUtilities { - private const string Pattern = @"([^ ]+) ([^ ]+) \(([^()]*)\)"; // PkgName Version Source - private static readonly Regex DependencyFormatRegex = new Regex(Pattern, RegexOptions.Compiled); - public const string CargoTomlSearchPattern = "Cargo.toml"; public const string CargoLockSearchPattern = "Cargo.lock"; @@ -25,12 +22,140 @@ public class RustCrateUtilities public static string[] DevDependencyKeys => new string[] { "dev-dependencies" }; + private const string Pattern = @"([^ ]+) ([^ ]+) \(([^()]*)\)"; // PkgName Version Source + private static readonly Regex DependencyFormatRegex = new Regex(Pattern, RegexOptions.Compiled); + private const string WorkspaceKey = "workspace"; private const string WorkspaceMemberKey = "members"; private const string WorkspaceExcludeKey = "exclude"; + public static DependencySpecification GenerateDependencySpecifications(TomlTable cargoToml, IEnumerable tomlDependencyKeys) + { + var dependencySpecifications = new DependencySpecification(); + var dependencyLocations = GetDependencies(cargoToml, tomlDependencyKeys); + foreach (var dependencies in dependencyLocations) + { + foreach (var dependency in dependencies.Keys) + { + string versionSpecifier; + if (dependencies[dependency].TomlType == TomlObjectType.String) + { + versionSpecifier = dependencies.Get(dependency); + } + else if (dependencies.Get(dependency).ContainsKey("version") && dependencies.Get(dependency).Get("version") != "0.0.0") + { + // We have a valid version that doesn't indicate 'internal' like 0.0.0 does. + versionSpecifier = dependencies.Get(dependency).Get("version"); + } + else if (dependencies.Get(dependency).ContainsKey("path")) + { + // If this is a workspace dependency specification that specifies a component by path reference, skip adding it directly here. + // Example: kubos-app = { path = "../../apis/app-api/rust" } + continue; + } + else + { + return null; + } + + // If the dependency is renamed, use the actual name of the package: + // https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#renaming-dependencies-in-cargotoml + string dependencyName; + if (dependencies[dependency].TomlType == TomlObjectType.Table && + dependencies.Get(dependency).ContainsKey("package")) + { + dependencyName = dependencies.Get(dependency).Get("package"); + } + else + { + dependencyName = dependency; + } + + dependencySpecifications.Add(dependencyName, versionSpecifier); + } + } + + return dependencySpecifications; + } + + public static IEnumerable ConvertCargoLockV2PackagesToV1(CargoLock cargoLock) + { + var packageMap = new Dictionary>(); + cargoLock.package.ToList().ForEach(package => + { + if (!packageMap.TryGetValue(package.name, out var packageList)) + { + packageMap[package.name] = new List() { package }; + } + else + { + packageList.Add(package); + } + }); + + return cargoLock.package.Select(package => + { + if (package.dependencies == null) + { + return package; + } + + try + { + // We're just formatting the v2 dependencies in the v1 way + package.dependencies = package.dependencies + .Select(dep => + { + // parts[0] => name + // parts[1] => version + var parts = dep.Split(' '); + + // Using v1 format, nothing to change + if (string.IsNullOrEmpty(dep) || parts.Length == 3) + { + return dep; + } + + // Below 2 cases use v2 format + else if (parts.Length == 1) + { + // There should only be 1 package in packageMap with this name since we don't specify a version + // We want this to throw if we find more than 1 package because it means there is ambiguity about which package is being used + var mappedPackage = packageMap[parts[0]].Single(); + + return MakeDependencyStringFromPackage(mappedPackage); + } + else if (parts.Length == 2) + { + // Search for the package name + version + // Throws if more than 1 for same reason as above + var mappedPackage = packageMap[parts[0]].Where(subPkg => subPkg.version == parts[1]).Single(); + + return MakeDependencyStringFromPackage(mappedPackage); + } + + throw new FormatException($"Did not expect the dependency string {dep} to have more than 3 parts"); + }).ToArray(); + } + catch + { + using var record = new RustCrateV2DetectorTelemetryRecord(); + + record.PackageInfo = $"{package.name}, {package.version}, {package.source}"; + record.Dependencies = string.Join(',', package.dependencies); + } + + return package; + }); + } + + public static string MakeDependencyStringFromPackage(CargoPackage package) + { + return $"{package.name} {package.version} ({package.source})"; + } + /// /// Given the project root Cargo.toml file, extract any workspaces specified and any root dependencies. /// @@ -91,6 +216,29 @@ public static CargoDependencyData ExtractRootDependencyAndWorkspaceSpecification return cargoDependencyData; } + /// + /// Generate a predicate which will be used to exclude directories which should not contain cargo.toml files. + /// + /// The FileInfo for the cargo.lock file found in the root directory. + /// A list of relative folder paths to include in search. + /// A list of relative folder paths to exclude from search. + /// + public static ExcludeDirectoryPredicate BuildExcludeDirectoryPredicateFromWorkspaces(FileInfo rootLockFileInfo, HashSet definedWorkspaces, HashSet definedExclusions) + { + var workspaceGlobs = BuildGlobMatchingFromWorkspaces(rootLockFileInfo, definedWorkspaces); + + // Since the paths come in as relative, make them fully qualified + var fullyQualifiedExclusions = definedExclusions.Select(x => Path.Combine(rootLockFileInfo.DirectoryName, x)).ToHashSet(); + + // The predicate will be evaluated with the current directory name to search and the full path of its parent. Return true when it should be excluded from search. + return (ReadOnlySpan nameOfDirectoryToConsider, ReadOnlySpan pathOfParentOfDirectoryToConsider) => + { + var currentPath = Path.Combine(pathOfParentOfDirectoryToConsider.ToString(), nameOfDirectoryToConsider.ToString()); + + return !workspaceGlobs.Values.Any(x => x.IsMatch(currentPath)) || fullyQualifiedExclusions.Contains(currentPath); + }; + } + /// /// Given a set of Cargo.toml files, extract development and non-development dependency lists for each. /// @@ -112,6 +260,28 @@ public static void ExtractDependencySpecifications(IEnumerable } } + public static void BuildGraph(HashSet cargoPackages, IList nonDevDependencies, IList devDependencies, ISingleFileComponentRecorder singleFileComponentRecorder) + { + // Get all root components that are not dev dependencies + // This is a bug: + // Say Cargo.toml defined async ^1.0 as a dependency + // Say Cargo.lock has async 1.0.0 and async 1.0.2 + // Both will be marked as root and there's no way to tell which one is "real" + IList nonDevRoots = cargoPackages + .Where(detectedComponent => IsCargoPackageInDependencySpecifications(detectedComponent, nonDevDependencies)) + .ToList(); + + // Get all roots that are dev deps + IList devRoots = cargoPackages + .Where(detectedComponent => IsCargoPackageInDependencySpecifications(detectedComponent, devDependencies)) + .ToList(); + + var packagesDict = cargoPackages.ToDictionary(cargoPackage => new CargoComponent(cargoPackage.name, cargoPackage.version).Id); + + FollowRoots(packagesDict, devRoots, singleFileComponentRecorder, true); + FollowRoots(packagesDict, nonDevRoots, singleFileComponentRecorder, false); + } + /// /// Extract development and non-development dependency lists from a given TomlTable. /// @@ -135,29 +305,6 @@ private static void GenerateDependencies(TomlTable cargoToml, IList - /// Generate a predicate which will be used to exclude directories which should not contain cargo.toml files. - /// - /// The FileInfo for the cargo.lock file found in the root directory. - /// A list of relative folder paths to include in search. - /// A list of relative folder paths to exclude from search. - /// - public static ExcludeDirectoryPredicate BuildExcludeDirectoryPredicateFromWorkspaces(FileInfo rootLockFileInfo, HashSet definedWorkspaces, HashSet definedExclusions) - { - var workspaceGlobs = BuildGlobMatchingFromWorkspaces(rootLockFileInfo, definedWorkspaces); - - // Since the paths come in as relative, make them fully qualified - var fullyQualifiedExclusions = definedExclusions.Select(x => Path.Combine(rootLockFileInfo.DirectoryName, x)).ToHashSet(); - - // The predicate will be evaluated with the current directory name to search and the full path of its parent. Return true when it should be excluded from search. - return (ReadOnlySpan nameOfDirectoryToConsider, ReadOnlySpan pathOfParentOfDirectoryToConsider) => - { - var currentPath = Path.Combine(pathOfParentOfDirectoryToConsider.ToString(), nameOfDirectoryToConsider.ToString()); - - return !workspaceGlobs.Values.Any(x => x.IsMatch(currentPath)) || fullyQualifiedExclusions.Contains(currentPath); - }; - } - /// /// Generates a list of Glob compatible Cargo workspace directories which will be searched. See https://docs.rs/glob/0.3.0/glob/struct.Pattern.html for glob patterns. /// @@ -191,28 +338,6 @@ private static Dictionary BuildGlobMatchingFromWorkspaces(FileInfo return directoryGlobs; } - public static void BuildGraph(HashSet cargoPackages, IList nonDevDependencies, IList devDependencies, ISingleFileComponentRecorder singleFileComponentRecorder) - { - // Get all root components that are not dev dependencies - // This is a bug: - // Say Cargo.toml defined async ^1.0 as a dependency - // Say Cargo.lock has async 1.0.0 and async 1.0.2 - // Both will be marked as root and there's no way to tell which one is "real" - IList nonDevRoots = cargoPackages - .Where(detectedComponent => IsCargoPackageInDependencySpecifications(detectedComponent, nonDevDependencies)) - .ToList(); - - // Get all roots that are dev deps - IList devRoots = cargoPackages - .Where(detectedComponent => IsCargoPackageInDependencySpecifications(detectedComponent, devDependencies)) - .ToList(); - - var packagesDict = cargoPackages.ToDictionary(cargoPackage => new CargoComponent(cargoPackage.name, cargoPackage.version).Id); - - FollowRoots(packagesDict, devRoots, singleFileComponentRecorder, true); - FollowRoots(packagesDict, nonDevRoots, singleFileComponentRecorder, false); - } - private static void FollowRoots(Dictionary packagesDict, IList roots, ISingleFileComponentRecorder singleFileComponentRecorder, bool isDevDependencies) { var componentQueue = new Queue<(string, CargoPackage)>(); @@ -288,131 +413,6 @@ private static DetectedComponent AddOrUpdateDetectedComponent( return recordedComponent; } - public static DependencySpecification GenerateDependencySpecifications(TomlTable cargoToml, IEnumerable tomlDependencyKeys) - { - var dependencySpecifications = new DependencySpecification(); - var dependencyLocations = GetDependencies(cargoToml, tomlDependencyKeys); - foreach (var dependencies in dependencyLocations) - { - foreach (var dependency in dependencies.Keys) - { - string versionSpecifier; - if (dependencies[dependency].TomlType == TomlObjectType.String) - { - versionSpecifier = dependencies.Get(dependency); - } - else if (dependencies.Get(dependency).ContainsKey("version") && dependencies.Get(dependency).Get("version") != "0.0.0") - { - // We have a valid version that doesn't indicate 'internal' like 0.0.0 does. - versionSpecifier = dependencies.Get(dependency).Get("version"); - } - else if (dependencies.Get(dependency).ContainsKey("path")) - { - // If this is a workspace dependency specification that specifies a component by path reference, skip adding it directly here. - // Example: kubos-app = { path = "../../apis/app-api/rust" } - continue; - } - else - { - return null; - } - - // If the dependency is renamed, use the actual name of the package: - // https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#renaming-dependencies-in-cargotoml - string dependencyName; - if (dependencies[dependency].TomlType == TomlObjectType.Table && - dependencies.Get(dependency).ContainsKey("package")) - { - dependencyName = dependencies.Get(dependency).Get("package"); - } - else - { - dependencyName = dependency; - } - - dependencySpecifications.Add(dependencyName, versionSpecifier); - } - } - - return dependencySpecifications; - } - - public static IEnumerable ConvertCargoLockV2PackagesToV1(CargoLock cargoLock) - { - var packageMap = new Dictionary>(); - cargoLock.package.ToList().ForEach(package => - { - if (!packageMap.TryGetValue(package.name, out var packageList)) - { - packageMap[package.name] = new List() { package }; - } - else - { - packageList.Add(package); - } - }); - - return cargoLock.package.Select(package => - { - if (package.dependencies == null) - { - return package; - } - - try - { - // We're just formatting the v2 dependencies in the v1 way - package.dependencies = package.dependencies - .Select(dep => - { - // parts[0] => name - // parts[1] => version - var parts = dep.Split(' '); - - // Using v1 format, nothing to change - if (string.IsNullOrEmpty(dep) || parts.Length == 3) - { - return dep; - } - - // Below 2 cases use v2 format - else if (parts.Length == 1) - { - // There should only be 1 package in packageMap with this name since we don't specify a version - // We want this to throw if we find more than 1 package because it means there is ambiguity about which package is being used - var mappedPackage = packageMap[parts[0]].Single(); - - return MakeDependencyStringFromPackage(mappedPackage); - } - else if (parts.Length == 2) - { - // Search for the package name + version - // Throws if more than 1 for same reason as above - var mappedPackage = packageMap[parts[0]].Where(subPkg => subPkg.version == parts[1]).Single(); - - return MakeDependencyStringFromPackage(mappedPackage); - } - - throw new FormatException($"Did not expect the dependency string {dep} to have more than 3 parts"); - }).ToArray(); - } - catch - { - using var record = new RustCrateV2DetectorTelemetryRecord(); - - record.PackageInfo = $"{package.name}, {package.version}, {package.source}"; - record.Dependencies = string.Join(',', package.dependencies); - } - - return package; - }); - } - - public static string MakeDependencyStringFromPackage(CargoPackage package) - { - return $"{package.name} {package.version} ({package.source})"; - } - private static CargoPackage DependencyStringToCargoPackage(string depString) { var regexMatch = DependencyFormatRegex.Match(depString); diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/SemVer/Comparator.cs b/src/Microsoft.ComponentDetection.Detectors/rust/SemVer/Comparator.cs index 5b6e1e7cc..fca953b99 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/SemVer/Comparator.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/SemVer/Comparator.cs @@ -1,4 +1,4 @@ -// This file was copied from the SemanticVersioning package found at https://github.com/adamreeve/semver.net. +// This file was copied from the SemanticVersioning package found at https://github.com/adamreeve/semver.net. // The range logic from SemanticVersioning is needed in the Rust detector to supplement the Semver versioning package // that is used elsewhere in this project. // @@ -123,26 +123,6 @@ public static Tuple TryParse(string input) : null; } - private static Operator ParseComparatorType(string input) - { - switch (input) - { - case "": - case "=": - return Operator.Equal; - case "<": - return Operator.LessThan; - case "<=": - return Operator.LessThanOrEqual; - case ">": - return Operator.GreaterThan; - case ">=": - return Operator.GreaterThanOrEqual; - default: - throw new ArgumentException(string.Format("Invalid comparator type: {0}", input)); - } - } - public bool IsSatisfied(SemVersion version) { switch (this.ComparatorType) @@ -251,5 +231,25 @@ public override int GetHashCode() { return this.ToString().GetHashCode(); } + + private static Operator ParseComparatorType(string input) + { + switch (input) + { + case "": + case "=": + return Operator.Equal; + case "<": + return Operator.LessThan; + case "<=": + return Operator.LessThanOrEqual; + case ">": + return Operator.GreaterThan; + case ">=": + return Operator.GreaterThanOrEqual; + default: + throw new ArgumentException(string.Format("Invalid comparator type: {0}", input)); + } + } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnBlockFile.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnBlockFile.cs index 2e2dc429d..65b3daeed 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnBlockFile.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnBlockFile.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections; using System.Collections.Generic; using System.IO; @@ -97,6 +97,11 @@ public IEnumerator GetEnumerator() yield break; } + IEnumerator IEnumerable.GetEnumerator() + { + return this.GetEnumerator(); + } + private void ReadVersionHeader() { this.YarnLockVersion = YarnLockVersion.Invalid; @@ -195,11 +200,6 @@ private YarnBlock ParseBlock(int level = 0) return block; } - IEnumerator IEnumerable.GetEnumerator() - { - return this.GetEnumerator(); - } - /// /// Increments the internal pointer so that it is at the next block. /// diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnLockParser.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnLockParser.cs index 49892f8e2..9365f73b8 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnLockParser.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnLockParser.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Composition; using System.Linq; @@ -18,6 +18,11 @@ public class YarnLockParser : IYarnLockParser private const string OptionalDependencies = "optionalDependencies"; + public static string NormalizeVersion(string version) + { + return version.StartsWith("npm:") ? version : $"npm:{version}"; + } + [Import] public ILogger Logger { get; set; } @@ -154,10 +159,5 @@ private bool TryReadNameAndSatisfiedVersion(string nameVersionPairing, out Tuple output = new Tuple(name, parts[1]); return true; } - - public static string NormalizeVersion(string version) - { - return version.StartsWith("npm:") ? version : $"npm:{version}"; - } } } diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/BcdeScanExecutionService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/BcdeScanExecutionService.cs index e4c1f58d8..92548ffd4 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/BcdeScanExecutionService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/BcdeScanExecutionService.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Collections.Immutable; using System.Composition; @@ -27,24 +27,6 @@ public class BcdeScanExecutionService : ServiceBase, IBcdeScanExecutionService [ImportMany] public IEnumerable> GraphTranslationServices { get; set; } - private DetectorRestrictions GetDetectorRestrictions(IDetectionArguments detectionArguments) - { - var detectorRestrictions = new DetectorRestrictions - { - AllowedDetectorIds = detectionArguments.DetectorsFilter, - AllowedDetectorCategories = detectionArguments.DetectorCategories, - }; - - if (detectionArguments.DetectorArgs != null && detectionArguments.DetectorArgs.Any()) - { - var args = ArgumentHelper.GetDetectorArgs(detectionArguments.DetectorArgs); - var allEnabledDetectorIds = args.Where(x => string.Equals("EnableIfDefaultOff", x.Value, StringComparison.OrdinalIgnoreCase) || string.Equals("Enable", x.Value, StringComparison.OrdinalIgnoreCase)); - detectorRestrictions.ExplicitlyEnabledDetectorIds = new HashSet(allEnabledDetectorIds.Select(x => x.Key), StringComparer.OrdinalIgnoreCase); - } - - return detectorRestrictions; - } - public async Task ExecuteScanAsync(IDetectionArguments detectionArguments) { this.Logger.LogCreateLoggingGroup(); @@ -83,5 +65,23 @@ private static Detector ConvertToContract(IComponentDetector detector) SupportedComponentTypes = detector.SupportedComponentTypes, }; } + + private DetectorRestrictions GetDetectorRestrictions(IDetectionArguments detectionArguments) + { + var detectorRestrictions = new DetectorRestrictions + { + AllowedDetectorIds = detectionArguments.DetectorsFilter, + AllowedDetectorCategories = detectionArguments.DetectorCategories, + }; + + if (detectionArguments.DetectorArgs != null && detectionArguments.DetectorArgs.Any()) + { + var args = ArgumentHelper.GetDetectorArgs(detectionArguments.DetectorArgs); + var allEnabledDetectorIds = args.Where(x => string.Equals("EnableIfDefaultOff", x.Value, StringComparison.OrdinalIgnoreCase) || string.Equals("Enable", x.Value, StringComparison.OrdinalIgnoreCase)); + detectorRestrictions.ExplicitlyEnabledDetectorIds = new HashSet(allEnabledDetectorIds.Select(x => x.Key), StringComparer.OrdinalIgnoreCase); + } + + return detectorRestrictions; + } } } diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs index 518ac8237..6a3631ff5 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs @@ -32,25 +32,6 @@ public class DetectorProcessingService : ServiceBase, IDetectorProcessingService [Import] public IObservableDirectoryWalkerFactory Scanner { get; set; } - private static IDictionary GetDetectorArgs(IEnumerable detectorArgsList) - { - var detectorArgs = new Dictionary(); - - foreach (var arg in detectorArgsList) - { - var keyValue = arg.Split('='); - - if (keyValue.Length != 2) - { - continue; - } - - detectorArgs.Add(keyValue[0], keyValue[1]); - } - - return detectorArgs; - } - public async Task ProcessDetectorsAsync(IDetectionArguments detectionArguments, IEnumerable detectors, DetectorRestrictions detectorRestrictions) { this.Logger.LogCreateLoggingGroup(); @@ -147,6 +128,105 @@ public async Task ProcessDetectorsAsync(IDetectionArgu return detectorProcessingResult; } + public ExcludeDirectoryPredicate GenerateDirectoryExclusionPredicate(string originalSourceDirectory, IEnumerable directoryExclusionList, IEnumerable directoryExclusionListObsolete, bool allowWindowsPaths, bool ignoreCase = true) + { + if (directoryExclusionListObsolete?.Any() != true && directoryExclusionList?.Any() != true) + { + return (ReadOnlySpan nameOfDirectoryToConsider, ReadOnlySpan pathOfParentOfDirectoryToConsider) => false; + } + + if (directoryExclusionListObsolete?.Any() == true) + { + var directories = directoryExclusionListObsolete + + // Note: directory info will *automatically* parent relative paths to the working directory of the current assembly. Hold on to your rear. + .Select(relativeOrAbsoluteExclusionPath => new DirectoryInfo(relativeOrAbsoluteExclusionPath)) + .Select(exclusionDirectoryInfo => new + { + nameOfExcludedDirectory = exclusionDirectoryInfo.Name, + pathOfParentOfDirectoryToExclude = exclusionDirectoryInfo.Parent.FullName, + rootedLinuxSymlinkCompatibleRelativePathToExclude = + Path.GetDirectoryName(// Get the parent of + Path.IsPathRooted(exclusionDirectoryInfo.ToString()) + ? exclusionDirectoryInfo.ToString() // If rooted, just use the natural path + : Path.Join(originalSourceDirectory, exclusionDirectoryInfo.ToString())), // If not rooted, join to sourceDir + }) + .Distinct(); + + return (ReadOnlySpan nameOfDirectoryToConsiderSpan, ReadOnlySpan pathOfParentOfDirectoryToConsiderSpan) => + { + var pathOfParentOfDirectoryToConsider = pathOfParentOfDirectoryToConsiderSpan.ToString(); + var nameOfDirectoryToConsider = nameOfDirectoryToConsiderSpan.ToString(); + + foreach (var valueTuple in directories) + { + var nameOfExcludedDirectory = valueTuple.nameOfExcludedDirectory; + var pathOfParentOfDirectoryToExclude = valueTuple.pathOfParentOfDirectoryToExclude; + + if (nameOfDirectoryToConsider.Equals(nameOfExcludedDirectory, StringComparison.Ordinal) + && (pathOfParentOfDirectoryToConsider.Equals(pathOfParentOfDirectoryToExclude, StringComparison.Ordinal) + || pathOfParentOfDirectoryToConsider.ToString().Equals(valueTuple.rootedLinuxSymlinkCompatibleRelativePathToExclude, StringComparison.Ordinal))) + { + this.Logger.LogVerbose($"Excluding folder {Path.Combine(pathOfParentOfDirectoryToConsider.ToString(), nameOfDirectoryToConsider.ToString())}."); + return true; + } + } + + return false; + }; + } + + var minimatchers = new Dictionary(); + + var globOptions = new GlobOptions() + { + Evaluation = new EvaluationOptions() + { + CaseInsensitive = ignoreCase, + }, + }; + + foreach (var directoryExclusion in directoryExclusionList) + { + minimatchers.Add(directoryExclusion, Glob.Parse(allowWindowsPaths ? directoryExclusion : /* [] escapes special chars */ directoryExclusion.Replace("\\", "[\\]"), globOptions)); + } + + return (name, directoryName) => + { + var path = Path.Combine(directoryName.ToString(), name.ToString()); + + return minimatchers.Any(minimatcherKeyValue => + { + if (minimatcherKeyValue.Value.IsMatch(path)) + { + this.Logger.LogVerbose($"Excluding folder {path} because it matched glob {minimatcherKeyValue.Key}."); + return true; + } + + return false; + }); + }; + } + + private static IDictionary GetDetectorArgs(IEnumerable detectorArgsList) + { + var detectorArgs = new Dictionary(); + + foreach (var arg in detectorArgsList) + { + var keyValue = arg.Split('='); + + if (keyValue.Length != 2) + { + continue; + } + + detectorArgs.Add(keyValue[0], keyValue[1]); + } + + return detectorArgs; + } + private IndividualDetectorScanResult CoalesceResult(IndividualDetectorScanResult individualDetectorScanResult) { if (individualDetectorScanResult == null) @@ -242,85 +322,5 @@ private void LogTabularOutput(ILogger logger, ConcurrentDictionary directoryExclusionList, IEnumerable directoryExclusionListObsolete, bool allowWindowsPaths, bool ignoreCase = true) - { - if (directoryExclusionListObsolete?.Any() != true && directoryExclusionList?.Any() != true) - { - return (ReadOnlySpan nameOfDirectoryToConsider, ReadOnlySpan pathOfParentOfDirectoryToConsider) => false; - } - - if (directoryExclusionListObsolete?.Any() == true) - { - var directories = directoryExclusionListObsolete - - // Note: directory info will *automatically* parent relative paths to the working directory of the current assembly. Hold on to your rear. - .Select(relativeOrAbsoluteExclusionPath => new DirectoryInfo(relativeOrAbsoluteExclusionPath)) - .Select(exclusionDirectoryInfo => new - { - nameOfExcludedDirectory = exclusionDirectoryInfo.Name, - pathOfParentOfDirectoryToExclude = exclusionDirectoryInfo.Parent.FullName, - rootedLinuxSymlinkCompatibleRelativePathToExclude = - Path.GetDirectoryName(// Get the parent of - Path.IsPathRooted(exclusionDirectoryInfo.ToString()) - ? exclusionDirectoryInfo.ToString() // If rooted, just use the natural path - : Path.Join(originalSourceDirectory, exclusionDirectoryInfo.ToString())), // If not rooted, join to sourceDir - }) - .Distinct(); - - return (ReadOnlySpan nameOfDirectoryToConsiderSpan, ReadOnlySpan pathOfParentOfDirectoryToConsiderSpan) => - { - var pathOfParentOfDirectoryToConsider = pathOfParentOfDirectoryToConsiderSpan.ToString(); - var nameOfDirectoryToConsider = nameOfDirectoryToConsiderSpan.ToString(); - - foreach (var valueTuple in directories) - { - var nameOfExcludedDirectory = valueTuple.nameOfExcludedDirectory; - var pathOfParentOfDirectoryToExclude = valueTuple.pathOfParentOfDirectoryToExclude; - - if (nameOfDirectoryToConsider.Equals(nameOfExcludedDirectory, StringComparison.Ordinal) - && (pathOfParentOfDirectoryToConsider.Equals(pathOfParentOfDirectoryToExclude, StringComparison.Ordinal) - || pathOfParentOfDirectoryToConsider.ToString().Equals(valueTuple.rootedLinuxSymlinkCompatibleRelativePathToExclude, StringComparison.Ordinal))) - { - this.Logger.LogVerbose($"Excluding folder {Path.Combine(pathOfParentOfDirectoryToConsider.ToString(), nameOfDirectoryToConsider.ToString())}."); - return true; - } - } - - return false; - }; - } - - var minimatchers = new Dictionary(); - - var globOptions = new GlobOptions() - { - Evaluation = new EvaluationOptions() - { - CaseInsensitive = ignoreCase, - }, - }; - - foreach (var directoryExclusion in directoryExclusionList) - { - minimatchers.Add(directoryExclusion, Glob.Parse(allowWindowsPaths ? directoryExclusion : /* [] escapes special chars */ directoryExclusion.Replace("\\", "[\\]"), globOptions)); - } - - return (name, directoryName) => - { - var path = Path.Combine(directoryName.ToString(), name.ToString()); - - return minimatchers.Any(minimatcherKeyValue => - { - if (minimatcherKeyValue.Value.IsMatch(path)) - { - this.Logger.LogVerbose($"Excluding folder {path} because it matched glob {minimatcherKeyValue.Key}."); - return true; - } - - return false; - }); - }; - } } } diff --git a/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs b/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs index 2e2dd71c2..82ac95332 100644 --- a/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs +++ b/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -26,22 +26,6 @@ public void TestCleanup() this.fileWritingServiceMock.VerifyAll(); } - private Logger CreateLogger(VerbosityMode verbosityMode) - { - var serviceUnderTest = new Logger - { - ConsoleWriter = this.consoleWritingServiceMock.Object, - FileWritingService = this.fileWritingServiceMock.Object, - }; - - serviceUnderTest.Init(verbosityMode); - - // We're not explicitly testing init behavior here, so we reset mock expecations. Another test should verify these. - this.consoleWritingServiceMock.Invocations.Clear(); - this.fileWritingServiceMock.Invocations.Clear(); - return serviceUnderTest; - } - [TestMethod] public void LogCreateLoggingGroup_HandlesFailedInit() { @@ -278,5 +262,21 @@ public void LogException_WritesEverythingIfNotErrorAndVerboseLogging() logger.LogException(error, false); } + + private Logger CreateLogger(VerbosityMode verbosityMode) + { + var serviceUnderTest = new Logger + { + ConsoleWriter = this.consoleWritingServiceMock.Object, + FileWritingService = this.fileWritingServiceMock.Object, + }; + + serviceUnderTest.Init(verbosityMode); + + // We're not explicitly testing init behavior here, so we reset mock expecations. Another test should verify these. + this.consoleWritingServiceMock.Invocations.Clear(); + this.fileWritingServiceMock.Invocations.Clear(); + return serviceUnderTest; + } } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/IvyDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/IvyDetectorTests.cs index 339c965ce..033697538 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/IvyDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/IvyDetectorTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -150,6 +150,12 @@ public async Task IvyDependencyGraph() dependencyGraph.IsDevelopmentDependency(d3Id).Should().BeFalse(); } + protected bool ShouldBeEquivalentTo(IEnumerable result, IEnumerable expected) + { + result.Should().BeEquivalentTo(expected); + return true; + } + private void IvyHappyPath(string content) { this.commandLineMock.Setup(x => x.CanCommandBeLocated(IvyDetector.PrimaryCommand, IvyDetector.AdditionalValidCommands, IvyDetector.AntVersionArgument)) @@ -179,11 +185,5 @@ private void IvyHappyPath(string content) ExitCode = 0, }); } - - protected bool ShouldBeEquivalentTo(IEnumerable result, IEnumerable expected) - { - result.Should().BeEquivalentTo(expected); - return true; - } } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/MvnCliDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/MvnCliDetectorTests.cs index a14dceaf7..fdaa9a3a1 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/MvnCliDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/MvnCliDetectorTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -176,6 +176,12 @@ public async Task MavenDependencyGraph() dependencyGraph.IsComponentExplicitlyReferenced(leafComponentId).Should().BeFalse(); } + protected bool ShouldBeEquivalentTo(IEnumerable result, IEnumerable expected) + { + result.Should().BeEquivalentTo(expected); + return true; + } + private void MvnCliHappyPath(string content) { this.commandLineMock.Setup(x => x.CanCommandBeLocated(MavenCommandService.PrimaryCommand, MavenCommandService.AdditionalValidCommands, MavenCommandService.MvnVersionArgument)).ReturnsAsync(true); @@ -197,11 +203,5 @@ private void MvnCliHappyPath(string content) ExitCode = 0, }); } - - protected bool ShouldBeEquivalentTo(IEnumerable result, IEnumerable expected) - { - result.Should().BeEquivalentTo(expected); - return true; - } } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/NuGetPackagesConfigDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/NuGetPackagesConfigDetectorTests.cs index a23719082..b3a655cf3 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/NuGetPackagesConfigDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/NuGetPackagesConfigDetectorTests.cs @@ -1,10 +1,10 @@ -namespace Microsoft.ComponentDetection.Detectors.Tests.NuGet +namespace Microsoft.ComponentDetection.Detectors.Tests.NuGet { using System.Threading.Tasks; + using FluentAssertions; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.NuGet; - using FluentAssertions; using Microsoft.ComponentDetection.TestsUtilities; using Microsoft.VisualStudio.TestTools.UnitTesting; diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs index 5dba71fc2..6662680b8 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -84,77 +84,6 @@ public void TestInit() this.isWin = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); } - private Mock SetupFileDetectorMock(string id) - { - var mockFileDetector = new Mock(); - mockFileDetector.SetupAllProperties(); - mockFileDetector.SetupGet(x => x.Id).Returns(id); - - var sourceDirectory = new DirectoryInfo(Path.Combine(Environment.CurrentDirectory, "Some", "Source", "Directory")); - this.componentDictionary.Should().ContainKey(id, $"MockDetector id:{id}, should be in mock dictionary"); - - var expectedResult = this.ExpectedResultForDetector(id); - - mockFileDetector.Setup(x => x.ExecuteDetectorAsync(It.Is(request => request.SourceDirectory == defaultArgs.SourceDirectory && request.ComponentRecorder != null))).ReturnsAsync( - (ScanRequest request) => - { - return mockFileDetector.Object.ExecuteDetectorAsync(request).Result; - }).Verifiable(); - mockFileDetector.Setup(x => x.ExecuteDetectorAsync(It.Is(request => request.SourceDirectory == defaultArgs.SourceDirectory && request.ComponentRecorder != null))).ReturnsAsync( - (ScanRequest request) => - { - this.serviceUnderTest.Scanner.Initialize(request.SourceDirectory, request.DirectoryExclusionPredicate, 1); - this.FillComponentRecorder(request.ComponentRecorder, id); - return expectedResult; - }).Verifiable(); - - return mockFileDetector; - } - - private IEnumerable GetDiscoveredComponentsFromDetectorProcessingResult(DetectorProcessingResult detectorProcessingResult) - { - return detectorProcessingResult - .ComponentRecorders - .Select(componentRecorder => componentRecorder.Item2.GetDetectedComponents()) - .SelectMany(x => x); - } - - private void FillComponentRecorder(IComponentRecorder componentRecorder, string id) - { - var singleFileRecorder = componentRecorder.CreateSingleFileComponentRecorder("/mock/location"); - singleFileRecorder.RegisterUsage(this.componentDictionary[id], false); - } - - private void ValidateExpectedComponents(DetectorProcessingResult result, IEnumerable detectorsRan) - { - var shouldBePresent = detectorsRan.Where(detector => !(detector is IExperimentalDetector)) - .Select(detector => this.componentDictionary[detector.Id]); - var isPresent = this.GetDiscoveredComponentsFromDetectorProcessingResult(result); - - var check = isPresent.Select(i => i.GetType()); - - isPresent.All(discovered => shouldBePresent.Contains(discovered)); - shouldBePresent.Should().HaveCount(isPresent.Count()); - } - - private Mock SetupCommandDetectorMock(string id) - { - var mockCommandDetector = new Mock(); - mockCommandDetector.SetupAllProperties(); - mockCommandDetector.SetupGet(x => x.Id).Returns(id); - - this.componentDictionary.Should().ContainKey(id, $"MockDetector id:{id}, should be in mock dictionary"); - - mockCommandDetector.Setup(x => x.ExecuteDetectorAsync(It.Is(request => request.SourceDirectory == defaultArgs.SourceDirectory && !request.DetectorArgs.Any()))).ReturnsAsync( - (ScanRequest request) => - { - this.FillComponentRecorder(request.ComponentRecorder, id); - return this.ExpectedResultForDetector(id); - }).Verifiable(); - - return mockCommandDetector; - } - [TestMethod] public void ProcessDetectorsAsync_HappyPathReturnsDetectedComponents() { @@ -576,5 +505,76 @@ public void ProcessDetectorsAsync_HandlesDetectorArgs() .And.NotContainKey("arg2") .And.Contain("arg3", "val3"); } + + private Mock SetupFileDetectorMock(string id) + { + var mockFileDetector = new Mock(); + mockFileDetector.SetupAllProperties(); + mockFileDetector.SetupGet(x => x.Id).Returns(id); + + var sourceDirectory = new DirectoryInfo(Path.Combine(Environment.CurrentDirectory, "Some", "Source", "Directory")); + this.componentDictionary.Should().ContainKey(id, $"MockDetector id:{id}, should be in mock dictionary"); + + var expectedResult = this.ExpectedResultForDetector(id); + + mockFileDetector.Setup(x => x.ExecuteDetectorAsync(It.Is(request => request.SourceDirectory == defaultArgs.SourceDirectory && request.ComponentRecorder != null))).ReturnsAsync( + (ScanRequest request) => + { + return mockFileDetector.Object.ExecuteDetectorAsync(request).Result; + }).Verifiable(); + mockFileDetector.Setup(x => x.ExecuteDetectorAsync(It.Is(request => request.SourceDirectory == defaultArgs.SourceDirectory && request.ComponentRecorder != null))).ReturnsAsync( + (ScanRequest request) => + { + this.serviceUnderTest.Scanner.Initialize(request.SourceDirectory, request.DirectoryExclusionPredicate, 1); + this.FillComponentRecorder(request.ComponentRecorder, id); + return expectedResult; + }).Verifiable(); + + return mockFileDetector; + } + + private IEnumerable GetDiscoveredComponentsFromDetectorProcessingResult(DetectorProcessingResult detectorProcessingResult) + { + return detectorProcessingResult + .ComponentRecorders + .Select(componentRecorder => componentRecorder.Item2.GetDetectedComponents()) + .SelectMany(x => x); + } + + private void FillComponentRecorder(IComponentRecorder componentRecorder, string id) + { + var singleFileRecorder = componentRecorder.CreateSingleFileComponentRecorder("/mock/location"); + singleFileRecorder.RegisterUsage(this.componentDictionary[id], false); + } + + private void ValidateExpectedComponents(DetectorProcessingResult result, IEnumerable detectorsRan) + { + var shouldBePresent = detectorsRan.Where(detector => !(detector is IExperimentalDetector)) + .Select(detector => this.componentDictionary[detector.Id]); + var isPresent = this.GetDiscoveredComponentsFromDetectorProcessingResult(result); + + var check = isPresent.Select(i => i.GetType()); + + isPresent.All(discovered => shouldBePresent.Contains(discovered)); + shouldBePresent.Should().HaveCount(isPresent.Count()); + } + + private Mock SetupCommandDetectorMock(string id) + { + var mockCommandDetector = new Mock(); + mockCommandDetector.SetupAllProperties(); + mockCommandDetector.SetupGet(x => x.Id).Returns(id); + + this.componentDictionary.Should().ContainKey(id, $"MockDetector id:{id}, should be in mock dictionary"); + + mockCommandDetector.Setup(x => x.ExecuteDetectorAsync(It.Is(request => request.SourceDirectory == defaultArgs.SourceDirectory && !request.DetectorArgs.Any()))).ReturnsAsync( + (ScanRequest request) => + { + this.FillComponentRecorder(request.ComponentRecorder, id); + return this.ExpectedResultForDetector(id); + }).Verifiable(); + + return mockCommandDetector; + } } } diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorRestrictionServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorRestrictionServiceTests.cs index ed2e41fee..808a1b942 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorRestrictionServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorRestrictionServiceTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Linq; using FluentAssertions; using Microsoft.ComponentDetection.Common; @@ -24,19 +24,6 @@ public class DetectorRestrictionServiceTests private IComponentDetector[] detectors; private DetectorRestrictionService serviceUnderTest; - private Mock GenerateDetector(string detectorName, string[] categories = null) - { - var mockDetector = new Mock(); - mockDetector.SetupGet(x => x.Id).Returns($"{detectorName}"); - if (categories == null) - { - categories = new[] { $"{detectorName}Category", "AllCategory" }; - } - - mockDetector.SetupGet(x => x.Categories).Returns(categories); - return mockDetector; - } - [TestInitialize] public void TestInitialize() { @@ -188,5 +175,18 @@ public void WithRestrictions_AlwaysIncludesDetectorsThatSpecifyAllCategory() .And.Contain(detectors[1]) .And.Contain(detectors[2]); } + + private Mock GenerateDetector(string detectorName, string[] categories = null) + { + var mockDetector = new Mock(); + mockDetector.SetupGet(x => x.Id).Returns($"{detectorName}"); + if (categories == null) + { + categories = new[] { $"{detectorName}Category", "AllCategory" }; + } + + mockDetector.SetupGet(x => x.Categories).Returns(categories); + return mockDetector; + } } } diff --git a/test/Microsoft.ComponentDetection.TestsUtilities/DetectorTestUtility.cs b/test/Microsoft.ComponentDetection.TestsUtilities/DetectorTestUtility.cs index 8c34fd91e..898b1052b 100644 --- a/test/Microsoft.ComponentDetection.TestsUtilities/DetectorTestUtility.cs +++ b/test/Microsoft.ComponentDetection.TestsUtilities/DetectorTestUtility.cs @@ -28,16 +28,6 @@ public class DetectorTestUtility private List<(string Name, Stream Contents, string Location, IEnumerable searchPatterns)> filesToAdd = new List<(string Name, Stream Contents, string Location, IEnumerable searchPatterns)>(); - private static IComponentStream CreateComponentStreamForFile(string pattern, string filePath, Stream content) - { - var getFileMock = new Mock(); - getFileMock.SetupGet(x => x.Stream).Returns(content); - getFileMock.SetupGet(x => x.Pattern).Returns(pattern); - getFileMock.SetupGet(x => x.Location).Returns(filePath); - - return getFileMock.Object; - } - public async Task<(IndividualDetectorScanResult, IComponentRecorder)> ExecuteDetector() { if (this.scanRequest == null) @@ -117,6 +107,16 @@ public DetectorTestUtility WithObservableDirectoryWalkerFactory(Mock(); + getFileMock.SetupGet(x => x.Stream).Returns(content); + getFileMock.SetupGet(x => x.Pattern).Returns(pattern); + getFileMock.SetupGet(x => x.Location).Returns(filePath); + + return getFileMock.Object; + } + private ProcessRequest CreateProcessRequest(string pattern, string filePath, Stream content) { return new ProcessRequest From 38c88e2bceb6b4cfac48c8d9eb9686f86711b4a8 Mon Sep 17 00:00:00 2001 From: amitla1 <46578839+amitla1@users.noreply.github.com> Date: Thu, 1 Sep 2022 11:25:06 -0700 Subject: [PATCH 02/12] removed whitespac --- .../ComponentDetectionIntegrationTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ComponentDetection.VerificationTests/ComponentDetectionIntegrationTests.cs b/test/Microsoft.ComponentDetection.VerificationTests/ComponentDetectionIntegrationTests.cs index 5bf8146d9..ee746f820 100644 --- a/test/Microsoft.ComponentDetection.VerificationTests/ComponentDetectionIntegrationTests.cs +++ b/test/Microsoft.ComponentDetection.VerificationTests/ComponentDetectionIntegrationTests.cs @@ -12,7 +12,7 @@ namespace Microsoft.ComponentDetection.VerificationTests { [TestClass] - public class ComponentDetectionIntegrationTests + public class ComponentDetectionIntegrationTests { private string oldLogFileContents; private string newLogFileContents; From c64a507bf23cdab4a757df44a01c947a3e891c5a Mon Sep 17 00:00:00 2001 From: amitla1 <46578839+amitla1@users.noreply.github.com> Date: Thu, 1 Sep 2022 11:34:05 -0700 Subject: [PATCH 03/12] reverted this file --- .../DetectorTestUtility.cs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/Microsoft.ComponentDetection.TestsUtilities/DetectorTestUtility.cs b/test/Microsoft.ComponentDetection.TestsUtilities/DetectorTestUtility.cs index 898b1052b..8c34fd91e 100644 --- a/test/Microsoft.ComponentDetection.TestsUtilities/DetectorTestUtility.cs +++ b/test/Microsoft.ComponentDetection.TestsUtilities/DetectorTestUtility.cs @@ -28,6 +28,16 @@ public class DetectorTestUtility private List<(string Name, Stream Contents, string Location, IEnumerable searchPatterns)> filesToAdd = new List<(string Name, Stream Contents, string Location, IEnumerable searchPatterns)>(); + private static IComponentStream CreateComponentStreamForFile(string pattern, string filePath, Stream content) + { + var getFileMock = new Mock(); + getFileMock.SetupGet(x => x.Stream).Returns(content); + getFileMock.SetupGet(x => x.Pattern).Returns(pattern); + getFileMock.SetupGet(x => x.Location).Returns(filePath); + + return getFileMock.Object; + } + public async Task<(IndividualDetectorScanResult, IComponentRecorder)> ExecuteDetector() { if (this.scanRequest == null) @@ -107,16 +117,6 @@ public DetectorTestUtility WithObservableDirectoryWalkerFactory(Mock(); - getFileMock.SetupGet(x => x.Stream).Returns(content); - getFileMock.SetupGet(x => x.Pattern).Returns(pattern); - getFileMock.SetupGet(x => x.Location).Returns(filePath); - - return getFileMock.Object; - } - private ProcessRequest CreateProcessRequest(string pattern, string filePath, Stream content) { return new ProcessRequest From 7780bd1ae0db32590853df27551f8a432d150258 Mon Sep 17 00:00:00 2001 From: amitla1 <46578839+amitla1@users.noreply.github.com> Date: Thu, 1 Sep 2022 11:36:04 -0700 Subject: [PATCH 04/12] reverted this file --- .editorconfig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.editorconfig b/.editorconfig index 631e60a7f..2cc2d88c2 100644 --- a/.editorconfig +++ b/.editorconfig @@ -479,6 +479,9 @@ dotnet_diagnostic.SA1200.severity = suggestion # A field should not follow a property dotnet_diagnostic.SA1201.severity = suggestion +# https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1202.md +# Constant fields should appear before non-constant fields +dotnet_diagnostic.SA1202.severity = suggestion # https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1203.md # 'public' members should come before 'private' members From 6019b2faf15d8b4dd8051850f99c99e9aee72c6f Mon Sep 17 00:00:00 2001 From: amitla1 <46578839+amitla1@users.noreply.github.com> Date: Thu, 1 Sep 2022 13:31:04 -0700 Subject: [PATCH 05/12] Reverted this file --- .../LoggerTests.cs | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs b/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs index 82ac95332..d81e59bd5 100644 --- a/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs +++ b/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs @@ -25,6 +25,22 @@ public void TestCleanup() this.consoleWritingServiceMock.VerifyAll(); this.fileWritingServiceMock.VerifyAll(); } + + private Logger CreateLogger(VerbosityMode verbosityMode) + { + var serviceUnderTest = new Logger + { + ConsoleWriter = this.consoleWritingServiceMock.Object, + FileWritingService = this.fileWritingServiceMock.Object, + }; + + serviceUnderTest.Init(verbosityMode); + + // We're not explicitly testing init behavior here, so we reset mock expecations. Another test should verify these. + this.consoleWritingServiceMock.Invocations.Clear(); + this.fileWritingServiceMock.Invocations.Clear(); + return serviceUnderTest; + } [TestMethod] public void LogCreateLoggingGroup_HandlesFailedInit() @@ -262,21 +278,5 @@ public void LogException_WritesEverythingIfNotErrorAndVerboseLogging() logger.LogException(error, false); } - - private Logger CreateLogger(VerbosityMode verbosityMode) - { - var serviceUnderTest = new Logger - { - ConsoleWriter = this.consoleWritingServiceMock.Object, - FileWritingService = this.fileWritingServiceMock.Object, - }; - - serviceUnderTest.Init(verbosityMode); - - // We're not explicitly testing init behavior here, so we reset mock expecations. Another test should verify these. - this.consoleWritingServiceMock.Invocations.Clear(); - this.fileWritingServiceMock.Invocations.Clear(); - return serviceUnderTest; - } } } From 72cd9758919fb5ba0f5aa68725a36fd57a43c8e4 Mon Sep 17 00:00:00 2001 From: amitla1 <46578839+amitla1@users.noreply.github.com> Date: Thu, 1 Sep 2022 13:46:24 -0700 Subject: [PATCH 06/12] Fixing indentation --- test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs b/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs index d81e59bd5..fa82875c6 100644 --- a/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs +++ b/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs @@ -26,7 +26,7 @@ public void TestCleanup() this.fileWritingServiceMock.VerifyAll(); } - private Logger CreateLogger(VerbosityMode verbosityMode) + private Logger CreateLogger(VerbosityMode verbosityMode) { var serviceUnderTest = new Logger { From 4d7d31e93e07fe33d4238421c17f2052082966cf Mon Sep 17 00:00:00 2001 From: amitla1 <46578839+amitla1@users.noreply.github.com> Date: Thu, 1 Sep 2022 13:52:10 -0700 Subject: [PATCH 07/12] Removed whitespace --- test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs b/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs index fa82875c6..5aa4d4939 100644 --- a/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs +++ b/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs @@ -25,7 +25,7 @@ public void TestCleanup() this.consoleWritingServiceMock.VerifyAll(); this.fileWritingServiceMock.VerifyAll(); } - + private Logger CreateLogger(VerbosityMode verbosityMode) { var serviceUnderTest = new Logger From 61e921ff3c1a1acb9930d18197977a3dd99d8ce4 Mon Sep 17 00:00:00 2001 From: amitla1 <46578839+amitla1@users.noreply.github.com> Date: Thu, 1 Sep 2022 13:59:28 -0700 Subject: [PATCH 08/12] Removed whitespace --- test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs b/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs index 5aa4d4939..7727d2293 100644 --- a/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs +++ b/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs @@ -25,7 +25,7 @@ public void TestCleanup() this.consoleWritingServiceMock.VerifyAll(); this.fileWritingServiceMock.VerifyAll(); } - + private Logger CreateLogger(VerbosityMode verbosityMode) { var serviceUnderTest = new Logger From b8d4f0a6ee5170390cc6cbc4be92d07f453df620 Mon Sep 17 00:00:00 2001 From: amitla1 <46578839+amitla1@users.noreply.github.com> Date: Thu, 1 Sep 2022 15:35:07 -0700 Subject: [PATCH 09/12] changed location of static variable --- src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs b/src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs index f91c96db9..cfd43bef3 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs @@ -33,11 +33,11 @@ public class PyPiClient : IPyPiClient [Import] public IEnvironmentVariableService EnvironmentVariableService { get; set; } + + private static HttpClientHandler httpClientHandler = new HttpClientHandler() { CheckCertificateRevocationList = true }; internal static HttpClient HttpClient = new HttpClient(httpClientHandler); - private static HttpClientHandler httpClientHandler = new HttpClientHandler() { CheckCertificateRevocationList = true }; - // time to wait before retrying a failed call to pypi.org private static readonly TimeSpan RETRYDELAY = TimeSpan.FromSeconds(1); From aaa43e8e81956251124bef3ea85298a7d907d7a4 Mon Sep 17 00:00:00 2001 From: amitla1 <46578839+amitla1@users.noreply.github.com> Date: Thu, 1 Sep 2022 15:45:11 -0700 Subject: [PATCH 10/12] took out the SA1202 suggestion --- .editorconfig | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.editorconfig b/.editorconfig index 2cc2d88c2..069286990 100644 --- a/.editorconfig +++ b/.editorconfig @@ -479,10 +479,6 @@ dotnet_diagnostic.SA1200.severity = suggestion # A field should not follow a property dotnet_diagnostic.SA1201.severity = suggestion -# https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1202.md -# Constant fields should appear before non-constant fields -dotnet_diagnostic.SA1202.severity = suggestion - # https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1203.md # 'public' members should come before 'private' members dotnet_diagnostic.SA1203.severity = suggestion From 5802ee6241f477d4842ab2499b30fb492068deb5 Mon Sep 17 00:00:00 2001 From: amitla1 <46578839+amitla1@users.noreply.github.com> Date: Thu, 1 Sep 2022 15:46:51 -0700 Subject: [PATCH 11/12] changed private method to come after public methods --- .../LoggerTests.cs | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs b/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs index 7727d2293..31381ebcb 100644 --- a/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs +++ b/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs @@ -25,22 +25,6 @@ public void TestCleanup() this.consoleWritingServiceMock.VerifyAll(); this.fileWritingServiceMock.VerifyAll(); } - - private Logger CreateLogger(VerbosityMode verbosityMode) - { - var serviceUnderTest = new Logger - { - ConsoleWriter = this.consoleWritingServiceMock.Object, - FileWritingService = this.fileWritingServiceMock.Object, - }; - - serviceUnderTest.Init(verbosityMode); - - // We're not explicitly testing init behavior here, so we reset mock expecations. Another test should verify these. - this.consoleWritingServiceMock.Invocations.Clear(); - this.fileWritingServiceMock.Invocations.Clear(); - return serviceUnderTest; - } [TestMethod] public void LogCreateLoggingGroup_HandlesFailedInit() @@ -278,5 +262,21 @@ public void LogException_WritesEverythingIfNotErrorAndVerboseLogging() logger.LogException(error, false); } + + private Logger CreateLogger(VerbosityMode verbosityMode) + { + var serviceUnderTest = new Logger + { + ConsoleWriter = this.consoleWritingServiceMock.Object, + FileWritingService = this.fileWritingServiceMock.Object, + }; + + serviceUnderTest.Init(verbosityMode); + + // We're not explicitly testing init behavior here, so we reset mock expecations. Another test should verify these. + this.consoleWritingServiceMock.Invocations.Clear(); + this.fileWritingServiceMock.Invocations.Clear(); + return serviceUnderTest; + } } } From 4af194cb0546b36e885c485172205af15d9b2f91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C2=8DAmitla=20Vannikumar?= Date: Fri, 2 Sep 2022 11:07:04 -0700 Subject: [PATCH 12/12] Added supression message for access level order --- .../pip/IPyPiClient.cs | 4 +++- .../LoggerTests.cs | 4 ++-- .../DetectorTestUtility.cs | 20 +++++++++---------- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs b/src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs index cfd43bef3..ad3774cfa 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Composition; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.IO.Compression; using System.Linq; @@ -33,9 +34,10 @@ public class PyPiClient : IPyPiClient [Import] public IEnvironmentVariableService EnvironmentVariableService { get; set; } - + private static HttpClientHandler httpClientHandler = new HttpClientHandler() { CheckCertificateRevocationList = true }; + [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1202:ElementMustBeAccessLevelOrder", Justification = "Field needs to be declared before use so order can not follow Access Levels.")] internal static HttpClient HttpClient = new HttpClient(httpClientHandler); // time to wait before retrying a failed call to pypi.org diff --git a/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs b/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs index 31381ebcb..82ac95332 100644 --- a/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs +++ b/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs @@ -262,8 +262,8 @@ public void LogException_WritesEverythingIfNotErrorAndVerboseLogging() logger.LogException(error, false); } - - private Logger CreateLogger(VerbosityMode verbosityMode) + + private Logger CreateLogger(VerbosityMode verbosityMode) { var serviceUnderTest = new Logger { diff --git a/test/Microsoft.ComponentDetection.TestsUtilities/DetectorTestUtility.cs b/test/Microsoft.ComponentDetection.TestsUtilities/DetectorTestUtility.cs index 8c34fd91e..898b1052b 100644 --- a/test/Microsoft.ComponentDetection.TestsUtilities/DetectorTestUtility.cs +++ b/test/Microsoft.ComponentDetection.TestsUtilities/DetectorTestUtility.cs @@ -28,16 +28,6 @@ public class DetectorTestUtility private List<(string Name, Stream Contents, string Location, IEnumerable searchPatterns)> filesToAdd = new List<(string Name, Stream Contents, string Location, IEnumerable searchPatterns)>(); - private static IComponentStream CreateComponentStreamForFile(string pattern, string filePath, Stream content) - { - var getFileMock = new Mock(); - getFileMock.SetupGet(x => x.Stream).Returns(content); - getFileMock.SetupGet(x => x.Pattern).Returns(pattern); - getFileMock.SetupGet(x => x.Location).Returns(filePath); - - return getFileMock.Object; - } - public async Task<(IndividualDetectorScanResult, IComponentRecorder)> ExecuteDetector() { if (this.scanRequest == null) @@ -117,6 +107,16 @@ public DetectorTestUtility WithObservableDirectoryWalkerFactory(Mock(); + getFileMock.SetupGet(x => x.Stream).Returns(content); + getFileMock.SetupGet(x => x.Pattern).Returns(pattern); + getFileMock.SetupGet(x => x.Location).Returns(filePath); + + return getFileMock.Object; + } + private ProcessRequest CreateProcessRequest(string pattern, string filePath, Stream content) { return new ProcessRequest