From f42f7ca15bc5fb663aafc96df23a34e04b3d8d56 Mon Sep 17 00:00:00 2001 From: Amitla Vannikumar Date: Fri, 11 Aug 2023 17:19:17 -0700 Subject: [PATCH 01/22] moved location of call to experiments to CD internal --- .../Services/DetectorProcessingService.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs index 2206606f8..572ba9708 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs @@ -122,7 +122,6 @@ public async Task ProcessDetectorsAsync(IDetectionArgu }).ToList(); var results = await Task.WhenAll(scanTasks); - await this.experimentService.FinishAsync(); var detectorProcessingResult = this.ConvertDetectorResultsIntoResult(results, exitCode); From 711bf0abfdc6a935c769806d14f751bd212b8eac Mon Sep 17 00:00:00 2001 From: Amitla Vannikumar Date: Fri, 11 Aug 2023 18:02:44 -0700 Subject: [PATCH 02/22] updated tests --- .../Services/DetectorProcessingServiceTests.cs | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs index 11e8ad74b..398cd0436 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs @@ -510,19 +510,6 @@ public async Task ProcessDetectorsAsync_HandlesDetectorArgsAsync() .And.Contain("arg3", "val3"); } - [TestMethod] - public async Task ProcessDetectorsAsync_FinishesExperimentsAsync() - { - this.detectorsToUse = new[] - { - this.firstFileComponentDetectorMock.Object, this.secondFileComponentDetectorMock.Object, - }; - - await this.serviceUnderTest.ProcessDetectorsAsync(DefaultArgs, this.detectorsToUse, new DetectorRestrictions()); - - this.experimentServiceMock.Verify(x => x.FinishAsync(), Times.Once()); - } - [TestMethod] public async Task ProcessDetectorsAsync_RecordsDetectorRunsAsync() { From dac8ea035349435c9edc931e586ce432b07084b5 Mon Sep 17 00:00:00 2001 From: Amitla Vannikumar Date: Tue, 15 Aug 2023 14:34:27 -0700 Subject: [PATCH 03/22] updated to call finishasync in CD --- .../Experiments/ExperimentService.cs | 7 ++++++- .../Experiments/IExperimentService.cs | 2 +- .../Orchestrator.cs | 2 ++ .../Services/DetectorProcessingService.cs | 7 ++++--- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs index 7a5a3c3a9..435c0f76a 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs @@ -109,13 +109,18 @@ private void FilterExperiments(IComponentDetector detector, int count) } /// - public async Task FinishAsync() + public async Task FinishAsync(bool shouldCheckAutomaticProcessFlag = false) { if (!DetectorExperiments.AreExperimentsEnabled) { return; } + if (!shouldCheckAutomaticProcessFlag && !Orchestrator.AutomaticallyProcessExperiments) + { + return; + } + foreach (var (config, experiment) in this.experiments) { var controlComponents = experiment.ControlGroupComponents; diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentService.cs index 958d45df7..5bdf03576 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentService.cs @@ -22,5 +22,5 @@ public interface IExperimentService /// Called when all detectors have finished executing. Processes the experiments and reports the results. /// /// A representing the asynchronous operation. - Task FinishAsync(); + Task FinishAsync(bool shouldCheckAutomaticProcessFlag = false); } diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Orchestrator.cs b/src/Microsoft.ComponentDetection.Orchestrator/Orchestrator.cs index bf3aeee9c..5776f6382 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Orchestrator.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Orchestrator.cs @@ -29,6 +29,8 @@ namespace Microsoft.ComponentDetection.Orchestrator; public class Orchestrator { + public const bool AutomaticallyProcessExperiments = false; + private static readonly bool IsLinux = RuntimeInformation.IsOSPlatform(OSPlatform.Linux); private readonly IServiceProvider serviceProvider; diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs index 572ba9708..e488fae42 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs @@ -86,9 +86,9 @@ public async Task ProcessDetectorsAsync(IDetectionArgu record.DetectedComponentCount = detectedComponents.Count(); var dependencyGraphs = componentRecorder.GetDependencyGraphsByLocation().Values; record.ExplicitlyReferencedComponentCount = dependencyGraphs.Select(dependencyGraph => - { - return dependencyGraph.GetAllExplicitlyReferencedComponents(); - }) + { + return dependencyGraph.GetAllExplicitlyReferencedComponents(); + }) .SelectMany(x => x) .Distinct() .Count(); @@ -122,6 +122,7 @@ public async Task ProcessDetectorsAsync(IDetectionArgu }).ToList(); var results = await Task.WhenAll(scanTasks); + await this.experimentService.FinishAsync(); var detectorProcessingResult = this.ConvertDetectorResultsIntoResult(results, exitCode); From ff6e14bb9f54c4bc00ce356f4e8676591cd62a06 Mon Sep 17 00:00:00 2001 From: Amitla Vannikumar Date: Tue, 15 Aug 2023 15:30:34 -0700 Subject: [PATCH 04/22] updated --- .../Experiments/DetectorExperiments.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs index 8ea8c0f16..da7acc5c8 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs @@ -7,6 +7,8 @@ namespace Microsoft.ComponentDetection.Orchestrator.Experiments; /// public static class DetectorExperiments { + public const bool AutomaticallyProcessExperiments = false; + /// /// Manually enables detector experiments. /// From 0f87b2afd6a5f77381706bab4fafb2b62d3ad6af Mon Sep 17 00:00:00 2001 From: Amitla Vannikumar Date: Tue, 15 Aug 2023 15:33:07 -0700 Subject: [PATCH 05/22] update documentation --- .../Experiments/IExperimentService.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentService.cs index 5bdf03576..04c9b7edc 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentService.cs @@ -19,7 +19,8 @@ public interface IExperimentService void RecordDetectorRun(IComponentDetector detector, ComponentRecorder componentRecorder, IDetectionArguments detectionArguments); /// - /// Called when all detectors have finished executing. Processes the experiments and reports the results. + /// Determines whether the the automatic processing flag should be checked. + /// The AutomaticallyProcessDetectors flag determins if experiments should be auotmatically processed. /// /// A representing the asynchronous operation. Task FinishAsync(bool shouldCheckAutomaticProcessFlag = false); From 7907d2b42fa00a02d3a257e74352616ba1f42504 Mon Sep 17 00:00:00 2001 From: Amitla Vannikumar Date: Tue, 15 Aug 2023 15:39:17 -0700 Subject: [PATCH 06/22] updated --- .../Experiments/ExperimentService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs index 435c0f76a..8ff6fcf26 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs @@ -116,7 +116,7 @@ public async Task FinishAsync(bool shouldCheckAutomaticProcessFlag = false) return; } - if (!shouldCheckAutomaticProcessFlag && !Orchestrator.AutomaticallyProcessExperiments) + if (!shouldCheckAutomaticProcessFlag && !DetectorExperiments.AutomaticallyProcessExperiments) { return; } From 3c308a5a09fb9ce6d98137195f23958d4865dee8 Mon Sep 17 00:00:00 2001 From: Amitla Vannikumar <46578839+amitla1@users.noreply.github.com> Date: Wed, 16 Aug 2023 11:32:31 -0700 Subject: [PATCH 07/22] Update DetectorProcessingServiceTests.cs --- .../Services/DetectorProcessingServiceTests.cs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs index 398cd0436..11e8ad74b 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs @@ -510,6 +510,19 @@ public async Task ProcessDetectorsAsync_HandlesDetectorArgsAsync() .And.Contain("arg3", "val3"); } + [TestMethod] + public async Task ProcessDetectorsAsync_FinishesExperimentsAsync() + { + this.detectorsToUse = new[] + { + this.firstFileComponentDetectorMock.Object, this.secondFileComponentDetectorMock.Object, + }; + + await this.serviceUnderTest.ProcessDetectorsAsync(DefaultArgs, this.detectorsToUse, new DetectorRestrictions()); + + this.experimentServiceMock.Verify(x => x.FinishAsync(), Times.Once()); + } + [TestMethod] public async Task ProcessDetectorsAsync_RecordsDetectorRunsAsync() { From 36dbd978d0b89342246fef6f636df2b727933ec1 Mon Sep 17 00:00:00 2001 From: Amitla Vannikumar Date: Wed, 16 Aug 2023 12:35:37 -0700 Subject: [PATCH 08/22] syntax --- .../Experiments/IExperimentService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentService.cs index 04c9b7edc..7b37f4f88 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentService.cs @@ -20,7 +20,7 @@ public interface IExperimentService /// /// Determines whether the the automatic processing flag should be checked. - /// The AutomaticallyProcessDetectors flag determins if experiments should be auotmatically processed. + /// The AutomaticallyProcessDetectors flag determins if experiments should be auotmatically processed. /// /// A representing the asynchronous operation. Task FinishAsync(bool shouldCheckAutomaticProcessFlag = false); From 8a78987004c5d65fbcd2e0598e2d1c367c468c52 Mon Sep 17 00:00:00 2001 From: Amitla Vannikumar Date: Wed, 16 Aug 2023 12:45:00 -0700 Subject: [PATCH 09/22] updated tests --- .../Services/DetectorProcessingServiceTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs index 11e8ad74b..8ab9716f1 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs @@ -520,7 +520,7 @@ public async Task ProcessDetectorsAsync_FinishesExperimentsAsync() await this.serviceUnderTest.ProcessDetectorsAsync(DefaultArgs, this.detectorsToUse, new DetectorRestrictions()); - this.experimentServiceMock.Verify(x => x.FinishAsync(), Times.Once()); + this.experimentServiceMock.Verify(x => x.FinishAsync(false), Times.Once()); } [TestMethod] From e250ba2b09590603d3b9a6f7f7bf210c77dd0010 Mon Sep 17 00:00:00 2001 From: Amitla Vannikumar Date: Wed, 16 Aug 2023 13:45:38 -0700 Subject: [PATCH 10/22] updated --- .../Experiments/DetectorExperiments.cs | 3 +++ .../Experiments/ExperimentServiceTests.cs | 4 ++-- .../Services/DetectorProcessingServiceTests.cs | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs index da7acc5c8..b774feaa2 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs @@ -7,6 +7,9 @@ namespace Microsoft.ComponentDetection.Orchestrator.Experiments; /// public static class DetectorExperiments { + /// + /// Check to automatically proccess experiments. + /// public const bool AutomaticallyProcessExperiments = false; /// diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs index a9d2372ac..1b4f63e0a 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs @@ -101,7 +101,7 @@ public async Task RecordDetectorRun_FiltersExperimentsAsync() this.loggerMock.Object); service.RecordDetectorRun(this.detectorMock.Object, this.componentRecorder, this.detectionArgsMock.Object); - await service.FinishAsync(); + await service.FinishAsync(It.Is(x => !x)); filterConfigMock.Verify(x => x.ShouldRecord(this.detectorMock.Object, components.Count), Times.Once()); this.experimentProcessorMock.Verify( @@ -151,7 +151,7 @@ public async Task FinishAsync_ProcessesExperimentsAsync() this.loggerMock.Object); service.RecordDetectorRun(this.detectorMock.Object, this.componentRecorder, this.detectionArgsMock.Object); - await service.FinishAsync(); + await service.FinishAsync(It.Is(x => x)); this.experimentProcessorMock.Verify( x => x.ProcessExperimentAsync(this.experimentConfigMock.Object, It.IsAny()), diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs index 8ab9716f1..2607b9e9c 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs @@ -520,7 +520,7 @@ public async Task ProcessDetectorsAsync_FinishesExperimentsAsync() await this.serviceUnderTest.ProcessDetectorsAsync(DefaultArgs, this.detectorsToUse, new DetectorRestrictions()); - this.experimentServiceMock.Verify(x => x.FinishAsync(false), Times.Once()); + this.experimentServiceMock.Verify(x => x.FinishAsync(It.Is(x => !x)), Times.Once()); } [TestMethod] From 8dc80ed5631591da8cba819ace877751d63d8749 Mon Sep 17 00:00:00 2001 From: Amitla Vannikumar Date: Wed, 16 Aug 2023 14:18:47 -0700 Subject: [PATCH 11/22] updated --- .../Experiments/DetectorExperiments.cs | 2 +- .../Experiments/ExperimentService.cs | 2 +- .../Services/DetectorProcessingService.cs | 1 + .../Experiments/ExperimentServiceTests.cs | 29 +++++++++++++++++-- 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs index b774feaa2..514829d0b 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs @@ -10,7 +10,7 @@ public static class DetectorExperiments /// /// Check to automatically proccess experiments. /// - public const bool AutomaticallyProcessExperiments = false; + public static bool AutomaticallyProcessExperiments { get; set; } /// /// Manually enables detector experiments. diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs index 8ff6fcf26..30049674c 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs @@ -116,7 +116,7 @@ public async Task FinishAsync(bool shouldCheckAutomaticProcessFlag = false) return; } - if (!shouldCheckAutomaticProcessFlag && !DetectorExperiments.AutomaticallyProcessExperiments) + if (!DetectorExperiments.AutomaticallyProcessExperiments) { return; } diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs index e488fae42..530d52af3 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs @@ -122,6 +122,7 @@ public async Task ProcessDetectorsAsync(IDetectionArgu }).ToList(); var results = await Task.WhenAll(scanTasks); + DetectorExperiments.AutomaticallyProcessExperiments = false; await this.experimentService.FinishAsync(); var detectorProcessingResult = this.ConvertDetectorResultsIntoResult(results, exitCode); diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs index 1b4f63e0a..8f7f080d8 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs @@ -101,7 +101,8 @@ public async Task RecordDetectorRun_FiltersExperimentsAsync() this.loggerMock.Object); service.RecordDetectorRun(this.detectorMock.Object, this.componentRecorder, this.detectionArgsMock.Object); - await service.FinishAsync(It.Is(x => !x)); + DetectorExperiments.AutomaticallyProcessExperiments = true; + await service.FinishAsync(); filterConfigMock.Verify(x => x.ShouldRecord(this.detectorMock.Object, components.Count), Times.Once()); this.experimentProcessorMock.Verify( @@ -151,13 +152,37 @@ public async Task FinishAsync_ProcessesExperimentsAsync() this.loggerMock.Object); service.RecordDetectorRun(this.detectorMock.Object, this.componentRecorder, this.detectionArgsMock.Object); - await service.FinishAsync(It.Is(x => x)); + DetectorExperiments.AutomaticallyProcessExperiments = true; + + await service.FinishAsync(); this.experimentProcessorMock.Verify( x => x.ProcessExperimentAsync(this.experimentConfigMock.Object, It.IsAny()), Times.Once()); } + [TestMethod] + public async Task FinishAsync_DoNotAutomaticallyProcess_ProcessesExperimentsAsync() + { + var components = ExperimentTestUtils.CreateRandomComponents(); + this.SetupGraphMock(components); + + var service = new ExperimentService( + new[] { this.experimentConfigMock.Object }, + new[] { this.experimentProcessorMock.Object }, + this.graphTranslationServiceMock.Object, + this.loggerMock.Object); + service.RecordDetectorRun(this.detectorMock.Object, this.componentRecorder, this.detectionArgsMock.Object); + + DetectorExperiments.AutomaticallyProcessExperiments = false; + + await service.FinishAsync(); + + this.experimentProcessorMock.Verify( + x => x.ProcessExperimentAsync(this.experimentConfigMock.Object, It.IsAny()), + Times.Never); + } + [TestMethod] public async Task FinishAsync_SwallowsExceptionsAsync() { From 4c67a6c41ee50accde7a39360b2c3604dd64cb1b Mon Sep 17 00:00:00 2001 From: Amitla Vannikumar Date: Wed, 16 Aug 2023 14:28:12 -0700 Subject: [PATCH 12/22] updated --- src/Microsoft.ComponentDetection.Orchestrator/Orchestrator.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Orchestrator.cs b/src/Microsoft.ComponentDetection.Orchestrator/Orchestrator.cs index 5776f6382..bf3aeee9c 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Orchestrator.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Orchestrator.cs @@ -29,8 +29,6 @@ namespace Microsoft.ComponentDetection.Orchestrator; public class Orchestrator { - public const bool AutomaticallyProcessExperiments = false; - private static readonly bool IsLinux = RuntimeInformation.IsOSPlatform(OSPlatform.Linux); private readonly IServiceProvider serviceProvider; From a3c7981d78b770b1a641dc73a7c7432b803b8d23 Mon Sep 17 00:00:00 2001 From: Amitla Vannikumar Date: Wed, 16 Aug 2023 14:28:54 -0700 Subject: [PATCH 13/22] updated --- .../Experiments/ExperimentService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs index 30049674c..46eef35c2 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs @@ -109,7 +109,7 @@ private void FilterExperiments(IComponentDetector detector, int count) } /// - public async Task FinishAsync(bool shouldCheckAutomaticProcessFlag = false) + public async Task FinishAsync() { if (!DetectorExperiments.AreExperimentsEnabled) { From c8157a92413763e2d211663bd5add7eb3363b94f Mon Sep 17 00:00:00 2001 From: Amitla Vannikumar Date: Wed, 16 Aug 2023 14:31:17 -0700 Subject: [PATCH 14/22] updated --- .../Experiments/IExperimentService.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentService.cs index 7b37f4f88..958d45df7 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentService.cs @@ -19,9 +19,8 @@ public interface IExperimentService void RecordDetectorRun(IComponentDetector detector, ComponentRecorder componentRecorder, IDetectionArguments detectionArguments); /// - /// Determines whether the the automatic processing flag should be checked. - /// The AutomaticallyProcessDetectors flag determins if experiments should be auotmatically processed. + /// Called when all detectors have finished executing. Processes the experiments and reports the results. /// /// A representing the asynchronous operation. - Task FinishAsync(bool shouldCheckAutomaticProcessFlag = false); + Task FinishAsync(); } From 350bc92e12947a30b83abaaf80ed9dd187691604 Mon Sep 17 00:00:00 2001 From: Amitla Vannikumar Date: Wed, 16 Aug 2023 14:34:23 -0700 Subject: [PATCH 15/22] updated --- .../Services/DetectorProcessingServiceTests.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs index 2607b9e9c..e23d0dfb2 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs @@ -520,7 +520,9 @@ public async Task ProcessDetectorsAsync_FinishesExperimentsAsync() await this.serviceUnderTest.ProcessDetectorsAsync(DefaultArgs, this.detectorsToUse, new DetectorRestrictions()); - this.experimentServiceMock.Verify(x => x.FinishAsync(It.Is(x => !x)), Times.Once()); + DetectorExperiments.AutomaticallyProcessExperiments = true; + + this.experimentServiceMock.Verify(x => x.FinishAsync(), Times.Once()); } [TestMethod] From aad5482a9bd73b23670e6e4c29b77fd8e629b104 Mon Sep 17 00:00:00 2001 From: Amitla Vannikumar Date: Wed, 16 Aug 2023 15:27:24 -0700 Subject: [PATCH 16/22] updated --- .../Experiments/DetectorExperiments.cs | 2 +- .../Services/DetectorProcessingService.cs | 1 - .../Experiments/ExperimentServiceTests.cs | 25 ------------------- .../DetectorProcessingServiceTests.cs | 2 -- 4 files changed, 1 insertion(+), 29 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs index 514829d0b..cd8433534 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs @@ -10,7 +10,7 @@ public static class DetectorExperiments /// /// Check to automatically proccess experiments. /// - public static bool AutomaticallyProcessExperiments { get; set; } + public static bool AutomaticallyProcessExperiments => true; /// /// Manually enables detector experiments. diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs index 530d52af3..e488fae42 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs @@ -122,7 +122,6 @@ public async Task ProcessDetectorsAsync(IDetectionArgu }).ToList(); var results = await Task.WhenAll(scanTasks); - DetectorExperiments.AutomaticallyProcessExperiments = false; await this.experimentService.FinishAsync(); var detectorProcessingResult = this.ConvertDetectorResultsIntoResult(results, exitCode); diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs index 8f7f080d8..a9d2372ac 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs @@ -101,7 +101,6 @@ public async Task RecordDetectorRun_FiltersExperimentsAsync() this.loggerMock.Object); service.RecordDetectorRun(this.detectorMock.Object, this.componentRecorder, this.detectionArgsMock.Object); - DetectorExperiments.AutomaticallyProcessExperiments = true; await service.FinishAsync(); filterConfigMock.Verify(x => x.ShouldRecord(this.detectorMock.Object, components.Count), Times.Once()); @@ -152,8 +151,6 @@ public async Task FinishAsync_ProcessesExperimentsAsync() this.loggerMock.Object); service.RecordDetectorRun(this.detectorMock.Object, this.componentRecorder, this.detectionArgsMock.Object); - DetectorExperiments.AutomaticallyProcessExperiments = true; - await service.FinishAsync(); this.experimentProcessorMock.Verify( @@ -161,28 +158,6 @@ public async Task FinishAsync_ProcessesExperimentsAsync() Times.Once()); } - [TestMethod] - public async Task FinishAsync_DoNotAutomaticallyProcess_ProcessesExperimentsAsync() - { - var components = ExperimentTestUtils.CreateRandomComponents(); - this.SetupGraphMock(components); - - var service = new ExperimentService( - new[] { this.experimentConfigMock.Object }, - new[] { this.experimentProcessorMock.Object }, - this.graphTranslationServiceMock.Object, - this.loggerMock.Object); - service.RecordDetectorRun(this.detectorMock.Object, this.componentRecorder, this.detectionArgsMock.Object); - - DetectorExperiments.AutomaticallyProcessExperiments = false; - - await service.FinishAsync(); - - this.experimentProcessorMock.Verify( - x => x.ProcessExperimentAsync(this.experimentConfigMock.Object, It.IsAny()), - Times.Never); - } - [TestMethod] public async Task FinishAsync_SwallowsExceptionsAsync() { diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs index e23d0dfb2..11e8ad74b 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs @@ -520,8 +520,6 @@ public async Task ProcessDetectorsAsync_FinishesExperimentsAsync() await this.serviceUnderTest.ProcessDetectorsAsync(DefaultArgs, this.detectorsToUse, new DetectorRestrictions()); - DetectorExperiments.AutomaticallyProcessExperiments = true; - this.experimentServiceMock.Verify(x => x.FinishAsync(), Times.Once()); } From 2e45389ae44bd994de3f51e5a951482ba5399354 Mon Sep 17 00:00:00 2001 From: Amitla Vannikumar Date: Wed, 16 Aug 2023 15:33:37 -0700 Subject: [PATCH 17/22] updated --- .../Experiments/DetectorExperiments.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs index cd8433534..10561cd28 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs @@ -10,7 +10,7 @@ public static class DetectorExperiments /// /// Check to automatically proccess experiments. /// - public static bool AutomaticallyProcessExperiments => true; + public static bool AutomaticallyProcessExperiments { get; set; } = true; /// /// Manually enables detector experiments. From 310cdab13afa51314835a8c1c23b8c763fdf10c6 Mon Sep 17 00:00:00 2001 From: Amitla Vannikumar Date: Wed, 16 Aug 2023 16:29:06 -0700 Subject: [PATCH 18/22] added test --- .../Experiments/ExperimentServiceTests.cs | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs index a9d2372ac..40ea6e808 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs @@ -138,6 +138,33 @@ public async Task RecordDetectorRun_Respects_DetectorExperiments_EnableAsync() Times.Never()); } + [TestMethod] + public async Task FinishAsync_DoesNotAutomaticallyProcessExperimentsAsync() + { + DetectorExperiments.AutomaticallyProcessExperiments = false; + + var filterConfigMock = new Mock(); + + var components = ExperimentTestUtils.CreateRandomComponents(); + + var service = new ExperimentService( + new[] { this.experimentConfigMock.Object, filterConfigMock.Object }, + new[] { this.experimentProcessorMock.Object }, + this.graphTranslationServiceMock.Object, + this.loggerMock.Object); + + service.RecordDetectorRun(this.detectorMock.Object, this.componentRecorder, this.detectionArgsMock.Object); + await service.FinishAsync(); + + filterConfigMock.Verify(x => x.ShouldRecord(this.detectorMock.Object, components.Count), Times.Never()); + this.experimentProcessorMock.Verify( + x => x.ProcessExperimentAsync(filterConfigMock.Object, It.IsAny()), + Times.Never()); + this.experimentProcessorMock.Verify( + x => x.ProcessExperimentAsync(this.experimentConfigMock.Object, It.IsAny()), + Times.Never()); + } + [TestMethod] public async Task FinishAsync_ProcessesExperimentsAsync() { From 8ad559197bf548ed61cc40c1c56718e2c54675db Mon Sep 17 00:00:00 2001 From: Amitla Vannikumar Date: Wed, 16 Aug 2023 16:30:28 -0700 Subject: [PATCH 19/22] added tests --- .../Experiments/ExperimentServiceTests.cs | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs index 40ea6e808..4d6084f78 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs @@ -138,6 +138,33 @@ public async Task RecordDetectorRun_Respects_DetectorExperiments_EnableAsync() Times.Never()); } + [TestMethod] + public async Task FinishAsync_AutomaticallyProcessesExperimentsAsync() + { + DetectorExperiments.AutomaticallyProcessExperiments = false; + + var filterConfigMock = new Mock(); + + var components = ExperimentTestUtils.CreateRandomComponents(); + + var service = new ExperimentService( + new[] { this.experimentConfigMock.Object, filterConfigMock.Object }, + new[] { this.experimentProcessorMock.Object }, + this.graphTranslationServiceMock.Object, + this.loggerMock.Object); + + service.RecordDetectorRun(this.detectorMock.Object, this.componentRecorder, this.detectionArgsMock.Object); + await service.FinishAsync(); + + filterConfigMock.Verify(x => x.ShouldRecord(this.detectorMock.Object, components.Count), Times.Never()); + this.experimentProcessorMock.Verify( + x => x.ProcessExperimentAsync(filterConfigMock.Object, It.IsAny()), + Times.Once()); + this.experimentProcessorMock.Verify( + x => x.ProcessExperimentAsync(this.experimentConfigMock.Object, It.IsAny()), + Times.Once()); + } + [TestMethod] public async Task FinishAsync_DoesNotAutomaticallyProcessExperimentsAsync() { From 5a2069b159de934d113b9fcf6cdd29f483fb92fc Mon Sep 17 00:00:00 2001 From: Amitla Vannikumar Date: Wed, 16 Aug 2023 16:47:05 -0700 Subject: [PATCH 20/22] updated tests --- .../Experiments/ExperimentServiceTests.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs index 4d6084f78..60a4f7d13 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs @@ -141,12 +141,14 @@ public async Task RecordDetectorRun_Respects_DetectorExperiments_EnableAsync() [TestMethod] public async Task FinishAsync_AutomaticallyProcessesExperimentsAsync() { - DetectorExperiments.AutomaticallyProcessExperiments = false; + DetectorExperiments.AutomaticallyProcessExperiments = true; var filterConfigMock = new Mock(); var components = ExperimentTestUtils.CreateRandomComponents(); + this.SetupGraphMock(components); + var service = new ExperimentService( new[] { this.experimentConfigMock.Object, filterConfigMock.Object }, new[] { this.experimentProcessorMock.Object }, @@ -156,10 +158,10 @@ public async Task FinishAsync_AutomaticallyProcessesExperimentsAsync() service.RecordDetectorRun(this.detectorMock.Object, this.componentRecorder, this.detectionArgsMock.Object); await service.FinishAsync(); - filterConfigMock.Verify(x => x.ShouldRecord(this.detectorMock.Object, components.Count), Times.Never()); + filterConfigMock.Verify(x => x.ShouldRecord(this.detectorMock.Object, components.Count), Times.Once()); this.experimentProcessorMock.Verify( x => x.ProcessExperimentAsync(filterConfigMock.Object, It.IsAny()), - Times.Once()); + Times.Never()); this.experimentProcessorMock.Verify( x => x.ProcessExperimentAsync(this.experimentConfigMock.Object, It.IsAny()), Times.Once()); From 9212630819c08f62116516bc493c658bf19bd293 Mon Sep 17 00:00:00 2001 From: Amitla Vannikumar Date: Wed, 16 Aug 2023 18:29:20 -0700 Subject: [PATCH 21/22] updated tests --- .../Experiments/ExperimentServiceTests.cs | 76 ++++++++----------- 1 file changed, 33 insertions(+), 43 deletions(-) diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs index 60a4f7d13..e399579f9 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs @@ -55,7 +55,11 @@ private void SetupGraphMock(IEnumerable components) } [TestInitialize] - public void EnableDetectorExperiments() => DetectorExperiments.Enable = true; + public void TestInitialize() + { + DetectorExperiments.Enable = true; + DetectorExperiments.AutomaticallyProcessExperiments = true; + } [TestMethod] public void RecordDetectorRun_AddsComponentsToControlAndExperimentGroup() @@ -139,63 +143,64 @@ public async Task RecordDetectorRun_Respects_DetectorExperiments_EnableAsync() } [TestMethod] - public async Task FinishAsync_AutomaticallyProcessesExperimentsAsync() + public async Task FinishAsync_ProcessesExperimentsAsync() { - DetectorExperiments.AutomaticallyProcessExperiments = true; - - var filterConfigMock = new Mock(); - var components = ExperimentTestUtils.CreateRandomComponents(); - this.SetupGraphMock(components); var service = new ExperimentService( - new[] { this.experimentConfigMock.Object, filterConfigMock.Object }, + new[] { this.experimentConfigMock.Object }, new[] { this.experimentProcessorMock.Object }, this.graphTranslationServiceMock.Object, this.loggerMock.Object); - service.RecordDetectorRun(this.detectorMock.Object, this.componentRecorder, this.detectionArgsMock.Object); + await service.FinishAsync(); - filterConfigMock.Verify(x => x.ShouldRecord(this.detectorMock.Object, components.Count), Times.Once()); - this.experimentProcessorMock.Verify( - x => x.ProcessExperimentAsync(filterConfigMock.Object, It.IsAny()), - Times.Never()); this.experimentProcessorMock.Verify( x => x.ProcessExperimentAsync(this.experimentConfigMock.Object, It.IsAny()), Times.Once()); } [TestMethod] - public async Task FinishAsync_DoesNotAutomaticallyProcessExperimentsAsync() + public async Task FinishAsync_SwallowsExceptionsAsync() { - DetectorExperiments.AutomaticallyProcessExperiments = false; - - var filterConfigMock = new Mock(); + this.experimentProcessorMock + .Setup(x => + x.ProcessExperimentAsync(It.IsAny(), It.IsAny())) + .ThrowsAsync(new IOException("test exception")); var components = ExperimentTestUtils.CreateRandomComponents(); + this.SetupGraphMock(components); var service = new ExperimentService( - new[] { this.experimentConfigMock.Object, filterConfigMock.Object }, + new[] { this.experimentConfigMock.Object }, new[] { this.experimentProcessorMock.Object }, this.graphTranslationServiceMock.Object, this.loggerMock.Object); - service.RecordDetectorRun(this.detectorMock.Object, this.componentRecorder, this.detectionArgsMock.Object); + + var act = async () => await service.FinishAsync(); + await act.Should().NotThrowAsync(); + } + + [TestMethod] + public async Task FinishAsync_SkipsEmptyExperimentsAsync() + { + var service = new ExperimentService( + new[] { this.experimentConfigMock.Object }, + new[] { this.experimentProcessorMock.Object }, + this.graphTranslationServiceMock.Object, + this.loggerMock.Object); await service.FinishAsync(); - filterConfigMock.Verify(x => x.ShouldRecord(this.detectorMock.Object, components.Count), Times.Never()); - this.experimentProcessorMock.Verify( - x => x.ProcessExperimentAsync(filterConfigMock.Object, It.IsAny()), - Times.Never()); this.experimentProcessorMock.Verify( - x => x.ProcessExperimentAsync(this.experimentConfigMock.Object, It.IsAny()), + x => x.ProcessExperimentAsync(It.IsAny(), It.IsAny()), Times.Never()); } [TestMethod] - public async Task FinishAsync_ProcessesExperimentsAsync() + public async Task FinishAsync_AutomaticallyProcessesExperimentsAsync() { var components = ExperimentTestUtils.CreateRandomComponents(); this.SetupGraphMock(components); @@ -215,12 +220,9 @@ public async Task FinishAsync_ProcessesExperimentsAsync() } [TestMethod] - public async Task FinishAsync_SwallowsExceptionsAsync() + public async Task FinishAsync_DoesNotAutomaticallyProcessExperimentsAsync() { - this.experimentProcessorMock - .Setup(x => - x.ProcessExperimentAsync(It.IsAny(), It.IsAny())) - .ThrowsAsync(new IOException("test exception")); + DetectorExperiments.AutomaticallyProcessExperiments = false; var components = ExperimentTestUtils.CreateRandomComponents(); this.SetupGraphMock(components); @@ -232,22 +234,10 @@ public async Task FinishAsync_SwallowsExceptionsAsync() this.loggerMock.Object); service.RecordDetectorRun(this.detectorMock.Object, this.componentRecorder, this.detectionArgsMock.Object); - var act = async () => await service.FinishAsync(); - await act.Should().NotThrowAsync(); - } - - [TestMethod] - public async Task FinishAsync_SkipsEmptyExperimentsAsync() - { - var service = new ExperimentService( - new[] { this.experimentConfigMock.Object }, - new[] { this.experimentProcessorMock.Object }, - this.graphTranslationServiceMock.Object, - this.loggerMock.Object); await service.FinishAsync(); this.experimentProcessorMock.Verify( - x => x.ProcessExperimentAsync(It.IsAny(), It.IsAny()), + x => x.ProcessExperimentAsync(this.experimentConfigMock.Object, It.IsAny()), Times.Never()); } From 6f6bb7e3f5755ad29ce58f2eda44371316005b06 Mon Sep 17 00:00:00 2001 From: Amitla Vannikumar <46578839+amitla1@users.noreply.github.com> Date: Wed, 16 Aug 2023 18:34:27 -0700 Subject: [PATCH 22/22] Update DetectorProcessingService.cs --- .../Services/DetectorProcessingService.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs index e488fae42..2206606f8 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs @@ -86,9 +86,9 @@ public async Task ProcessDetectorsAsync(IDetectionArgu record.DetectedComponentCount = detectedComponents.Count(); var dependencyGraphs = componentRecorder.GetDependencyGraphsByLocation().Values; record.ExplicitlyReferencedComponentCount = dependencyGraphs.Select(dependencyGraph => - { - return dependencyGraph.GetAllExplicitlyReferencedComponents(); - }) + { + return dependencyGraph.GetAllExplicitlyReferencedComponents(); + }) .SelectMany(x => x) .Distinct() .Count();