diff --git a/src/Microsoft.ComponentDetection.Common/FileWritingService.cs b/src/Microsoft.ComponentDetection.Common/FileWritingService.cs index d24ee2002..9539bb9e6 100644 --- a/src/Microsoft.ComponentDetection.Common/FileWritingService.cs +++ b/src/Microsoft.ComponentDetection.Common/FileWritingService.cs @@ -48,6 +48,13 @@ public void WriteFile(string relativeFilePath, string text) } } + public async Task WriteFileAsync(string relativeFilePath, string text) + { + relativeFilePath = this.ResolveFilePath(relativeFilePath); + + await File.WriteAllTextAsync(relativeFilePath, text); + } + public void WriteFile(FileInfo relativeFilePath, string text) { File.WriteAllText(relativeFilePath.FullName, text); diff --git a/src/Microsoft.ComponentDetection.Common/IFileWritingService.cs b/src/Microsoft.ComponentDetection.Common/IFileWritingService.cs index e9a9341b4..8ef32b030 100644 --- a/src/Microsoft.ComponentDetection.Common/IFileWritingService.cs +++ b/src/Microsoft.ComponentDetection.Common/IFileWritingService.cs @@ -2,6 +2,7 @@ using System; using System.IO; +using System.Threading.Tasks; // All file paths are relative and will replace occurrences of {timestamp} with the shared file timestamp. public interface IFileWritingService : IDisposable, IAsyncDisposable @@ -12,6 +13,8 @@ public interface IFileWritingService : IDisposable, IAsyncDisposable void WriteFile(string relativeFilePath, string text); + Task WriteFileAsync(string relativeFilePath, string text); + void WriteFile(FileInfo relativeFilePath, string text); string ResolveFilePath(string relativeFilePath); diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/IExperimentConfiguration.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/IExperimentConfiguration.cs new file mode 100644 index 000000000..ff328e440 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/IExperimentConfiguration.cs @@ -0,0 +1,31 @@ +namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Configs; + +using Microsoft.ComponentDetection.Contracts; + +/// +/// Defines the configuration for an experiment. An experiment is a set of detectors that are grouped into control and +/// experiment groups. The control group is used to determine the baseline for the experiment. The experiment group is +/// used to determine the impact of the experiment on the baseline. The unique set of components from the two sets of +/// detectors is compared and differences are reported via telemetry. +/// +public interface IExperimentConfiguration +{ + /// + /// The name of the experiment. + /// + string Name { get; } + + /// + /// Specifies if the detector is in the control group. + /// + /// The detector. + /// true if the detector is in the control group; otherwise, false. + bool IsInControlGroup(IComponentDetector componentDetector); + + /// + /// Specifies if the detector is in the control group. + /// + /// The detector. + /// true if the detector is in the experiment group; otherwise, false. + bool IsInExperimentGroup(IComponentDetector componentDetector); +} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/NewNugetExperiment.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/NewNugetExperiment.cs new file mode 100644 index 000000000..65c57f006 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/NewNugetExperiment.cs @@ -0,0 +1,21 @@ +namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Configs; + +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Detectors.NuGet; + +/// +/// Comparing the new NuGet detector approach to the old one. +/// +public class NewNugetExperiment : IExperimentConfiguration +{ + /// + public string Name => "NewNugetDetector"; + + /// + public bool IsInControlGroup(IComponentDetector componentDetector) => + componentDetector is NuGetComponentDetector or NuGetProjectModelProjectCentricComponentDetector; + + /// + public bool IsInExperimentGroup(IComponentDetector componentDetector) => + componentDetector is NuGetProjectModelProjectCentricComponentDetector or NuGetPackagesConfigDetector; +} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/NpmLockfile3Experiment.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/NpmLockfile3Experiment.cs new file mode 100644 index 000000000..82eaea030 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/NpmLockfile3Experiment.cs @@ -0,0 +1,19 @@ +namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Configs; + +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Detectors.Npm; + +/// +/// Validating the . +/// +public class NpmLockfile3Experiment : IExperimentConfiguration +{ + /// + public string Name => "LockfileVersion3"; + + /// + public bool IsInControlGroup(IComponentDetector componentDetector) => componentDetector is NpmComponentDetector; + + /// + public bool IsInExperimentGroup(IComponentDetector componentDetector) => componentDetector is NpmLockfile3Detector; +} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DefaultExperimentProcessor.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DefaultExperimentProcessor.cs new file mode 100644 index 000000000..dbe47107f --- /dev/null +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DefaultExperimentProcessor.cs @@ -0,0 +1,40 @@ +namespace Microsoft.ComponentDetection.Orchestrator.Experiments; + +using System; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.ComponentDetection.Common; +using Microsoft.ComponentDetection.Orchestrator.Experiments.Configs; +using Microsoft.ComponentDetection.Orchestrator.Experiments.Models; +using Microsoft.Extensions.Logging; + +/// +/// The default experiment processor. Writes a JSON output file to a temporary directory. +/// +public class DefaultExperimentProcessor : IExperimentProcessor +{ + private readonly IFileWritingService fileWritingService; + private readonly ILogger logger; + + /// + /// Initializes a new instance of the class. + /// + /// The file writing service. + /// The logger. + public DefaultExperimentProcessor(IFileWritingService fileWritingService, ILogger logger) + { + this.fileWritingService = fileWritingService; + this.logger = logger; + } + + /// + public async Task ProcessExperimentAsync(IExperimentConfiguration config, ExperimentDiff diff) + { + var filename = $"Experiment_{config.Name}_{{timestamp}}_{Environment.ProcessId}.json"; + + this.logger.LogInformation("Writing experiment {Name} results to {Filename}", config.Name, this.fileWritingService.ResolveFilePath(filename)); + + var serializedDiff = JsonSerializer.Serialize(diff, new JsonSerializerOptions { WriteIndented = true }); + await this.fileWritingService.WriteFileAsync(filename, serializedDiff); + } +} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs new file mode 100644 index 000000000..017780a7d --- /dev/null +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs @@ -0,0 +1,83 @@ +namespace Microsoft.ComponentDetection.Orchestrator.Experiments; + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Orchestrator.Experiments.Configs; +using Microsoft.ComponentDetection.Orchestrator.Experiments.Models; +using Microsoft.Extensions.Logging; + +/// +public class ExperimentService : IExperimentService +{ + private readonly List<(IExperimentConfiguration Config, ExperimentResults ExperimentResults)> experiments; + private readonly IEnumerable experimentProcessors; + private readonly ILogger logger; + + /// + /// Initializes a new instance of the class. + /// + /// The experiment configurations. + /// The experiment processors. + /// The logger. + public ExperimentService( + IEnumerable configs, + IEnumerable experimentProcessors, + ILogger logger) + { + this.experiments = configs.Select(x => (x, new ExperimentResults())).ToList(); + this.experimentProcessors = experimentProcessors; + this.logger = logger; + } + + /// + public void RecordDetectorRun(IComponentDetector detector, IEnumerable components) + { + foreach (var (config, experimentResults) in this.experiments) + { + if (config.IsInControlGroup(detector)) + { + experimentResults.AddComponentsToControlGroup(components); + this.logger.LogDebug( + "Adding {Count} Components from {Id} to Control Group for {Experiment}", + components.Count(), + detector.Id, + config.Name); + } + + if (config.IsInExperimentGroup(detector)) + { + experimentResults.AddComponentsToExperimentalGroup(components); + this.logger.LogDebug( + "Adding {Count} Components from {Id} to Experiment Group for {Experiment}", + components.Count(), + detector.Id, + config.Name); + } + } + } + + /// + public async Task FinishAsync() + { + foreach (var (config, experiment) in this.experiments) + { + var oldComponents = experiment.ControlGroupComponents; + var newComponents = experiment.ExperimentGroupComponents; + + this.logger.LogInformation( + "Experiment {Experiment} finished and has {Count} components in the control group and {Count} components in the experiment group.", + config.Name, + oldComponents.Count, + newComponents.Count); + + var diff = new ExperimentDiff(experiment.ControlGroupComponents, experiment.ExperimentGroupComponents); + + foreach (var processor in this.experimentProcessors) + { + await processor.ProcessExperimentAsync(config, diff); + } + } + } +} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentProcessor.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentProcessor.cs new file mode 100644 index 000000000..0762affc5 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentProcessor.cs @@ -0,0 +1,19 @@ +namespace Microsoft.ComponentDetection.Orchestrator.Experiments; + +using System.Threading.Tasks; +using Microsoft.ComponentDetection.Orchestrator.Experiments.Configs; +using Microsoft.ComponentDetection.Orchestrator.Experiments.Models; + +/// +/// Processes the results of an experiment. Used to report the results of an experiment, such as by writing to a file. +/// +public interface IExperimentProcessor +{ + /// + /// Asynchronously processes the results of an experiment. + /// + /// The experiment configuration. + /// The difference in components between two sets of detectors. + /// A representing the asynchronous operation. + Task ProcessExperimentAsync(IExperimentConfiguration config, ExperimentDiff diff); +} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentService.cs new file mode 100644 index 000000000..abd35eed4 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentService.cs @@ -0,0 +1,24 @@ +namespace Microsoft.ComponentDetection.Orchestrator.Experiments; + +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.ComponentDetection.Contracts; + +/// +/// Service for recording detector results and processing the results for any active experiments. +/// +public interface IExperimentService +{ + /// + /// Records the results of a detector execution and processes the results for any active experiments. + /// + /// The detector. + /// The detected components from the . + void RecordDetectorRun(IComponentDetector detector, IEnumerable components); + + /// + /// Called when all detectors have finished executing. Processes the experiments and reports the results. + /// + /// A representing the asynchronous operation. + Task FinishAsync(); +} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Models/ExperimentComponent.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Models/ExperimentComponent.cs new file mode 100644 index 000000000..850beeaea --- /dev/null +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Models/ExperimentComponent.cs @@ -0,0 +1,37 @@ +namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Models; + +using System.Collections.Generic; +using System.Linq; +using Microsoft.ComponentDetection.Contracts; + +/// +/// A model representing a component detected by a detector, as relevant to an experiment. +/// +public record ExperimentComponent +{ + /// + /// Creates a new from a . + /// + /// The detected component. + public ExperimentComponent(DetectedComponent detectedComponent) + { + this.Id = detectedComponent.Component.Id; + this.DevelopmentDependency = detectedComponent.DevelopmentDependency ?? false; + this.RootIds = detectedComponent.DependencyRoots?.Select(x => x.Id).ToHashSet() ?? new HashSet(); + } + + /// + /// The component ID. + /// + public string Id { get; } + + /// + /// true if the component is a development dependency; otherwise, false. + /// + public bool DevelopmentDependency { get; } + + /// + /// The set of root component IDs for this component. + /// + public HashSet RootIds { get; } +} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Models/ExperimentDiff.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Models/ExperimentDiff.cs new file mode 100644 index 000000000..496426d51 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Models/ExperimentDiff.cs @@ -0,0 +1,119 @@ +namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Models; + +using System.Collections.Generic; +using System.Linq; + +/// +/// A model for the difference between two sets of instances. +/// +public class ExperimentDiff +{ + /// + /// Creates a new . + /// + /// A set of components from the control group. + /// A set of components from the experimental group. + public ExperimentDiff( + IEnumerable controlGroupComponents, + IEnumerable experimentGroupComponents) + { + var oldComponentDictionary = controlGroupComponents.ToDictionary(x => x.Id); + var newComponentDictionary = experimentGroupComponents.ToDictionary(x => x.Id); + + this.AddedIds = newComponentDictionary.Keys.Except(oldComponentDictionary.Keys).ToList(); + this.RemovedIds = oldComponentDictionary.Keys.Except(newComponentDictionary.Keys).ToList(); + + this.DevelopmentDependencyChanges = new List(); + this.AddedRootIds = new Dictionary>(); + this.RemovedRootIds = new Dictionary>(); + + // Need performance benchmark to see if this is worth parallelization + foreach (var id in newComponentDictionary.Keys.Intersect(oldComponentDictionary.Keys)) + { + var oldComponent = oldComponentDictionary[id]; + var newComponent = newComponentDictionary[id]; + + if (oldComponent.DevelopmentDependency != newComponent.DevelopmentDependency) + { + this.DevelopmentDependencyChanges.Add(new DevelopmentDependencyChange( + id, + oldComponent.DevelopmentDependency, + newComponent.DevelopmentDependency)); + } + + var addedRootIds = newComponent.RootIds.Except(oldComponent.RootIds).ToHashSet(); + var removedRootIds = oldComponent.RootIds.Except(newComponent.RootIds).ToHashSet(); + + if (addedRootIds.Count > 0) + { + this.AddedRootIds[id] = addedRootIds; + } + + if (removedRootIds.Count > 0) + { + this.RemovedRootIds[id] = removedRootIds; + } + } + } + + /// + /// Gets a list of component IDs that were present in the experimental group but not the control group. + /// + public List AddedIds { get; } + + /// + /// Gets a list of component IDs that were present in the control group but not the experimental group. + /// + public List RemovedIds { get; } + + /// + /// Gets a list of changes to the development dependency status of components. + /// + public List DevelopmentDependencyChanges { get; } + + /// + /// Gets a dictionary of component IDs to the set of root IDs that were added to the component. The component ID + /// is the key. + /// + public Dictionary> AddedRootIds { get; } + + /// + /// Gets a dictionary of component IDs to the set of root IDs that were removed from the component. The component + /// ID is the key. + /// + public Dictionary> RemovedRootIds { get; } + + /// + /// Stores information about a change to the development dependency status of a component. + /// + public class DevelopmentDependencyChange + { + /// + /// Creates a new . + /// + /// The component ID. + /// The old value of the development dependency status. + /// The new value of the development dependency status. + public DevelopmentDependencyChange(string id, bool oldValue, bool newValue) + { + this.Id = id; + this.OldValue = oldValue; + this.NewValue = newValue; + } + + /// + /// Gets the component ID. + /// + public string Id { get; } + + /// + /// Gets the old value of the development dependency status. + /// + public bool OldValue { get; } + + /// + /// Gets the new value of the development dependency status. + /// + public bool NewValue { get; } + } +} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Models/ExperimentResults.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Models/ExperimentResults.cs new file mode 100644 index 000000000..cf5927e31 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Models/ExperimentResults.cs @@ -0,0 +1,51 @@ +namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Models; + +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using Microsoft.ComponentDetection.Contracts; + +/// +/// Stores the results of a detector execution for an experiment. Buckets components into a control group and an +/// experimental group. +/// +public class ExperimentResults +{ + private readonly HashSet controlGroupComponents = new(); + + private readonly HashSet experimentGroupComponents = new(); + + /// + /// The set of components in the control group. + /// + public IImmutableSet ControlGroupComponents => + this.controlGroupComponents.ToImmutableHashSet(); + + /// + /// The set of components in the experimental group. + /// + public IImmutableSet ExperimentGroupComponents => + this.experimentGroupComponents.ToImmutableHashSet(); + + /// + /// Adds the components to the control group. + /// + /// The components. + public void AddComponentsToControlGroup(IEnumerable components) => + AddComponents(this.controlGroupComponents, components); + + /// + /// Adds the components to the experimental group. + /// + /// The components. + public void AddComponentsToExperimentalGroup(IEnumerable components) => + AddComponents(this.experimentGroupComponents, components); + + private static void AddComponents(ISet group, IEnumerable components) + { + foreach (var experimentComponent in components.Select(x => new ExperimentComponent(x))) + { + group.Add(experimentComponent); + } + } +} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs index 9c9ee170d..9ee129ee3 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs @@ -22,6 +22,8 @@ using Microsoft.ComponentDetection.Detectors.Yarn; using Microsoft.ComponentDetection.Detectors.Yarn.Parsers; using Microsoft.ComponentDetection.Orchestrator.ArgumentSets; +using Microsoft.ComponentDetection.Orchestrator.Experiments; +using Microsoft.ComponentDetection.Orchestrator.Experiments.Configs; using Microsoft.ComponentDetection.Orchestrator.Services; using Microsoft.ComponentDetection.Orchestrator.Services.GraphTranslation; using Microsoft.Extensions.DependencyInjection; @@ -65,6 +67,12 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s services.AddSingleton(); services.AddSingleton(); + // Experiments + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + // Detectors // CocoaPods services.AddSingleton(); diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs index c32d5585b..0b241b01b 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Orchestrator.Services; + using System; using System.Collections.Concurrent; using System.Collections.Generic; @@ -14,6 +15,7 @@ using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.BcdeModels; using Microsoft.ComponentDetection.Orchestrator.ArgumentSets; +using Microsoft.ComponentDetection.Orchestrator.Experiments; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using static System.Environment; @@ -22,12 +24,15 @@ public class DetectorProcessingService : IDetectorProcessingService { private readonly IObservableDirectoryWalkerFactory scanner; private readonly ILogger logger; + private readonly IExperimentService experimentService; public DetectorProcessingService( IObservableDirectoryWalkerFactory scanner, + IExperimentService experimentService, ILogger logger) { this.scanner = scanner; + this.experimentService = experimentService; this.logger = logger; } @@ -104,6 +109,8 @@ public async Task ProcessDetectorsAsync(IDetectionArgu exitCode = resultCode; } + this.experimentService.RecordDetectorRun(detector, detectedComponents); + if (isExperimentalDetector) { return (new IndividualDetectorScanResult(), new ComponentRecorder(), detector); @@ -115,6 +122,7 @@ public async Task ProcessDetectorsAsync(IDetectionArgu }).ToList(); var results = await Task.WhenAll(scanTasks); + await this.experimentService.FinishAsync(); var detectorProcessingResult = this.ConvertDetectorResultsIntoResult(results, exitCode); diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/DefaultExperimentProcessorTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/DefaultExperimentProcessorTests.cs new file mode 100644 index 000000000..c719d4a59 --- /dev/null +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/DefaultExperimentProcessorTests.cs @@ -0,0 +1,48 @@ +namespace Microsoft.ComponentDetection.Orchestrator.Tests.Experiments; + +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.ComponentDetection.Common; +using Microsoft.ComponentDetection.Orchestrator.Experiments; +using Microsoft.ComponentDetection.Orchestrator.Experiments.Configs; +using Microsoft.ComponentDetection.Orchestrator.Experiments.Models; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +[TestClass] +[TestCategory("Governance/All")] +[TestCategory("Governance/ComponentDetection")] +public class DefaultExperimentProcessorTests +{ + private readonly Mock fileWritingServiceMock; + private readonly DefaultExperimentProcessor processor; + + public DefaultExperimentProcessorTests() + { + var loggerMock = new Mock>(); + this.fileWritingServiceMock = new Mock(); + this.processor = new DefaultExperimentProcessor(this.fileWritingServiceMock.Object, loggerMock.Object); + } + + [TestMethod] + public async Task ProcessExperimentAsync_WritesSerializedExperimentDiffToFileAsync() + { + var config = new Mock(); + config.Setup(c => c.Name).Returns("TestExperiment"); + + var diff = new ExperimentDiff( + ExperimentTestUtils.CreateRandomExperimentComponents(), + ExperimentTestUtils.CreateRandomExperimentComponents()); + + var serializedDiff = JsonSerializer.Serialize(diff, new JsonSerializerOptions { WriteIndented = true }); + + await this.processor.ProcessExperimentAsync(config.Object, diff); + + this.fileWritingServiceMock.Verify( + f => f.WriteFileAsync( + It.Is(s => s.StartsWith($"Experiment_{config.Object.Name}_")), + serializedDiff), + Times.Once); + } +} diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentDiffTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentDiffTests.cs new file mode 100644 index 000000000..3af3dcec9 --- /dev/null +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentDiffTests.cs @@ -0,0 +1,126 @@ +namespace Microsoft.ComponentDetection.Orchestrator.Tests.Experiments; + +using System.Collections.Generic; +using System.Linq; +using FluentAssertions; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.ComponentDetection.Orchestrator.Experiments.Models; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +[TestCategory("Governance/All")] +[TestCategory("Governance/ComponentDetection")] +public class ExperimentDiffTests +{ + [TestMethod] + public void ExperimentDiff_DiffsAddedIds() + { + var testComponents = ExperimentTestUtils.CreateRandomExperimentComponents(); + var diff = new ExperimentDiff(Enumerable.Empty(), testComponents); + + diff.AddedIds.Should().BeEquivalentTo(testComponents.Select(x => x.Id)); + diff.RemovedIds.Should().BeEmpty(); + + diff.DevelopmentDependencyChanges.Should().BeEmpty(); + diff.AddedRootIds.Should().BeEmpty(); + diff.RemovedRootIds.Should().BeEmpty(); + } + + [TestMethod] + public void ExperimentDiff_DiffsRemovedIds() + { + var testComponents = ExperimentTestUtils.CreateRandomExperimentComponents(); + var diff = new ExperimentDiff(testComponents, Enumerable.Empty()); + + diff.RemovedIds.Should().BeEquivalentTo(testComponents.Select(x => x.Id)); + diff.AddedIds.Should().BeEmpty(); + + diff.DevelopmentDependencyChanges.Should().BeEmpty(); + diff.AddedRootIds.Should().BeEmpty(); + diff.RemovedRootIds.Should().BeEmpty(); + } + + [TestMethod] + public void ExperimentDiff_DiffsDevDependencies() + { + var detectedComponent = ExperimentTestUtils.CreateRandomComponent(); + var componentA = new DetectedComponent(detectedComponent.Component); + var componentB = new DetectedComponent(detectedComponent.Component); + + componentA.DevelopmentDependency = false; + componentB.DevelopmentDependency = true; + + var diff = new ExperimentDiff( + new[] { new ExperimentComponent(componentA) }, + new[] { new ExperimentComponent(componentB) }); + + diff.DevelopmentDependencyChanges.Should().HaveCount(1); + + var change = diff.DevelopmentDependencyChanges.First(); + change.Id.Should().Be(detectedComponent.Component.Id); + change.OldValue.Should().BeFalse(); + change.NewValue.Should().BeTrue(); + + diff.AddedIds.Should().BeEmpty(); + diff.RemovedIds.Should().BeEmpty(); + diff.AddedRootIds.Should().BeEmpty(); + diff.RemovedRootIds.Should().BeEmpty(); + } + + [TestMethod] + public void ExperimentDiff_DiffsAddedRootIds() + { + var rootComponent = ExperimentTestUtils.CreateRandomComponent(); + var component = ExperimentTestUtils.CreateRandomComponent(); + + var componentA = new DetectedComponent(component.Component); + var componentB = new DetectedComponent(component.Component) + { + DependencyRoots = new HashSet { rootComponent.Component }, + }; + + var diff = new ExperimentDiff( + new[] { new ExperimentComponent(componentA), }, + new[] { new ExperimentComponent(componentB), }); + + diff.AddedRootIds.Should().HaveCount(1); + diff.RemovedRootIds.Should().BeEmpty(); + + var addedRoot = diff.AddedRootIds[component.Component.Id]; + addedRoot.Should().HaveCount(1); + addedRoot.Should().BeEquivalentTo(rootComponent.Component.Id); + + diff.AddedIds.Should().BeEmpty(); + diff.RemovedIds.Should().BeEmpty(); + diff.DevelopmentDependencyChanges.Should().BeEmpty(); + } + + [TestMethod] + public void ExperimentDiff_DiffsRemovedRootIds() + { + var rootComponent = ExperimentTestUtils.CreateRandomComponent(); + var component = ExperimentTestUtils.CreateRandomComponent(); + + var componentA = new DetectedComponent(component.Component) + { + DependencyRoots = new HashSet { rootComponent.Component }, + }; + var componentB = new DetectedComponent(component.Component); + + var diff = new ExperimentDiff( + new[] { new ExperimentComponent(componentA), }, + new[] { new ExperimentComponent(componentB), }); + + diff.RemovedRootIds.Should().HaveCount(1); + diff.AddedRootIds.Should().BeEmpty(); + + var removedRoot = diff.RemovedRootIds[component.Component.Id]; + removedRoot.Should().HaveCount(1); + removedRoot.Should().BeEquivalentTo(rootComponent.Component.Id); + + diff.AddedIds.Should().BeEmpty(); + diff.RemovedIds.Should().BeEmpty(); + diff.DevelopmentDependencyChanges.Should().BeEmpty(); + } +} diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentResultsTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentResultsTests.cs new file mode 100644 index 000000000..7e841ea5e --- /dev/null +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentResultsTests.cs @@ -0,0 +1,35 @@ +namespace Microsoft.ComponentDetection.Orchestrator.Tests.Experiments; + +using FluentAssertions; +using Microsoft.ComponentDetection.Orchestrator.Experiments.Models; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +[TestCategory("Governance/All")] +[TestCategory("Governance/ComponentDetection")] +public class ExperimentResultsTests +{ + [TestMethod] + public void ExperimentResults_AddsToOnlyControlGroup() + { + var experiment = new ExperimentResults(); + var testComponents = ExperimentTestUtils.CreateRandomComponents(); + + experiment.AddComponentsToControlGroup(testComponents); + + experiment.ControlGroupComponents.Should().HaveCount(testComponents.Count); + experiment.ExperimentGroupComponents.Should().BeEmpty(); + } + + [TestMethod] + public void ExperimentResults_AddsToOnlyExperimentGroup() + { + var experiment = new ExperimentResults(); + var testComponents = ExperimentTestUtils.CreateRandomComponents(); + + experiment.AddComponentsToExperimentalGroup(testComponents); + + experiment.ControlGroupComponents.Should().BeEmpty(); + experiment.ExperimentGroupComponents.Should().HaveCount(testComponents.Count); + } +} diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs new file mode 100644 index 000000000..4939ad536 --- /dev/null +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs @@ -0,0 +1,70 @@ +namespace Microsoft.ComponentDetection.Orchestrator.Tests.Experiments; + +using System.Linq; +using System.Threading.Tasks; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Orchestrator.Experiments; +using Microsoft.ComponentDetection.Orchestrator.Experiments.Configs; +using Microsoft.ComponentDetection.Orchestrator.Experiments.Models; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +[TestClass] +[TestCategory("Governance/All")] +[TestCategory("Governance/ComponentDetection")] +public class ExperimentServiceTests +{ + private readonly Mock experimentConfigMock; + private readonly Mock experimentProcessorMock; + private readonly Mock> loggerMock; + private readonly Mock detectorMock; + + public ExperimentServiceTests() + { + this.experimentConfigMock = new Mock(); + this.experimentProcessorMock = new Mock(); + this.loggerMock = new Mock>(); + this.detectorMock = new Mock(); + } + + [TestMethod] + public void RecordDetectorRun_AddsComponentsToControlAndExperimentGroup() + { + var components = ExperimentTestUtils.CreateRandomComponents(); + + this.experimentConfigMock.Setup(x => x.IsInControlGroup(this.detectorMock.Object)).Returns(true); + this.experimentConfigMock.Setup(x => x.IsInExperimentGroup(this.detectorMock.Object)).Returns(true); + + var service = new ExperimentService( + new[] { this.experimentConfigMock.Object }, + Enumerable.Empty(), + this.loggerMock.Object); + + service.RecordDetectorRun(this.detectorMock.Object, components); + + this.experimentConfigMock.Verify(x => x.IsInControlGroup(this.detectorMock.Object), Times.Once()); + this.experimentConfigMock.Verify(x => x.IsInExperimentGroup(this.detectorMock.Object), Times.Once()); + } + + [TestMethod] + public async Task FinishAsync_ProcessesExperimentsAsync() + { + var components = ExperimentTestUtils.CreateRandomComponents(); + + this.experimentConfigMock.Setup(x => x.IsInControlGroup(this.detectorMock.Object)).Returns(true); + this.experimentConfigMock.Setup(x => x.IsInExperimentGroup(this.detectorMock.Object)).Returns(true); + + var service = new ExperimentService( + new[] { this.experimentConfigMock.Object }, + new[] { this.experimentProcessorMock.Object }, + this.loggerMock.Object); + service.RecordDetectorRun(this.detectorMock.Object, components); + + await service.FinishAsync(); + + this.experimentProcessorMock.Verify( + x => x.ProcessExperimentAsync(this.experimentConfigMock.Object, It.IsAny()), + Times.Once()); + } +} diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentTestUtils.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentTestUtils.cs new file mode 100644 index 000000000..eaf29ed3d --- /dev/null +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentTestUtils.cs @@ -0,0 +1,23 @@ +namespace Microsoft.ComponentDetection.Orchestrator.Tests.Experiments; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.ComponentDetection.Orchestrator.Experiments.Models; + +public static class ExperimentTestUtils +{ + public static DetectedComponent CreateRandomComponent() => new(new NpmComponent(Guid.NewGuid().ToString(), CreateRandomVersion())); + + public static List CreateRandomComponents(int length = 5) => + Enumerable.Range(0, length).Select(_ => CreateRandomComponent()).ToList(); + + public static List CreateRandomExperimentComponents(int length = 5) => + CreateRandomComponents(length).Select(x => new ExperimentComponent(x)).ToList(); + + private static string CreateRandomVersion() => + $"{RandomNumberGenerator.GetInt32(0, 100)}.{RandomNumberGenerator.GetInt32(0, 100)}.{RandomNumberGenerator.GetInt32(0, 100)}"; +} diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs index 675e339f8..37eb40229 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Orchestrator.Tests.Services; + using System; using System.Collections.Generic; using System.IO; @@ -10,6 +11,7 @@ namespace Microsoft.ComponentDetection.Orchestrator.Tests.Services; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Orchestrator.ArgumentSets; +using Microsoft.ComponentDetection.Orchestrator.Experiments; using Microsoft.ComponentDetection.Orchestrator.Services; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -36,6 +38,7 @@ public class DetectorProcessingServiceTests private readonly Mock> loggerMock; private readonly DetectorProcessingService serviceUnderTest; private readonly Mock directoryWalkerFactory; + private readonly Mock experimentServiceMock; private readonly Mock firstFileComponentDetectorMock; private readonly Mock secondFileComponentDetectorMock; @@ -49,10 +52,11 @@ public class DetectorProcessingServiceTests public DetectorProcessingServiceTests() { + this.experimentServiceMock = new Mock(); this.loggerMock = new Mock>(); this.directoryWalkerFactory = new Mock(); this.serviceUnderTest = - new DetectorProcessingService(this.directoryWalkerFactory.Object, this.loggerMock.Object); + new DetectorProcessingService(this.directoryWalkerFactory.Object, this.experimentServiceMock.Object, this.loggerMock.Object); this.firstFileComponentDetectorMock = this.SetupFileDetectorMock("firstFileDetectorId"); this.secondFileComponentDetectorMock = this.SetupFileDetectorMock("secondFileDetectorId"); @@ -505,6 +509,47 @@ public async Task ProcessDetectorsAsync_HandlesDetectorArgsAsync() .And.Contain("arg3", "val3"); } + [TestMethod] + public async Task ProcessDetectorsAsync_FinishesExperimentsAsync() + { + this.detectorsToUse = new[] + { + this.firstFileComponentDetectorMock.Object, this.secondFileComponentDetectorMock.Object, + }; + + await this.serviceUnderTest.ProcessDetectorsAsync(DefaultArgs, this.detectorsToUse, new DetectorRestrictions()); + + this.experimentServiceMock.Verify(x => x.FinishAsync(), Times.Once()); + } + + [TestMethod] + public async Task ProcessDetectorsAsync_RecordsDetectorRunsAsync() + { + this.detectorsToUse = new[] + { + this.firstFileComponentDetectorMock.Object, this.secondFileComponentDetectorMock.Object, + }; + + var firstComponents = new[] { this.componentDictionary[this.firstFileComponentDetectorMock.Object.Id] }; + var secondComponents = new[] { this.componentDictionary[this.secondFileComponentDetectorMock.Object.Id] }; + + await this.serviceUnderTest.ProcessDetectorsAsync(DefaultArgs, this.detectorsToUse, new DetectorRestrictions()); + + this.experimentServiceMock.Verify( + x => + x.RecordDetectorRun( + It.Is(detector => detector == this.firstFileComponentDetectorMock.Object), + It.Is>(components => components.SequenceEqual(firstComponents))), + Times.Once()); + + this.experimentServiceMock.Verify( + x => + x.RecordDetectorRun( + It.Is(detector => detector == this.secondFileComponentDetectorMock.Object), + It.Is>(components => components.SequenceEqual(secondComponents))), + Times.Once()); + } + private Mock SetupFileDetectorMock(string id) { var mockFileDetector = new Mock();