feat: add detector experiments

src/Microsoft.ComponentDetection.Common/FileWritingService.cs

@@ -48,6 +48,13 @@ public void WriteFile(string relativeFilePath, string text)
         }
     }
 
+    public async Task WriteFileAsync(string relativeFilePath, string text)
+    {
+        relativeFilePath = this.ResolveFilePath(relativeFilePath);
+
+        await File.WriteAllTextAsync(relativeFilePath, text);
+    }
+
     public void WriteFile(FileInfo relativeFilePath, string text)
     {
         File.WriteAllText(relativeFilePath.FullName, text);

src/Microsoft.ComponentDetection.Common/IFileWritingService.cs

@@ -2,6 +2,7 @@
 
 using System;
 using System.IO;
+using System.Threading.Tasks;
 
 // All file paths are relative and will replace occurrences of {timestamp} with the shared file timestamp.
 public interface IFileWritingService : IDisposable, IAsyncDisposable
@@ -12,6 +13,8 @@ public interface IFileWritingService : IDisposable, IAsyncDisposable
 
     void WriteFile(string relativeFilePath, string text);
 
+    Task WriteFileAsync(string relativeFilePath, string text);
+
     void WriteFile(FileInfo relativeFilePath, string text);
 
     string ResolveFilePath(string relativeFilePath);

src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/IExperimentConfiguration.cs

@@ -0,0 +1,31 @@
+namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Configs;
+
+using Microsoft.ComponentDetection.Contracts;
+
+/// <summary>
+/// Defines the configuration for an experiment. An experiment is a set of detectors that are grouped into control and
+/// experiment groups. The control group is used to determine the baseline for the experiment. The experiment group is
+/// used to determine the impact of the experiment on the baseline. The unique set of components from the two sets of
+/// detectors is compared and differences are reported via telemetry.
+/// </summary>
+public interface IExperimentConfiguration
+{
+    /// <summary>
+    /// The name of the experiment.
+    /// </summary>
+    string Name { get; }
+
+    /// <summary>
+    /// Specifies if the detector is in the control group.
+    /// </summary>
+    /// <param name="componentDetector">The detector.</param>
+    /// <returns><c>true</c> if the detector is in the control group; otherwise, <c>false</c>.</returns>
+    bool IsInControlGroup(IComponentDetector componentDetector);
+
+    /// <summary>
+    /// Specifies if the detector is in the control group.
+    /// </summary>
+    /// <param name="componentDetector">The detector.</param>
+    /// <returns><c>true</c> if the detector is in the experiment group; otherwise, <c>false</c>.</returns>
+    bool IsInExperimentGroup(IComponentDetector componentDetector);
+}

src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/NewNugetExperiment.cs

@@ -0,0 +1,21 @@
+namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Configs;
+
+using Microsoft.ComponentDetection.Contracts;
+using Microsoft.ComponentDetection.Detectors.NuGet;
+
+/// <summary>
+/// Comparing the new NuGet detector approach to the old one.
+/// </summary>
+public class NewNugetExperiment : IExperimentConfiguration
+{
+    /// <inheritdoc />
+    public string Name => "NewNugetDetector";
+
+    /// <inheritdoc />
+    public bool IsInControlGroup(IComponentDetector componentDetector) =>
+        componentDetector is NuGetComponentDetector or NuGetProjectModelProjectCentricComponentDetector;
+
+    /// <inheritdoc />
+    public bool IsInExperimentGroup(IComponentDetector componentDetector) =>
+        componentDetector is NuGetProjectModelProjectCentricComponentDetector or NuGetPackagesConfigDetector;
+}

src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/NpmLockfile3Experiment.cs

@@ -0,0 +1,19 @@
+namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Configs;
+
+using Microsoft.ComponentDetection.Contracts;
+using Microsoft.ComponentDetection.Detectors.Npm;
+
+/// <summary>
+/// Validating the <see cref="NpmLockfile3Detector"/>.
+/// </summary>
+public class NpmLockfile3Experiment : IExperimentConfiguration
+{
+    /// <inheritdoc />
+    public string Name => "LockfileVersion3";
+
+    /// <inheritdoc />
+    public bool IsInControlGroup(IComponentDetector componentDetector) => componentDetector is NpmComponentDetector;
+
+    /// <inheritdoc />
+    public bool IsInExperimentGroup(IComponentDetector componentDetector) => componentDetector is NpmLockfile3Detector;
+}

src/Microsoft.ComponentDetection.Orchestrator/Experiments/DefaultExperimentProcessor.cs

@@ -0,0 +1,40 @@
+namespace Microsoft.ComponentDetection.Orchestrator.Experiments;
+
+using System;
+using System.Text.Json;
+using System.Threading.Tasks;
+using Microsoft.ComponentDetection.Common;
+using Microsoft.ComponentDetection.Orchestrator.Experiments.Configs;
+using Microsoft.ComponentDetection.Orchestrator.Experiments.Models;
+using Microsoft.Extensions.Logging;
+
+/// <summary>
+/// The default experiment processor. Writes a JSON output file to a temporary directory.
+/// </summary>
+public class DefaultExperimentProcessor : IExperimentProcessor
+{
+    private readonly IFileWritingService fileWritingService;
+    private readonly ILogger<DefaultExperimentProcessor> logger;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="DefaultExperimentProcessor"/> class.
+    /// </summary>
+    /// <param name="fileWritingService">The file writing service.</param>
+    /// <param name="logger">The logger.</param>
+    public DefaultExperimentProcessor(IFileWritingService fileWritingService, ILogger<DefaultExperimentProcessor> logger)
+    {
+        this.fileWritingService = fileWritingService;
+        this.logger = logger;
+    }
+
+    /// <inheritdoc />
+    public async Task ProcessExperimentAsync(IExperimentConfiguration config, ExperimentDiff diff)
+    {
+        var filename = $"Experiment_{config.Name}_{{timestamp}}_{Environment.ProcessId}.json";
+
+        this.logger.LogInformation("Writing experiment {Name} results to {Filename}", config.Name, this.fileWritingService.ResolveFilePath(filename));
+
+        var serializedDiff = JsonSerializer.Serialize(diff, new JsonSerializerOptions { WriteIndented = true });
+        await this.fileWritingService.WriteFileAsync(filename, serializedDiff);
+    }
+}

src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs

@@ -0,0 +1,83 @@
+namespace Microsoft.ComponentDetection.Orchestrator.Experiments;
+
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.ComponentDetection.Contracts;
+using Microsoft.ComponentDetection.Orchestrator.Experiments.Configs;
+using Microsoft.ComponentDetection.Orchestrator.Experiments.Models;
+using Microsoft.Extensions.Logging;
+
+/// <inheritdoc />
+public class ExperimentService : IExperimentService
+{
+    private readonly List<(IExperimentConfiguration Config, ExperimentResults ExperimentResults)> experiments;
+    private readonly IEnumerable<IExperimentProcessor> experimentProcessors;
+    private readonly ILogger<ExperimentService> logger;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="ExperimentService"/> class.
+    /// </summary>
+    /// <param name="configs">The experiment configurations.</param>
+    /// <param name="experimentProcessors">The experiment processors.</param>
+    /// <param name="logger">The logger.</param>
+    public ExperimentService(
+        IEnumerable<IExperimentConfiguration> configs,
+        IEnumerable<IExperimentProcessor> experimentProcessors,
+        ILogger<ExperimentService> logger)
+    {
+        this.experiments = configs.Select(x => (x, new ExperimentResults())).ToList();
+        this.experimentProcessors = experimentProcessors;
+        this.logger = logger;
+    }
+
+    /// <inheritdoc />
+    public void RecordDetectorRun(IComponentDetector detector, IEnumerable<DetectedComponent> components)
+    {
+        foreach (var (config, experimentResults) in this.experiments)
+        {
+            if (config.IsInControlGroup(detector))
+            {
+                experimentResults.AddComponentsToControlGroup(components);
+                this.logger.LogDebug(
+                    "Adding {Count} Components from {Id} to Control Group for {Experiment}",
+                    components.Count(),
+                    detector.Id,
+                    config.Name);
+            }
+
+            if (config.IsInExperimentGroup(detector))
+            {
+                experimentResults.AddComponentsToExperimentalGroup(components);
+                this.logger.LogDebug(
+                    "Adding {Count} Components from {Id} to Experiment Group for {Experiment}",
+                    components.Count(),
+                    detector.Id,
+                    config.Name);
+            }
+        }
+    }
+
+    /// <inheritdoc />
+    public async Task FinishAsync()
+    {
+        foreach (var (config, experiment) in this.experiments)
+        {
+            var oldComponents = experiment.ControlGroupComponents;
+            var newComponents = experiment.ExperimentGroupComponents;
+
+            this.logger.LogInformation(
+                "Experiment {Experiment} finished and has {Count} components in the control group and {Count} components in the experiment group.",
+                config.Name,
+                oldComponents.Count,
+                newComponents.Count);
+
+            var diff = new ExperimentDiff(experiment.ControlGroupComponents, experiment.ExperimentGroupComponents);
+
+            foreach (var processor in this.experimentProcessors)
+            {
+                await processor.ProcessExperimentAsync(config, diff);
+            }
+        }
+    }
+}

src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentProcessor.cs

@@ -0,0 +1,19 @@
+namespace Microsoft.ComponentDetection.Orchestrator.Experiments;
+
+using System.Threading.Tasks;
+using Microsoft.ComponentDetection.Orchestrator.Experiments.Configs;
+using Microsoft.ComponentDetection.Orchestrator.Experiments.Models;
+
+/// <summary>
+/// Processes the results of an experiment. Used to report the results of an experiment, such as by writing to a file.
+/// </summary>
+public interface IExperimentProcessor
+{
+    /// <summary>
+    /// Asynchronously processes the results of an experiment.
+    /// </summary>
+    /// <param name="config">The experiment configuration.</param>
+    /// <param name="diff">The difference in components between two sets of detectors.</param>
+    /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
+    Task ProcessExperimentAsync(IExperimentConfiguration config, ExperimentDiff diff);
+}

src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentService.cs

@@ -0,0 +1,24 @@
+namespace Microsoft.ComponentDetection.Orchestrator.Experiments;
+
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Microsoft.ComponentDetection.Contracts;
+
+/// <summary>
+/// Service for recording detector results and processing the results for any active experiments.
+/// </summary>
+public interface IExperimentService
+{
+    /// <summary>
+    /// Records the results of a detector execution and processes the results for any active experiments.
+    /// </summary>
+    /// <param name="detector">The detector.</param>
+    /// <param name="components">The detected components from the <paramref name="detector"/>.</param>
+    void RecordDetectorRun(IComponentDetector detector, IEnumerable<DetectedComponent> components);
+
+    /// <summary>
+    /// Called when all detectors have finished executing. Processes the experiments and reports the results.
+    /// </summary>
+    /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
+    Task FinishAsync();
+}

src/Microsoft.ComponentDetection.Orchestrator/Experiments/Models/ExperimentComponent.cs

@@ -0,0 +1,37 @@
+namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Models;
+
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.ComponentDetection.Contracts;
+
+/// <summary>
+/// A model representing a component detected by a detector, as relevant to an experiment.
+/// </summary>
+public record ExperimentComponent
+{
+    /// <summary>
+    /// Creates a new <see cref="ExperimentComponent"/> from a <see cref="DetectedComponent"/>.
+    /// </summary>
+    /// <param name="detectedComponent">The detected component.</param>
+    public ExperimentComponent(DetectedComponent detectedComponent)
+    {
+        this.Id = detectedComponent.Component.Id;
+        this.DevelopmentDependency = detectedComponent.DevelopmentDependency ?? false;
+        this.RootIds = detectedComponent.DependencyRoots?.Select(x => x.Id).ToHashSet() ?? new HashSet<string>();
+    }
+
+    /// <summary>
+    /// The component ID.
+    /// </summary>
+    public string Id { get; }
+
+    /// <summary>
+    /// <c>true</c> if the component is a development dependency; otherwise, <c>false</c>.
+    /// </summary>
+    public bool DevelopmentDependency { get; }
+
+    /// <summary>
+    /// The set of root component IDs for this component.
+    /// </summary>
+    public HashSet<string> RootIds { get; }
+}