Merge pull request #63088 from jasonmalinowski/add-telemetry-for-gene…

…rators

Add telemetry for the time and files outputted for source generators

src/Features/Core/Portable/Diagnostics/DiagnosticAnalyzerTelemetry.cs

@@ -106,7 +106,7 @@ public void ReportAndClear(int correlationId)
                     else
                     {
                         // annonymize analyzer and exception names:
-                        m["Analyzer.NameHashCode"] = ComputeSha256Hash(analyzerName);
+                        m["Analyzer.NameHashCode"] = AnalyzerNameForTelemetry.ComputeSha256Hash(analyzerName);
                     }
 
                     m["Analyzer.CodeBlock"] = analyzerInfo.CodeBlockActionsCount;
@@ -130,11 +130,5 @@ public void ReportAndClear(int correlationId)
                 }));
             }
         }
-
-        private static string ComputeSha256Hash(string name)
-        {
-            using var sha256 = SHA256.Create();
-            return Convert.ToBase64String(sha256.ComputeHash(Encoding.UTF8.GetBytes(name)));
-        }
     }
 }

src/VisualStudio/Core/Def/RoslynPackage.cs

@@ -311,13 +311,14 @@ protected override void Dispose(bool disposing)
             base.Dispose(disposing);
         }
 
-        private static void ReportSessionWideTelemetry()
+        private void ReportSessionWideTelemetry()
         {
             SolutionLogger.ReportTelemetry();
             AsyncCompletionLogger.ReportTelemetry();
             CompletionProvidersLogger.ReportTelemetry();
             ChangeSignatureLogger.ReportTelemetry();
             InheritanceMarginLogger.ReportTelemetry();
+            ComponentModel.GetService<VisualStudioSourceGeneratorTelemetryCollectorWorkspaceServiceFactory>().ReportOtherWorkspaceTelemetry();
         }
 
         private void DisposeVisualStudioServices()

src/VisualStudio/Core/Def/Workspace/VisualStudioSourceGeneratorTelemetryCollectorWorkspaceServiceFactory.cs

@@ -0,0 +1,105 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Composition;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.CodeAnalysis.Editor.Shared.Utilities;
+using Microsoft.CodeAnalysis.Host;
+using Microsoft.CodeAnalysis.Host.Mef;
+using Microsoft.CodeAnalysis.Internal.Log;
+using Microsoft.CodeAnalysis.SourceGeneratorTelemetry;
+using Microsoft.VisualStudio.Shell;
+using Microsoft.VisualStudio.Shell.Interop;
+
+namespace Microsoft.VisualStudio.LanguageServices
+{
+    /// <summary>
+    /// Exports a <see cref="ISourceGeneratorTelemetryCollectorWorkspaceService"/> which is watched across all workspaces. This lets us collect
+    /// statistics for all workspaces (including things like interactive, preview, etc.) so we can get the overall counts to report.
+    /// </summary>
+    [Export]
+    [ExportWorkspaceServiceFactory(typeof(ISourceGeneratorTelemetryCollectorWorkspaceService)), Shared]
+    internal class VisualStudioSourceGeneratorTelemetryCollectorWorkspaceServiceFactory : IWorkspaceServiceFactory, IVsSolutionEvents
+    {
+        /// <summary>
+        /// The collector that's used to collect all the telemetry for operations within <see cref="VisualStudioWorkspace"/>. We'll report this
+        /// when the solution is closed, so the telemetry is linked to that.
+        /// </summary>
+        private readonly SourceGeneratorTelemetryCollectorWorkspaceService _visualStudioWorkspaceInstance = new SourceGeneratorTelemetryCollectorWorkspaceService();
+
+        /// <summary>
+        /// The collector used to collect telemetry for any other workspaces that might be created; we'll report this at the end of the session since nothing here is necessarily
+        /// linked to a specific solution. The expectation is this may be empty for many/most sessions, but we don't want a hole in our reporting and discover that the hard way.
+        /// </summary>
+        private readonly SourceGeneratorTelemetryCollectorWorkspaceService _otherWorkspacesInstance = new SourceGeneratorTelemetryCollectorWorkspaceService();
+
+        private readonly IThreadingContext _threadingContext;
+        private readonly IAsyncServiceProvider _serviceProvider;
+        private volatile int _subscribedToSolutionEvents;
+
+        [ImportingConstructor]
+        [Obsolete(MefConstruction.ImportingConstructorMessage, error: true)]
+        public VisualStudioSourceGeneratorTelemetryCollectorWorkspaceServiceFactory(IThreadingContext threadingContext, SVsServiceProvider serviceProvider)
+        {
+            _threadingContext = threadingContext;
+            _serviceProvider = (IAsyncServiceProvider)serviceProvider;
+        }
+
+        public IWorkspaceService CreateService(HostWorkspaceServices workspaceServices)
+        {
+            // We will record all generators for the main workspace in one bucket, and any other generators running in other
+            // workspaces (interactive, for example) will be put in a different bucket. This allows us to report the telemetry
+            // from the primary workspace on solution closed, while not letting the unrelated runs pollute those numbers.
+            if (workspaceServices.Workspace is VisualStudioWorkspace)
+            {
+                EnsureSubscribedToSolutionEvents();
+                return _visualStudioWorkspaceInstance;
+            }
+            else
+            {
+                return _otherWorkspacesInstance;
+            }
+        }
+
+        private void EnsureSubscribedToSolutionEvents()
+        {
+            if (Interlocked.CompareExchange(ref _subscribedToSolutionEvents, 1, 0) == 0)
+            {
+                Task.Run(async () =>
+                {
+                    var shellService = await _serviceProvider.GetServiceAsync<SVsSolution, IVsSolution>(_threadingContext.JoinableTaskFactory).ConfigureAwait(true);
+                    await _threadingContext.JoinableTaskFactory.SwitchToMainThreadAsync(_threadingContext.DisposalToken);
+                    shellService.AdviseSolutionEvents(this, out _);
+                }, _threadingContext.DisposalToken);
+            }
+        }
+
+        public void ReportOtherWorkspaceTelemetry()
+        {
+            _otherWorkspacesInstance.ReportStatisticsAndClear(FunctionId.SourceGenerator_OtherWorkspaceSessionStatistics);
+        }
+
+        int IVsSolutionEvents.OnAfterOpenProject(IVsHierarchy pHierarchy, int fAdded) => VSConstants.E_NOTIMPL;
+        int IVsSolutionEvents.OnQueryCloseProject(IVsHierarchy pHierarchy, int fRemoving, ref int pfCancel) => VSConstants.E_NOTIMPL;
+        int IVsSolutionEvents.OnBeforeCloseProject(IVsHierarchy pHierarchy, int fRemoved) => VSConstants.E_NOTIMPL;
+        int IVsSolutionEvents.OnAfterLoadProject(IVsHierarchy pStubHierarchy, IVsHierarchy pRealHierarchy) => VSConstants.E_NOTIMPL;
+        int IVsSolutionEvents.OnQueryUnloadProject(IVsHierarchy pRealHierarchy, ref int pfCancel) => VSConstants.E_NOTIMPL;
+        int IVsSolutionEvents.OnBeforeUnloadProject(IVsHierarchy pRealHierarchy, IVsHierarchy pStubHierarchy) => VSConstants.E_NOTIMPL;
+        int IVsSolutionEvents.OnAfterOpenSolution(object pUnkReserved, int fNewSolution) => VSConstants.E_NOTIMPL;
+        int IVsSolutionEvents.OnQueryCloseSolution(object pUnkReserved, ref int pfCancel) => VSConstants.E_NOTIMPL;
+
+        int IVsSolutionEvents.OnBeforeCloseSolution(object pUnkReserved)
+        {
+            // Report the telemetry now before the solution is closed; since this will be reported per solution session ID, it means
+            // we can distinguish how many solutions have generators versus just overall sessions.
+            _visualStudioWorkspaceInstance.ReportStatisticsAndClear(FunctionId.SourceGenerator_SolutionStatistics);
+
+            return VSConstants.S_OK;
+        }
+
+        int IVsSolutionEvents.OnAfterCloseSolution(object pUnkReserved) => VSConstants.E_NOTIMPL;
+    }
+}

src/Workspaces/SharedUtilitiesAndExtensions/Workspace/Core/Log/AbstractLogAggregator.cs → src/Workspaces/Core/Portable/Log/AbstractLogAggregator.cs

@@ -2,12 +2,11 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-#nullable disable
-
 using System;
 using System.Collections;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 using System.Threading;
 using Roslyn.Utilities;
@@ -17,7 +16,7 @@ namespace Microsoft.CodeAnalysis.Internal.Log
     /// <summary>
     /// helper class to aggregate some numeric value log in client side
     /// </summary>
-    internal abstract class AbstractLogAggregator<TKey, TValue> : IEnumerable<KeyValuePair<TKey, TValue>>
+    internal abstract class AbstractLogAggregator<TKey, TValue> : IEnumerable<KeyValuePair<TKey, TValue>> where TKey : notnull
     {
         private readonly ConcurrentDictionary<TKey, TValue> _map = new(concurrencyLevel: 2, capacity: 2);
         private readonly Func<TKey, TValue> _createCounter;
@@ -31,6 +30,8 @@ protected AbstractLogAggregator()
 
         public bool IsEmpty => _map.IsEmpty;
 
+        public void Clear() => _map.Clear();
+
         public IEnumerator<KeyValuePair<TKey, TValue>> GetEnumerator()
             => _map.GetEnumerator();
 
@@ -41,7 +42,7 @@ IEnumerator IEnumerable.GetEnumerator()
         protected TValue GetCounter(TKey key)
             => _map.GetOrAdd(key, _createCounter);
 
-        protected bool TryGetCounter(TKey key, out TValue counter)
+        protected bool TryGetCounter(TKey key, [MaybeNullWhen(false)] out TValue counter)
         {
             if (_map.TryGetValue(key, out counter))
             {

src/Workspaces/Core/Portable/Log/AnalyzerNameForTelemetry.cs

@@ -0,0 +1,20 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Security.Cryptography;
+using System.Text;
+
+namespace Microsoft.CodeAnalysis.Internal.Log
+{
+    internal static class AnalyzerNameForTelemetry
+    {
+        public static string ComputeSha256Hash(string name)
+        {
+            using var sha256 = SHA256.Create();
+            return Convert.ToBase64String(sha256.ComputeHash(Encoding.UTF8.GetBytes(name)));
+        }
+    }
+}

src/Workspaces/SharedUtilitiesAndExtensions/Workspace/Core/Log/CountLogAggregator.cs → src/Workspaces/Core/Portable/Log/CountLogAggregator.cs

@@ -2,13 +2,11 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-#nullable disable
-
 using System.Threading;
 
 namespace Microsoft.CodeAnalysis.Internal.Log
 {
-    internal class CountLogAggregator<TKey> : AbstractLogAggregator<TKey, CountLogAggregator<TKey>.Counter>
+    internal class CountLogAggregator<TKey> : AbstractLogAggregator<TKey, CountLogAggregator<TKey>.Counter> where TKey : notnull
     {
         protected override Counter CreateCounter()
             => new();

src/Workspaces/Core/Portable/Log/HistogramLogAggregator.cs

@@ -12,7 +12,7 @@ namespace Microsoft.CodeAnalysis.Internal.Log
     /// <summary>
     /// Defines a log aggregator to create a histogram
     /// </summary>
-    internal sealed class HistogramLogAggregator<TKey> : AbstractLogAggregator<TKey, HistogramLogAggregator<TKey>.HistogramCounter>
+    internal sealed class HistogramLogAggregator<TKey> : AbstractLogAggregator<TKey, HistogramLogAggregator<TKey>.HistogramCounter> where TKey : notnull
     {
         private readonly int _bucketSize;
         private readonly int _maxBucketValue;

src/Workspaces/Core/Portable/Log/StatisticLogAggregator.cs

@@ -2,13 +2,11 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-#nullable disable
-
 using System;
 
 namespace Microsoft.CodeAnalysis.Internal.Log
 {
-    internal sealed class StatisticLogAggregator<TKey> : AbstractLogAggregator<TKey, StatisticLogAggregator<TKey>.StatisticCounter>
+    internal sealed class StatisticLogAggregator<TKey> : AbstractLogAggregator<TKey, StatisticLogAggregator<TKey>.StatisticCounter> where TKey : notnull
     {
         protected override StatisticCounter CreateCounter()
             => new();

src/Workspaces/Core/Portable/SourceGeneratorTelemetry/ISourceGeneratorTelemetryCollectorWorkspaceService.cs

@@ -0,0 +1,16 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Microsoft.CodeAnalysis.Host;
+
+namespace Microsoft.CodeAnalysis.SourceGeneratorTelemetry
+{
+    internal interface ISourceGeneratorTelemetryCollectorWorkspaceService : IWorkspaceService
+    {
+        void CollectRunResult(GeneratorDriverRunResult driverRunResult, GeneratorDriverTimingInfo driverTimingInfo);
+    }
+}

src/Workspaces/Core/Portable/SourceGeneratorTelemetry/SourceGeneratorTelemetryCollectorWorkspaceService.cs

@@ -0,0 +1,75 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Diagnostics;
+using System.Runtime.CompilerServices;
+using Microsoft.CodeAnalysis.Internal.Log;
+using Roslyn.Utilities;
+
+namespace Microsoft.CodeAnalysis.SourceGeneratorTelemetry
+{
+    internal class SourceGeneratorTelemetryCollectorWorkspaceService : ISourceGeneratorTelemetryCollectorWorkspaceService
+    {
+        private sealed record GeneratorTelemetryKey
+        {
+            public GeneratorTelemetryKey(ISourceGenerator generator)
+            {
+                Identity = new SourceGeneratorIdentity(generator);
+                FileVersion = FileVersionInfo.GetVersionInfo(generator.GetGeneratorType().Assembly.Location).FileVersion ?? "(null)";
+            }
+
+            // TODO: mark these 'required' when we have the attributes in place
+            public SourceGeneratorIdentity Identity { get; }
+            public string FileVersion { get; }
+        }
+
+        /// <summary>
+        /// Cache of the <see cref="GeneratorTelemetryKey"/> for a generator to avoid repeatedly reading version information from disk;
+        /// this is a ConditionalWeakTable so having telemetry for older runs doesn't keep the generator itself alive.
+        /// </summary>
+        private readonly ConditionalWeakTable<ISourceGenerator, GeneratorTelemetryKey> _generatorTelemetryKeys = new();
+
+        private readonly StatisticLogAggregator<GeneratorTelemetryKey> _elapsedTimeByGenerator = new();
+        private readonly StatisticLogAggregator<GeneratorTelemetryKey> _producedFilesByGenerator = new();
+
+        private GeneratorTelemetryKey GetTelemetryKey(ISourceGenerator generator)
+            => _generatorTelemetryKeys.GetValue(generator, static g => new GeneratorTelemetryKey(g));
+
+        public void CollectRunResult(GeneratorDriverRunResult driverRunResult, GeneratorDriverTimingInfo driverTimingInfo)
+        {
+            foreach (var generatorTime in driverTimingInfo.GeneratorTimes)
+            {
+                _elapsedTimeByGenerator.AddDataPoint(GetTelemetryKey(generatorTime.Generator), generatorTime.ElapsedTime);
+            }
+
+            foreach (var generatorResult in driverRunResult.Results)
+            {
+                _producedFilesByGenerator.AddDataPoint(GetTelemetryKey(generatorResult.Generator), generatorResult.GeneratedSources.Length);
+            }
+        }
+
+        public void ReportStatisticsAndClear(FunctionId functionId)
+        {
+            foreach (var (telemetryKey, elapsedTimeCounter) in _elapsedTimeByGenerator)
+            {
+                // We'll log one event per generator
+                Logger.Log(functionId, KeyValueLogMessage.Create(map =>
+                {
+                    // TODO: have a policy for when we don't have to hash them
+                    map[nameof(telemetryKey.Identity.AssemblyName) + "Hashed"] = AnalyzerNameForTelemetry.ComputeSha256Hash(telemetryKey.Identity.AssemblyName);
+                    map[nameof(telemetryKey.Identity.AssemblyVersion)] = telemetryKey.Identity.AssemblyVersion.ToString();
+                    map[nameof(telemetryKey.Identity.TypeName) + "Hashed"] = AnalyzerNameForTelemetry.ComputeSha256Hash(telemetryKey.Identity.TypeName);
+                    map[nameof(telemetryKey.FileVersion)] = telemetryKey.FileVersion;
+
+                    var result = elapsedTimeCounter.GetStatisticResult();
+                    result.WriteTelemetryPropertiesTo(map, prefix: "ElapsedTimePerRun.");
+
+                    var producedFileCount = _producedFilesByGenerator.GetStatisticResult(telemetryKey);
+                    producedFileCount.WriteTelemetryPropertiesTo(map, prefix: "GeneratedFileCountPerRun.");
+                }));
+            }
+        }
+    }
+}

src/Workspaces/Core/Portable/Workspace/Solution/SolutionState.CompilationTracker.cs

@@ -18,6 +18,7 @@
 using Microsoft.CodeAnalysis.Logging;
 using Microsoft.CodeAnalysis.PooledObjects;
 using Microsoft.CodeAnalysis.Shared.Collections;
+using Microsoft.CodeAnalysis.SourceGeneratorTelemetry;
 using Roslyn.Utilities;
 
 namespace Microsoft.CodeAnalysis
@@ -861,6 +862,9 @@ public CompilationInfo(Compilation compilation, bool hasSuccessfullyLoaded, Comp
                             // END HACK HACK HACK HACK.
 
                             generatorInfo = generatorInfo.WithDriver(generatorInfo.Driver!.RunGenerators(compilationToRunGeneratorsOn, cancellationToken));
+
+                            solution.Services.GetService<ISourceGeneratorTelemetryCollectorWorkspaceService>()?.CollectRunResult(generatorInfo.Driver!.GetRunResult(), generatorInfo.Driver!.GetTimingInfo());
+
                             var runResult = generatorInfo.Driver!.GetRunResult();
 
                             // We may be able to reuse compilationWithStaleGeneratedTrees if the generated trees are identical. We will assign null

src/Workspaces/Core/Portable/Workspace/Solution/SourceGeneratedDocumentIdentity.cs

@@ -61,6 +61,7 @@ public void WriteTo(ObjectWriter writer)
 
             writer.WriteString(HintName);
             writer.WriteString(Generator.AssemblyName);
+            writer.WriteString(Generator.AssemblyVersion.ToString());
             writer.WriteString(Generator.TypeName);
             writer.WriteString(FilePath);
         }
@@ -71,10 +72,15 @@ internal static SourceGeneratedDocumentIdentity ReadFrom(ObjectReader reader)
 
             var hintName = reader.ReadString();
             var generatorAssemblyName = reader.ReadString();
+            var generatorAssemblyVersion = Version.Parse(reader.ReadString());
             var generatorTypeName = reader.ReadString();
             var filePath = reader.ReadString();
 
-            return new SourceGeneratedDocumentIdentity(documentId, hintName, new SourceGeneratorIdentity(generatorAssemblyName, generatorTypeName), filePath);
+            return new SourceGeneratedDocumentIdentity(
+                documentId,
+                hintName,
+                new SourceGeneratorIdentity(generatorAssemblyName, generatorAssemblyVersion, generatorTypeName),
+                filePath);
         }
     }
 }

src/Workspaces/Core/Portable/Workspace/Solution/SourceGeneratorIdentity.cs

@@ -8,16 +8,16 @@
 
 namespace Microsoft.CodeAnalysis
 {
-    internal record struct SourceGeneratorIdentity(string AssemblyName, string TypeName)
+    internal record struct SourceGeneratorIdentity(string AssemblyName, Version AssemblyVersion, string TypeName)
     {
         public SourceGeneratorIdentity(ISourceGenerator generator)
-            : this(GetGeneratorAssemblyName(generator), GetGeneratorTypeName(generator))
+            : this(GetGeneratorAssemblyName(generator), generator.GetGeneratorType().Assembly.GetName().Version!, GetGeneratorTypeName(generator))
         {
         }
 
         public static string GetGeneratorAssemblyName(ISourceGenerator generator)
         {
-            return generator.GetGeneratorType().Assembly.FullName!;
+            return generator.GetGeneratorType().Assembly.GetName().Name!;
         }
 
         public static string GetGeneratorTypeName(ISourceGenerator generator)