Reapply "track total content length and files indexed (#3318)" (#3346) (

#3349)

* Reapply "track total content length and files indexed (#3318)" (#3346)

This reverts commit 8a0247d.

* add FunctionsPreservedDependencies so openTelemetry can be used / DiagnosticSource loaded as needed by functions host

src/Microsoft.Health.Dicom.Core/Features/Common/IndexedFileProperties.cs

@@ -0,0 +1,38 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System;
+
+namespace Microsoft.Health.Dicom.Core.Features.Common;
+
+/// <summary>
+/// Metadata on FileProperty table in database
+/// </summary>
+public readonly struct IndexedFileProperties : IEquatable<IndexedFileProperties>
+{
+    /// <summary>
+    /// Total indexed FileProperty in database
+    /// </summary>
+    public long TotalIndexed { get; init; }
+
+    /// <summary>
+    /// Total sum of all ContentLength rows in FileProperty table
+    /// </summary>
+    public long TotalSum { get; init; }
+
+    public override bool Equals(object obj) => obj is IndexedFileProperties other && Equals(other);
+
+    public bool Equals(IndexedFileProperties other)
+        => TotalIndexed == other.TotalIndexed && TotalSum == other.TotalSum;
+
+    public override int GetHashCode()
+        => HashCode.Combine(TotalIndexed, TotalSum);
+
+    public static bool operator ==(IndexedFileProperties left, IndexedFileProperties right)
+        => left.Equals(right);
+
+    public static bool operator !=(IndexedFileProperties left, IndexedFileProperties right)
+        => !(left == right);
+}

src/Microsoft.Health.Dicom.Core/Features/Store/IIndexDataStore.cs

@@ -185,4 +185,11 @@ public interface IIndexDataStore
     /// <param name="cancellationToken">The cancellation token.</param>
     /// <returns>A task that with list of instance metadata with new watermark.</returns>
     Task UpdateFilePropertiesContentLengthAsync(IReadOnlyDictionary<long, FileProperties> filePropertiesByWatermark, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Retrieves total count in FileProperty table and summation of all content length values across FileProperty table.
+    /// </summary>
+    /// <param name="cancellationToken">The cancellation token.</param>
+    /// <returns>A task that gets the count</returns>
+    Task<IndexedFileProperties> GetIndexedFileMetricsAsync(CancellationToken cancellationToken = default);
 }

src/Microsoft.Health.Dicom.Core/Features/Telemetry/MetricPointExtensions.cs

@@ -0,0 +1,51 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+using OpenTelemetry.Metrics;
+
+namespace Microsoft.Health.Dicom.Core.Features.Telemetry;
+
+/// <summary>
+/// Since only enumerators are exposed publicly for working with tags or getting the collection of
+/// metrics, these extension facilitate getting both.
+/// </summary>
+public static class MetricPointExtensions
+{
+    /// <summary>
+    /// Get tags key value pairs from metric point.
+    /// </summary>
+    public static Dictionary<string, object> GetTags(this MetricPoint metricPoint)
+    {
+        var tags = new Dictionary<string, object>();
+        foreach (var pair in metricPoint.Tags)
+        {
+            tags.Add(pair.Key, pair.Value);
+        }
+
+        return tags;
+    }
+
+    /// <summary>
+    /// Get all metrics emitted after flushing.
+    /// </summary>
+    [SuppressMessage("Performance", "CA1859: Use concrete types when possible for improved performance", Justification = "Result should be read-only.")]
+    public static IReadOnlyList<MetricPoint> GetMetricPoints(this ICollection<Metric> exportedItems, string metricName)
+    {
+        MetricPointsAccessor accessor = exportedItems
+            .Single(item => item.Name.Equals(metricName, StringComparison.Ordinal))
+            .GetMetricPoints();
+        var metrics = new List<MetricPoint>();
+        foreach (MetricPoint metricPoint in accessor)
+        {
+            metrics.Add(metricPoint);
+        }
+
+        return metrics;
+    }
+}

src/Microsoft.Health.Dicom.Core/Microsoft.Health.Dicom.Core.csproj

@@ -32,6 +32,7 @@
     <PackageReference Include="Microsoft.Health.Operations" />
     <PackageReference Include="Microsoft.IO.RecyclableMemoryStream" />
     <PackageReference Include="Newtonsoft.Json" />
+    <PackageReference Include="OpenTelemetry" />
     <PackageReference Include="SixLabors.ImageSharp" />
     <PackageReference Include="System.Linq.Async" />
   </ItemGroup>

src/Microsoft.Health.Dicom.Functions.App/Microsoft.Health.Dicom.Functions.App.csproj

@@ -26,6 +26,29 @@
     <ProjectReference Include="..\Microsoft.Health.Dicom.Functions\Microsoft.Health.Dicom.Functions.csproj" />
   </ItemGroup>
 
+  <!--
+  The Azure Functions host includes its own assemblies and by default will attempt to prevent additional versions from
+  being added to its app domain by the function app. However, we can export our own versions of common assemblies by using the build flags
+  _FunctionsSkipCleanOutput or FunctionsPreservedDependencies. For the list of host dependencies, see here:
+  https://github.com/Azure/azure-functions-host/blob/dev/tools/ExtensionsMetadataGenerator/test/ExtensionsMetadataGeneratorTests/ExistingRuntimeAssemblies.txt
+  For the list of assemblies and whether extensions receive their own version by default, see here:
+  https://github.com/Azure/azure-functions-host/blob/dev/src/WebJobs.Script/runtimeassemblies.json
+-->
+  <ItemGroup>
+    <FunctionsPreservedDependencies Include="Microsoft.ApplicationInsights.dll" />
+    <FunctionsPreservedDependencies Include="Microsoft.Azure.WebJobs.Extensions.dll" />
+    <FunctionsPreservedDependencies Include="Microsoft.IdentityModel.Logging.dll" />
+    <FunctionsPreservedDependencies Include="Microsoft.IdentityModel.Protocols.dll" />
+    <FunctionsPreservedDependencies Include="Microsoft.IdentityModel.Protocols.OpenIdConnect.dll" />
+    <FunctionsPreservedDependencies Include="Microsoft.IdentityModel.Tokens.dll" />
+    <FunctionsPreservedDependencies Include="System.Diagnostics.DiagnosticSource.dll" />
+    <FunctionsPreservedDependencies Include="System.IdentityModel.Tokens.Jwt.dll" />
+    <FunctionsPreservedDependencies Include="System.Memory.Data.dll" />
+    <FunctionsPreservedDependencies Include="System.Text.Encodings.Web.dll" />
+    <FunctionsPreservedDependencies Include="System.Text.Json.dll" />
+  </ItemGroup>
+
+
   <ItemGroup>
     <None Update="host.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>

src/Microsoft.Health.Dicom.Functions.App/host.json

@@ -75,6 +75,9 @@
         "FirstRetryInterval": "00:01:00",
         "MaxNumberOfAttempts": 4
       }
+    },
+    "IndexMetricsCollection": {
+      "Frequency": "0 0 * * *"
     }
   },
   "Extensions": {

src/Microsoft.Health.Dicom.Functions.App/local.settings.json

@@ -2,6 +2,7 @@
   "IsEncrypted": false,
   "Values": {
     "AzureFunctionsJobHost__DicomFunctions__Indexing__MaxParallelBatches": "1",
+    "AzureFunctionsJobHost__DicomFunctions__IndexMetricsCollection__Frequency": "0 0 * * *",
     "AzureFunctionsJobHost__Logging__Console__IsEnabled": "true",
     "AzureFunctionsJobHost__SqlServer__ConnectionString": "server=(local);Initial Catalog=Dicom;Integrated Security=true;TrustServerCertificate=true",
     "AzureFunctionsJobHost__BlobStore__ConnectionString": "UseDevelopmentStorage=true",

src/Microsoft.Health.Dicom.Functions.UnitTests/IndexMetricsCollection/IndexMetricsCollectionFunctionTests.cs

@@ -0,0 +1,56 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System.Threading.Tasks;
+using Microsoft.Azure.WebJobs;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using Microsoft.Health.Dicom.Core.Configs;
+using Microsoft.Health.Dicom.Core.Features.Common;
+using Microsoft.Health.Dicom.Core.Features.Store;
+using Microsoft.Health.Dicom.Functions.MetricsCollection;
+using NSubstitute;
+using Xunit;
+
+namespace Microsoft.Health.Dicom.Functions.UnitTests.IndexMetricsCollection;
+
+public class IndexMetricsCollectionFunctionTests
+{
+    private readonly IndexMetricsCollectionFunction _collectionFunction;
+    private readonly IIndexDataStore _indexStore;
+    private readonly TimerInfo _timer;
+
+    public IndexMetricsCollectionFunctionTests()
+    {
+        _indexStore = Substitute.For<IIndexDataStore>();
+        _collectionFunction = new IndexMetricsCollectionFunction(
+            _indexStore,
+            Options.Create(new FeatureConfiguration { EnableExternalStore = true, }));
+        _timer = Substitute.For<TimerInfo>(default, default, default);
+    }
+
+    [Fact]
+    public async Task GivenIndexMetricsCollectionFunction_WhenRun_CollectionExecutedWhenExternalStoreEnabled()
+    {
+        _indexStore.GetIndexedFileMetricsAsync().ReturnsForAnyArgs(new IndexedFileProperties());
+
+        await _collectionFunction.Run(_timer, NullLogger.Instance);
+
+        await _indexStore.ReceivedWithAnyArgs(1).GetIndexedFileMetricsAsync();
+    }
+
+    [Fact]
+    public async Task GivenIndexMetricsCollectionFunction_WhenRun_CollectionNotExecutedWhenExternalStoreNotEnabled()
+    {
+        _indexStore.GetIndexedFileMetricsAsync().ReturnsForAnyArgs(new IndexedFileProperties());
+        var collectionFunctionWihtoutExternalStore = new IndexMetricsCollectionFunction(
+            _indexStore,
+            Options.Create(new FeatureConfiguration { EnableExternalStore = false, }));
+
+        await collectionFunctionWihtoutExternalStore.Run(_timer, NullLogger.Instance);
+
+        await _indexStore.DidNotReceiveWithAnyArgs().GetIndexedFileMetricsAsync();
+    }
+}

src/Microsoft.Health.Dicom.Functions/MetricsCollection/IndexMetricsCollectionFunction.cs

@@ -0,0 +1,70 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System.Threading.Tasks;
+using EnsureThat;
+using Microsoft.Azure.WebJobs;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Microsoft.Health.Dicom.Core.Configs;
+using Microsoft.Health.Dicom.Core.Features.Common;
+using Microsoft.Health.Dicom.Core.Features.Store;
+using Microsoft.Health.Functions.Extensions;
+
+namespace Microsoft.Health.Dicom.Functions.MetricsCollection;
+
+/// <summary>
+///  A function for collecting index metrics
+/// </summary>
+public class IndexMetricsCollectionFunction
+{
+    private readonly IIndexDataStore _indexDataStore;
+    private readonly bool _externalStoreEnabled;
+    private readonly bool _enableDataPartitions;
+    private const string RunFrequencyVariable = $"%{AzureFunctionsJobHost.RootSectionName}:DicomFunctions:{IndexMetricsCollectionOptions.SectionName}:{nameof(IndexMetricsCollectionOptions.Frequency)}%";
+
+    public IndexMetricsCollectionFunction(
+        IIndexDataStore indexDataStore,
+        IOptions<FeatureConfiguration> featureConfiguration)
+    {
+        EnsureArg.IsNotNull(featureConfiguration, nameof(featureConfiguration));
+        _indexDataStore = EnsureArg.IsNotNull(indexDataStore, nameof(indexDataStore));
+        _externalStoreEnabled = featureConfiguration.Value.EnableExternalStore;
+        _enableDataPartitions = featureConfiguration.Value.EnableDataPartitions;
+    }
+
+    /// <summary>
+    /// Asynchronously collects index metrics.
+    /// </summary>
+    /// <param name="invocationTimer">The timer which tracks the invocation schedule.</param>
+    /// <param name="log">A diagnostic logger.</param>
+    /// <returns>A task that represents the asynchronous metrics collection operation.</returns>
+    [FunctionName(nameof(IndexMetricsCollectionFunction))]
+    public async Task Run(
+        [TimerTrigger(RunFrequencyVariable)] TimerInfo invocationTimer,
+        ILogger log)
+    {
+        EnsureArg.IsNotNull(invocationTimer, nameof(invocationTimer));
+        EnsureArg.IsNotNull(log, nameof(log));
+        if (!_externalStoreEnabled)
+        {
+            log.LogInformation("External store is not enabled. Skipping index metrics collection.");
+            return;
+        }
+
+        if (invocationTimer.IsPastDue)
+        {
+            log.LogWarning("Current function invocation is running late.");
+        }
+        IndexedFileProperties indexedFileProperties = await _indexDataStore.GetIndexedFileMetricsAsync();
+
+        log.LogInformation(
+            "DICOM telemetry - TotalFilesIndexed: {TotalFilesIndexed} , TotalByesIndexed: {TotalContentLengthIndexed} ,  with ExternalStoreEnabled: {ExternalStoreEnabled} and DataPartitionsEnabled: {PartitionsEnabled}",
+            indexedFileProperties.TotalIndexed,
+            indexedFileProperties.TotalSum,
+            _externalStoreEnabled,
+            _enableDataPartitions);
+    }
+}

src/Microsoft.Health.Dicom.Functions/MetricsCollection/IndexMetricsCollectionOptions.cs

@@ -0,0 +1,26 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System.ComponentModel.DataAnnotations;
+
+namespace Microsoft.Health.Dicom.Functions.MetricsCollection;
+
+/// <summary>
+/// Options on collecting indexing metrics
+/// </summary>
+public class IndexMetricsCollectionOptions
+{
+    /// <summary>
+    /// The default section name for <see cref="IndexMetricsCollectionOptions"/> in a configuration.
+    /// </summary>
+    public const string SectionName = "IndexMetricsCollection";
+
+    /// <summary>
+    /// Gets or sets the cron expression that indicates how frequently to run the index metrics collection function.
+    /// </summary>
+    /// <value>A value cron expression</value>
+    [Required]
+    public string Frequency { get; set; }
+}

src/Microsoft.Health.Dicom.Functions/Registration/ServiceCollectionExtensions.cs

@@ -24,6 +24,7 @@
 using Microsoft.Health.Dicom.Functions.DataCleanup;
 using Microsoft.Health.Dicom.Functions.Export;
 using Microsoft.Health.Dicom.Functions.Indexing;
+using Microsoft.Health.Dicom.Functions.MetricsCollection;
 using Microsoft.Health.Dicom.Functions.Update;
 using Microsoft.Health.Dicom.SqlServer.Registration;
 using Microsoft.Health.Extensions.DependencyInjection;
@@ -67,6 +68,7 @@ public static IDicomFunctionsBuilder ConfigureFunctions(
             .AddFunctionsOptions<PurgeHistoryOptions>(configuration, PurgeHistoryOptions.SectionName, isDicomFunction: false)
             .AddFunctionsOptions<FeatureConfiguration>(configuration, "DicomServer:Features", isDicomFunction: false)
             .AddFunctionsOptions<UpdateOptions>(configuration, UpdateOptions.SectionName)
+            .AddFunctionsOptions<IndexMetricsCollectionOptions>(configuration, IndexMetricsCollectionOptions.SectionName)
             .ConfigureDurableFunctionSerialization()
             .AddJsonSerializerOptions(o => o.ConfigureDefaultDicomSettings())
             .AddSingleton<UpdateMeter>()

src/Microsoft.Health.Dicom.SqlServer/Features/Schema/Migrations/57.diff.sql

@@ -0,0 +1,31 @@
+SET XACT_ABORT ON
+
+BEGIN TRANSACTION
+GO
+CREATE OR ALTER PROCEDURE dbo.GetIndexedFileMetrics
+    AS
+BEGIN
+    SET NOCOUNT ON;
+    SET XACT_ABORT ON;
+SELECT TotalIndexedFileCount=COUNT_BIG(*),
+       TotalIndexedBytes=SUM(ContentLength)
+FROM   dbo.FileProperty;
+END
+GO
+
+COMMIT TRANSACTION
+
+IF NOT EXISTS 
+(
+    SELECT *
+    FROM    sys.indexes
+    WHERE   NAME = 'IXC_FileProperty_ContentLength'
+        AND Object_id = OBJECT_ID('dbo.FileProperty')
+)
+BEGIN
+    CREATE NONCLUSTERED INDEX IXC_FileProperty_ContentLength ON dbo.FileProperty
+    (
+    ContentLength
+    ) WITH (DATA_COMPRESSION = PAGE, ONLINE = ON)
+END
+GO