Skip to content

Commit 0f09dd6

Browse files
esmadauesmadau
authored
Reapply "track total content length and files indexed (#3318)" (#3346) (#3349)
* Reapply "track total content length and files indexed (#3318)" (#3346) This reverts commit 8a0247d. * add FunctionsPreservedDependencies so openTelemetry can be used / DiagnosticSource loaded as needed by functions host
1 parent 25dc7a6 commit 0f09dd6

File tree

27 files changed

+7165
-3
lines changed

27 files changed

+7165
-3
lines changed

‎src/Microsoft.Health.Dicom.Core/Features/Common/IndexedFileProperties.cs

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
// -------------------------------------------------------------------------------------------------
2+
// Copyright (c) Microsoft Corporation. All rights reserved.
3+
// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
4+
// -------------------------------------------------------------------------------------------------
5+
6+
using System;
7+
8+
namespace Microsoft.Health.Dicom.Core.Features.Common;
9+
10+
/// <summary>
11+
/// Metadata on FileProperty table in database
12+
/// </summary>
13+
public readonly struct IndexedFileProperties : IEquatable<IndexedFileProperties>
14+
{
15+
/// <summary>
16+
/// Total indexed FileProperty in database
17+
/// </summary>
18+
public long TotalIndexed { get; init; }
19+
20+
/// <summary>
21+
/// Total sum of all ContentLength rows in FileProperty table
22+
/// </summary>
23+
public long TotalSum { get; init; }
24+
25+
public override bool Equals(object obj) => obj is IndexedFileProperties other && Equals(other);
26+
27+
public bool Equals(IndexedFileProperties other)
28+
=> TotalIndexed == other.TotalIndexed && TotalSum == other.TotalSum;
29+
30+
public override int GetHashCode()
31+
=> HashCode.Combine(TotalIndexed, TotalSum);
32+
33+
public static bool operator ==(IndexedFileProperties left, IndexedFileProperties right)
34+
=> left.Equals(right);
35+
36+
public static bool operator !=(IndexedFileProperties left, IndexedFileProperties right)
37+
=> !(left == right);
38+
}

‎src/Microsoft.Health.Dicom.Core/Features/Store/IIndexDataStore.cs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,4 +185,11 @@ public interface IIndexDataStore
185185
/// <param name="cancellationToken">The cancellation token.</param>
186186
/// <returns>A task that with list of instance metadata with new watermark.</returns>
187187
Task UpdateFilePropertiesContentLengthAsync(IReadOnlyDictionary<long, FileProperties> filePropertiesByWatermark, CancellationToken cancellationToken = default);
188+
189+
/// <summary>
190+
/// Retrieves total count in FileProperty table and summation of all content length values across FileProperty table.
191+
/// </summary>
192+
/// <param name="cancellationToken">The cancellation token.</param>
193+
/// <returns>A task that gets the count</returns>
194+
Task<IndexedFileProperties> GetIndexedFileMetricsAsync(CancellationToken cancellationToken = default);
188195
}

‎src/Microsoft.Health.Dicom.Core/Features/Telemetry/MetricPointExtensions.cs

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
// -------------------------------------------------------------------------------------------------
2+
// Copyright (c) Microsoft Corporation. All rights reserved.
3+
// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
4+
// -------------------------------------------------------------------------------------------------
5+
6+
using System;
7+
using System.Collections.Generic;
8+
using System.Diagnostics.CodeAnalysis;
9+
using System.Linq;
10+
using OpenTelemetry.Metrics;
11+
12+
namespace Microsoft.Health.Dicom.Core.Features.Telemetry;
13+
14+
/// <summary>
15+
/// Since only enumerators are exposed publicly for working with tags or getting the collection of
16+
/// metrics, these extension facilitate getting both.
17+
/// </summary>
18+
public static class MetricPointExtensions
19+
{
20+
/// <summary>
21+
/// Get tags key value pairs from metric point.
22+
/// </summary>
23+
public static Dictionary<string, object> GetTags(this MetricPoint metricPoint)
24+
{
25+
var tags = new Dictionary<string, object>();
26+
foreach (var pair in metricPoint.Tags)
27+
{
28+
tags.Add(pair.Key, pair.Value);
29+
}
30+
31+
return tags;
32+
}
33+
34+
/// <summary>
35+
/// Get all metrics emitted after flushing.
36+
/// </summary>
37+
[SuppressMessage("Performance", "CA1859: Use concrete types when possible for improved performance", Justification = "Result should be read-only.")]
38+
public static IReadOnlyList<MetricPoint> GetMetricPoints(this ICollection<Metric> exportedItems, string metricName)
39+
{
40+
MetricPointsAccessor accessor = exportedItems
41+
.Single(item => item.Name.Equals(metricName, StringComparison.Ordinal))
42+
.GetMetricPoints();
43+
var metrics = new List<MetricPoint>();
44+
foreach (MetricPoint metricPoint in accessor)
45+
{
46+
metrics.Add(metricPoint);
47+
}
48+
49+
return metrics;
50+
}
51+
}

‎src/Microsoft.Health.Dicom.Core/Microsoft.Health.Dicom.Core.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
<PackageReference Include="Microsoft.Health.Operations" />
3333
<PackageReference Include="Microsoft.IO.RecyclableMemoryStream" />
3434
<PackageReference Include="Newtonsoft.Json" />
35+
<PackageReference Include="OpenTelemetry" />
3536
<PackageReference Include="SixLabors.ImageSharp" />
3637
<PackageReference Include="System.Linq.Async" />
3738
</ItemGroup>

‎src/Microsoft.Health.Dicom.Functions.App/Microsoft.Health.Dicom.Functions.App.csproj

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,29 @@
2626
<ProjectReference Include="..\Microsoft.Health.Dicom.Functions\Microsoft.Health.Dicom.Functions.csproj" />
2727
</ItemGroup>
2828

29+
<!--
30+
The Azure Functions host includes its own assemblies and by default will attempt to prevent additional versions from
31+
being added to its app domain by the function app. However, we can export our own versions of common assemblies by using the build flags
32+
_FunctionsSkipCleanOutput or FunctionsPreservedDependencies. For the list of host dependencies, see here:
33+
https://github.com/Azure/azure-functions-host/blob/dev/tools/ExtensionsMetadataGenerator/test/ExtensionsMetadataGeneratorTests/ExistingRuntimeAssemblies.txt
34+
For the list of assemblies and whether extensions receive their own version by default, see here:
35+
https://github.com/Azure/azure-functions-host/blob/dev/src/WebJobs.Script/runtimeassemblies.json
36+
-->
37+
<ItemGroup>
38+
<FunctionsPreservedDependencies Include="Microsoft.ApplicationInsights.dll" />
39+
<FunctionsPreservedDependencies Include="Microsoft.Azure.WebJobs.Extensions.dll" />
40+
<FunctionsPreservedDependencies Include="Microsoft.IdentityModel.Logging.dll" />
41+
<FunctionsPreservedDependencies Include="Microsoft.IdentityModel.Protocols.dll" />
42+
<FunctionsPreservedDependencies Include="Microsoft.IdentityModel.Protocols.OpenIdConnect.dll" />
43+
<FunctionsPreservedDependencies Include="Microsoft.IdentityModel.Tokens.dll" />
44+
<FunctionsPreservedDependencies Include="System.Diagnostics.DiagnosticSource.dll" />
45+
<FunctionsPreservedDependencies Include="System.IdentityModel.Tokens.Jwt.dll" />
46+
<FunctionsPreservedDependencies Include="System.Memory.Data.dll" />
47+
<FunctionsPreservedDependencies Include="System.Text.Encodings.Web.dll" />
48+
<FunctionsPreservedDependencies Include="System.Text.Json.dll" />
49+
</ItemGroup>
50+
51+
2952
<ItemGroup>
3053
<None Update="host.json">
3154
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>

‎src/Microsoft.Health.Dicom.Functions.App/host.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@
7575
"FirstRetryInterval": "00:01:00",
7676
"MaxNumberOfAttempts": 4
7777
}
78+
},
79+
"IndexMetricsCollection": {
80+
"Frequency": "0 0 * * *"
7881
}
7982
},
8083
"Extensions": {

‎src/Microsoft.Health.Dicom.Functions.App/local.settings.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
"IsEncrypted": false,
33
"Values": {
44
"AzureFunctionsJobHost__DicomFunctions__Indexing__MaxParallelBatches": "1",
5+
"AzureFunctionsJobHost__DicomFunctions__IndexMetricsCollection__Frequency": "0 0 * * *",
56
"AzureFunctionsJobHost__Logging__Console__IsEnabled": "true",
67
"AzureFunctionsJobHost__SqlServer__ConnectionString": "server=(local);Initial Catalog=Dicom;Integrated Security=true;TrustServerCertificate=true",
78
"AzureFunctionsJobHost__BlobStore__ConnectionString": "UseDevelopmentStorage=true",

‎src/Microsoft.Health.Dicom.Functions.UnitTests/IndexMetricsCollection/IndexMetricsCollectionFunctionTests.cs

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
// -------------------------------------------------------------------------------------------------
2+
// Copyright (c) Microsoft Corporation. All rights reserved.
3+
// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
4+
// -------------------------------------------------------------------------------------------------
5+
6+
using System.Threading.Tasks;
7+
using Microsoft.Azure.WebJobs;
8+
using Microsoft.Extensions.Logging.Abstractions;
9+
using Microsoft.Extensions.Options;
10+
using Microsoft.Health.Dicom.Core.Configs;
11+
using Microsoft.Health.Dicom.Core.Features.Common;
12+
using Microsoft.Health.Dicom.Core.Features.Store;
13+
using Microsoft.Health.Dicom.Functions.MetricsCollection;
14+
using NSubstitute;
15+
using Xunit;
16+
17+
namespace Microsoft.Health.Dicom.Functions.UnitTests.IndexMetricsCollection;
18+
19+
public class IndexMetricsCollectionFunctionTests
20+
{
21+
private readonly IndexMetricsCollectionFunction _collectionFunction;
22+
private readonly IIndexDataStore _indexStore;
23+
private readonly TimerInfo _timer;
24+
25+
public IndexMetricsCollectionFunctionTests()
26+
{
27+
_indexStore = Substitute.For<IIndexDataStore>();
28+
_collectionFunction = new IndexMetricsCollectionFunction(
29+
_indexStore,
30+
Options.Create(new FeatureConfiguration { EnableExternalStore = true, }));
31+
_timer = Substitute.For<TimerInfo>(default, default, default);
32+
}
33+
34+
[Fact]
35+
public async Task GivenIndexMetricsCollectionFunction_WhenRun_CollectionExecutedWhenExternalStoreEnabled()
36+
{
37+
_indexStore.GetIndexedFileMetricsAsync().ReturnsForAnyArgs(new IndexedFileProperties());
38+
39+
await _collectionFunction.Run(_timer, NullLogger.Instance);
40+
41+
await _indexStore.ReceivedWithAnyArgs(1).GetIndexedFileMetricsAsync();
42+
}
43+
44+
[Fact]
45+
public async Task GivenIndexMetricsCollectionFunction_WhenRun_CollectionNotExecutedWhenExternalStoreNotEnabled()
46+
{
47+
_indexStore.GetIndexedFileMetricsAsync().ReturnsForAnyArgs(new IndexedFileProperties());
48+
var collectionFunctionWihtoutExternalStore = new IndexMetricsCollectionFunction(
49+
_indexStore,
50+
Options.Create(new FeatureConfiguration { EnableExternalStore = false, }));
51+
52+
await collectionFunctionWihtoutExternalStore.Run(_timer, NullLogger.Instance);
53+
54+
await _indexStore.DidNotReceiveWithAnyArgs().GetIndexedFileMetricsAsync();
55+
}
56+
}

‎src/Microsoft.Health.Dicom.Functions/MetricsCollection/IndexMetricsCollectionFunction.cs

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
// -------------------------------------------------------------------------------------------------
2+
// Copyright (c) Microsoft Corporation. All rights reserved.
3+
// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
4+
// -------------------------------------------------------------------------------------------------
5+
6+
using System.Threading.Tasks;
7+
using EnsureThat;
8+
using Microsoft.Azure.WebJobs;
9+
using Microsoft.Extensions.Logging;
10+
using Microsoft.Extensions.Options;
11+
using Microsoft.Health.Dicom.Core.Configs;
12+
using Microsoft.Health.Dicom.Core.Features.Common;
13+
using Microsoft.Health.Dicom.Core.Features.Store;
14+
using Microsoft.Health.Functions.Extensions;
15+
16+
namespace Microsoft.Health.Dicom.Functions.MetricsCollection;
17+
18+
/// <summary>
19+
/// A function for collecting index metrics
20+
/// </summary>
21+
public class IndexMetricsCollectionFunction
22+
{
23+
private readonly IIndexDataStore _indexDataStore;
24+
private readonly bool _externalStoreEnabled;
25+
private readonly bool _enableDataPartitions;
26+
private const string RunFrequencyVariable = $"%{AzureFunctionsJobHost.RootSectionName}:DicomFunctions:{IndexMetricsCollectionOptions.SectionName}:{nameof(IndexMetricsCollectionOptions.Frequency)}%";
27+
28+
public IndexMetricsCollectionFunction(
29+
IIndexDataStore indexDataStore,
30+
IOptions<FeatureConfiguration> featureConfiguration)
31+
{
32+
EnsureArg.IsNotNull(featureConfiguration, nameof(featureConfiguration));
33+
_indexDataStore = EnsureArg.IsNotNull(indexDataStore, nameof(indexDataStore));
34+
_externalStoreEnabled = featureConfiguration.Value.EnableExternalStore;
35+
_enableDataPartitions = featureConfiguration.Value.EnableDataPartitions;
36+
}
37+
38+
/// <summary>
39+
/// Asynchronously collects index metrics.
40+
/// </summary>
41+
/// <param name="invocationTimer">The timer which tracks the invocation schedule.</param>
42+
/// <param name="log">A diagnostic logger.</param>
43+
/// <returns>A task that represents the asynchronous metrics collection operation.</returns>
44+
[FunctionName(nameof(IndexMetricsCollectionFunction))]
45+
public async Task Run(
46+
[TimerTrigger(RunFrequencyVariable)] TimerInfo invocationTimer,
47+
ILogger log)
48+
{
49+
EnsureArg.IsNotNull(invocationTimer, nameof(invocationTimer));
50+
EnsureArg.IsNotNull(log, nameof(log));
51+
if (!_externalStoreEnabled)
52+
{
53+
log.LogInformation("External store is not enabled. Skipping index metrics collection.");
54+
return;
55+
}
56+
57+
if (invocationTimer.IsPastDue)
58+
{
59+
log.LogWarning("Current function invocation is running late.");
60+
}
61+
IndexedFileProperties indexedFileProperties = await _indexDataStore.GetIndexedFileMetricsAsync();
62+
63+
log.LogInformation(
64+
"DICOM telemetry - TotalFilesIndexed: {TotalFilesIndexed} , TotalByesIndexed: {TotalContentLengthIndexed} , with ExternalStoreEnabled: {ExternalStoreEnabled} and DataPartitionsEnabled: {PartitionsEnabled}",
65+
indexedFileProperties.TotalIndexed,
66+
indexedFileProperties.TotalSum,
67+
_externalStoreEnabled,
68+
_enableDataPartitions);
69+
}
70+
}

‎src/Microsoft.Health.Dicom.Functions/MetricsCollection/IndexMetricsCollectionOptions.cs

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
// -------------------------------------------------------------------------------------------------
2+
// Copyright (c) Microsoft Corporation. All rights reserved.
3+
// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
4+
// -------------------------------------------------------------------------------------------------
5+
6+
using System.ComponentModel.DataAnnotations;
7+
8+
namespace Microsoft.Health.Dicom.Functions.MetricsCollection;
9+
10+
/// <summary>
11+
/// Options on collecting indexing metrics
12+
/// </summary>
13+
public class IndexMetricsCollectionOptions
14+
{
15+
/// <summary>
16+
/// The default section name for <see cref="IndexMetricsCollectionOptions"/> in a configuration.
17+
/// </summary>
18+
public const string SectionName = "IndexMetricsCollection";
19+
20+
/// <summary>
21+
/// Gets or sets the cron expression that indicates how frequently to run the index metrics collection function.
22+
/// </summary>
23+
/// <value>A value cron expression</value>
24+
[Required]
25+
public string Frequency { get; set; }
26+
}

‎src/Microsoft.Health.Dicom.Functions/Registration/ServiceCollectionExtensions.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
using Microsoft.Health.Dicom.Functions.DataCleanup;
2525
using Microsoft.Health.Dicom.Functions.Export;
2626
using Microsoft.Health.Dicom.Functions.Indexing;
27+
using Microsoft.Health.Dicom.Functions.MetricsCollection;
2728
using Microsoft.Health.Dicom.Functions.Update;
2829
using Microsoft.Health.Dicom.SqlServer.Registration;
2930
using Microsoft.Health.Extensions.DependencyInjection;
@@ -67,6 +68,7 @@ public static IDicomFunctionsBuilder ConfigureFunctions(
6768
.AddFunctionsOptions<PurgeHistoryOptions>(configuration, PurgeHistoryOptions.SectionName, isDicomFunction: false)
6869
.AddFunctionsOptions<FeatureConfiguration>(configuration, "DicomServer:Features", isDicomFunction: false)
6970
.AddFunctionsOptions<UpdateOptions>(configuration, UpdateOptions.SectionName)
71+
.AddFunctionsOptions<IndexMetricsCollectionOptions>(configuration, IndexMetricsCollectionOptions.SectionName)
7072
.ConfigureDurableFunctionSerialization()
7173
.AddJsonSerializerOptions(o => o.ConfigureDefaultDicomSettings())
7274
.AddSingleton<UpdateMeter>()

‎src/Microsoft.Health.Dicom.SqlServer/Features/Schema/Migrations/57.diff.sql

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
SET XACT_ABORT ON
2+
3+
BEGIN TRANSACTION
4+
GO
5+
CREATE OR ALTER PROCEDURE dbo.GetIndexedFileMetrics
6+
AS
7+
BEGIN
8+
SET NOCOUNT ON;
9+
SET XACT_ABORT ON;
10+
SELECT TotalIndexedFileCount=COUNT_BIG(*),
11+
TotalIndexedBytes=SUM(ContentLength)
12+
FROM dbo.FileProperty;
13+
END
14+
GO
15+
16+
COMMIT TRANSACTION
17+
18+
IF NOT EXISTS
19+
(
20+
SELECT *
21+
FROM sys.indexes
22+
WHERE NAME = 'IXC_FileProperty_ContentLength'
23+
AND Object_id = OBJECT_ID('dbo.FileProperty')
24+
)
25+
BEGIN
26+
CREATE NONCLUSTERED INDEX IXC_FileProperty_ContentLength ON dbo.FileProperty
27+
(
28+
ContentLength
29+
) WITH (DATA_COMPRESSION = PAGE, ONLINE = ON)
30+
END
31+
GO

0 commit comments

Comments
 (0)