Merge pull request #910 from NuGet/master

Merge branch master into dev
NuGet · Aug 11, 2020 · 46ebdab · 46ebdab
2 parents 693f255 + 02defbb
commit 46ebdab
Showing 1 changed file with 18 additions and 15 deletions.
diff --git a/src/Stats.ImportAzureCdnStatistics/Warehouse.cs b/src/Stats.ImportAzureCdnStatistics/Warehouse.cs
@@ -17,7 +17,7 @@ namespace Stats.ImportAzureCdnStatistics
     internal class Warehouse
         : IStatisticsWarehouse
     {
-        private const int _defaultCommandTimeout = 1800; // 30 minutes max
+        private const int _defaultCommandTimeout = 3600; // 60 minutes max
         private const int _maxRetryCount = 3;
         private readonly TimeSpan _retryDelay = TimeSpan.FromSeconds(5);
         private readonly ILogger _logger;
@@ -149,6 +149,7 @@ public async Task<DataTable> CreateAsync(IReadOnlyCollection<PackageStatistics>
             }
 
             _logger.LogDebug("Creating facts...");
+            var stopwatchCreatingFacts = Stopwatch.StartNew();
             foreach (var groupedByPackageId in sourceData.GroupBy(e => e.PackageId, StringComparer.OrdinalIgnoreCase))
             {
                 var packagesForId = packages.Where(e => string.Equals(e.PackageId, groupedByPackageId.Key, StringComparison.OrdinalIgnoreCase)).ToList();
@@ -167,7 +168,7 @@ public async Task<DataTable> CreateAsync(IReadOnlyCollection<PackageStatistics>
                     }
 
                     var packageId = package.Id;
-                    var dimensionIdsDictionary = new Dictionary<Tuple<int, int, int, int, int, int>, int>();
+                    var dimensionIdsDictionary = new Dictionary<(int, int, int, int, int, int), int>();
                     foreach (var element in groupedByPackageIdAndVersion)
                     {
                         // required dimensions
@@ -176,6 +177,7 @@ public async Task<DataTable> CreateAsync(IReadOnlyCollection<PackageStatistics>
 
                         // dimensions that could be "(unknown)"
                         int operationId = DimensionId.Unknown;
+
                         if (knownOperationsAvailable && operations.ContainsKey(element.Operation))
                         {
                             operationId = operations[element.Operation];
@@ -203,7 +205,7 @@ public async Task<DataTable> CreateAsync(IReadOnlyCollection<PackageStatistics>
                             }
                         }
 
-                        var dimensionIds = Tuple.Create(dateId, timeId, operationId, platformId, clientId, userAgentId);
+                        var dimensionIds = (dateId, timeId, operationId, platformId, clientId, userAgentId);
                         if (dimensionIdsDictionary.ContainsKey(dimensionIds))
                         {
                             dimensionIdsDictionary[dimensionIds] += 1;
@@ -214,27 +216,28 @@ public async Task<DataTable> CreateAsync(IReadOnlyCollection<PackageStatistics>
                         }
                     }
 
-                    foreach (var dimensionIds in dimensionIdsDictionary.Keys)
+                    foreach (var dimensionIds in dimensionIdsDictionary)
                     {
+                        // create fact
                         var dataRow = factsDataTable.NewRow();
-                        var dateId = dimensionIds.Item1;
-                        var timeId = dimensionIds.Item2;
-                        var operationId = dimensionIds.Item3;
-                        var platformId = dimensionIds.Item4;
-                        var clientId = dimensionIds.Item5;
-                        var userAgentId = dimensionIds.Item6;
-                        var downloadCount = dimensionIdsDictionary[dimensionIds];
-                        FillDataRow(dataRow, dateId, timeId, packageId, operationId, platformId, clientId, userAgentId, logFileNameId, downloadCount);
+
+                        (int dateId, int timeId, int operationId, int platformId, int clientId, int userAgentId) key = dimensionIds.Key;
+                        var downloadCount = dimensionIds.Value;
+
+                        FillDataRow(dataRow, key.dateId, key.timeId, packageId, key.operationId, key.platformId, key.clientId, key.userAgentId, logFileNameId, downloadCount);
                         factsDataTable.Rows.Add(dataRow);
 
-                        _logger.LogDebug("Inserted 1 row into factsDataTable, which counts for {downloadCount} downloads, with the dimension Ids (" +
-                            "dateId: {dateId}, timeId: {timeId}, packageId: {packageId}, operationId: {operationId}, platformId: {platformId}, clientId: {clientId}, " +
-                            "userAgentId: {userAgentId}, logFileNameId: {logFileNameId}).", downloadCount, dateId, timeId, packageId, operationId, platformId, clientId, userAgentId, logFileNameId);
+                        _logger.LogDebug("Inserted 1 row into factsDataTable, which counts for {DownloadCount} downloads, with the dimension Ids (" +
+                            "dateId: {DateId}, timeId: {TimeId}, packageId: {PackageId}, operationId: {OperationId}, platformId: {PlatformId}, clientId: {ClientId}, " +
+                            "userAgentId: {UserAgentId}, logFileNameId: {LogFileNameId}).", downloadCount, key.dateId, key.timeId, packageId, key.operationId,
+                            key.platformId, key.clientId, key.userAgentId, logFileNameId);
                     }
                 }
             }
+            stopwatchCreatingFacts.Stop();
             stopwatch.Stop();
             _logger.LogDebug("  DONE (" + factsDataTable.Rows.Count + " facts, " + stopwatch.ElapsedMilliseconds + "ms)");
+            _applicationInsightsHelper.TrackMetric("Facts creation time (ms)", stopwatchCreatingFacts.ElapsedMilliseconds, logFileName);
             _applicationInsightsHelper.TrackMetric("Blob record count", factsDataTable.Rows.Count, logFileName);
 
             return factsDataTable;