Increment stack event count with separate job and run every 5 seconds…

… instead of every event processed.

Exceptionless.sln.DotSettings

@@ -1,4 +1,5 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+    <s:String x:Key="/Default/CodeInspection/CSharpLanguageProject/LanguageLevel/@EntryValue">CSharp71</s:String>
 	<s:Boolean x:Key="/Default/CodeEditing/GenerateMemberBody/CopyXmlDocumentation/@EntryValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/CodeEditing/Intellisense/CodeCompletion/AutoCompleteBasicCompletion/@EntryValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/CodeEditing/Intellisense/CodeCompletion/AutoCompleteImportCompletion/@EntryValue">True</s:Boolean>

src/Exceptionless.Core/Bootstrapper.cs

@@ -171,6 +171,7 @@ public class Bootstrapper {
 
             container.AddSingleton<UsageService>();
             container.AddSingleton<SlackService>();
+            container.AddSingleton<StackService>();
 
             container.AddTransient<IDomainLoginProvider, ActiveDirectoryLoginProvider>();
 
@@ -256,6 +257,7 @@ public class Bootstrapper {
             new JobRunner(container.GetRequiredService<RetentionLimitsJob>(), loggerFactory, initialDelay: TimeSpan.FromMinutes(15), interval: TimeSpan.FromHours(1)).RunInBackground(token);
             new JobRunner(container.GetRequiredService<WorkItemJob>(), loggerFactory, initialDelay: TimeSpan.FromSeconds(2), instanceCount: 2).RunInBackground(token);
             new JobRunner(container.GetRequiredService<MaintainIndexesJob>(), loggerFactory, initialDelay: SystemClock.UtcNow.Ceiling(TimeSpan.FromHours(1)) - SystemClock.UtcNow, interval: TimeSpan.FromHours(1)).RunInBackground(token);
+            new JobRunner(container.GetRequiredService<StackEventCountJob>(), loggerFactory, initialDelay: TimeSpan.FromSeconds(2), interval: TimeSpan.FromSeconds(5)).RunInBackground(token);
 
             logger.LogWarning("Jobs running in process.");
         }

src/Exceptionless.Core/Jobs/StackEventCountJob.cs

@@ -0,0 +1,33 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Exceptionless.Core.Extensions;
+using Exceptionless.Core.Services;
+using Foundatio.Caching;
+using Foundatio.Jobs;
+using Foundatio.Lock;
+using Microsoft.Extensions.Logging;
+
+namespace Exceptionless.Core.Jobs {
+    [Job(Description = "Update event occurrence count for stacks.", InitialDelay = "2s", Interval = "5s")]
+    public class StackEventCountJob : JobWithLockBase {
+        private readonly StackService _stackService;
+        private readonly ILockProvider _lockProvider;
+
+        public StackEventCountJob(StackService stackService, ICacheClient cacheClient, ILoggerFactory loggerFactory = null) : base(loggerFactory) {
+            _stackService = stackService;
+            _lockProvider = new ThrottlingLockProvider(cacheClient, 1, TimeSpan.FromSeconds(5));
+        }
+
+        protected override async Task<JobResult> RunInternalAsync(JobContext context) {
+            _logger.LogTrace("Start save stack event counts.");
+            await _stackService.SaveStackUsagesAsync(cancellationToken: context.CancellationToken).AnyContext();
+            _logger.LogTrace("Finished save stack event counts.");
+            return JobResult.Success;
+        }
+
+        protected override Task<ILock> GetLockAsync(CancellationToken cancellationToken = default) {
+            return _lockProvider.AcquireAsync(nameof(StackEventCountJob), TimeSpan.FromSeconds(5), new CancellationToken(true));
+        }
+    }
+}

src/Exceptionless.Core/Pipeline/060_UpdateStatsAction.cs

@@ -4,16 +4,16 @@
 using System.Threading.Tasks;
 using Exceptionless.Core.Extensions;
 using Exceptionless.Core.Plugins.EventProcessor;
-using Exceptionless.Core.Repositories;
+using Exceptionless.Core.Services;
 using Microsoft.Extensions.Logging;
 
 namespace Exceptionless.Core.Pipeline {
     [Priority(60)]
     public class UpdateStatsAction : EventPipelineActionBase {
-        private readonly IStackRepository _stackRepository;
+        private readonly StackService _stackService;
 
-        public UpdateStatsAction(IStackRepository stackRepository, ILoggerFactory loggerFactory = null) : base(loggerFactory) {
-            _stackRepository = stackRepository;
+        public UpdateStatsAction(StackService stackService, ILoggerFactory loggerFactory = null) : base(loggerFactory) {
+            _stackService = stackService;
         }
 
         protected override bool IsCritical => true;
@@ -36,7 +36,7 @@ public class UpdateStatsAction : EventPipelineActionBase {
                 int count = stackContexts.Count;
                 DateTime minDate = stackContexts.Min(s => s.Event.Date.UtcDateTime);
                 DateTime maxDate = stackContexts.Max(s => s.Event.Date.UtcDateTime);
-                await _stackRepository.IncrementEventCounterAsync(stackContexts[0].Event.OrganizationId, stackContexts[0].Event.ProjectId, stackGroup.Key, minDate, maxDate, count).AnyContext();
+                await _stackService.IncrementStackUsageAsync(stackContexts[0].Event.OrganizationId, stackContexts[0].Event.ProjectId, stackGroup.Key, minDate, maxDate, count).AnyContext();
 
                 // Update stacks in memory since they are used in notifications.
                 foreach (var ctx in stackContexts) {

src/Exceptionless.Core/Repositories/StackRepository.cs

@@ -76,7 +76,6 @@ public StackRepository(ExceptionlessElasticConfiguration configuration, IEventRe
 ctx._source.total_occurrences += params.count;";
 
             var request = new UpdateRequest<Stack, Stack>(GetIndexById(stackId), ElasticType.Type, stackId) {
-                RetryOnConflict = 1,
                 Script = new InlineScript(script.Replace("\r\n", String.Empty).Replace("    ", " ")) {
                     Params = new Dictionary<string, object>(3) {
                         { "minOccurrenceDateUtc", minOccurrenceDateUtc },

src/Exceptionless.Core/Services/StackService.cs

@@ -0,0 +1,94 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Exceptionless.Core.Extensions;
+using Exceptionless.Core.Repositories;
+using Foundatio.Caching;
+using Foundatio.Utility;
+using Microsoft.Extensions.Logging;
+
+namespace Exceptionless.Core.Services {
+    public class StackService {
+        private readonly ILogger<UsageService> _logger;
+        private readonly IStackRepository _stackRepository;
+        private readonly ICacheClient _cache;
+
+        public StackService(IStackRepository stackRepository, ICacheClient cache, ILoggerFactory loggerFactory = null) {
+            _stackRepository = stackRepository;
+            _cache = cache;
+            _logger = loggerFactory.CreateLogger<UsageService>();
+        }
+
+        public async Task IncrementStackUsageAsync(string organizationId, string projectId, string stackId, DateTime minOccurrenceDateUtc, DateTime maxOccurrenceDateUtc, int count) {
+            if (String.IsNullOrEmpty(organizationId) || String.IsNullOrEmpty(projectId) || String.IsNullOrEmpty(stackId) || count == 0)
+                return;
+            var expireTimeout = TimeSpan.FromDays(1);
+            var tasks = new List<Task>(4);
+
+            string occurenceCountCacheKey = GetStackOccurrenceCountCacheKey(organizationId, projectId, stackId),
+                occurrenceMinDateCacheKey = GetStackOccurrenceMinDateCacheKey(organizationId, projectId, stackId),
+                occurrenceMaxDateCacheKey = GetStackOccurrenceMaxDateCacheKey(organizationId, projectId, stackId),
+                occurrenceSetCacheKey = GetStackOccurrenceSetCacheKey();
+
+            var cachedOccurrenceMinDateUtc = await _cache.GetAsync<DateTime>(occurrenceMinDateCacheKey).AnyContext();
+            if (!cachedOccurrenceMinDateUtc.HasValue || cachedOccurrenceMinDateUtc.IsNull || cachedOccurrenceMinDateUtc.Value > minOccurrenceDateUtc)
+                tasks.Add(_cache.SetAsync(occurrenceMinDateCacheKey, minOccurrenceDateUtc, expireTimeout));
+
+            var cachedOccurrenceMaxDateUtc = await _cache.GetAsync<DateTime>(occurrenceMaxDateCacheKey).AnyContext();
+            if (!cachedOccurrenceMaxDateUtc.HasValue || cachedOccurrenceMinDateUtc.IsNull || cachedOccurrenceMaxDateUtc.Value < maxOccurrenceDateUtc)
+                tasks.Add(_cache.SetAsync(occurrenceMaxDateCacheKey, maxOccurrenceDateUtc, expireTimeout));
+
+            tasks.Add(_cache.IncrementAsync(occurenceCountCacheKey, count, expireTimeout));
+            tasks.Add(_cache.SetAddAsync(occurrenceSetCacheKey, Tuple.Create(organizationId, projectId, stackId), expireTimeout));
+
+            await Task.WhenAll(tasks).AnyContext();
+        }
+
+        public async Task SaveStackUsagesAsync(bool sendNotifications = true, CancellationToken cancellationToken = default) {
+            var occurrenceSetCacheKey = GetStackOccurrenceSetCacheKey();
+            var stackUsageSet = await _cache.GetSetAsync<Tuple<string, string, string>>(occurrenceSetCacheKey).AnyContext();
+            if (!stackUsageSet.HasValue || stackUsageSet.IsNull) return;
+            foreach (var tuple in stackUsageSet.Value) {
+                if (cancellationToken.IsCancellationRequested) break;
+
+                string organizationId = tuple.Item1, projectId = tuple.Item2, stackId = tuple.Item3;
+                string occurenceCountCacheKey = GetStackOccurrenceCountCacheKey(organizationId, projectId, stackId),
+                    occurrenceMinDateCacheKey = GetStackOccurrenceMinDateCacheKey(organizationId, projectId, stackId),
+                    occurrenceMaxDateCacheKey = GetStackOccurrenceMaxDateCacheKey(organizationId, projectId, stackId);
+                var occurrenceCount = await _cache.GetAsync<long>(occurenceCountCacheKey, 0).AnyContext();
+                if (occurrenceCount == 0) return;
+                var occurrenceMinDate = _cache.GetAsync(occurrenceMinDateCacheKey, SystemClock.UtcNow);
+                var occurrenceMaxDate = _cache.GetAsync(occurrenceMaxDateCacheKey, SystemClock.UtcNow);
+
+                await Task.WhenAll(occurrenceMinDate, occurrenceMaxDate).AnyContext();
+
+                var tasks = new List<Task> {
+                    _stackRepository.IncrementEventCounterAsync(organizationId, projectId, stackId, occurrenceMinDate.Result, occurrenceMaxDate.Result, (int)occurrenceCount, sendNotifications),
+                    _cache.RemoveAllAsync(new[] { occurenceCountCacheKey, occurrenceMinDateCacheKey, occurrenceMaxDateCacheKey }),
+                    _cache.SetRemoveAsync(occurrenceSetCacheKey, tuple, TimeSpan.FromHours(24))
+                };
+                await Task.WhenAll(tasks).AnyContext();
+
+                _logger.LogTrace($"Increment event count {occurrenceCount} for organization:{organizationId} project:{projectId} stack:{stackId} with occurrenceMinDate:{occurrenceMinDate.Result} occurrenceMaxDate:{occurrenceMaxDate.Result}");
+            }
+        }
+
+        private string GetStackOccurrenceSetCacheKey() {
+            return "usage:occurrences";
+        }
+
+        private string GetStackOccurrenceCountCacheKey(string organizationId, string projectId, string stackId) {
+            return $"usage:occurrences:count:{organizationId}:{projectId}:{stackId}";
+        }
+
+        private string GetStackOccurrenceMinDateCacheKey(string organizationId, string projectId, string stackId) {
+            return $"usage:occurrences:mindate:{organizationId}:{projectId}:{stackId}";
+        }
+
+        private string GetStackOccurrenceMaxDateCacheKey(string organizationId, string projectId, string stackId) {
+            return $"usage:occurrences:maxdate:{organizationId}:{projectId}:{stackId}";
+        }
+    }
+}

src/Jobs/StackEventCountJob/Program.cs

@@ -0,0 +1,33 @@
+using System;
+using System.Threading.Tasks;
+using Exceptionless;
+using Exceptionless.Insulation.Jobs;
+using Foundatio.Jobs;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Serilog;
+
+namespace StackEventCountJob {
+    public class Program {
+        public static async Task<int> Main() {
+            try {
+                var serviceProvider = JobServiceProvider.GetServiceProvider();
+                var job = serviceProvider.GetService<Exceptionless.Core.Jobs.StackEventCountJob>();
+                return await new JobRunner(
+                    serviceProvider.GetService<Exceptionless.Core.Jobs.StackEventCountJob>(), 
+                    serviceProvider.GetRequiredService<ILoggerFactory>(), 
+                    initialDelay: TimeSpan.FromSeconds(2), 
+                    interval: TimeSpan.FromSeconds(5)
+                ).RunInConsoleAsync();
+            }
+            catch (Exception ex) {
+                Log.Fatal(ex, "Job terminated unexpectedly");
+                return 1;
+            }
+            finally {
+                Log.CloseAndFlush();
+                await ExceptionlessClient.Default.ProcessQueueAsync();
+            }
+        }
+    }
+}

src/Jobs/StackEventCountJob/Properties/launchSettings.json

@@ -0,0 +1,10 @@
+{
+  "profiles": {
+    "StackEventCountJob": {
+      "commandName": "Project",
+      "environmentVariables": {
+        "AppMode": "Development"
+      }
+    }
+  }
+}

src/Jobs/StackEventCountJob/StackEventCountJob.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <Import Project="..\..\..\build\common.props" />
+  <PropertyGroup>
+    <TargetFramework>netcoreapp2.0</TargetFramework>
+    <OutputType>Exe</OutputType>
+    <LangVersion>latest</LangVersion>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\Exceptionless.Core\Exceptionless.Core.csproj" />
+    <ProjectReference Include="..\..\Exceptionless.Insulation\Exceptionless.Insulation.csproj" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="..\run.bat" Link="run.bat" CopyToOutputDirectory="Always" />
+
+    <None Include="..\appsettings.yml" Link="appsettings.yml" CopyToOutputDirectory="Always" />
+    <None Include="..\appsettings.*.yml" Link="%(Filename).yml" DependentUpon="appsettings.yml" CopyToOutputDirectory="Always" />
+  </ItemGroup>
+
+</Project>