Enable reliable submission scenarios using ReferenceId #144

Source/Core/Exceptionless.Core.csproj

@@ -256,6 +256,7 @@
     <Compile Include="Models\WorkItems\ThrottleBotsWorkItem.cs" />
     <Compile Include="Pipeline\035_CopySimpleDataToIdxAction.cs" />
     <Compile Include="Pipeline\050_MarkProjectConfiguredAction.cs" />
+    <Compile Include="Plugins\EventProcessor\Default\05_CheckForDuplicateReferenceIdPlugin.cs" />
     <Compile Include="Repositories\Base\DocumentChangeEventArgs.cs" />
     <Compile Include="Repositories\Configuration\ElasticSearchConfiguration.cs" />
     <Compile Include="Repositories\Configuration\Indexes\EventIndex.cs" />
@@ -377,11 +378,11 @@
     <Compile Include="Repositories\Options\ElasticSearchOptions.cs" />
     <Compile Include="Repositories\Options\ElasticSearchOptionsExtensions.cs" />
     <Compile Include="Repositories\Options\ElasticSearchPagingOptions.cs" />
-    <Compile Include="Repositories\Base\ElasticSearchReadOnlyRepository.cs" />
-    <Compile Include="Repositories\Base\ElasticSearchRepository.cs" />
-    <Compile Include="Repositories\Base\ElasticSearchRepositoryOwnedByOrganizationAndProject.cs" />
-    <Compile Include="Repositories\Base\ElasticSearchRepositoryOwnedByOrganization.cs" />
-    <Compile Include="Repositories\Base\ElasticSearchRepositoryOwnedByOrganizationAndProjectAndStack.cs" />
+    <Compile Include="Repositories\Base\ReadOnlyRepository.cs" />
+    <Compile Include="Repositories\Base\Repository.cs" />
+    <Compile Include="Repositories\Base\RepositoryOwnedByOrganizationAndProject.cs" />
+    <Compile Include="Repositories\Base\RepositoryOwnedByOrganization.cs" />
+    <Compile Include="Repositories\Base\RepositoryOwnedByOrganizationAndProjectAndStack.cs" />
     <Compile Include="Repositories\EventRepository.cs" />
     <Compile Include="Serialization\DataObjectConverter.cs" />
     <Compile Include="Utility\GenericEventArgs.cs" />

Source/Core/Plugins/EventProcessor/Default/05_CheckForDuplicateReferenceIdPlugin.cs

@@ -0,0 +1,39 @@
+using System;
+using System.Threading.Tasks;
+using Exceptionless.Core.Component;
+using Exceptionless.Core.Extensions;
+using Exceptionless.Core.Pipeline;
+using Foundatio.Caching;
+
+namespace Exceptionless.Core.Plugins.EventProcessor {
+    [Priority(5)]
+    public class CheckForDuplicateReferenceIdPlugin : EventProcessorPluginBase {
+        private readonly ICacheClient _cacheClient;
+
+        public CheckForDuplicateReferenceIdPlugin(ICacheClient cacheClient) {
+            _cacheClient = cacheClient;
+        }
+
+        public override async Task EventProcessingAsync(EventContext context) {
+            if (String.IsNullOrEmpty(context.Event.ReferenceId))
+                return;
+
+            // TODO: Look into using a lock on reference id so we can ensure there is no race conditions with setting keys
+            if (await _cacheClient.AddAsync(GetCacheKey(context), true, TimeSpan.FromMinutes(1)).AnyContext())
+                return;
+
+            context.IsCancelled = true;
+        }
+
+        public override Task EventProcessedAsync(EventContext context) {
+            if (String.IsNullOrEmpty(context.Event.ReferenceId))
+                return TaskHelper.Completed();
+
+            return _cacheClient.SetAsync(GetCacheKey(context), true, TimeSpan.FromDays(1));
+        }
+
+        private string GetCacheKey(EventContext context) {
+            return String.Concat("project:", context.Project.Id, ":", context.Event.ReferenceId);
+        }
+    }
+}

Source/Core/Plugins/EventProcessor/Default/0_ThrottleBotsPlugin.cs

@@ -3,38 +3,29 @@
 using Exceptionless.Core.Extensions;
 using Exceptionless.Core.Models.WorkItems;
 using Exceptionless.Core.Pipeline;
-using Exceptionless.Core.Repositories;
 using Exceptionless.DateTimeExtensions;
 using Foundatio.Caching;
 using Foundatio.Jobs;
 using Foundatio.Logging;
-using Foundatio.Metrics;
 using Foundatio.Queues;
 
 namespace Exceptionless.Core.Plugins.EventProcessor {
     [Priority(0)]
     public class ThrottleBotsPlugin : EventProcessorPluginBase {
         private readonly ICacheClient _cacheClient;
-        private readonly IMetricsClient _metricsClient;
-        private readonly IEventRepository _eventRepository;
-        private readonly IProjectRepository _projectRepository;
         private readonly IQueue<WorkItemData> _workItemQueue;
         private readonly TimeSpan _throttlingPeriod = TimeSpan.FromMinutes(5);
 
-        public ThrottleBotsPlugin(ICacheClient cacheClient, IEventRepository eventRepository, IProjectRepository projectRepository, IMetricsClient metricsClient, IQueue<WorkItemData> workItemQueue) {
+        public ThrottleBotsPlugin(ICacheClient cacheClient, IQueue<WorkItemData> workItemQueue) {
             _cacheClient = cacheClient;
-            _metricsClient = metricsClient;
-            _eventRepository = eventRepository;
-            _projectRepository = projectRepository;
             _workItemQueue = workItemQueue;
         }
 
         public override async Task EventProcessingAsync(EventContext context) {
             if (Settings.Current.WebsiteMode == WebsiteMode.Dev)
                 return;
-
-            var project = await _projectRepository.GetByIdAsync(context.Event.ProjectId).AnyContext();
-            if (project == null || !project.DeleteBotDataEnabled)
+
+            if (!context.Project.DeleteBotDataEnabled)
                 return;
 
             // Throttle errors by client ip address to no more than X every 5 minutes.

Source/Core/Plugins/EventProcessor/EventPluginManager.cs

@@ -29,6 +29,10 @@ public async Task EventProcessingAsync(EventContext context) {
             foreach (var plugin in Plugins.Values) {
                 try {
                     await plugin.EventProcessingAsync(context).AnyContext();
+                    if (context.IsCancelled) {
+                        Logger.Trace().Message($"Event processing was cancelled by plugin: {plugin.GetType().FullName}").Write();
+                        break;
+                    }
                 } catch (Exception ex) {
                     Logger.Error().Message("Error calling event processing in plugin \"{0}\": {1}", plugin.GetType().FullName, ex.Message).Exception(ex).Write();
                 }
@@ -42,6 +46,10 @@ public async Task EventProcessedAsync(EventContext context) {
             foreach (var plugin in Plugins.Values) {
                 try {
                     await plugin.EventProcessedAsync(context).AnyContext();
+                    if (context.IsCancelled) {
+                        Logger.Trace().Message($"Event processed was cancelled by plugin: {plugin.GetType().FullName}").Write();
+                        break;
+                    }
                 } catch (Exception ex) {
                     Logger.Error().Message("Error calling event processed in plugin \"{0}\": {1}", plugin.GetType().FullName, ex.Message).Exception(ex).Write();
                 }

Source/Core/Plugins/WebHook/Default/LoadDefaults.cs

@@ -28,7 +28,7 @@ public override async Task<object> CreateFromEventAsync(WebHookDataContext ctx)
                 throw new ArgumentException("Project not found.");
 
             if (ctx.Organization == null)
-                ctx.Organization = await _organizationRepository.GetByIdAsync(ctx.Event.OrganizationId).AnyContext();
+                ctx.Organization = await _organizationRepository.GetByIdAsync(ctx.Event.OrganizationId, true).AnyContext();
 
             if (ctx.Organization == null)
                 throw new ArgumentException("Organization not found.");
@@ -53,7 +53,7 @@ public override async Task<object> CreateFromStackAsync(WebHookDataContext ctx)
                 throw new ArgumentException("Project not found.");
 
             if (ctx.Organization == null)
-                ctx.Organization = await _organizationRepository.GetByIdAsync(ctx.Stack.OrganizationId).AnyContext();
+                ctx.Organization = await _organizationRepository.GetByIdAsync(ctx.Stack.OrganizationId, true).AnyContext();
 
             if (ctx.Organization == null)
                 throw new ArgumentException("Organization not found.");

Source/Core/Repositories/ApplicationRepository.cs

@@ -7,7 +7,7 @@
 using Nest;
 
 namespace Exceptionless.Core.Repositories {
-    public class ApplicationRepository : ElasticSearchRepositoryOwnedByOrganization<Application>, IApplicationRepository {
+    public class ApplicationRepository : RepositoryOwnedByOrganization<Application>, IApplicationRepository {
         public ApplicationRepository(IElasticClient elasticClient, OrganizationIndex index, IValidator<Application> validator = null, ICacheClient cacheClient = null, IMessagePublisher messagePublisher = null) 
             : base(elasticClient, index, validator, cacheClient, messagePublisher) {}
     }

Source/Core/Repositories/Base/ElasticSearchReadOnlyRepository.cs → Source/Core/Repositories/Base/ReadOnlyRepository.cs

@@ -23,7 +23,7 @@ public FindResults() {
         public long Total { get; set; }
     }
 
-    public abstract class ElasticSearchReadOnlyRepository<T> : IReadOnlyRepository<T> where T : class, IIdentity, new() {
+    public abstract class ReadOnlyRepository<T> : IReadOnlyRepository<T> where T : class, IIdentity, new() {
         protected readonly static bool _supportsSoftDeletes = typeof(ISupportSoftDeletes).IsAssignableFrom(typeof(T));
         private static readonly DateTime MIN_OBJECTID_DATE = new DateTime(2000, 1, 1);
         protected static readonly string _entityType = typeof(T).Name;
@@ -33,7 +33,7 @@ public FindResults() {
         protected readonly IElasticClient _elasticClient;
         protected readonly IElasticSearchIndex _index;
 
-        protected ElasticSearchReadOnlyRepository(IElasticClient elasticClient, IElasticSearchIndex index, ICacheClient cacheClient = null) {
+        protected ReadOnlyRepository(IElasticClient elasticClient, IElasticSearchIndex index, ICacheClient cacheClient = null) {
             _elasticClient = elasticClient;
             _index = index;
             Cache = cacheClient;
@@ -79,7 +79,7 @@ public Task InvalidateCacheAsync(ICollection<T> documents) {
         }
 
         protected string GetScopedCacheKey(string cacheKey) {
-            return String.Concat(GetTypeName(), "-", cacheKey);
+            return String.Concat(GetTypeName(), ":", cacheKey);
         }
 
         protected async Task<FindResults<T>> FindAsync(ElasticSearchOptions<T> options) {
@@ -226,7 +226,7 @@ protected async Task<long> CountAsync(ElasticSearchOptions<T> options) {
                 throw new ArgumentNullException(nameof(options));
 
             if (EnableCache && options.UseCache) {
-                var cachedValue = await Cache.GetAsync<long>(GetScopedCacheKey("count-" + options.CacheKey)).AnyContext();
+                var cachedValue = await Cache.GetAsync<long>(GetScopedCacheKey("count:" + options.CacheKey)).AnyContext();
                 if (cachedValue.HasValue)
                     return cachedValue.Value;
             }
@@ -253,7 +253,7 @@ protected async Task<long> CountAsync(ElasticSearchOptions<T> options) {
                 throw new ApplicationException($"ElasticSearch error code \"{results.ConnectionStatus.HttpStatusCode}\".", results.ConnectionStatus.OriginalException);
 
             if (EnableCache && options.UseCache)
-                await Cache.SetAsync(GetScopedCacheKey("count-" + options.CacheKey), results.Count, options.GetCacheExpirationDate()).AnyContext();
+                await Cache.SetAsync(GetScopedCacheKey("count:" + options.CacheKey), results.Count, options.GetCacheExpirationDate()).AnyContext();
 
             return results.Count;
         }

Source/Core/Repositories/Base/ElasticSearchRepository.cs → Source/Core/Repositories/Base/Repository.cs

@@ -16,7 +16,7 @@
 using DataDictionary = Exceptionless.Core.Models.DataDictionary;
 
 namespace Exceptionless.Core.Repositories {
-    public abstract class ElasticSearchRepository<T> : ElasticSearchReadOnlyRepository<T>, IRepository<T> where T : class, IIdentity, new() {
+    public abstract class Repository<T> : ReadOnlyRepository<T>, IRepository<T> where T : class, IIdentity, new() {
         protected readonly IValidator<T> _validator;
         protected readonly IMessagePublisher _messagePublisher;
         protected readonly static bool _isOwnedByOrganization = typeof(IOwnedByOrganization).IsAssignableFrom(typeof(T));
@@ -25,7 +25,7 @@ namespace Exceptionless.Core.Repositories {
         protected readonly static bool _hasDates = typeof(IHaveDates).IsAssignableFrom(typeof(T));
         protected readonly static bool _hasCreatedDate = typeof(IHaveCreatedDate).IsAssignableFrom(typeof(T));
 
-        protected ElasticSearchRepository(IElasticClient elasticClient, IElasticSearchIndex index, IValidator<T> validator = null, ICacheClient cacheClient = null, IMessagePublisher messagePublisher = null) : base(elasticClient, index, cacheClient) {
+        protected Repository(IElasticClient elasticClient, IElasticSearchIndex index, IValidator<T> validator = null, ICacheClient cacheClient = null, IMessagePublisher messagePublisher = null) : base(elasticClient, index, cacheClient) {
             _validator = validator;
             _messagePublisher = messagePublisher;
         }

Source/Core/Repositories/Base/ElasticSearchRepositoryOwnedByOrganization.cs → Source/Core/Repositories/Base/RepositoryOwnedByOrganization.cs

@@ -11,8 +11,8 @@
 using Nest;
 
 namespace Exceptionless.Core.Repositories {
-    public abstract class ElasticSearchRepositoryOwnedByOrganization<T> : ElasticSearchRepository<T>, IRepositoryOwnedByOrganization<T> where T : class, IOwnedByOrganization, IIdentity, new() {
-        public ElasticSearchRepositoryOwnedByOrganization(IElasticClient elasticClient, IElasticSearchIndex index, IValidator<T> validator = null, ICacheClient cacheClient = null, IMessagePublisher messagePublisher = null) : base(elasticClient, index, validator, cacheClient, messagePublisher) { }
+    public abstract class RepositoryOwnedByOrganization<T> : Repository<T>, IRepositoryOwnedByOrganization<T> where T : class, IOwnedByOrganization, IIdentity, new() {
+        public RepositoryOwnedByOrganization(IElasticClient elasticClient, IElasticSearchIndex index, IValidator<T> validator = null, ICacheClient cacheClient = null, IMessagePublisher messagePublisher = null) : base(elasticClient, index, validator, cacheClient, messagePublisher) { }
 
         public Task<long> CountByOrganizationIdAsync(string organizationId) {
             var options = new ElasticSearchOptions<T>().WithOrganizationId(organizationId);

Source/Core/Repositories/Base/ElasticSearchRepositoryOwnedByOrganizationAndProject.cs → Source/Core/Repositories/Base/RepositoryOwnedByOrganizationAndProject.cs

@@ -10,8 +10,8 @@
 using Nest;
 
 namespace Exceptionless.Core.Repositories {
-    public abstract class ElasticSearchRepositoryOwnedByOrganizationAndProject<T> : ElasticSearchRepositoryOwnedByOrganization<T>, IRepositoryOwnedByProject<T> where T : class, IOwnedByProject, IIdentity, IOwnedByOrganization, new() {
-        public ElasticSearchRepositoryOwnedByOrganizationAndProject(IElasticClient elasticClient, IElasticSearchIndex index, IValidator<T> validator = null, ICacheClient cacheClient = null, IMessagePublisher messagePublisher = null) : base(elasticClient, index, validator, cacheClient, messagePublisher) { }
+    public abstract class RepositoryOwnedByOrganizationAndProject<T> : RepositoryOwnedByOrganization<T>, IRepositoryOwnedByProject<T> where T : class, IOwnedByProject, IIdentity, IOwnedByOrganization, new() {
+        public RepositoryOwnedByOrganizationAndProject(IElasticClient elasticClient, IElasticSearchIndex index, IValidator<T> validator = null, ICacheClient cacheClient = null, IMessagePublisher messagePublisher = null) : base(elasticClient, index, validator, cacheClient, messagePublisher) { }
 
         public virtual Task<FindResults<T>> GetByProjectIdAsync(string projectId, PagingOptions paging = null, bool useCache = false, TimeSpan? expiresIn = null) {
             return FindAsync(new ElasticSearchOptions<T>()

Source/Core/Repositories/Base/ElasticSearchRepositoryOwnedByOrganizationAndProjectAndStack.cs → Source/Core/Repositories/Base/RepositoryOwnedByOrganizationAndProjectAndStack.cs

@@ -10,8 +10,8 @@
 using Nest;
 
 namespace Exceptionless.Core.Repositories {
-    public abstract class ElasticSearchRepositoryOwnedByOrganizationAndProjectAndStack<T> : ElasticSearchRepositoryOwnedByOrganizationAndProject<T>, IRepositoryOwnedByStack<T> where T : class, IOwnedByProject, IIdentity, IOwnedByStack, IOwnedByOrganization, new() {
-        public ElasticSearchRepositoryOwnedByOrganizationAndProjectAndStack(IElasticClient elasticClient, IElasticSearchIndex index, IValidator<T> validator = null, ICacheClient cacheClient = null, IMessagePublisher messagePublisher = null) : base(elasticClient, index, validator, cacheClient, messagePublisher) {}
+    public abstract class RepositoryOwnedByOrganizationAndProjectAndStack<T> : RepositoryOwnedByOrganizationAndProject<T>, IRepositoryOwnedByStack<T> where T : class, IOwnedByProject, IIdentity, IOwnedByStack, IOwnedByOrganization, new() {
+        public RepositoryOwnedByOrganizationAndProjectAndStack(IElasticClient elasticClient, IElasticSearchIndex index, IValidator<T> validator = null, ICacheClient cacheClient = null, IMessagePublisher messagePublisher = null) : base(elasticClient, index, validator, cacheClient, messagePublisher) {}
 
         public virtual Task<FindResults<T>> GetByStackIdAsync(string stackId, PagingOptions paging = null, bool useCache = false, TimeSpan? expiresIn = null) {
             return FindAsync(new ElasticSearchOptions<T>()

Source/Core/Repositories/EventRepository.cs

@@ -11,7 +11,7 @@
 using Nest;
 
 namespace Exceptionless.Core.Repositories {
-    public class EventRepository : ElasticSearchRepositoryOwnedByOrganizationAndProjectAndStack<PersistentEvent>, IEventRepository {
+    public class EventRepository : RepositoryOwnedByOrganizationAndProjectAndStack<PersistentEvent>, IEventRepository {
         public EventRepository(IElasticClient elasticClient, EventIndex index,  IValidator<PersistentEvent> validator = null, IMessagePublisher messagePublisher = null)
             : base(elasticClient, index, validator, null, messagePublisher) {
             EnableCache = false;

Source/Core/Repositories/OrganizationRepository.cs

@@ -15,7 +15,7 @@
 using Nest;
 
 namespace Exceptionless.Core.Repositories {
-    public class OrganizationRepository : ElasticSearchRepository<Organization>, IOrganizationRepository {
+    public class OrganizationRepository : Repository<Organization>, IOrganizationRepository {
         public OrganizationRepository(IElasticClient elasticClient, OrganizationIndex index, IValidator<Organization> validator = null, ICacheClient cacheClient = null, IMessagePublisher messagePublisher = null) : base(elasticClient, index, validator, cacheClient, messagePublisher) { }
 
         public Task<Organization> GetByInviteTokenAsync(string token) {

Source/Core/Repositories/ProjectRepository.cs

@@ -10,7 +10,7 @@
 using Nest;
 
 namespace Exceptionless.Core.Repositories {
-    public class ProjectRepository : ElasticSearchRepositoryOwnedByOrganization<Project>, IProjectRepository {
+    public class ProjectRepository : RepositoryOwnedByOrganization<Project>, IProjectRepository {
         public ProjectRepository(IElasticClient elasticClient, OrganizationIndex index, IValidator<Project> validator = null, ICacheClient cacheClient = null, IMessagePublisher messagePublisher = null) 
             : base(elasticClient, index, validator, cacheClient, messagePublisher) {}
 

Source/Core/Repositories/StackRepository.cs

@@ -12,7 +12,7 @@
 using Nest;
 
 namespace Exceptionless.Core.Repositories {
-    public class StackRepository : ElasticSearchRepositoryOwnedByOrganizationAndProject<Stack>, IStackRepository {
+    public class StackRepository : RepositoryOwnedByOrganizationAndProject<Stack>, IStackRepository {
         private const string STACKING_VERSION = "v2";
         private readonly IEventRepository _eventRepository;
 
@@ -43,7 +43,7 @@ private string GetStackSignatureCacheKey(Stack stack) {
         }
 
         private string GetStackSignatureCacheKey(string projectId, string signatureHash) {
-            return String.Concat(projectId, "-", signatureHash, "-", STACKING_VERSION);
+            return String.Concat(projectId, ":", signatureHash, ":", STACKING_VERSION);
         }
 
         public async Task IncrementEventCounterAsync(string organizationId, string projectId, string stackId, DateTime minOccurrenceDateUtc, DateTime maxOccurrenceDateUtc, int count, bool sendNotifications = true) {

Source/Core/Repositories/TokenRepository.cs

@@ -11,7 +11,7 @@
 using Token = Exceptionless.Core.Models.Token;
 
 namespace Exceptionless.Core.Repositories {
-    public class TokenRepository : ElasticSearchRepositoryOwnedByOrganizationAndProject<Token>, ITokenRepository {
+    public class TokenRepository : RepositoryOwnedByOrganizationAndProject<Token>, ITokenRepository {
         public TokenRepository(IElasticClient elasticClient, OrganizationIndex index, IValidator<Token> validator = null, ICacheClient cacheClient = null, IMessagePublisher messagePublisher = null) 
             : base(elasticClient, index, validator, cacheClient, messagePublisher) {}