From 321fab90978c81876872e80766e734f2db8d39ec Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 21 Jan 2016 15:22:43 -0600 Subject: [PATCH 1/5] #178 Added the ability to pass the clients submission ip address into the events request info --- .../Extensions/PersistentEventExtensions.cs | 19 +++--- Source/Core/Extensions/StringExtensions.cs | 9 ++- Source/Core/Jobs/EventPostsJob.cs | 2 +- Source/Core/Pipeline/EventPipeline.cs | 9 +-- .../Default/40_RequestInfoPlugin.cs | 59 ++++++++++++------- .../Plugins/EventProcessor/EventContext.cs | 5 +- 6 files changed, 65 insertions(+), 38 deletions(-) diff --git a/Source/Core/Extensions/PersistentEventExtensions.cs b/Source/Core/Extensions/PersistentEventExtensions.cs index 9364badfee..4f14eb5c47 100644 --- a/Source/Core/Extensions/PersistentEventExtensions.cs +++ b/Source/Core/Extensions/PersistentEventExtensions.cs @@ -223,16 +223,17 @@ public static IEnumerable GetIpAddresses(this PersistentEvent ev) { if (!String.IsNullOrEmpty(ev.Geo) && (ev.Geo.Contains(".") || ev.Geo.Contains(":"))) yield return ev.Geo; - var request = ev.GetRequestInfo(); - if (!String.IsNullOrEmpty(request?.ClientIpAddress)) - yield return request.ClientIpAddress; - - var environmentInfo = ev.GetEnvironmentInfo(); - if (String.IsNullOrEmpty(environmentInfo?.IpAddress)) - yield break; + var ri = ev.GetRequestInfo(); + if (!String.IsNullOrEmpty(ri?.ClientIpAddress)) { + foreach (var ip in ri.ClientIpAddress.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)) + yield return ip; + } - foreach (var ip in environmentInfo.IpAddress.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)) - yield return ip; + var ei = ev.GetEnvironmentInfo(); + if (!String.IsNullOrEmpty(ei?.IpAddress)) { + foreach (var ip in ei.IpAddress.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)) + yield return ip; + } } private static bool IsValidIdentifier(string value) { diff --git a/Source/Core/Extensions/StringExtensions.cs b/Source/Core/Extensions/StringExtensions.cs index d9eaa478b4..8ef5411838 100644 --- a/Source/Core/Extensions/StringExtensions.cs +++ b/Source/Core/Extensions/StringExtensions.cs @@ -8,11 +8,18 @@ namespace Exceptionless.Core.Extensions { public static class StringExtensions { + public static bool IsLocalHost(this string ip) { + if (String.IsNullOrEmpty(ip)) + return false; + + return String.Equals(ip, "::1") || String.Equals(ip, "127.0.0.1"); + } + public static bool IsPrivateNetwork(this string ip) { if (String.IsNullOrEmpty(ip)) return false; - if (String.Equals(ip, "::1") || String.Equals(ip, "127.0.0.1")) + if (ip.IsLocalHost()) return true; // 10.0.0.0 – 10.255.255.255 (Class A) diff --git a/Source/Core/Jobs/EventPostsJob.cs b/Source/Core/Jobs/EventPostsJob.cs index d5341fb593..135e8adc31 100644 --- a/Source/Core/Jobs/EventPostsJob.cs +++ b/Source/Core/Jobs/EventPostsJob.cs @@ -102,7 +102,7 @@ protected override async Task ProcessQueueEntryAsync(JobQueueEntryCon var created = DateTime.UtcNow; try { events.ForEach(e => e.CreatedUtc = created); - var results = await _eventPipeline.RunAsync(events.Take(eventsToProcess).ToList()).AnyContext(); + var results = await _eventPipeline.RunAsync(events.Take(eventsToProcess).ToList(), eventPostInfo).AnyContext(); Logger.Info().Message("Ran {0} events through the pipeline: id={1} project={2} success={3} error={4}", results.Count, queueEntry.Id, eventPostInfo.ProjectId, results.Count(r => r.IsProcessed), results.Count(r => r.HasError)).WriteIf(!isInternalProject); foreach (var eventContext in results) { if (eventContext.IsCancelled) diff --git a/Source/Core/Pipeline/EventPipeline.cs b/Source/Core/Pipeline/EventPipeline.cs index 1412e0c954..cd0daeadff 100644 --- a/Source/Core/Pipeline/EventPipeline.cs +++ b/Source/Core/Pipeline/EventPipeline.cs @@ -9,6 +9,7 @@ using Exceptionless.Core.Repositories; using Exceptionless.Core.Models; using Exceptionless.Core.Helpers; +using Exceptionless.Core.Queues.Models; using Exceptionless.Core.Repositories.Base; using Foundatio.Metrics; @@ -24,12 +25,12 @@ public EventPipeline(IDependencyResolver dependencyResolver, IOrganizationReposi _metricsClient = metricsClient; } - public Task RunAsync(PersistentEvent ev) { - return RunAsync(new EventContext(ev)); + public Task RunAsync(PersistentEvent ev, EventPostInfo epi = null) { + return RunAsync(new EventContext(ev, epi)); } - public Task> RunAsync(IEnumerable events) { - return RunAsync(events.Select(ev => new EventContext(ev)).ToList()); + public Task> RunAsync(IEnumerable events, EventPostInfo epi = null) { + return RunAsync(events.Select(ev => new EventContext(ev, epi)).ToList()); } public override async Task> RunAsync(ICollection contexts) { diff --git a/Source/Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs b/Source/Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs index 4e4fdf038b..61fa234a9c 100644 --- a/Source/Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs +++ b/Source/Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs @@ -40,36 +40,51 @@ public override async Task EventBatchProcessingAsync(ICollection c if (request == null) continue; - var info = await _parser.ParseAsync(request.UserAgent, context.Project.Id).AnyContext(); - if (info != null) { - if (!String.Equals(info.UserAgent.Family, "Other")) { - request.Data[RequestInfo.KnownDataKeys.Browser] = info.UserAgent.Family; - if (!String.IsNullOrEmpty(info.UserAgent.Major)) { - request.Data[RequestInfo.KnownDataKeys.BrowserVersion] = String.Join(".", new[] { info.UserAgent.Major, info.UserAgent.Minor, info.UserAgent.Patch }.Where(v => !String.IsNullOrEmpty(v))); - request.Data[RequestInfo.KnownDataKeys.BrowserMajorVersion] = info.UserAgent.Major; - } - } + SetClientIPAddress(request, context.EventPostInfo?.IpAddress); + await SetBrowserOsAndDeviceFromUserAgent(request, context); + + context.Event.AddRequestInfo(request.ApplyDataExclusions(exclusions, MAX_VALUE_LENGTH)); + } + } + + private void SetClientIPAddress(RequestInfo request, string clientIPAddress) { + if (String.IsNullOrEmpty(clientIPAddress)) + return; - if (!String.Equals(info.Device.Family, "Other")) - request.Data[RequestInfo.KnownDataKeys.Device] = info.Device.Family; + if (clientIPAddress.IsLocalHost()) + return; + if (String.IsNullOrWhiteSpace(request.ClientIpAddress)) + request.ClientIpAddress = clientIPAddress; + else if (!request.ClientIpAddress.Contains(clientIPAddress)) + request.ClientIpAddress += String.Concat(",", clientIPAddress); + } - if (!String.Equals(info.OS.Family, "Other")) { - request.Data[RequestInfo.KnownDataKeys.OS] = info.OS.Family; - if (!String.IsNullOrEmpty(info.OS.Major)) { - request.Data[RequestInfo.KnownDataKeys.OSVersion] = String.Join(".", new[] { info.OS.Major, info.OS.Minor, info.OS.Patch }.Where(v => !String.IsNullOrEmpty(v))); - request.Data[RequestInfo.KnownDataKeys.OSMajorVersion] = info.OS.Major; - } + private async Task SetBrowserOsAndDeviceFromUserAgent(RequestInfo request, EventContext context) { + var info = await _parser.ParseAsync(request.UserAgent, context.Project.Id).AnyContext(); + if (info != null) { + if (!String.Equals(info.UserAgent.Family, "Other")) { + request.Data[RequestInfo.KnownDataKeys.Browser] = info.UserAgent.Family; + if (!String.IsNullOrEmpty(info.UserAgent.Major)) { + request.Data[RequestInfo.KnownDataKeys.BrowserVersion] = String.Join(".", new[] { info.UserAgent.Major, info.UserAgent.Minor, info.UserAgent.Patch }.Where(v => !String.IsNullOrEmpty(v))); + request.Data[RequestInfo.KnownDataKeys.BrowserMajorVersion] = info.UserAgent.Major; } + } - var botPatterns = context.Project.Configuration.Settings.ContainsKey(SettingsDictionary.KnownKeys.UserAgentBotPatterns) - ? context.Project.Configuration.Settings.GetStringCollection(SettingsDictionary.KnownKeys.UserAgentBotPatterns).ToList() - : new List(); + if (!String.Equals(info.Device.Family, "Other")) + request.Data[RequestInfo.KnownDataKeys.Device] = info.Device.Family; - request.Data[RequestInfo.KnownDataKeys.IsBot] = info.Device.IsSpider || request.UserAgent.AnyWildcardMatches(botPatterns); + if (!String.Equals(info.OS.Family, "Other")) { + request.Data[RequestInfo.KnownDataKeys.OS] = info.OS.Family; + if (!String.IsNullOrEmpty(info.OS.Major)) { + request.Data[RequestInfo.KnownDataKeys.OSVersion] = String.Join(".", new[] { info.OS.Major, info.OS.Minor, info.OS.Patch }.Where(v => !String.IsNullOrEmpty(v))); + request.Data[RequestInfo.KnownDataKeys.OSMajorVersion] = info.OS.Major; + } } - context.Event.AddRequestInfo(request.ApplyDataExclusions(exclusions, MAX_VALUE_LENGTH)); + var botPatterns = context.Project.Configuration.Settings.ContainsKey(SettingsDictionary.KnownKeys.UserAgentBotPatterns) ? context.Project.Configuration.Settings.GetStringCollection(SettingsDictionary.KnownKeys.UserAgentBotPatterns).ToList() : new List(); + + request.Data[RequestInfo.KnownDataKeys.IsBot] = info.Device.IsSpider || request.UserAgent.AnyWildcardMatches(botPatterns); } } } diff --git a/Source/Core/Plugins/EventProcessor/EventContext.cs b/Source/Core/Plugins/EventProcessor/EventContext.cs index 8f0bd871fa..7e3e197616 100644 --- a/Source/Core/Plugins/EventProcessor/EventContext.cs +++ b/Source/Core/Plugins/EventProcessor/EventContext.cs @@ -3,15 +3,18 @@ using Exceptionless.Core.Pipeline; using Exceptionless.Core.Utility; using Exceptionless.Core.Models; +using Exceptionless.Core.Queues.Models; namespace Exceptionless.Core.Plugins.EventProcessor { public class EventContext : ExtensibleObject, IPipelineContext { - public EventContext(PersistentEvent ev) { + public EventContext(PersistentEvent ev, EventPostInfo epi = null) { Event = ev; + EventPostInfo = epi; StackSignatureData = new Dictionary(); } public PersistentEvent Event { get; set; } + public EventPostInfo EventPostInfo { get; set; } public Stack Stack { get; set; } public Project Project { get; set; } public Organization Organization { get; set; } From 24ea3ed0e0de152df7faa5a35a28cd51b15d1d62 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 21 Jan 2016 17:15:14 -0600 Subject: [PATCH 2/5] Flow the submission client ip into the location details (incase there is no request info) --- .../Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs | 2 +- Source/Core/Plugins/EventProcessor/Default/50_GeoPlugin.cs | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Source/Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs b/Source/Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs index 61fa234a9c..7991b070ee 100644 --- a/Source/Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs +++ b/Source/Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs @@ -52,7 +52,7 @@ private void SetClientIPAddress(RequestInfo request, string clientIPAddress) { return; if (clientIPAddress.IsLocalHost()) - return; + clientIPAddress = "127.0.0.1"; if (String.IsNullOrWhiteSpace(request.ClientIpAddress)) request.ClientIpAddress = clientIPAddress; diff --git a/Source/Core/Plugins/EventProcessor/Default/50_GeoPlugin.cs b/Source/Core/Plugins/EventProcessor/Default/50_GeoPlugin.cs index ca926f9b57..520fdbe6a6 100644 --- a/Source/Core/Plugins/EventProcessor/Default/50_GeoPlugin.cs +++ b/Source/Core/Plugins/EventProcessor/Default/50_GeoPlugin.cs @@ -29,14 +29,16 @@ public override async Task EventBatchProcessingAsync(ICollection c // The geo coordinates are all the same, set the location from the result of any of the ip addresses. if (!String.IsNullOrEmpty(group.Key)) { - result = await GetGeoFromIPAddressesAsync(group.SelectMany(c => c.Event.GetIpAddresses()).Distinct()).AnyContext(); + var ips = group.SelectMany(c => c.Event.GetIpAddresses()).Union(new[] { group.First().EventPostInfo?.IpAddress }).Distinct(); + result = await GetGeoFromIPAddressesAsync(ips).AnyContext(); group.ForEach(c => UpdateGeoAndlocation(c.Event, result)); continue; } // Each event could be a different user; foreach (var context in group) { - result = await GetGeoFromIPAddressesAsync(context.Event.GetIpAddresses()).AnyContext(); + var ips = context.Event.GetIpAddresses().Union(new[] { context.EventPostInfo?.IpAddress }); + result = await GetGeoFromIPAddressesAsync(ips).AnyContext(); UpdateGeoAndlocation(context.Event, result); } } From a5ea45aaeaedc516908288d71d46e0deeb9e1077 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 21 Jan 2016 17:53:21 -0600 Subject: [PATCH 3/5] Trim ip addresses and ensure no duplicates are added to request info --- .../Extensions/PersistentEventExtensions.cs | 6 +++--- .../Default/40_RequestInfoPlugin.cs | 18 ++++++++++++------ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/Source/Core/Extensions/PersistentEventExtensions.cs b/Source/Core/Extensions/PersistentEventExtensions.cs index 4f14eb5c47..e73b52c56e 100644 --- a/Source/Core/Extensions/PersistentEventExtensions.cs +++ b/Source/Core/Extensions/PersistentEventExtensions.cs @@ -221,18 +221,18 @@ public static IEnumerable GetIpAddresses(this PersistentEvent ev) { yield break; if (!String.IsNullOrEmpty(ev.Geo) && (ev.Geo.Contains(".") || ev.Geo.Contains(":"))) - yield return ev.Geo; + yield return ev.Geo.Trim(); var ri = ev.GetRequestInfo(); if (!String.IsNullOrEmpty(ri?.ClientIpAddress)) { foreach (var ip in ri.ClientIpAddress.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)) - yield return ip; + yield return ip.Trim(); } var ei = ev.GetEnvironmentInfo(); if (!String.IsNullOrEmpty(ei?.IpAddress)) { foreach (var ip in ei.IpAddress.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)) - yield return ip; + yield return ip.Trim(); } } diff --git a/Source/Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs b/Source/Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs index 7991b070ee..819967132b 100644 --- a/Source/Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs +++ b/Source/Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs @@ -40,24 +40,30 @@ public override async Task EventBatchProcessingAsync(ICollection c if (request == null) continue; - SetClientIPAddress(request, context.EventPostInfo?.IpAddress); + AddClientIPAddress(request, context.EventPostInfo?.IpAddress); await SetBrowserOsAndDeviceFromUserAgent(request, context); context.Event.AddRequestInfo(request.ApplyDataExclusions(exclusions, MAX_VALUE_LENGTH)); } } - private void SetClientIPAddress(RequestInfo request, string clientIPAddress) { + private void AddClientIPAddress(RequestInfo request, string clientIPAddress) { if (String.IsNullOrEmpty(clientIPAddress)) return; if (clientIPAddress.IsLocalHost()) clientIPAddress = "127.0.0.1"; - if (String.IsNullOrWhiteSpace(request.ClientIpAddress)) - request.ClientIpAddress = clientIPAddress; - else if (!request.ClientIpAddress.Contains(clientIPAddress)) - request.ClientIpAddress += String.Concat(",", clientIPAddress); + var ips = (request.ClientIpAddress ?? String.Empty) + .Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries) + .Select(ip => ip.Trim()) + .Where(ip => !ip.IsLocalHost()) + .ToList(); + + if (ips.Count == 0 || !clientIPAddress.IsLocalHost()) + ips.Add(clientIPAddress); + + request.ClientIpAddress = ips.Distinct().ToDelimitedString(); } private async Task SetBrowserOsAndDeviceFromUserAgent(RequestInfo request, EventContext context) { From c5a1e647d9dd30ffc119febf22eeb7475bf9c8fc Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 21 Jan 2016 19:27:17 -0600 Subject: [PATCH 4/5] Fixes #119 Can't create project or organization that ends with a dot --- Source/Api/Controllers/OrganizationController.cs | 2 +- Source/Api/Controllers/ProjectController.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/Api/Controllers/OrganizationController.cs b/Source/Api/Controllers/OrganizationController.cs index e49580e1b8..9cc1cdf975 100644 --- a/Source/Api/Controllers/OrganizationController.cs +++ b/Source/Api/Controllers/OrganizationController.cs @@ -610,7 +610,7 @@ public async Task DeleteDataAsync(string id, string key) { /// The organization name is available. /// The organization name is not available. [HttpGet] - [Route("check-name/{*name:minlength(1)}")] + [Route("check-name")] public async Task IsNameAvailableAsync(string name) { if (await IsOrganizationNameAvailableInternalAsync(name)) return StatusCode(HttpStatusCode.NoContent); diff --git a/Source/Api/Controllers/ProjectController.cs b/Source/Api/Controllers/ProjectController.cs index daa4d23df3..7e5a605852 100644 --- a/Source/Api/Controllers/ProjectController.cs +++ b/Source/Api/Controllers/ProjectController.cs @@ -382,8 +382,8 @@ public async Task DemoteTabAsync(string id, string name) { /// The project name is available. /// The project name is not available. [HttpGet] - [Route("check-name/{*name:minlength(1)}")] - [Route("~/" + API_PREFIX + "/organizations/{organizationId:objectid}/projects/check-name/{*name:minlength(1)}")] + [Route("check-name")] + [Route("~/" + API_PREFIX + "/organizations/{organizationId:objectid}/projects/check-name")] public async Task IsNameAvailableAsync(string name, string organizationId = null) { if (await IsProjectNameAvailableInternalAsync(organizationId, name)) return StatusCode(HttpStatusCode.NoContent); From b4b8f8ad1abc4324d7884b3660d4cad3a4d48b22 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 21 Jan 2016 19:27:27 -0600 Subject: [PATCH 5/5] Revert "Fixes #119 Can't create project or organization that ends with a dot" This reverts commit c5a1e647d9dd30ffc119febf22eeb7475bf9c8fc. --- Source/Api/Controllers/OrganizationController.cs | 2 +- Source/Api/Controllers/ProjectController.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/Api/Controllers/OrganizationController.cs b/Source/Api/Controllers/OrganizationController.cs index 9cc1cdf975..e49580e1b8 100644 --- a/Source/Api/Controllers/OrganizationController.cs +++ b/Source/Api/Controllers/OrganizationController.cs @@ -610,7 +610,7 @@ public async Task DeleteDataAsync(string id, string key) { /// The organization name is available. /// The organization name is not available. [HttpGet] - [Route("check-name")] + [Route("check-name/{*name:minlength(1)}")] public async Task IsNameAvailableAsync(string name) { if (await IsOrganizationNameAvailableInternalAsync(name)) return StatusCode(HttpStatusCode.NoContent); diff --git a/Source/Api/Controllers/ProjectController.cs b/Source/Api/Controllers/ProjectController.cs index 7e5a605852..daa4d23df3 100644 --- a/Source/Api/Controllers/ProjectController.cs +++ b/Source/Api/Controllers/ProjectController.cs @@ -382,8 +382,8 @@ public async Task DemoteTabAsync(string id, string name) { /// The project name is available. /// The project name is not available. [HttpGet] - [Route("check-name")] - [Route("~/" + API_PREFIX + "/organizations/{organizationId:objectid}/projects/check-name")] + [Route("check-name/{*name:minlength(1)}")] + [Route("~/" + API_PREFIX + "/organizations/{organizationId:objectid}/projects/check-name/{*name:minlength(1)}")] public async Task IsNameAvailableAsync(string name, string organizationId = null) { if (await IsProjectNameAvailableInternalAsync(organizationId, name)) return StatusCode(HttpStatusCode.NoContent);