diff --git a/Source/Core/Extensions/PersistentEventExtensions.cs b/Source/Core/Extensions/PersistentEventExtensions.cs index 9364badfee..e73b52c56e 100644 --- a/Source/Core/Extensions/PersistentEventExtensions.cs +++ b/Source/Core/Extensions/PersistentEventExtensions.cs @@ -221,18 +221,19 @@ public static IEnumerable GetIpAddresses(this PersistentEvent ev) { yield break; if (!String.IsNullOrEmpty(ev.Geo) && (ev.Geo.Contains(".") || ev.Geo.Contains(":"))) - yield return ev.Geo; + yield return ev.Geo.Trim(); - var request = ev.GetRequestInfo(); - if (!String.IsNullOrEmpty(request?.ClientIpAddress)) - yield return request.ClientIpAddress; - - var environmentInfo = ev.GetEnvironmentInfo(); - if (String.IsNullOrEmpty(environmentInfo?.IpAddress)) - yield break; + var ri = ev.GetRequestInfo(); + if (!String.IsNullOrEmpty(ri?.ClientIpAddress)) { + foreach (var ip in ri.ClientIpAddress.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)) + yield return ip.Trim(); + } - foreach (var ip in environmentInfo.IpAddress.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)) - yield return ip; + var ei = ev.GetEnvironmentInfo(); + if (!String.IsNullOrEmpty(ei?.IpAddress)) { + foreach (var ip in ei.IpAddress.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)) + yield return ip.Trim(); + } } private static bool IsValidIdentifier(string value) { diff --git a/Source/Core/Extensions/StringExtensions.cs b/Source/Core/Extensions/StringExtensions.cs index d9eaa478b4..8ef5411838 100644 --- a/Source/Core/Extensions/StringExtensions.cs +++ b/Source/Core/Extensions/StringExtensions.cs @@ -8,11 +8,18 @@ namespace Exceptionless.Core.Extensions { public static class StringExtensions { + public static bool IsLocalHost(this string ip) { + if (String.IsNullOrEmpty(ip)) + return false; + + return String.Equals(ip, "::1") || String.Equals(ip, "127.0.0.1"); + } + public static bool IsPrivateNetwork(this string ip) { if (String.IsNullOrEmpty(ip)) return false; - if (String.Equals(ip, "::1") || String.Equals(ip, "127.0.0.1")) + if (ip.IsLocalHost()) return true; // 10.0.0.0 – 10.255.255.255 (Class A) diff --git a/Source/Core/Jobs/EventPostsJob.cs b/Source/Core/Jobs/EventPostsJob.cs index d5341fb593..135e8adc31 100644 --- a/Source/Core/Jobs/EventPostsJob.cs +++ b/Source/Core/Jobs/EventPostsJob.cs @@ -102,7 +102,7 @@ protected override async Task ProcessQueueEntryAsync(JobQueueEntryCon var created = DateTime.UtcNow; try { events.ForEach(e => e.CreatedUtc = created); - var results = await _eventPipeline.RunAsync(events.Take(eventsToProcess).ToList()).AnyContext(); + var results = await _eventPipeline.RunAsync(events.Take(eventsToProcess).ToList(), eventPostInfo).AnyContext(); Logger.Info().Message("Ran {0} events through the pipeline: id={1} project={2} success={3} error={4}", results.Count, queueEntry.Id, eventPostInfo.ProjectId, results.Count(r => r.IsProcessed), results.Count(r => r.HasError)).WriteIf(!isInternalProject); foreach (var eventContext in results) { if (eventContext.IsCancelled) diff --git a/Source/Core/Pipeline/EventPipeline.cs b/Source/Core/Pipeline/EventPipeline.cs index 1412e0c954..cd0daeadff 100644 --- a/Source/Core/Pipeline/EventPipeline.cs +++ b/Source/Core/Pipeline/EventPipeline.cs @@ -9,6 +9,7 @@ using Exceptionless.Core.Repositories; using Exceptionless.Core.Models; using Exceptionless.Core.Helpers; +using Exceptionless.Core.Queues.Models; using Exceptionless.Core.Repositories.Base; using Foundatio.Metrics; @@ -24,12 +25,12 @@ public EventPipeline(IDependencyResolver dependencyResolver, IOrganizationReposi _metricsClient = metricsClient; } - public Task RunAsync(PersistentEvent ev) { - return RunAsync(new EventContext(ev)); + public Task RunAsync(PersistentEvent ev, EventPostInfo epi = null) { + return RunAsync(new EventContext(ev, epi)); } - public Task> RunAsync(IEnumerable events) { - return RunAsync(events.Select(ev => new EventContext(ev)).ToList()); + public Task> RunAsync(IEnumerable events, EventPostInfo epi = null) { + return RunAsync(events.Select(ev => new EventContext(ev, epi)).ToList()); } public override async Task> RunAsync(ICollection contexts) { diff --git a/Source/Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs b/Source/Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs index 4e4fdf038b..819967132b 100644 --- a/Source/Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs +++ b/Source/Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs @@ -40,36 +40,57 @@ public override async Task EventBatchProcessingAsync(ICollection c if (request == null) continue; - var info = await _parser.ParseAsync(request.UserAgent, context.Project.Id).AnyContext(); - if (info != null) { - if (!String.Equals(info.UserAgent.Family, "Other")) { - request.Data[RequestInfo.KnownDataKeys.Browser] = info.UserAgent.Family; - if (!String.IsNullOrEmpty(info.UserAgent.Major)) { - request.Data[RequestInfo.KnownDataKeys.BrowserVersion] = String.Join(".", new[] { info.UserAgent.Major, info.UserAgent.Minor, info.UserAgent.Patch }.Where(v => !String.IsNullOrEmpty(v))); - request.Data[RequestInfo.KnownDataKeys.BrowserMajorVersion] = info.UserAgent.Major; - } - } + AddClientIPAddress(request, context.EventPostInfo?.IpAddress); + await SetBrowserOsAndDeviceFromUserAgent(request, context); + + context.Event.AddRequestInfo(request.ApplyDataExclusions(exclusions, MAX_VALUE_LENGTH)); + } + } + + private void AddClientIPAddress(RequestInfo request, string clientIPAddress) { + if (String.IsNullOrEmpty(clientIPAddress)) + return; + + if (clientIPAddress.IsLocalHost()) + clientIPAddress = "127.0.0.1"; - if (!String.Equals(info.Device.Family, "Other")) - request.Data[RequestInfo.KnownDataKeys.Device] = info.Device.Family; + var ips = (request.ClientIpAddress ?? String.Empty) + .Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries) + .Select(ip => ip.Trim()) + .Where(ip => !ip.IsLocalHost()) + .ToList(); + if (ips.Count == 0 || !clientIPAddress.IsLocalHost()) + ips.Add(clientIPAddress); - if (!String.Equals(info.OS.Family, "Other")) { - request.Data[RequestInfo.KnownDataKeys.OS] = info.OS.Family; - if (!String.IsNullOrEmpty(info.OS.Major)) { - request.Data[RequestInfo.KnownDataKeys.OSVersion] = String.Join(".", new[] { info.OS.Major, info.OS.Minor, info.OS.Patch }.Where(v => !String.IsNullOrEmpty(v))); - request.Data[RequestInfo.KnownDataKeys.OSMajorVersion] = info.OS.Major; - } + request.ClientIpAddress = ips.Distinct().ToDelimitedString(); + } + + private async Task SetBrowserOsAndDeviceFromUserAgent(RequestInfo request, EventContext context) { + var info = await _parser.ParseAsync(request.UserAgent, context.Project.Id).AnyContext(); + if (info != null) { + if (!String.Equals(info.UserAgent.Family, "Other")) { + request.Data[RequestInfo.KnownDataKeys.Browser] = info.UserAgent.Family; + if (!String.IsNullOrEmpty(info.UserAgent.Major)) { + request.Data[RequestInfo.KnownDataKeys.BrowserVersion] = String.Join(".", new[] { info.UserAgent.Major, info.UserAgent.Minor, info.UserAgent.Patch }.Where(v => !String.IsNullOrEmpty(v))); + request.Data[RequestInfo.KnownDataKeys.BrowserMajorVersion] = info.UserAgent.Major; } + } - var botPatterns = context.Project.Configuration.Settings.ContainsKey(SettingsDictionary.KnownKeys.UserAgentBotPatterns) - ? context.Project.Configuration.Settings.GetStringCollection(SettingsDictionary.KnownKeys.UserAgentBotPatterns).ToList() - : new List(); + if (!String.Equals(info.Device.Family, "Other")) + request.Data[RequestInfo.KnownDataKeys.Device] = info.Device.Family; - request.Data[RequestInfo.KnownDataKeys.IsBot] = info.Device.IsSpider || request.UserAgent.AnyWildcardMatches(botPatterns); + if (!String.Equals(info.OS.Family, "Other")) { + request.Data[RequestInfo.KnownDataKeys.OS] = info.OS.Family; + if (!String.IsNullOrEmpty(info.OS.Major)) { + request.Data[RequestInfo.KnownDataKeys.OSVersion] = String.Join(".", new[] { info.OS.Major, info.OS.Minor, info.OS.Patch }.Where(v => !String.IsNullOrEmpty(v))); + request.Data[RequestInfo.KnownDataKeys.OSMajorVersion] = info.OS.Major; + } } - context.Event.AddRequestInfo(request.ApplyDataExclusions(exclusions, MAX_VALUE_LENGTH)); + var botPatterns = context.Project.Configuration.Settings.ContainsKey(SettingsDictionary.KnownKeys.UserAgentBotPatterns) ? context.Project.Configuration.Settings.GetStringCollection(SettingsDictionary.KnownKeys.UserAgentBotPatterns).ToList() : new List(); + + request.Data[RequestInfo.KnownDataKeys.IsBot] = info.Device.IsSpider || request.UserAgent.AnyWildcardMatches(botPatterns); } } } diff --git a/Source/Core/Plugins/EventProcessor/Default/50_GeoPlugin.cs b/Source/Core/Plugins/EventProcessor/Default/50_GeoPlugin.cs index ca926f9b57..520fdbe6a6 100644 --- a/Source/Core/Plugins/EventProcessor/Default/50_GeoPlugin.cs +++ b/Source/Core/Plugins/EventProcessor/Default/50_GeoPlugin.cs @@ -29,14 +29,16 @@ public override async Task EventBatchProcessingAsync(ICollection c // The geo coordinates are all the same, set the location from the result of any of the ip addresses. if (!String.IsNullOrEmpty(group.Key)) { - result = await GetGeoFromIPAddressesAsync(group.SelectMany(c => c.Event.GetIpAddresses()).Distinct()).AnyContext(); + var ips = group.SelectMany(c => c.Event.GetIpAddresses()).Union(new[] { group.First().EventPostInfo?.IpAddress }).Distinct(); + result = await GetGeoFromIPAddressesAsync(ips).AnyContext(); group.ForEach(c => UpdateGeoAndlocation(c.Event, result)); continue; } // Each event could be a different user; foreach (var context in group) { - result = await GetGeoFromIPAddressesAsync(context.Event.GetIpAddresses()).AnyContext(); + var ips = context.Event.GetIpAddresses().Union(new[] { context.EventPostInfo?.IpAddress }); + result = await GetGeoFromIPAddressesAsync(ips).AnyContext(); UpdateGeoAndlocation(context.Event, result); } } diff --git a/Source/Core/Plugins/EventProcessor/EventContext.cs b/Source/Core/Plugins/EventProcessor/EventContext.cs index 8f0bd871fa..7e3e197616 100644 --- a/Source/Core/Plugins/EventProcessor/EventContext.cs +++ b/Source/Core/Plugins/EventProcessor/EventContext.cs @@ -3,15 +3,18 @@ using Exceptionless.Core.Pipeline; using Exceptionless.Core.Utility; using Exceptionless.Core.Models; +using Exceptionless.Core.Queues.Models; namespace Exceptionless.Core.Plugins.EventProcessor { public class EventContext : ExtensibleObject, IPipelineContext { - public EventContext(PersistentEvent ev) { + public EventContext(PersistentEvent ev, EventPostInfo epi = null) { Event = ev; + EventPostInfo = epi; StackSignatureData = new Dictionary(); } public PersistentEvent Event { get; set; } + public EventPostInfo EventPostInfo { get; set; } public Stack Stack { get; set; } public Project Project { get; set; } public Organization Organization { get; set; }