diff --git a/.editorconfig b/.editorconfig index 28e37c6f..9bdbb34e 100644 --- a/.editorconfig +++ b/.editorconfig @@ -2,4 +2,7 @@ root=true [*] indent_style = space -indent_size = 4 \ No newline at end of file +indent_size = 4 + +[*.csproj] +indent_size = 2 diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ClientTestClusterBase.cs b/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ClientTestClusterBase.cs new file mode 100644 index 00000000..c355ffbb --- /dev/null +++ b/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ClientTestClusterBase.cs @@ -0,0 +1,41 @@ +using Elastic.Managed.Ephemeral; +using Elastic.Managed.Ephemeral.Plugins; +using Elastic.Xunit; +using Nest; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap +{ + public abstract class ClientTestClusterBase : XunitClusterBase + { + protected ClientTestClusterBase(ClientTestClusterConfiguration configuration) : base(configuration) { } + + public IElasticClient Client => this.GetOrAddClient(s => ConnectionSettings(s)); + + protected virtual ConnectionSettings ConnectionSettings(ConnectionSettings s) => s; + } + + public class ClientTestClusterConfiguration : XunitClusterConfiguration + { + public ClientTestClusterConfiguration( + string elasticsearchVersion, + ClusterFeatures features = ClusterFeatures.None, + int numberOfNodes = 1, + params ElasticsearchPlugin[] plugins + ) + : base(elasticsearchVersion, features, new ElasticsearchPlugins(plugins), numberOfNodes) + { + HttpFiddlerAware = true; + CacheEsHomeInstallation = true; + + Add(AttributeKey("testingcluster"), "true"); + + Add($"script.max_compilations_per_minute", "10000", "<6.0.0-rc1"); + Add($"script.max_compilations_rate", "10000/1m", ">=6.0.0-rc1"); + + Add($"script.inline", "true", "<5.5.0"); + Add($"script.stored", "true", ">5.0.0-alpha1 <5.5.0"); + Add($"script.indexed", "true", "<5.0.0-alpha1"); + Add($"script.allowed_types", "inline,stored", ">=5.5.0"); + } + } +} diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/EphemeralClusterExtensions.cs b/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/EphemeralClusterExtensions.cs new file mode 100644 index 00000000..6db19ae3 --- /dev/null +++ b/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/EphemeralClusterExtensions.cs @@ -0,0 +1,54 @@ +using System; +using System.Diagnostics; +using System.Linq; +using System.Security.Cryptography.X509Certificates; +using Elastic.Managed.Ephemeral; +using Elastic.Xunit; +using Elasticsearch.Net; +using Nest; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap +{ + public static class EphemeralClusterExtensions + { + private static readonly bool RunningMitmProxy = Process.GetProcessesByName("mitmproxy").Any(); + private static readonly bool RunningFiddler = Process.GetProcessesByName("fiddler").Any(); + private static string LocalOrProxyHost => RunningFiddler || RunningMitmProxy ? "ipv4.fiddler" : "localhost"; + + public static ConnectionSettings CreateConnectionSettings(this IEphemeralCluster cluster) + where TConfig : EphemeralClusterConfiguration + { + var clusterNodes = cluster.NodesUris(LocalOrProxyHost); + return new ConnectionSettings(new StaticConnectionPool(clusterNodes)); + } + + public static IElasticClient GetOrAddClient( + this IEphemeralCluster cluster, + Func modifySettings = null + ) + where TConfig : EphemeralClusterConfiguration + { + modifySettings = modifySettings ?? (s => s); + return cluster.GetOrAddClient(c => + { + var settings = modifySettings(cluster.CreateConnectionSettings()); + + var current = (IConnectionConfigurationValues)settings; + var notAlreadyAuthenticated = current.BasicAuthenticationCredentials == null && current.ClientCertificates == null; + var noCertValidation = current.ServerCertificateValidationCallback == null; + + if (cluster.ClusterConfiguration.EnableSecurity && notAlreadyAuthenticated) + settings = settings.BasicAuthentication(ClusterAuthentication.Admin.Username, ClusterAuthentication.Admin.Password); + if (cluster.ClusterConfiguration.EnableSsl && noCertValidation) + { + //todo use CA callback instead of allow all + // ReSharper disable once UnusedVariable + var ca = new X509Certificate2(cluster.ClusterConfiguration.FileSystem.CaCertificate); + settings = settings.ServerCertificateValidationCallback(CertificateValidations.AllowAll); + } + var client = new ElasticClient(settings); + return client; + }); + } + } +} \ No newline at end of file diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/SerilogSinkElasticsearchXunitRunOptions.cs b/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/SerilogSinkElasticsearchXunitRunOptions.cs new file mode 100644 index 00000000..ea48e4c4 --- /dev/null +++ b/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/SerilogSinkElasticsearchXunitRunOptions.cs @@ -0,0 +1,107 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using Elastic.Xunit; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap +{ + /// Feeding TestClient.Configuration options to the runner + public class SerilogSinkElasticsearchXunitRunOptions : ElasticXunitRunOptions + { + public SerilogSinkElasticsearchXunitRunOptions() + { + RunIntegrationTests = true; + RunUnitTests = false; + ClusterFilter = null; + TestFilter = null; + IntegrationTestsMayUseAlreadyRunningNode = false; + } + + public override void OnBeforeTestsRun() { } + + public override void OnTestsFinished(Dictionary clusterTotals, ConcurrentBag> failedCollections) + { + Console.Out.Flush(); + DumpClusterTotals(clusterTotals); + DumpFailedCollections(failedCollections); + } + + private static void DumpClusterTotals(Dictionary clusterTotals) + { + Console.WriteLine("--------"); + Console.WriteLine("Individual cluster running times:"); + foreach (var kv in clusterTotals) Console.WriteLine($"- {kv.Key}: {kv.Value.Elapsed}"); + Console.WriteLine("--------"); + } + + private static void DumpFailedCollections(ConcurrentBag> failedCollections) + { + if (failedCollections.Count <= 0) return; + + Console.ForegroundColor = ConsoleColor.Red; + Console.WriteLine("Failed collections:"); + foreach (var t in failedCollections.OrderBy(p => p.Item1).ThenBy(t => t.Item2)) + + { + var cluster = t.Item1; + Console.WriteLine($" - {cluster}: {t.Item2}"); + } + DumpReproduceFilters(failedCollections); + Console.ResetColor(); + } + + private static void DumpReproduceFilters(ConcurrentBag> failedCollections) + { + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("---Reproduce: -----"); + var reproduceLine = ReproduceCommandLine(failedCollections); + Console.WriteLine(reproduceLine); + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TEAMCITY_VERSION"))) + Console.WriteLine($"##teamcity[buildProblem description='{reproduceLine}']"); + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TF_BUILD"))) + { + var count = failedCollections.Count; + Console.WriteLine($"##vso[task.logissue type=error;]{count} test failures"); + Console.WriteLine($"##vso[task.logissue type=error;]{reproduceLine}"); + } + Console.WriteLine("--------"); + } + + private static string ReproduceCommandLine(ConcurrentBag> failedCollections) + { + var sb = new StringBuilder("build.bat "); + + if (failedCollections.Count > 0) + { + var clusters = string.Join(",", failedCollections + .Select(c => c.Item1.ToLowerInvariant()) + .Distinct()); + sb.Append(" \""); + sb.Append(clusters); + sb.Append("\""); + } + + if ((failedCollections.Count < 30) && failedCollections.Count > 0) + { + sb.Append(" \""); + var tests = string.Join(",", failedCollections + .OrderBy(t => t.Item2) + .Select(c => c.Item2.ToLowerInvariant() + .Split('.') + .Last() + .Replace("apitests", "") + .Replace("usagetests", "") + .Replace("tests", "") + )); + sb.Append(tests); + sb.Append("\""); + } + + var reproduceLine = sb.ToString(); + return reproduceLine; + } + } +} diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/XunitBootstrap.cs b/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/XunitBootstrap.cs new file mode 100644 index 00000000..65de3e39 --- /dev/null +++ b/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/XunitBootstrap.cs @@ -0,0 +1,6 @@ +using Elastic.Xunit; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap; +using Xunit; + +[assembly: TestFramework("Elastic.Xunit.Sdk.ElasticTestFramework", "Elastic.Xunit")] +[assembly: ElasticXunitConfiguration(typeof(SerilogSinkElasticsearchXunitRunOptions))] diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Clusters/Elasticsearch7xCluster.cs b/Serilog.Sinks.Elasticsearch.IntegrationTests/Clusters/Elasticsearch7xCluster.cs new file mode 100644 index 00000000..8961e016 --- /dev/null +++ b/Serilog.Sinks.Elasticsearch.IntegrationTests/Clusters/Elasticsearch7xCluster.cs @@ -0,0 +1,23 @@ +using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Clusters +{ + /// + /// Use this cluster for APIs that do writes. If they are however intrusive or long running consider IntrusiveOperationCluster + /// instead. + /// + public class Elasticsearch7XCluster : ClientTestClusterBase + { + public Elasticsearch7XCluster() : base(CreateConfiguration()) { } + + private static ClientTestClusterConfiguration CreateConfiguration() + { + return new ClientTestClusterConfiguration("7.0.0") + { + MaxConcurrency = 1 + }; + } + + protected override void SeedCluster() { } + } +} diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7X.cs b/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7X.cs new file mode 100644 index 00000000..dd9d3249 --- /dev/null +++ b/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7X.cs @@ -0,0 +1,79 @@ +using System.Linq; +using Elastic.Managed.Ephemeral; +using Elastic.Xunit; +using Elastic.Xunit.XunitPlumbing; +using FluentAssertions; +using Nest; +using Serilog.Core; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Clusters; +using Xunit; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests +{ + public class Elasticsearch7XSetup + { + public const string IndexPrefix = "logs-7x-"; + public const string TemplateName = "serilog-logs-7x"; + public Elasticsearch7XSetup() + { + var loggerConfig = new LoggerConfiguration() + .MinimumLevel.Information() + .WriteTo.ColoredConsole() + .WriteTo.Elasticsearch(new ElasticsearchSinkOptions + { + + IndexFormat = IndexPrefix + "{0:yyyy.MM.dd}", + TemplateName = TemplateName, + AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7, + AutoRegisterTemplate = true + }); + var logger = loggerConfig.CreateLogger(); + + logger.Information("Hello Information"); + logger.Debug("Hello Debug"); + logger.Warning("Hello Warning"); + logger.Error("Hello Error"); + logger.Fatal("Hello Fatal"); + + logger.Dispose(); + } + + } + + + public class Elasticsearch7X : IClusterFixture, IClassFixture + { + private readonly Elasticsearch7XCluster _cluster; + private IElasticClient _client; + + public Elasticsearch7X(Elasticsearch7XCluster cluster, Elasticsearch7XSetup setup) + { + _cluster = cluster; + _client = cluster.Client; + } + + + [I] public void AssertTemplate() + { + var templateResponse = _client.Indices.GetTemplate(Elasticsearch7XSetup.TemplateName); + templateResponse.TemplateMappings.Should().NotBeEmpty(); + templateResponse.TemplateMappings.Keys.Should().Contain(Elasticsearch7XSetup.TemplateName); + + var template = templateResponse.TemplateMappings[Elasticsearch7XSetup.TemplateName]; + + template.IndexPatterns.Should().Contain(pattern => pattern.StartsWith(Elasticsearch7XSetup.IndexPrefix)); + + } + [I] public void AssertLogs() + { + var refreshed = _client.Indices.Refresh(Elasticsearch7XSetup.IndexPrefix + "*"); + + var search = _client.Search(s => s.Index(Elasticsearch7XSetup.IndexPrefix + "*")); + + // Informational should be filtered + search.Documents.Count().Should().Be(4); + + + } + } +} \ No newline at end of file diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj b/Serilog.Sinks.Elasticsearch.IntegrationTests/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj new file mode 100644 index 00000000..a6a57003 --- /dev/null +++ b/Serilog.Sinks.Elasticsearch.IntegrationTests/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj @@ -0,0 +1,26 @@ + + + + netcoreapp2.1;net461 + $(NoWarn);xUnit1013 + True + latest + True + + + + + + + + + + + + + + + + + + diff --git a/serilog-sinks-elasticsearch.sln b/serilog-sinks-elasticsearch.sln index 1f08366a..bb9350f1 100644 --- a/serilog-sinks-elasticsearch.sln +++ b/serilog-sinks-elasticsearch.sln @@ -24,6 +24,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serilog.Sinks.Elasticsearch EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serilog.Formatting.Elasticsearch", "src\Serilog.Formatting.Elasticsearch\Serilog.Formatting.Elasticsearch.csproj", "{0E6D34BF-322A-4803-94D1-355F6D5024BE}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serilog.Sinks.Elasticsearch.IntegrationTests", "Serilog.Sinks.Elasticsearch.IntegrationTests\Serilog.Sinks.Elasticsearch.IntegrationTests.csproj", "{23BC3821-E028-48B4-8F2C-83BB1B8B5525}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -46,6 +48,10 @@ Global {0E6D34BF-322A-4803-94D1-355F6D5024BE}.Debug|Any CPU.Build.0 = Debug|Any CPU {0E6D34BF-322A-4803-94D1-355F6D5024BE}.Release|Any CPU.ActiveCfg = Release|Any CPU {0E6D34BF-322A-4803-94D1-355F6D5024BE}.Release|Any CPU.Build.0 = Release|Any CPU + {23BC3821-E028-48B4-8F2C-83BB1B8B5525}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {23BC3821-E028-48B4-8F2C-83BB1B8B5525}.Debug|Any CPU.Build.0 = Debug|Any CPU + {23BC3821-E028-48B4-8F2C-83BB1B8B5525}.Release|Any CPU.ActiveCfg = Release|Any CPU + {23BC3821-E028-48B4-8F2C-83BB1B8B5525}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/serilog-sinks-elasticsearch.sln.DotSettings b/serilog-sinks-elasticsearch.sln.DotSettings new file mode 100644 index 00000000..95859094 --- /dev/null +++ b/serilog-sinks-elasticsearch.sln.DotSettings @@ -0,0 +1,4 @@ + + True + True + True \ No newline at end of file diff --git a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.Symbols.nuspec b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.Symbols.nuspec index 289c659a..9932b8fb 100644 --- a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.Symbols.nuspec +++ b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.Symbols.nuspec @@ -12,7 +12,7 @@ serilog logging elasticsearch - + diff --git a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.nuspec b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.nuspec index 5c6c8148..c6ba9cc7 100644 --- a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.nuspec +++ b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.nuspec @@ -13,7 +13,7 @@ serilog logging elasticsearch - + diff --git a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj index d2e3559f..5826a20f 100644 --- a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj +++ b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj @@ -4,7 +4,7 @@ 6.0.0 alpha Michiel van Oudheusden, Martijn Laarman, Mogens Heller Grabe, Serilog Contributors - net45;netstandard1.3;netstandard2.0 + net461;netstandard2.0 true true Serilog.Sinks.Elasticsearch @@ -28,31 +28,6 @@ false - - - - - - - - - - - - - - - - - - - - - - - - - 1591;1701;1702 $(DefineConstants);DURABLE;THREADING_TIMER @@ -63,7 +38,7 @@ $(DefineConstants);DURABLE;THREADING_TIMER - + 1591;1701;1702 $(DefineConstants);DURABLE;THREADING_TIMER;HRESULTS @@ -75,10 +50,18 @@ - - + + + + + + + + + + diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchLogClient.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchLogClient.cs deleted file mode 100644 index 6b2d45bf..00000000 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchLogClient.cs +++ /dev/null @@ -1,125 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.ExceptionServices; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using Elasticsearch.Net; -using Serilog.Debugging; - -namespace Serilog.Sinks.Elasticsearch.Durable -{ - /// - /// - /// - public class ElasticsearchLogClient : ILogClient> - { - private readonly IElasticLowLevelClient _elasticLowLevelClient; - private readonly Func _cleanPayload; - - /// - /// - /// - /// - /// - public ElasticsearchLogClient(IElasticLowLevelClient elasticLowLevelClient, - Func cleanPayload) - { - _elasticLowLevelClient = elasticLowLevelClient; - _cleanPayload = cleanPayload; - } - - public async Task SendPayloadAsync(List payload) - { - return await SendPayloadAsync(payload, true); - } - - public async Task SendPayloadAsync(List payload,bool first) - { - try - { - if (payload == null || !payload.Any()) return new SentPayloadResult(null, true); - var response = await _elasticLowLevelClient.BulkAsync(PostData.MultiJson(payload)); - - if (response.Success) - { - var cleanPayload = new List(); - var invalidPayload = GetInvalidPayloadAsync(response, payload,out cleanPayload); - if ((cleanPayload?.Any() ?? false) && first) - { - await SendPayloadAsync(cleanPayload,false); - } - - return new SentPayloadResult(response, true, invalidPayload); - } - else - { - SelfLog.WriteLine("Received failed ElasticSearch shipping result {0}: {1}", response.HttpStatusCode, - response.OriginalException); - return new SentPayloadResult(response, false, - new InvalidResult() - { - StatusCode = response.HttpStatusCode ?? 500, - Content = response.OriginalException.ToString() - }); - } - } - catch (Exception ex) - { - SelfLog.WriteLine("Exception while emitting periodic batch from {0}: {1}", this, ex); - return new SentPayloadResult(null, false, null, ex); - } - - - } - - private InvalidResult GetInvalidPayloadAsync(DynamicResponse baseResult, List payload, out List cleanPayload) - { - int i = 0; - cleanPayload = new List(); - var items = baseResult.Body["items"]; - if (items == null) return null; - List badPayload = new List(); - - bool hasErrors = false; - foreach (dynamic item in items) - { - long? status = item.index?.status; - i++; - if (!status.HasValue || status < 300) - { - continue; - } - - hasErrors = true; - var id = item.index?._id; - var error = item.index?.error; - if (int.TryParse(id.Split('_')[0], out int index)) - { - SelfLog.WriteLine("Received failed ElasticSearch shipping result {0}: {1}. Failed payload : {2}.", status, error?.ToString(), payload.ElementAt(index * 2 + 1)); - badPayload.Add(payload.ElementAt(index * 2)); - badPayload.Add(payload.ElementAt(index * 2 + 1)); - if (_cleanPayload != null) - { - cleanPayload.Add(payload.ElementAt(index * 2)); - cleanPayload.Add(_cleanPayload(payload.ElementAt(index * 2 + 1), status, error?.ToString())); - } - } - else - { - SelfLog.WriteLine($"Received failed ElasticSearch shipping result {status}: {error?.ToString()}."); - } - } - - if (!hasErrors) - return null; - return new InvalidResult() - { - StatusCode = baseResult.HttpStatusCode ?? 500, - Content = baseResult.ToString(), - BadPayLoad = String.Join(Environment.NewLine, badPayload) - }; - } - } -} diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchLogShipper.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchLogShipper.cs deleted file mode 100644 index 6a6b8b26..00000000 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchLogShipper.cs +++ /dev/null @@ -1,71 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Serilog.Core; -using Serilog.Debugging; -using Serilog.Sinks.Elasticsearch.Durable; - -namespace Serilog.Sinks.Elasticsearch.Durable -{ - /// - /// - /// - public class ElasticsearchLogShipper : LogShipper> - { - private readonly Action _registerTemplateIfNeeded; - bool _didRegisterTemplateIfNeeded = false; - - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - public ElasticsearchLogShipper(string bufferBaseFilename, int batchPostingLimit, TimeSpan period, - long? eventBodyLimitBytes, LoggingLevelSwitch levelControlSwitch, ILogClient> logClient, - IPayloadReader> payloadReader, long? retainedInvalidPayloadsLimitBytes, - long? bufferSizeLimitBytes, Action registerTemplateIfNeeded) - : base(bufferBaseFilename, batchPostingLimit, period, eventBodyLimitBytes, - levelControlSwitch, logClient, payloadReader, retainedInvalidPayloadsLimitBytes, bufferSizeLimitBytes) - { - _registerTemplateIfNeeded = registerTemplateIfNeeded; - } - - /// - /// - /// - /// - protected override async Task OnTick() - { - bool success = true; - try - { - if (!_didRegisterTemplateIfNeeded) - { - if (_registerTemplateIfNeeded != null) - { - _registerTemplateIfNeeded(); - _didRegisterTemplateIfNeeded = true; - } - } - } - catch (Exception ex) - { - SelfLog.WriteLine("Exception while emitting periodic batch from {0}: {1}", this, ex); - _connectionSchedule.MarkFailure(); - success = false; - } - if (success) - await base.OnTick(); - } - } -} diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchPayloadReader.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchPayloadReader.cs deleted file mode 100644 index a31b253e..00000000 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchPayloadReader.cs +++ /dev/null @@ -1,100 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Serilog.Sinks.Elasticsearch.Durable -{ - /// - /// - /// - public class ElasticsearchPayloadReader: APayloadReader> - { - private readonly string _pipelineName; - private readonly string _typeName; - private readonly Func _serialize; - private readonly Func _getIndexForEvent; - private List _payload; - private int _count; - private DateTime _date; - - /// - /// - /// - /// - /// - /// - /// - public ElasticsearchPayloadReader(string pipelineName,string typeName, Func serialize,Func getIndexForEvent) - { - _pipelineName = pipelineName; - _typeName = typeName; - _serialize = serialize; - _getIndexForEvent = getIndexForEvent; - } - - /// - /// - /// - /// - public override List GetNoPayload() - { - return new List(); - } - - /// - /// - /// - /// - protected override void InitPayLoad(string filename) - { - _payload = new List(); - _count = 0; - var lastToken = filename.Split('-').Last(); - - // lastToken should be something like 20150218.json or 20150218_3.json now - if (!lastToken.ToLowerInvariant().EndsWith(".json")) - { - throw new FormatException(string.Format("The file name '{0}' does not seem to follow the right file pattern - it must be named [whatever]-{{Date}}[_n].json", Path.GetFileName(filename))); - } - - var dateString = lastToken.Substring(0, 8); - _date = DateTime.ParseExact(dateString, "yyyyMMdd", CultureInfo.InvariantCulture); - } - /// - /// - /// - /// - protected override List FinishPayLoad() - { - return _payload; - } - - /// - /// - /// - /// - protected override void AddToPayLoad(string nextLine) - { - var indexName = _getIndexForEvent(nextLine, _date); - var action = default(object); - - if (string.IsNullOrWhiteSpace(_pipelineName)) - { - action = new { index = new { _index = indexName, _type = _typeName, _id = _count + "_" + Guid.NewGuid() } }; - } - else - { - action = new { index = new { _index = indexName, _type = _typeName, _id = _count + "_" + Guid.NewGuid(), pipeline = _pipelineName } }; - } - - var actionJson = _serialize(action); - _payload.Add(actionJson); - _payload.Add(nextLine); - _count++; - } - } -} diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs index 0325992b..e247e8b7 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs @@ -118,7 +118,7 @@ public class ElasticsearchTemplateProvider }, properties = new Dictionary { - {"message", new {type = "text", index = "true"}}, + {"message", new {type = "text", index = true}}, { "exceptions", new { @@ -128,8 +128,8 @@ public class ElasticsearchTemplateProvider {"Depth", new {type = "integer"}}, {"RemoteStackIndex", new {type = "integer"}}, {"HResult", new {type = "integer"}}, - {"StackTraceString", new {type = "text", index = "true"}}, - {"RemoteStackTraceString", new {type = "text", index = "true"}}, + {"StackTraceString", new {type = "text", index = true}}, + {"RemoteStackTraceString", new {type = "text", index = true}}, { "ExceptionMessage", new { @@ -210,7 +210,7 @@ public class ElasticsearchTemplateProvider }, properties = new Dictionary { - {"message", new {type = "text", index = "true"}}, + {"message", new {type = "text", index = true}}, { "exceptions", new { @@ -220,8 +220,8 @@ public class ElasticsearchTemplateProvider {"Depth", new {type = "integer"}}, {"RemoteStackIndex", new {type = "integer"}}, {"HResult", new {type = "integer"}}, - {"StackTraceString", new {type = "text", index = "true"}}, - {"RemoteStackTraceString", new {type = "text", index = "true"}}, + {"StackTraceString", new {type = "text", index = true}}, + {"RemoteStackTraceString", new {type = "text", index = true}}, { "ExceptionMessage", new { diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs index 8397d403..29ccb503 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs @@ -271,6 +271,7 @@ public ElasticsearchSinkOptions() this.BufferFileCountLimit = 31; this.BufferFileSizeLimitBytes = 100L * 1024 * 1024; this.FormatStackTraceAsArray = false; + this.ConnectionPool = new SingleNodeConnectionPool(_defaultNode); } /// @@ -290,13 +291,13 @@ public ElasticsearchSinkOptions(IConnectionPool connectionPool) public ElasticsearchSinkOptions(IEnumerable nodes) : this() { - nodes = nodes != null && nodes.Any(n => n != null) - ? nodes.Where(n => n != null) - : new[] { new Uri("http://localhost:9200") }; - if (nodes.Count() == 1) - ConnectionPool = new SingleNodeConnectionPool(nodes.First()); + var materialized = nodes?.Where(n => n != null).ToArray(); + if (materialized == null || materialized.Length == 0) + materialized = new[] { _defaultNode }; + if (materialized.Length == 1) + ConnectionPool = new SingleNodeConnectionPool(materialized.First()); else - ConnectionPool = new StaticConnectionPool(nodes); + ConnectionPool = new StaticConnectionPool(materialized); } /// @@ -304,6 +305,8 @@ public ElasticsearchSinkOptions(IEnumerable nodes) /// /// The node to write to public ElasticsearchSinkOptions(Uri node) : this(new[] { node }) { } + + private readonly Uri _defaultNode = new Uri("http://localhost:9200"); } /// diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs index 679857c9..1b0249e9 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs @@ -17,6 +17,7 @@ using System.Text; using System.Text.RegularExpressions; using Elasticsearch.Net; +using Elasticsearch.Net.Specification.IndicesApi; using Serilog.Debugging; using Serilog.Events; using Serilog.Formatting; @@ -150,7 +151,10 @@ public void RegisterTemplateIfNeeded() { if (!_options.OverwriteTemplate) { - var templateExistsResponse = _client.IndicesExistsTemplateForAll(_templateName); + var templateExistsResponse = _client.Indices.TemplateExistsForAll(_templateName, new IndexTemplateExistsRequestParameters() + { + RequestConfiguration = new RequestConfiguration() { AllowedStatusCodes = new [] {200, 404} } + }); if (templateExistsResponse.HttpStatusCode == 200) { TemplateRegistrationSuccess = true; @@ -160,7 +164,7 @@ public void RegisterTemplateIfNeeded() } Console.WriteLine(_client.Serializer.SerializeToString(GetTemplateData())); - var result = _client.IndicesPutTemplateForAll(_templateName, GetTempatePostData()); + var result = _client.Indices.PutTemplateForAll(_templateName, GetTempatePostData()); if (!result.Success) { diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj b/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj index ed9333dd..a4c073c3 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj @@ -1,7 +1,7 @@  - netcoreapp1.1;netcoreapp2.0;net451 + netcoreapp2.0;net461 Serilog.Sinks.Elasticsearch.Tests Serilog.Sinks.Elasticsearch.Tests true @@ -41,14 +41,17 @@ + + - - + + + @@ -62,31 +65,17 @@ - - - - - - - - $(DefineConstants);DOTNETCORE;NO_SERIALIZATION - - $(DefineConstants);DOTNETCORE;PARTIALLY_SERIALIZATION - - - -