From d6f1ae510b6bab273df091646bc073704946a499 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Thu, 27 Sep 2018 09:31:48 +0200 Subject: [PATCH] re-arch the way we do anlysis tests by making it easier for test classes to do full blown endpoint testing --- src/Nest/Analysis/Tokenizers/Tokenizers.cs | 2 +- .../Clusters/ReadOnlyCluster.cs | 6 +- src/Tests/Tests.Core/Tests.Core.csproj | 2 +- src/Tests/Tests.Domain/Tests.Domain.csproj | 2 +- src/Tests/Tests/Analysis/AnalysisCrudTests.cs | 9 +- .../Tests/Analysis/AnalysisUsageTests.cs | 58 + .../Tests/Analysis/IAnalysisAssertion.cs | 8 + .../AnalysisWithNormalizerCrudTests.cs | 9 +- .../TokenFilters/ITokenFilterAssertion.cs | 12 + .../TokenFilters/TokenFilterAssertionBase.cs | 88 + .../TokenFilters/TokenFilterUsageTests.cs | 1486 ++++++++++------- .../Tokenizers/ITokenizerAssertion.cs | 12 + .../Tokenizers/TokenizerAssertionBase.cs | 88 + .../Tokenizers/TokenizerUsageTests.cs | 361 ++-- .../EndpointTests/TestState/AsyncLazy.cs | 27 +- .../TestState/CallUniqueValues.cs | 42 +- .../EndpointTests/TestState/EndpointUsage.cs | 124 +- .../EndpointTests/TestState/LazyResponses.cs | 17 + .../Core/Keyword/KeywordPropertyTests.cs | 3 +- 19 files changed, 1500 insertions(+), 856 deletions(-) create mode 100644 src/Tests/Tests/Analysis/AnalysisUsageTests.cs create mode 100644 src/Tests/Tests/Analysis/IAnalysisAssertion.cs create mode 100644 src/Tests/Tests/Analysis/TokenFilters/ITokenFilterAssertion.cs create mode 100644 src/Tests/Tests/Analysis/TokenFilters/TokenFilterAssertionBase.cs create mode 100644 src/Tests/Tests/Analysis/Tokenizers/ITokenizerAssertion.cs create mode 100644 src/Tests/Tests/Analysis/Tokenizers/TokenizerAssertionBase.cs create mode 100644 src/Tests/Tests/Framework/EndpointTests/TestState/LazyResponses.cs diff --git a/src/Nest/Analysis/Tokenizers/Tokenizers.cs b/src/Nest/Analysis/Tokenizers/Tokenizers.cs index eaf6fe86dc..7853699465 100644 --- a/src/Nest/Analysis/Tokenizers/Tokenizers.cs +++ b/src/Nest/Analysis/Tokenizers/Tokenizers.cs @@ -19,7 +19,7 @@ public Tokenizers(Dictionary container) public void Add(string name, ITokenizer analyzer) => BackingDictionary.Add(name, analyzer); } - public class TokenizersDescriptor :IsADictionaryDescriptorBase + public class TokenizersDescriptor : IsADictionaryDescriptorBase { public TokenizersDescriptor() : base(new Tokenizers()) { } diff --git a/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/ReadOnlyCluster.cs b/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/ReadOnlyCluster.cs index 4026cf2b61..653ccec7e5 100644 --- a/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/ReadOnlyCluster.cs +++ b/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/ReadOnlyCluster.cs @@ -1,11 +1,11 @@ -using Elastic.Managed.Ephemeral.Plugins; -using Tests.Core.ManagedElasticsearch.NodeSeeders; +using Tests.Core.ManagedElasticsearch.NodeSeeders; +using static Elastic.Managed.Ephemeral.Plugins.ElasticsearchPlugin; namespace Tests.Core.ManagedElasticsearch.Clusters { public class ReadOnlyCluster : ClientTestClusterBase { - public ReadOnlyCluster() : base(ElasticsearchPlugin.MapperMurmur3) { } + public ReadOnlyCluster() : base(MapperMurmur3, AnalysisKuromoji, AnalysisIcu) { } protected override void SeedCluster() => new DefaultSeeder(this.Client).SeedNode(); } diff --git a/src/Tests/Tests.Core/Tests.Core.csproj b/src/Tests/Tests.Core/Tests.Core.csproj index 50aa941ee6..107a57626a 100644 --- a/src/Tests/Tests.Core/Tests.Core.csproj +++ b/src/Tests/Tests.Core/Tests.Core.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Tests/Tests.Domain/Tests.Domain.csproj b/src/Tests/Tests.Domain/Tests.Domain.csproj index 2fd21b9925..5caa065105 100644 --- a/src/Tests/Tests.Domain/Tests.Domain.csproj +++ b/src/Tests/Tests.Domain/Tests.Domain.csproj @@ -12,7 +12,7 @@ - + \ No newline at end of file diff --git a/src/Tests/Tests/Analysis/AnalysisCrudTests.cs b/src/Tests/Tests/Analysis/AnalysisCrudTests.cs index af05d17443..b445fd0b5f 100644 --- a/src/Tests/Tests/Analysis/AnalysisCrudTests.cs +++ b/src/Tests/Tests/Analysis/AnalysisCrudTests.cs @@ -2,6 +2,7 @@ using Elastic.Xunit.XunitPlumbing; using FluentAssertions; using Nest; +using Tests.Analysis.Tokenizers; using Tests.Core.Extensions; using Tests.Core.ManagedElasticsearch.Clusters; using Tests.Framework; @@ -48,8 +49,8 @@ protected virtual CreateIndexRequest CreateInitializer(string indexName) => new { Analyzers = Analyzers.AnalyzerUsageTests.InitializerExample.Analysis.Analyzers, CharFilters = CharFilters.CharFilterUsageTests.InitializerExample.Analysis.CharFilters, - Tokenizers = Tokenizers.TokenizerUsageTests.InitializerExample.Analysis.Tokenizers, - TokenFilters = TokenFilters.TokenFilterUsageTests.InitializerExample.Analysis.TokenFilters, + Tokenizers = AnalysisUsageTests.TokenizersInitializer.Analysis.Tokenizers, + TokenFilters = AnalysisUsageTests.TokenFiltersInitializer.Analysis.TokenFilters, } } }; @@ -59,8 +60,8 @@ protected virtual CreateIndexRequest CreateInitializer(string indexName) => new .Analysis(a => a .Analyzers(t => Promise(Analyzers.AnalyzerUsageTests.FluentExample(s).Value.Analysis.Analyzers)) .CharFilters(t => Promise(CharFilters.CharFilterUsageTests.FluentExample(s).Value.Analysis.CharFilters)) - .Tokenizers(t => Promise(Tokenizers.TokenizerUsageTests.FluentExample(s).Value.Analysis.Tokenizers)) - .TokenFilters(t => Promise(TokenFilters.TokenFilterUsageTests.FluentExample(s).Value.Analysis.TokenFilters)) + .Tokenizers(t => Promise(AnalysisUsageTests.TokenizersFluent.Analysis.Tokenizers)) + .TokenFilters(t => Promise(AnalysisUsageTests.TokenFiltersFluent.Analysis.TokenFilters)) ) ); diff --git a/src/Tests/Tests/Analysis/AnalysisUsageTests.cs b/src/Tests/Tests/Analysis/AnalysisUsageTests.cs new file mode 100644 index 0000000000..4f054ae92e --- /dev/null +++ b/src/Tests/Tests/Analysis/AnalysisUsageTests.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using Elastic.Xunit.XunitPlumbing; +using Nest; +using Tests.Analysis.TokenFilters; +using Tests.Core.Client; +using Tests.Search; + +namespace Tests.Analysis.Tokenizers +{ + public static class AnalysisUsageTests + { + public static IndexSettings TokenizersFluent => Fluent(i => i.Fluent, (a, v) => a.Tokenizers = v.Value); + + public static IndexSettings TokenFiltersFluent => Fluent(i => i.Fluent, (a, v) => a.TokenFilters = v.Value); + + public static IndexSettings TokenizersInitializer => Init(i => i.Initializer, (a, v) => a.Tokenizers = v); + + public static IndexSettings TokenFiltersInitializer => Init(i => i.Initializer, (a, v) => a.TokenFilters = v); + + private static IndexSettings Fluent(Func>> fluent, Action> set) + where TAssertion : IAnalysisAssertion + where TContainer : IPromise, new() + where TValue : class => Wrap(an => set(an, Apply((t, a) => fluent(a)(a.Name, t)))); + + private static IndexSettings Init(Func value, Action set) + where TAssertion : IAnalysisAssertion + where TContainer : IDictionary, new() => Wrap(an => set(an, Apply((t, a) => t[a.Name] = value(a)))); + + private static TContainer Apply(Action act) + where TAssertion : IAnalysisAssertion + where TContainer : new() => All().Aggregate(new TContainer() , (t,a) => { act(t,a); return t; }, t=>t); + + private static IndexSettings Wrap(Action set) + { + var a = new Nest.Analysis(); + var s =new IndexSettings { Analysis = a }; + set(a); + return s; + } + + private static List All() + where TAssertion : IAnalysisAssertion + { + var types = + from t in typeof(TokenizerTests).GetNestedTypes() + where typeof(TAssertion).IsAssignableFrom(t) && t.IsClass + let a = t.GetCustomAttributes(typeof(SkipVersionAttribute)).FirstOrDefault() as SkipVersionAttribute + where a != null && !a.Ranges.Any(r=>r.IsSatisfied(TestClient.Configuration.ElasticsearchVersion)) + select (TAssertion) Activator.CreateInstance(t); + return types.ToList(); + } + + + } +} diff --git a/src/Tests/Tests/Analysis/IAnalysisAssertion.cs b/src/Tests/Tests/Analysis/IAnalysisAssertion.cs new file mode 100644 index 0000000000..9aaa65a6c9 --- /dev/null +++ b/src/Tests/Tests/Analysis/IAnalysisAssertion.cs @@ -0,0 +1,8 @@ +namespace Tests.Analysis.Tokenizers +{ + public interface IAnalysisAssertion + { + string Name { get; } + object Json { get; } + } +} \ No newline at end of file diff --git a/src/Tests/Tests/Analysis/Normalizers/AnalysisWithNormalizerCrudTests.cs b/src/Tests/Tests/Analysis/Normalizers/AnalysisWithNormalizerCrudTests.cs index 5e8decd87d..b6d7cb1359 100644 --- a/src/Tests/Tests/Analysis/Normalizers/AnalysisWithNormalizerCrudTests.cs +++ b/src/Tests/Tests/Analysis/Normalizers/AnalysisWithNormalizerCrudTests.cs @@ -2,6 +2,7 @@ using Elastic.Xunit.XunitPlumbing; using FluentAssertions; using Nest; +using Tests.Analysis.Tokenizers; using Tests.Core.ManagedElasticsearch.Clusters; using Tests.Framework; using Tests.Framework.Integration; @@ -25,8 +26,8 @@ protected override CreateIndexRequest CreateInitializer(string indexName) => new { Analyzers = Analyzers.AnalyzerUsageTests.InitializerExample.Analysis.Analyzers, CharFilters = CharFilters.CharFilterUsageTests.InitializerExample.Analysis.CharFilters, - Tokenizers = Tokenizers.TokenizerUsageTests.InitializerExample.Analysis.Tokenizers, - TokenFilters = TokenFilters.TokenFilterUsageTests.InitializerExample.Analysis.TokenFilters, + Tokenizers = AnalysisUsageTests.TokenizersInitializer.Analysis.Tokenizers, + TokenFilters = AnalysisUsageTests.TokenFiltersInitializer.Analysis.TokenFilters, Normalizers = Normalizers.NormalizerUsageTests.InitializerExample.Analysis.Normalizers, } } @@ -37,8 +38,8 @@ protected override CreateIndexRequest CreateInitializer(string indexName) => new .Analysis(a => a .Analyzers(t => Promise(Analyzers.AnalyzerUsageTests.FluentExample(s).Value.Analysis.Analyzers)) .CharFilters(t => Promise(CharFilters.CharFilterUsageTests.FluentExample(s).Value.Analysis.CharFilters)) - .Tokenizers(t => Promise(Tokenizers.TokenizerUsageTests.FluentExample(s).Value.Analysis.Tokenizers)) - .TokenFilters(t => Promise(TokenFilters.TokenFilterUsageTests.FluentExample(s).Value.Analysis.TokenFilters)) + .Tokenizers(t => Promise(AnalysisUsageTests.TokenizersFluent.Analysis.Tokenizers)) + .TokenFilters(t => Promise(AnalysisUsageTests.TokenFiltersFluent.Analysis.TokenFilters)) .Normalizers(t => Promise(Normalizers.NormalizerUsageTests.FluentExample(s).Value.Analysis.Normalizers)) ) ); diff --git a/src/Tests/Tests/Analysis/TokenFilters/ITokenFilterAssertion.cs b/src/Tests/Tests/Analysis/TokenFilters/ITokenFilterAssertion.cs new file mode 100644 index 0000000000..6346ecd7ac --- /dev/null +++ b/src/Tests/Tests/Analysis/TokenFilters/ITokenFilterAssertion.cs @@ -0,0 +1,12 @@ +using System; +using Nest; + +namespace Tests.Analysis.Tokenizers +{ + + public interface ITokenFilterAssertion : IAnalysisAssertion + { + ITokenFilter Initializer { get; } + Func> Fluent { get; } + } +} diff --git a/src/Tests/Tests/Analysis/TokenFilters/TokenFilterAssertionBase.cs b/src/Tests/Tests/Analysis/TokenFilters/TokenFilterAssertionBase.cs new file mode 100644 index 0000000000..580815b0fe --- /dev/null +++ b/src/Tests/Tests/Analysis/TokenFilters/TokenFilterAssertionBase.cs @@ -0,0 +1,88 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Elastic.Xunit; +using Elastic.Xunit.XunitPlumbing; +using FluentAssertions; +using Nest; +using Tests.Core.Client; +using Tests.Core.ManagedElasticsearch.Clusters; +using Tests.Core.Serialization; +using Tests.Framework.Integration; + +namespace Tests.Analysis.TokenFilters +{ + + [IntegrationTestCluster(typeof(ReadOnlyCluster))] + public abstract class TokenFilterAssertionBase where TAssertion : TokenFilterAssertionBase, new() + { + private static readonly SingleEndpointUsage Usage = new SingleEndpointUsage + ( + fluent: (s, c) => c.CreateIndex(s, FluentCall), + fluentAsync: (s, c) => c.CreateIndexAsync(s, FluentCall), + request: (s, c) => c.CreateIndex(InitializerCall(s)), + requestAsync: (s, c) => c.CreateIndexAsync(InitializerCall(s)), + valuePrefix: $"test-{typeof(TAssertion).Name.ToLowerInvariant()}" + ) + { + OnAfterCall = c=> c.DeleteIndex(Usage.CallUniqueValues.Value) + }; + private static TAssertion AssertionSetup { get; } = new TAssertion(); + + protected TokenFilterAssertionBase() + { + this.Client = (ElasticXunitRunner.CurrentCluster as ReadOnlyCluster)?.Client ?? TestClient.DefaultInMemoryClient; + Usage.KickOffOnce(this.Client); + } + + private IElasticClient Client { get; } + + protected abstract string Name { get; } + protected abstract ITokenFilter Initializer { get; } + protected abstract Func> Fluent { get; } + protected abstract object Json { get; } + + [U] public async Task TestPutSettingsRequest() => await Usage.AssertOnAllResponses(r => + { + var json = new + { + settings = new + { + analysis = new + { + tokenizer = new Dictionary + { + { AssertionSetup.Name, AssertionSetup.Json} + } + } + } + }; + SerializationTestHelper.Expect(json).FromRequest(r); + }); + + [I] public async Task TestPutSettingsResponse() => await Usage.AssertOnAllResponses(r => + { + r.ApiCall.HttpStatusCode.Should().Be(200); + }); + + private static CreateIndexRequest InitializerCall(string index) => new CreateIndexRequest(index) + { + Settings = new IndexSettings + { + Analysis = new Nest.Analysis + { + TokenFilters = new Nest.TokenFilters { { AssertionSetup.Name, AssertionSetup.Initializer } } + + } + } + }; + + private static Func FluentCall => i => i + .Settings(s => s + .Analysis(a => a + .TokenFilters(d => AssertionSetup.Fluent(AssertionSetup.Name, d)) + ) + ); + + } +} diff --git a/src/Tests/Tests/Analysis/TokenFilters/TokenFilterUsageTests.cs b/src/Tests/Tests/Analysis/TokenFilters/TokenFilterUsageTests.cs index 2f647f5490..3c4d9df958 100644 --- a/src/Tests/Tests/Analysis/TokenFilters/TokenFilterUsageTests.cs +++ b/src/Tests/Tests/Analysis/TokenFilters/TokenFilterUsageTests.cs @@ -4,638 +4,878 @@ namespace Tests.Analysis.TokenFilters { - /** - */ + using FuncTokenFilters = Func>; - public class TokenFilterUsageTests : PromiseUsageTestBase + public static class TokenFilterTests { - protected override object ExpectJson => new + public class AsciiFoldingTests : TokenFilterAssertionBase { - analysis = new + protected override string Name => "ascii"; + protected override ITokenFilter Initializer => new AsciiFoldingTokenFilter {PreserveOriginal = true}; + protected override FuncTokenFilters Fluent => (n, tf) => tf.AsciiFolding(n, t => t.PreserveOriginal()); + protected override object Json => new {type = "asciifolding", preserve_original = true}; + } + + public class CommonGramsTests : TokenFilterAssertionBase + { + protected override string Name => "mycomgram"; + + protected override ITokenFilter Initializer => + new CommonGramsTokenFilter {QueryMode = true, IgnoreCase = true, CommonWords = new[] {"x", "y", "z"}}; + + protected override FuncTokenFilters Fluent => (n, tf) => tf + .CommonGrams(n, t => t.CommonWords("x", "y", "z").IgnoreCase().QueryMode()); + + protected override object Json => new { - filter = new + type = "common_grams", + common_words = new[] {"x", "y", "z"}, + ignore_case = true, + query_mode = true + }; + } + + public class DelimitedPayloadFilterTests : TokenFilterAssertionBase + { + protected override string Name => "mydp"; + + protected override ITokenFilter Initializer => + new DelimitedPayloadTokenFilter {Delimiter = '-', Encoding = DelimitedPayloadEncoding.Identity}; + + protected override FuncTokenFilters Fluent => (n, tf) => tf + .DelimitedPayload(n, t => t.Delimiter('-').Encoding(DelimitedPayloadEncoding.Identity)); + + protected override object Json => new { type = "delimited_payload_filter", delimiter = "-", encoding = "identity" }; + } + + public class DictionaryDecompounderTests : TokenFilterAssertionBase + { + protected override string Name => "dcc"; + + protected override ITokenFilter Initializer => + new DictionaryDecompounderTokenFilter { - myAscii = new - { - type = "asciifolding", - preserve_original = true - }, - myCommonGrams = new - { - type = "common_grams", - common_words = new[] {"x", "y", "z"}, - ignore_case = true, - query_mode = true - }, - mydp = new - { - type = "delimited_payload_filter", - delimiter = "-", - encoding = "identity" - }, - dcc = new - { - type = "dictionary_decompounder", - word_list = new[] {"x", "y", "z"}, - min_word_size = 2, - min_subword_size = 2, - max_subword_size = 2, - only_longest_match = true - }, - etf = new - { - type = "edge_ngram", - min_gram = 1, - max_gram = 2 - }, - elision = new - { - type = "elision", - articles = new[] {"a", "b", "c"} - }, - hunspell = new - { - type = "hunspell", - locale = "en_US", - dictionary = "path_to_dict", - dedup = true, - longest_only = true - }, - hypdecomp = new - { - type = "hyphenation_decompounder", - word_list = new[] {"x", "y", "z"}, - min_word_size = 2, - min_subword_size = 2, - max_subword_size = 2, - only_longest_match = true, - hyphenation_patterns_path = "analysis/fop.xml" - }, - keeptypes = new - { - type = "keep_types", - types = new[] {"", ""} - }, - icuc = new { - alternate = "non-ignorable", - caseFirst = "lower", - caseLevel = true, - country = "DE", - decomposition = "no", - hiraganaQuaternaryMode = true, - language = "de", - numeric = true, - strength = "tertiary", - type = "icu_collation", - variant = "@collation=phonebook" - }, - icuf = new { - type = "icu_folding", - unicodeSetFilter = "[^åäöÅÄÖ]" - }, - icun = new { - name = "nfc", - type = "icu_normalizer" - }, - icut = new { - dir = "forward", - id = "Any-Latin; NFD; [:Nonspacing Mark:] Remove; NFC", - type = "icu_transform" - }, - keepwords = new - { - type = "keep", - keep_words = new[] {"a", "b", "c"}, - keep_words_case = true - }, - marker = new - { - type = "keyword_marker", - keywords = new[] {"a", "b"}, - ignore_case = true - }, - kfr = new - { - type = "kuromoji_readingform", - use_romaji = true - }, - kpos = new - { - stoptags = new[] - { - "# verb-main:", - "動詞-自立" - }, - type = "kuromoji_part_of_speech" - }, - ks = new - { - minimum_length = 4, - type = "kuromoji_stemmer" - }, - kstem = new - { - type = "kstem" - }, - length = new - { - type = "length", - min = 10, - max = 200 - }, - limit = new - { - type = "limit", - max_token_count = 12, - consume_all_tokens = true - }, - lc = new - { - type = "lowercase" - }, - ngram = new - { - type = "ngram", - min_gram = 3, - max_gram = 4 - }, - pc = new - { - type = "pattern_capture", - patterns = new[] {"\\d", "\\w"}, - preserve_original = true - }, - pr = new - { - type = "pattern_replace", - pattern = "(\\d|\\w)", - replacement = "replacement" - }, - porter = new - { - type = "porter_stem" - }, - rev = new - { - type = "reverse" - }, - shing = new - { - type = "shingle", - min_shingle_size = 8, - max_shingle_size = 10, - output_unigrams = true, - output_unigrams_if_no_shingles = true, - token_separator = "|", - filler_token = "x" - }, - snow = new - { - type = "snowball", - language = "Dutch" - }, - standard = new - { - type = "standard" - }, - stem = new - { - type = "stemmer", - language = "arabic" - }, - stemo = new - { - type = "stemmer_override", - rules_path = "analysis/custom_stems.txt" - }, - stop = new - { - type = "stop", - stopwords = new[] {"x", "y", "z"}, - ignore_case = true, - remove_trailing = true - }, - syn = new - { - type = "synonym", - synonyms_path = "analysis/stopwords.txt", - format = "wordnet", - synonyms = new[] {"x=>y", "z=>s"}, - expand = true, - tokenizer = "whitespace" - }, - syn_graph = new - { - type = "synonym_graph", - synonyms_path = "analysis/stopwords.txt", - format = "wordnet", - synonyms = new[] {"x=>y", "z=>s"}, - expand = true, - tokenizer = "whitespace" - }, - trimmer = new - { - type = "trim" - }, - truncer = new - { - type = "truncate", - length = 100 - }, - uq = new - { - type = "unique", - only_on_same_position = true - }, - upper = new - { - type = "uppercase" - }, - wd = new - { - type = "word_delimiter", - generate_word_parts = true, - generate_number_parts = true, - catenate_words = true, - catenate_numbers = true, - catenate_all = true, - split_on_case_change = true, - preserve_original = true, - split_on_numerics = true, - stem_english_possessive = true, - protected_words = new[] {"x", "y", "z"} - }, - wdg = new - { - type = "word_delimiter_graph", - generate_word_parts = true, - generate_number_parts = true, - catenate_words = true, - catenate_numbers = true, - catenate_all = true, - split_on_case_change = true, - preserve_original = true, - split_on_numerics = true, - stem_english_possessive = true, - protected_words = new[] {"x", "y", "z"} - }, - phonetic = new - { - type = "phonetic", - encoder = "beider_morse", - rule_type = "exact", - name_type = "sephardic", - languageset = new [] { "cyrillic", "english", "hebrew" } - } - } - } - }; - - /** - * - */ - protected override Func> Fluent => FluentExample; - - public static Func> FluentExample => s => s - .Analysis(analysis => analysis - .TokenFilters(tf => tf - .AsciiFolding("myAscii", t => t.PreserveOriginal()) - .CommonGrams("myCommonGrams", t => t - .CommonWords("x", "y", "z") - .IgnoreCase() - .QueryMode() - ) - .DelimitedPayload("mydp", t => t - .Delimiter('-') - .Encoding(DelimitedPayloadEncoding.Identity) - ) - .DictionaryDecompounder("dcc", t => t - .MaxSubwordSize(2) - .MinSubwordSize(2) - .MinWordSize(2) - .OnlyLongestMatch() - .WordList("x", "y", "z") - ) - .EdgeNGram("etf", t => t - .MaxGram(2) - .MinGram(1) - ) - .Elision("elision", t => t - .Articles("a", "b", "c") - ) - .Hunspell("hunspell", t => t - .Dedup() - .Dictionary("path_to_dict") - .Locale("en_US") - .LongestOnly() - ) - .HyphenationDecompounder("hypdecomp", t => t - .MaxSubwordSize(2) - .MinSubwordSize(2) - .MinWordSize(2) - .OnlyLongestMatch() - .WordList("x", "y", "z") - .HyphenationPatternsPath("analysis/fop.xml") - ) - .KeepTypes("keeptypes", t => t - .Types("", "") - ) - .KeepWords("keepwords", t => t - .KeepWords("a", "b", "c") - .KeepWordsCase() - ) - .KeywordMarker("marker", t => t - .IgnoreCase() - .Keywords("a", "b") - ) - .KStem("kstem") - .Length("length", t => t - .Max(200) - .Min(10) - ) - .LimitTokenCount("limit", t => t - .ConsumeAllToken() - .MaxTokenCount(12) - ) - .Lowercase("lc") - .NGram("ngram", t => t - .MinGram(3) - .MaxGram(4) - ) - .PatternCapture("pc", t => t - .Patterns(@"\d", @"\w") - .PreserveOriginal() - ) - .PatternReplace("pr", t => t - .Pattern(@"(\d|\w)") - .Replacement("replacement") - ) - .PorterStem("porter") - .Reverse("rev") - .Shingle("shing", t => t - .FillerToken("x") - .MaxShingleSize(10) - .MinShingleSize(8) - .OutputUnigrams() - .OutputUnigramsIfNoShingles() - .TokenSeparator("|") - ) - .Snowball("snow", t => t.Language(SnowballLanguage.Dutch)) - .Standard("standard") - .Stemmer("stem", t => t.Language("arabic")) - .StemmerOverride("stemo", t => t.RulesPath("analysis/custom_stems.txt")) - .Stop("stop", t => t - .IgnoreCase() - .RemoveTrailing() - .StopWords("x", "y", "z") - ) - .Synonym("syn", t => t - .Expand() - .Format(SynonymFormat.WordNet) - .SynonymsPath("analysis/stopwords.txt") - .Synonyms("x=>y", "z=>s") - .Tokenizer("whitespace") - ) - .SynonymGraph("syn_graph", t => t - .Expand() - .Format(SynonymFormat.WordNet) - .SynonymsPath("analysis/stopwords.txt") - .Synonyms("x=>y", "z=>s") - .Tokenizer("whitespace") - ) - .Trim("trimmer") - .Truncate("truncer", t => t.Length(100)) - .Unique("uq", t => t.OnlyOnSamePosition()) - .Uppercase("upper") - .WordDelimiter("wd", t => t - .CatenateAll() - .CatenateNumbers() - .CatenateWords() - .GenerateNumberParts() - .GenerateWordParts() - .PreserveOriginal() - .ProtectedWords("x", "y", "z") - .SplitOnCaseChange() - .SplitOnNumerics() - .StemEnglishPossessive() - ) - .WordDelimiterGraph("wdg", t => t - .CatenateAll() - .CatenateNumbers() - .CatenateWords() - .GenerateNumberParts() - .GenerateWordParts() - .PreserveOriginal() - .ProtectedWords("x", "y", "z") - .SplitOnCaseChange() - .SplitOnNumerics() - .StemEnglishPossessive() - ) - .KuromojiPartOfSpeech("kpos", t => t - .StopTags("# verb-main:", "動詞-自立") - ) - .KuromojiReadingForm("kfr", t => t - .UseRomaji() - ) - .KuromojiStemmer("ks", t => t - .MinimumLength(4) - ) - .IcuCollation("icuc", t => t - .Alternate(IcuCollationAlternate.NonIgnorable) - .CaseFirst(IcuCollationCaseFirst.Lower) - .HiraganaQuaternaryMode() - .Decomposition(IcuCollationDecomposition.No) - .Numeric() - .CaseLevel() - .Country("DE") - .Language("de") - .Strength(IcuCollationStrength.Tertiary) - .Variant("@collation=phonebook") - ) - .IcuFolding("icuf", t=>t.UnicodeSetFilter("[^åäöÅÄÖ]")) - .IcuNormalization("icun", t=>t.Name(IcuNormalizationType.Canonical)) - .IcuTransform("icut", t=>t - .Direction(IcuTransformDirection.Forward) - .Id("Any-Latin; NFD; [:Nonspacing Mark:] Remove; NFC") - ) - .Phonetic("phonetic", t => t - .Encoder(PhoneticEncoder.Beidermorse) - .RuleType(PhoneticRuleType.Exact) - .NameType(PhoneticNameType.Sephardic) - .LanguageSet(PhoneticLanguage.Cyrillic, PhoneticLanguage.English, PhoneticLanguage.Hebrew) - ) - ) - ); + MinWordSize = 2, + MinSubwordSize = 2, + MaxSubwordSize = 2, + OnlyLongestMatch = true, + WordList = new[] {"x", "y", "z"} + }; + + protected override FuncTokenFilters Fluent => (n, tf) => tf + .DictionaryDecompounder(n, t => t + .MaxSubwordSize(2) + .MinSubwordSize(2) + .MinWordSize(2) + .OnlyLongestMatch() + .WordList("x", "y", "z") + ); + + protected override object Json => new + { + type = "dictionary_decompounder", + word_list = new[] {"x", "y", "z"}, + min_word_size = 2, + min_subword_size = 2, + max_subword_size = 2, + only_longest_match = true + }; + + } + + public class EdgeNgramTests : TokenFilterAssertionBase + { + protected override string Name => "etf"; + + protected override ITokenFilter Initializer => new EdgeNGramTokenFilter {MaxGram = 2, MinGram = 1}; + + protected override FuncTokenFilters Fluent => (n, tf) => tf + .EdgeNGram(n, t => t.MaxGram(2).MinGram(1)); + protected override object Json => new { type = "edge_ngram", min_gram = 1, max_gram = 2 }; + + } + + public class ElisionTests : TokenFilterAssertionBase + { + protected override string Name => "el"; + + protected override ITokenFilter Initializer => new ElisionTokenFilter {Articles = new[] {"a", "b", "c"}}; + + protected override FuncTokenFilters Fluent => (n, tf) => tf.Elision(n, t => t.Articles("a", "b", "c")); + + protected override object Json => new { type = "elision", articles = new[] {"a", "b", "c"} }; + } + + public class HunspellTests : TokenFilterAssertionBase + { + protected override string Name => "huns"; + + protected override ITokenFilter Initializer => + new HunspellTokenFilter + { + Dedup = true, + Dictionary = "path_to_dict", + Locale = "en_US", + LongestOnly = true + }; - /** - */ - protected override IndexSettings Initializer => InitializerExample; + protected override FuncTokenFilters Fluent => (n, tf) => tf + .Hunspell(n, t => t + .Dedup() + .Dictionary("path_to_dict") + .Locale("en_US") + .LongestOnly() + ); - public static IndexSettings InitializerExample => - new IndexSettings + protected override object Json => new { - Analysis = new Nest.Analysis + type = "hunspell", + locale = "en_US", + dictionary = "path_to_dict", + dedup = true, + longest_only = true + }; + + } + + public class HyphenationDecompounderTests : TokenFilterAssertionBase + { + protected override string Name => "hyphdecomp"; + + protected override ITokenFilter Initializer => + new HyphenationDecompounderTokenFilter { - TokenFilters = new Nest.TokenFilters - { - {"myAscii", new AsciiFoldingTokenFilter {PreserveOriginal = true}}, - {"myCommonGrams", new CommonGramsTokenFilter {QueryMode = true, IgnoreCase = true, CommonWords = new[] {"x", "y", "z"}}}, - {"mydp", new DelimitedPayloadTokenFilter {Delimiter = '-', Encoding = DelimitedPayloadEncoding.Identity}}, - { - "dcc", new DictionaryDecompounderTokenFilter - { - MinWordSize = 2, - MinSubwordSize = 2, - MaxSubwordSize = 2, - OnlyLongestMatch = true, - WordList = new[] {"x", "y", "z"} - } - }, - {"etf", new EdgeNGramTokenFilter {MaxGram = 2, MinGram = 1}}, - {"elision", new ElisionTokenFilter {Articles = new[] {"a", "b", "c"}}}, - { - "hunspell", new HunspellTokenFilter - { - Dedup = true, - Dictionary = "path_to_dict", - Locale = "en_US", - LongestOnly = true - } - }, - { - "hypdecomp", new HyphenationDecompounderTokenFilter - { - MaxSubwordSize = 2, - MinSubwordSize = 2, - MinWordSize = 2, - OnlyLongestMatch = true, - WordList = new[] {"x", "y", "z"}, - HyphenationPatternsPath = "analysis/fop.xml" - } - }, - {"keeptypes", new KeepTypesTokenFilter {Types = new[] {"", ""}}}, - {"keepwords", new KeepWordsTokenFilter {KeepWordsCase = true, KeepWords = new[] {"a", "b", "c"}}}, - {"marker", new KeywordMarkerTokenFilter {IgnoreCase = true, Keywords = new[] {"a", "b"}}}, - {"kstem", new KStemTokenFilter { }}, - {"length", new LengthTokenFilter {Min = 10, Max = 200}}, - {"limit", new LimitTokenCountTokenFilter {ConsumeAllTokens = true, MaxTokenCount = 12}}, - {"lc", new LowercaseTokenFilter()}, - {"ngram", new NGramTokenFilter {MinGram = 3, MaxGram = 4}}, - {"pc", new PatternCaptureTokenFilter {Patterns = new[] {@"\d", @"\w"}, PreserveOriginal = true}}, - {"pr", new PatternReplaceTokenFilter {Pattern = @"(\d|\w)", Replacement = "replacement"}}, - {"porter", new PorterStemTokenFilter()}, - {"rev", new ReverseTokenFilter()}, - { - "shing", new ShingleTokenFilter - { - FillerToken = "x", - MaxShingleSize = 10, - MinShingleSize = 8, - OutputUnigrams = true, - OutputUnigramsIfNoShingles = true, - TokenSeparator = "|" - } - }, - {"snow", new SnowballTokenFilter {Language = SnowballLanguage.Dutch}}, - {"standard", new StandardTokenFilter()}, - {"stem", new StemmerTokenFilter {Language = "arabic"}}, - {"stemo", new StemmerOverrideTokenFilter {RulesPath = "analysis/custom_stems.txt"}}, - {"stop", new StopTokenFilter {IgnoreCase = true, RemoveTrailing = true, StopWords = new[] {"x", "y", "z"}}}, - { - "syn", new SynonymTokenFilter - { - Expand = true, - Format = SynonymFormat.WordNet, - SynonymsPath = "analysis/stopwords.txt", - Synonyms = new[] {"x=>y", "z=>s"}, - Tokenizer = "whitespace" - } - }, - { - "syn_graph", new SynonymGraphTokenFilter - { - Expand = true, - Format = SynonymFormat.WordNet, - SynonymsPath = "analysis/stopwords.txt", - Synonyms = new[] {"x=>y", "z=>s"}, - Tokenizer = "whitespace" - } - }, - {"trimmer", new TrimTokenFilter()}, - {"truncer", new TruncateTokenFilter {Length = 100}}, - {"uq", new UniqueTokenFilter {OnlyOnSamePosition = true,}}, - {"upper", new UppercaseTokenFilter()}, - { - "wd", new WordDelimiterTokenFilter - { - CatenateAll = true, - CatenateNumbers = true, - CatenateWords = true, - GenerateNumberParts = true, - GenerateWordParts = true, - PreserveOriginal = true, - ProtectedWords = new[] {"x", "y", "z"}, - SplitOnCaseChange = true, - SplitOnNumerics = true, - StemEnglishPossessive = true - } - }, - { - "wdg", new WordDelimiterGraphTokenFilter - { - CatenateAll = true, - CatenateNumbers = true, - CatenateWords = true, - GenerateNumberParts = true, - GenerateWordParts = true, - PreserveOriginal = true, - ProtectedWords = new[] {"x", "y", "z"}, - SplitOnCaseChange = true, - SplitOnNumerics = true, - StemEnglishPossessive = true - } - }, - {"kpos", new KuromojiPartOfSpeechTokenFilter {StopTags = new[] {"# verb-main:", "動詞-自立"}}}, - {"kfr", new KuromojiReadingFormTokenFilter {UseRomaji = true}}, - {"ks", new KuromojiStemmerTokenFilter {MinimumLength = 4}}, - - { - "icuc", new IcuCollationTokenFilter - { - Alternate = IcuCollationAlternate.NonIgnorable, - CaseFirst = IcuCollationCaseFirst.Lower, - HiraganaQuaternaryMode = true, - Decomposition = IcuCollationDecomposition.No, - Numeric = true, - CaseLevel = true, - Country = "DE", - Language = "de", - Strength = IcuCollationStrength.Tertiary, - Variant = "@collation=phonebook" - } - }, - {"icuf", new IcuFoldingTokenFilter - { - UnicodeSetFilter = "[^åäöÅÄÖ]" - }}, - {"icun", new IcuNormalizationTokenFilter - { - Name = IcuNormalizationType.Canonical - }}, - {"icut", new IcuTransformTokenFilter - { - Direction = IcuTransformDirection.Forward, - Id = "Any-Latin; NFD; [:Nonspacing Mark:] Remove; NFC" - }}, - { "phonetic", new PhoneticTokenFilter - { - Encoder = PhoneticEncoder.Beidermorse, - RuleType = PhoneticRuleType.Exact, - NameType = PhoneticNameType.Sephardic, - LanguageSet = new [] { PhoneticLanguage.Cyrillic, PhoneticLanguage.English, PhoneticLanguage.Hebrew } - }}, - } - } + MaxSubwordSize = 2, + MinSubwordSize = 2, + MinWordSize = 2, + OnlyLongestMatch = true, + WordList = new[] {"x", "y", "z"}, + HyphenationPatternsPath = "analysis/fop.xml" + }; + + protected override FuncTokenFilters Fluent => (n, tf) => tf + .HyphenationDecompounder(n, t => t + .MaxSubwordSize(2) + .MinSubwordSize(2) + .MinWordSize(2) + .OnlyLongestMatch() + .WordList("x", "y", "z") + .HyphenationPatternsPath("analysis/fop.xml") + ); + + protected override object Json => new + { + type = "hyphenation_decompounder", + word_list = new[] {"x", "y", "z"}, + min_word_size = 2, + min_subword_size = 2, + max_subword_size = 2, + only_longest_match = true, + hyphenation_patterns_path = "analysis/fop.xml" + }; + + } + + public class KeepTypesTests : TokenFilterAssertionBase + { + protected override string Name => "keeptypes"; + + protected override ITokenFilter Initializer => + new KeepTypesTokenFilter {Types = new[] {"", ""}}; + + protected override FuncTokenFilters Fluent => (n, tf) => tf + .KeepTypes(n, t => t + .Types("", "") + ); + + protected override object Json => new + { + type = "keep_types", + types = new[] {"", ""} }; + + } + + public class IcuCollationTests : TokenFilterAssertionBase + { + protected override string Name => "icuc"; + + protected override ITokenFilter Initializer => + new IcuCollationTokenFilter + { + Alternate = IcuCollationAlternate.NonIgnorable, + CaseFirst = IcuCollationCaseFirst.Lower, + HiraganaQuaternaryMode = true, + Decomposition = IcuCollationDecomposition.No, + Numeric = true, + CaseLevel = true, + Country = "DE", + Language = "de", + Strength = IcuCollationStrength.Tertiary, + Variant = "@collation=phonebook" + }; + + protected override FuncTokenFilters Fluent => (n, tf) => tf + .IcuCollation(n, t => t + .Alternate(IcuCollationAlternate.NonIgnorable) + .CaseFirst(IcuCollationCaseFirst.Lower) + .HiraganaQuaternaryMode() + .Decomposition(IcuCollationDecomposition.No) + .Numeric() + .CaseLevel() + .Country("DE") + .Language("de") + .Strength(IcuCollationStrength.Tertiary) + .Variant("@collation=phonebook") + ); + + protected override object Json => new + { + alternate = "non-ignorable", + caseFirst = "lower", + caseLevel = true, + country = "DE", + decomposition = "no", + hiraganaQuaternaryMode = true, + language = "de", + numeric = true, + strength = "tertiary", + type = "icu_collation", + variant = "@collation=phonebook" + }; + + } + + public class IcuFoldingTests : TokenFilterAssertionBase + { + protected override string Name => "icuf"; + + protected override ITokenFilter Initializer => + new IcuFoldingTokenFilter { UnicodeSetFilter = "[^åäöÅÄÖ]" }; + + protected override FuncTokenFilters Fluent => (n, tf) => tf.IcuFolding(n, t => t.UnicodeSetFilter("[^åäöÅÄÖ]")); + + protected override object Json => new + { + type = "icu_folding", + unicodeSetFilter = "[^åäöÅÄÖ]" + }; + + } + + public class IcuNormalizerTests : TokenFilterAssertionBase + { + protected override string Name => "icun"; + + protected override ITokenFilter Initializer => new IcuNormalizationTokenFilter { Name = IcuNormalizationType.Canonical }; + + protected override FuncTokenFilters Fluent => (n, tf) => tf .IcuNormalization(n, t => t.Name(IcuNormalizationType.Canonical)); + + protected override object Json => new + { + name = "nfc", + type = "icu_normalizer" + }; + + } + + public class IcuTransformTests : TokenFilterAssertionBase + { + protected override string Name => "icut"; + + protected override ITokenFilter Initializer => + new IcuTransformTokenFilter + { + Direction = IcuTransformDirection.Forward, + Id = "Any-Latin; NFD; [:Nonspacing Mark:] Remove; NFC" + }; + + protected override FuncTokenFilters Fluent => (n, tf) => tf + .IcuTransform(n, t => t + .Direction(IcuTransformDirection.Forward) + .Id("Any-Latin; NFD; [:Nonspacing Mark:] Remove; NFC") + ); + + protected override object Json => new + { + dir = "forward", + id = "Any-Latin; NFD; [:Nonspacing Mark:] Remove; NFC", + type = "icu_transform" + }; + + } + + public class KeepwordsTests : TokenFilterAssertionBase + { + protected override string Name => "keepwords"; + + protected override ITokenFilter Initializer => + new KeepWordsTokenFilter {KeepWordsCase = true, KeepWords = new[] {"a", "b", "c"}}; + + protected override FuncTokenFilters Fluent => (n, tf) => tf + .KeepWords(n, t => t + .KeepWords("a", "b", "c") + .KeepWordsCase() + ); + + protected override object Json => new + { + type = "keep", + keep_words = new[] {"a", "b", "c"}, + keep_words_case = true + }; + + } + + public class MarkerTests : TokenFilterAssertionBase + { + protected override string Name => "marker"; + + protected override ITokenFilter Initializer => new KeywordMarkerTokenFilter {IgnoreCase = true, Keywords = new[] {"a", "b"}}; + + protected override FuncTokenFilters Fluent => (n, tf) => tf + .KeywordMarker("marker", t => t + .IgnoreCase() + .Keywords("a", "b") + ); + + protected override object Json => new + { + type = "keyword_marker", + keywords = new[] {"a", "b"}, + ignore_case = true + }; + + + } + + public class KuromojiReadingFormTests : TokenFilterAssertionBase + { + protected override string Name => "kfr"; + + protected override ITokenFilter Initializer => new KuromojiReadingFormTokenFilter {UseRomaji = true}; + + protected override FuncTokenFilters Fluent => (n, tf) => tf.KuromojiReadingForm(n, t => t.UseRomaji()); + + protected override object Json => new + { + type = "kuromoji_readingform", + use_romaji = true + }; + + } + + public class KuromojiPartOfSpeechTests : TokenFilterAssertionBase + { + protected override string Name => "kpos"; + + protected override ITokenFilter Initializer => + new KuromojiPartOfSpeechTokenFilter {StopTags = new[] {"# verb-main:", "動詞-自立"}}; + + protected override FuncTokenFilters Fluent => (n, tf) => tf + .KuromojiPartOfSpeech(n, t => t.StopTags("# verb-main:", "動詞-自立")); + + protected override object Json => new + { + stoptags = new[] + { + "# verb-main:", + "動詞-自立" + }, + type = "kuromoji_part_of_speech" + }; + + } + + public class KuromojiStemmerTests : TokenFilterAssertionBase + { + protected override string Name => "ks"; + + protected override ITokenFilter Initializer => new KuromojiStemmerTokenFilter {MinimumLength = 4}; + + protected override FuncTokenFilters Fluent => (n, tf) => tf.KuromojiStemmer(n, t => t.MinimumLength(4)); + + protected override object Json => new + { + minimum_length = 4, + type = "kuromoji_stemmer" + }; + + } + + public class KStemTests : TokenFilterAssertionBase + { + protected override string Name => "kstem"; + protected override ITokenFilter Initializer => new KStemTokenFilter { }; + protected override FuncTokenFilters Fluent => (n, tf) => tf.KStem(n); + protected override object Json => new {type = "kstem"}; + } + + public class LengthTests : TokenFilterAssertionBase + { + protected override string Name => "length"; + protected override ITokenFilter Initializer => new LengthTokenFilter {Min = 10, Max = 200}; + + protected override FuncTokenFilters Fluent => (n, tf) => tf.Length(n, t => t.Max(200).Min(10)); + protected override object Json => new {type = "length", min = 10, max = 200}; + + } + + public class LimitTests : TokenFilterAssertionBase + { + protected override string Name => "limit"; + + protected override ITokenFilter Initializer => new LimitTokenCountTokenFilter {ConsumeAllTokens = true, MaxTokenCount = 12}; + + protected override FuncTokenFilters Fluent => (n, tf) => tf.LimitTokenCount(n, t => t.ConsumeAllToken().MaxTokenCount(12)); + + protected override object Json => new + { + type = "limit", + max_token_count = 12, + consume_all_tokens = true + }; + + } + + public class LowercaseTests : TokenFilterAssertionBase + { + protected override string Name => "lc"; + + protected override ITokenFilter Initializer => new LowercaseTokenFilter(); + + protected override FuncTokenFilters Fluent => (n, tf) => tf.Lowercase(n); + + protected override object Json => new {type = "lowercase"}; + + } + + public class NGramTests : TokenFilterAssertionBase + { + protected override string Name => "ngram"; + + protected override ITokenFilter Initializer => new NGramTokenFilter {MinGram = 3, MaxGram = 4}; + + protected override FuncTokenFilters Fluent => (n, tf) => tf.NGram(n, t => t.MinGram(3).MaxGram(4)); + + protected override object Json => new {type = "ngram", min_gram = 3, max_gram = 4}; + + } + + public class PatternCaptureTests : TokenFilterAssertionBase + { + protected override string Name => "pc"; + + protected override ITokenFilter Initializer => + new PatternCaptureTokenFilter {Patterns = new[] {@"\d", @"\w"}, PreserveOriginal = true}; + + protected override FuncTokenFilters Fluent => (n, tf) => tf + .PatternCapture(n, t => t.Patterns(@"\d", @"\w").PreserveOriginal()); + + protected override object Json => new + { + type = "pattern_capture", + patterns = new[] {"\\d", "\\w"}, + preserve_original = true + }; + } + + public class PatternReplaceTests : TokenFilterAssertionBase + { + protected override string Name => "pr"; + + protected override ITokenFilter Initializer => + new PatternReplaceTokenFilter {Pattern = @"(\d|\w)", Replacement = "replacement"}; + + protected override FuncTokenFilters Fluent => (n, tf) => tf + .PatternReplace(n, t => t + .Pattern(@"(\d|\w)") + .Replacement("replacement") + ); + + protected override object Json => new + { + type = "pattern_replace", + pattern = "(\\d|\\w)", + replacement = "replacement" + }; + + } + + public class PorterStemTests : TokenFilterAssertionBase + { + protected override string Name => "porter"; + protected override ITokenFilter Initializer => new PorterStemTokenFilter(); + protected override FuncTokenFilters Fluent => (n, tf) => tf.PorterStem(n); + protected override object Json => new { type = "porter_stem" }; + } + + public class ReverseTests : TokenFilterAssertionBase + { + protected override string Name => "rev"; + protected override ITokenFilter Initializer => new ReverseTokenFilter(); + protected override FuncTokenFilters Fluent => (n, tf) => tf.Reverse(n); + protected override object Json => new {type = "reverse"}; + + } + + public class ShingleTests : TokenFilterAssertionBase + { + protected override string Name => "shing"; + + protected override ITokenFilter Initializer => new ShingleTokenFilter + { + FillerToken = "x", + MaxShingleSize = 10, + MinShingleSize = 8, + OutputUnigrams = true, + OutputUnigramsIfNoShingles = true, + TokenSeparator = "|" + }; + + protected override FuncTokenFilters Fluent => (n, tf) => tf + .Shingle(n, t => t + .FillerToken("x") + .MaxShingleSize(10) + .MinShingleSize(8) + .OutputUnigrams() + .OutputUnigramsIfNoShingles() + .TokenSeparator("|") + ); + + protected override object Json => new + { + type = "shingle", + min_shingle_size = 8, + max_shingle_size = 10, + output_unigrams = true, + output_unigrams_if_no_shingles = true, + token_separator = "|", + filler_token = "x" + }; + + } + + public class SnowballTests : TokenFilterAssertionBase + { + protected override string Name => "snow"; + + protected override ITokenFilter Initializer => new SnowballTokenFilter {Language = SnowballLanguage.Dutch}; + + protected override FuncTokenFilters Fluent => (n, tf) => tf.Snowball(n, t => t.Language(SnowballLanguage.Dutch)); + + protected override object Json => new + { + type = "snowball", + language = "Dutch" + }; + + } + + public class StandardTests : TokenFilterAssertionBase + { + protected override string Name => "standard"; + + protected override ITokenFilter Initializer => new StandardTokenFilter(); + + protected override FuncTokenFilters Fluent => (n, tf) => tf.Standard(n); + + protected override object Json => new { type = "standard" }; + + } + + public class StemmerTests : TokenFilterAssertionBase + { + protected override string Name => "stem"; + + protected override ITokenFilter Initializer => new StemmerTokenFilter {Language = "arabic"}; + + protected override FuncTokenFilters Fluent => (n, tf) => tf.Stemmer(n, t => t.Language("arabic")); + + protected override object Json => new + { + type = "stemmer", + language = "arabic" + }; + + } + + public class StemmerOverrideTests : TokenFilterAssertionBase + { + protected override string Name => "stemo"; + + protected override ITokenFilter Initializer => new StemmerOverrideTokenFilter {RulesPath = "analysis/custom_stems.txt"}; + + protected override FuncTokenFilters Fluent => (n, tf) => tf.StemmerOverride(n, t => t.RulesPath("analysis/custom_stems.txt")); + + protected override object Json => new + { + type = "stemmer_override", + rules_path = "analysis/custom_stems.txt" + }; + + } + + public class StopTests : TokenFilterAssertionBase + { + protected override string Name => "stop"; + + protected override ITokenFilter Initializer => + new StopTokenFilter {IgnoreCase = true, RemoveTrailing = true, StopWords = new[] {"x", "y", "z"}}; + + protected override FuncTokenFilters Fluent => (n, tf) => tf + .Stop(n, t => t + .IgnoreCase() + .RemoveTrailing() + .StopWords("x", "y", "z") + ); + + protected override object Json => new + { + type = "stop", + stopwords = new[] {"x", "y", "z"}, + ignore_case = true, + remove_trailing = true + }; + + } + + public class SynonymTests : TokenFilterAssertionBase + { + protected override string Name => "syn"; + + protected override ITokenFilter Initializer => + new SynonymTokenFilter + { + Expand = true, + Format = SynonymFormat.WordNet, + SynonymsPath = "analysis/stopwords.txt", + Synonyms = new[] {"x=>y", "z=>s"}, + Tokenizer = "whitespace" + }; + + protected override FuncTokenFilters Fluent => (n, tf) => tf + .Synonym(n, t => t + .Expand() + .Format(SynonymFormat.WordNet) + .SynonymsPath("analysis/stopwords.txt") + .Synonyms("x=>y", "z=>s") + .Tokenizer("whitespace") + ); + + protected override object Json => new + { + type = "synonym", + synonyms_path = "analysis/stopwords.txt", + format = "wordnet", + synonyms = new[] {"x=>y", "z=>s"}, + expand = true, + tokenizer = "whitespace" + }; + + } + + public class SynonymGraphTests : TokenFilterAssertionBase + { + protected override string Name => "syn_graph"; + + protected override ITokenFilter Initializer => + new SynonymGraphTokenFilter + { + Expand = true, + Format = SynonymFormat.WordNet, + SynonymsPath = "analysis/stopwords.txt", + Synonyms = new[] {"x=>y", "z=>s"}, + Tokenizer = "whitespace" + }; + + protected override FuncTokenFilters Fluent => (n, tf) => tf + .SynonymGraph(n, t => t + .Expand() + .Format(SynonymFormat.WordNet) + .SynonymsPath("analysis/stopwords.txt") + .Synonyms("x=>y", "z=>s") + .Tokenizer("whitespace") + ); + + protected override object Json => new + { + type = "synonym_graph", + synonyms_path = "analysis/stopwords.txt", + format = "wordnet", + synonyms = new[] {"x=>y", "z=>s"}, + expand = true, + tokenizer = "whitespace" + }; + + } + + public class TrimTests : TokenFilterAssertionBase + { + protected override string Name => "trimmer"; + protected override ITokenFilter Initializer => new TrimTokenFilter(); + protected override FuncTokenFilters Fluent => (n, tf) => tf.Trim(n); + protected override object Json => new {type = "trim"}; + } + + public class TruncateTests : TokenFilterAssertionBase + { + protected override string Name => "truncer"; + protected override ITokenFilter Initializer => new TruncateTokenFilter {Length = 100}; + protected override FuncTokenFilters Fluent => (n, tf) => tf.Truncate(n, t => t.Length(100)); + protected override object Json => new {type = "truncate", length = 100}; + } + + public class UniqueTests : TokenFilterAssertionBase + { + protected override string Name => "uq"; + protected override ITokenFilter Initializer => new UniqueTokenFilter {OnlyOnSamePosition = true,}; + protected override FuncTokenFilters Fluent => (n, tf) => tf.Unique(n, t => t.OnlyOnSamePosition()); + protected override object Json => new {type = "unique", only_on_same_position = true}; + + } + public class UppercaseTests : TokenFilterAssertionBase + { + protected override string Name => "upper"; + protected override ITokenFilter Initializer => new UppercaseTokenFilter(); + protected override FuncTokenFilters Fluent => (n, tf) => tf.Uppercase(n); + protected override object Json => new {type = "uppercase"}; + + } + public class WordDelimiterTests : TokenFilterAssertionBase + { + protected override string Name => "wd"; + + protected override ITokenFilter Initializer => + new WordDelimiterTokenFilter + { + CatenateAll = true, + CatenateNumbers = true, + CatenateWords = true, + GenerateNumberParts = true, + GenerateWordParts = true, + PreserveOriginal = true, + ProtectedWords = new[] {"x", "y", "z"}, + SplitOnCaseChange = true, + SplitOnNumerics = true, + StemEnglishPossessive = true + }; + + protected override FuncTokenFilters Fluent => (n, tf) => tf + .WordDelimiter(n, t => t + .CatenateAll() + .CatenateNumbers() + .CatenateWords() + .GenerateNumberParts() + .GenerateWordParts() + .PreserveOriginal() + .ProtectedWords("x", "y", "z") + .SplitOnCaseChange() + .SplitOnNumerics() + .StemEnglishPossessive() + ); + + protected override object Json => new + { + type = "word_delimiter", + generate_word_parts = true, + generate_number_parts = true, + catenate_words = true, + catenate_numbers = true, + catenate_all = true, + split_on_case_change = true, + preserve_original = true, + split_on_numerics = true, + stem_english_possessive = true, + protected_words = new[] {"x", "y", "z"} + }; + + } + + public class WordDelimiterGraphTests : TokenFilterAssertionBase + { + protected override string Name => "wdg"; + + protected override ITokenFilter Initializer => + new WordDelimiterGraphTokenFilter + { + CatenateAll = true, + CatenateNumbers = true, + CatenateWords = true, + GenerateNumberParts = true, + GenerateWordParts = true, + PreserveOriginal = true, + ProtectedWords = new[] {"x", "y", "z"}, + SplitOnCaseChange = true, + SplitOnNumerics = true, + StemEnglishPossessive = true + }; + + protected override FuncTokenFilters Fluent => (n, tf) => tf + .WordDelimiterGraph(n, t => t + .CatenateAll() + .CatenateNumbers() + .CatenateWords() + .GenerateNumberParts() + .GenerateWordParts() + .PreserveOriginal() + .ProtectedWords("x", "y", "z") + .SplitOnCaseChange() + .SplitOnNumerics() + .StemEnglishPossessive() + ); + + protected override object Json => new + { + type = "word_delimiter_graph", + generate_word_parts = true, + generate_number_parts = true, + catenate_words = true, + catenate_numbers = true, + catenate_all = true, + split_on_case_change = true, + preserve_original = true, + split_on_numerics = true, + stem_english_possessive = true, + protected_words = new[] {"x", "y", "z"} + }; + + } + + public class PhoneticTests : TokenFilterAssertionBase + { + protected override string Name => "phonetic"; + + protected override ITokenFilter Initializer => + new PhoneticTokenFilter + { + Encoder = PhoneticEncoder.Beidermorse, + RuleType = PhoneticRuleType.Exact, + NameType = PhoneticNameType.Sephardic, + LanguageSet = new[] {PhoneticLanguage.Cyrillic, PhoneticLanguage.English, PhoneticLanguage.Hebrew} + }; + + protected override FuncTokenFilters Fluent => (n, tf) => tf + .Phonetic(n, t => t + .Encoder(PhoneticEncoder.Beidermorse) + .RuleType(PhoneticRuleType.Exact) + .NameType(PhoneticNameType.Sephardic) + .LanguageSet( + PhoneticLanguage.Cyrillic, + PhoneticLanguage.English, + PhoneticLanguage.Hebrew + ) + ); + + protected override object Json => new + { + type = "phonetic", + encoder = "beider_morse", + rule_type = "exact", + name_type = "sephardic", + languageset = new[] {"cyrillic", "english", "hebrew"} + }; + + } } } diff --git a/src/Tests/Tests/Analysis/Tokenizers/ITokenizerAssertion.cs b/src/Tests/Tests/Analysis/Tokenizers/ITokenizerAssertion.cs new file mode 100644 index 0000000000..d882ed8e6b --- /dev/null +++ b/src/Tests/Tests/Analysis/Tokenizers/ITokenizerAssertion.cs @@ -0,0 +1,12 @@ +using System; +using Nest; + +namespace Tests.Analysis.Tokenizers +{ + + public interface ITokenizerAssertion : IAnalysisAssertion + { + ITokenizer Initializer { get; } + Func> Fluent { get; } + } +} diff --git a/src/Tests/Tests/Analysis/Tokenizers/TokenizerAssertionBase.cs b/src/Tests/Tests/Analysis/Tokenizers/TokenizerAssertionBase.cs new file mode 100644 index 0000000000..638efa8c1f --- /dev/null +++ b/src/Tests/Tests/Analysis/Tokenizers/TokenizerAssertionBase.cs @@ -0,0 +1,88 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Elastic.Xunit; +using Elastic.Xunit.XunitPlumbing; +using FluentAssertions; +using Nest; +using Tests.Core.Client; +using Tests.Core.ManagedElasticsearch.Clusters; +using Tests.Core.Serialization; +using Tests.Framework.Integration; + +namespace Tests.Analysis.Tokenizers +{ + + [IntegrationTestCluster(typeof(ReadOnlyCluster))] + public abstract class TokenizerAssertionBase where TAssertion : TokenizerAssertionBase, new() + { + private static readonly SingleEndpointUsage Usage = new SingleEndpointUsage + ( + fluent: (s, c) => c.CreateIndex(s, FluentCall), + fluentAsync: (s, c) => c.CreateIndexAsync(s, FluentCall), + request: (s, c) => c.CreateIndex(InitializerCall(s)), + requestAsync: (s, c) => c.CreateIndexAsync(InitializerCall(s)), + valuePrefix: $"test-{typeof(TAssertion).Name.ToLowerInvariant()}" + ) + { + OnAfterCall = c=> c.DeleteIndex(Usage.CallUniqueValues.Value) + }; + private static TAssertion AssertionSetup { get; } = new TAssertion(); + + protected TokenizerAssertionBase() + { + this.Client = (ElasticXunitRunner.CurrentCluster as ReadOnlyCluster)?.Client ?? TestClient.DefaultInMemoryClient; + Usage.KickOffOnce(this.Client); + } + + private IElasticClient Client { get; } + + protected abstract string Name { get; } + protected abstract ITokenizer Initializer { get; } + protected abstract Func> Fluent { get; } + protected abstract object Json { get; } + + [U] public async Task TestPutSettingsRequest() => await Usage.AssertOnAllResponses(r => + { + var json = new + { + settings = new + { + analysis = new + { + tokenizer = new Dictionary + { + { AssertionSetup.Name, AssertionSetup.Json} + } + } + } + }; + SerializationTestHelper.Expect(json).FromRequest(r); + }); + + [I] public async Task TestPutSettingsResponse() => await Usage.AssertOnAllResponses(r => + { + r.ApiCall.HttpStatusCode.Should().Be(200); + }); + + private static CreateIndexRequest InitializerCall(string index) => new CreateIndexRequest(index) + { + Settings = new IndexSettings + { + Analysis = new Nest.Analysis + { + Tokenizers = new Nest.Tokenizers { { AssertionSetup.Name, AssertionSetup.Initializer } } + + } + } + }; + + private static Func FluentCall => i => i + .Settings(s => s + .Analysis(a => a + .Tokenizers(d => AssertionSetup.Fluent(AssertionSetup.Name, d)) + ) + ); + + } +} diff --git a/src/Tests/Tests/Analysis/Tokenizers/TokenizerUsageTests.cs b/src/Tests/Tests/Analysis/Tokenizers/TokenizerUsageTests.cs index d89658685e..613ecc2d12 100644 --- a/src/Tests/Tests/Analysis/Tokenizers/TokenizerUsageTests.cs +++ b/src/Tests/Tests/Analysis/Tokenizers/TokenizerUsageTests.cs @@ -1,186 +1,207 @@ using System; using Nest; -using Tests.Framework; namespace Tests.Analysis.Tokenizers { - /** - */ + using FuncTokenizer = Func>; - public class TokenizerUsageTests : PromiseUsageTestBase + public static class TokenizerTests { - protected override object ExpectJson => new + public class EdgeNGramTests : TokenizerAssertionBase { - analysis = new + protected override string Name => "endgen"; + + protected override ITokenizer Initializer => new EdgeNGramTokenizer + { + MaxGram = 2, + MinGram = 1, + TokenChars = new[] {TokenChar.Digit, TokenChar.Letter} + }; + + protected override FuncTokenizer Fluent => (n, t) => t.EdgeNGram(n, e => e + .MaxGram(2) + .MinGram(1) + .TokenChars(TokenChar.Digit, TokenChar.Letter) + ); + + protected override object Json => new + { + min_gram = 1, + max_gram = 2, + token_chars = new[] {"digit", "letter"}, + type = "edge_ngram" + }; + } + + public class NGramTests : TokenizerAssertionBase + { + protected override string Name => "ng"; + + protected override ITokenizer Initializer => new NGramTokenizer + { + MaxGram = 2, + MinGram = 1, + TokenChars = new[] {TokenChar.Digit, TokenChar.Letter} + }; + + protected override FuncTokenizer Fluent => (n, t) => t.NGram(n, e => e + .MaxGram(2) + .MinGram(1) + .TokenChars(TokenChar.Digit, TokenChar.Letter) + ); + + protected override object Json => new + { + min_gram = 1, + max_gram = 2, + token_chars = new[] {"digit", "letter"}, + type = "ngram" + }; + } + + public class PathHierarchyTests : TokenizerAssertionBase + { + protected override string Name => "path"; + + protected override ITokenizer Initializer => new PathHierarchyTokenizer { - tokenizer = new - { - endgen = new - { - min_gram = 1, - max_gram = 2, - token_chars = new[] {"digit", "letter"}, - type = "edge_ngram" - }, - icu = new - { - rule_files = "Latn:icu-files/KeywordTokenizer.rbbi", - type = "icu_tokenizer" - }, - kuromoji = new - { - discard_punctuation = true, - mode = "extended", - nbest_cost = 1000, - nbest_examples = "/箱根山-箱根/成田空港-成田/", - type = "kuromoji_tokenizer" - }, - ng = new - { - min_gram = 1, - max_gram = 2, - token_chars = new[] {"digit", "letter"}, - type = "ngram" - }, - path = new - { - delimiter = "|", - replacement = "-", - buffer_size = 2048, - reverse = true, - skip = 1, - type = "path_hierarchy" - }, - pattern = new - { - pattern = @"\W+", - flags = "CASE_INSENSITIVE", - group = 1, - type = "pattern" - }, - standard = new - { - type = "standard" - }, - uax = new - { - max_token_length = 12, - type = "uax_url_email" - }, - whitespace = new - { - type = "whitespace" - } - } - } - }; - - /** - * - */ - protected override Func> Fluent => FluentExample; - - public static Func> FluentExample => s => s - .Analysis(analysis => analysis - .Tokenizers(tokenizer => tokenizer - .EdgeNGram("endgen", t => t - .MaxGram(2) - .MinGram(1) - .TokenChars(TokenChar.Digit, TokenChar.Letter) - ) - .NGram("ng", t => t - .MaxGram(2) - .MinGram(1) - .TokenChars(TokenChar.Digit, TokenChar.Letter) - ) - .PathHierarchy("path", t => t - .BufferSize(2048) - .Delimiter('|') - .Replacement('-') - .Reverse() - .Skip(1) - ) - .Pattern("pattern", t => t - .Flags("CASE_INSENSITIVE") - .Group(1) - .Pattern(@"\W+") - ) - .Standard("standard") - .UaxEmailUrl("uax", t => t.MaxTokenLength(12)) - .Whitespace("whitespace") - .Kuromoji("kuromoji", t => t - .Mode(KuromojiTokenizationMode.Extended) - .DiscardPunctuation() - .NBestExamples("/箱根山-箱根/成田空港-成田/") - .NBestCost(1000) - ) - .Icu("icu", t => t.RuleFiles("Latn:icu-files/KeywordTokenizer.rbbi")) - ) + BufferSize = 2048, + Delimiter = '|', + Replacement = '-', + Reverse = true, + Skip = 1 + }; + + protected override FuncTokenizer Fluent => (n, t) => t.PathHierarchy(n, e => e + .BufferSize(2048) + .Delimiter('|') + .Replacement('-') + .Reverse() + .Skip(1) ); - /** - */ - protected override IndexSettings Initializer => InitializerExample; + protected override object Json => new + { + delimiter = "|", + replacement = "-", + buffer_size = 2048, + reverse = true, + skip = 1, + type = "path_hierarchy" + }; + } - public static IndexSettings InitializerExample => - new IndexSettings + public class IcuTests : TokenizerAssertionBase + { + protected override string Name => "icu"; + private const string RuleFiles = "Latn:icu-files/KeywordTokenizer.rbbi"; + + protected override ITokenizer Initializer => new IcuTokenizer + { + RuleFiles = RuleFiles, + }; + + protected override FuncTokenizer Fluent => (n, t) => t.Icu(n, e => e + .RuleFiles(RuleFiles) + ); + + protected override object Json => new { - Analysis = new Nest.Analysis - { - Tokenizers = new Nest.Tokenizers - { - { - "endgen", new EdgeNGramTokenizer - { - MaxGram = 2, - MinGram = 1, - TokenChars = new[] {TokenChar.Digit, TokenChar.Letter} - } - }, - { - "ng", new NGramTokenizer - { - MaxGram = 2, - MinGram = 1, - TokenChars = new[] {TokenChar.Digit, TokenChar.Letter} - } - }, - { - "path", new PathHierarchyTokenizer - { - BufferSize = 2048, - Delimiter = '|', - Replacement = '-', - Reverse = true, - Skip = 1 - } - }, - { - "pattern", new PatternTokenizer - { - Flags = "CASE_INSENSITIVE", - Group = 1, - Pattern = @"\W+" - } - }, - {"standard", new StandardTokenizer()}, - {"uax", new UaxEmailUrlTokenizer {MaxTokenLength = 12}}, - {"icu", new IcuTokenizer - { - RuleFiles = "Latn:icu-files/KeywordTokenizer.rbbi", - }}, - {"whitespace", new WhitespaceTokenizer()}, - { - "kuromoji", new KuromojiTokenizer - { - Mode = KuromojiTokenizationMode.Extended, - DiscardPunctuation = true, - NBestExamples = "/箱根山-箱根/成田空港-成田/", - NBestCost = 1000 - } - }, - } - } + rule_files = RuleFiles, + type = "icu_tokenizer" }; + } + + public class KuromojiTests : TokenizerAssertionBase + { + protected override string Name => "kuro"; + private const string Example = "/箱根山-箱根/成田空港-成田/"; + + protected override ITokenizer Initializer => new KuromojiTokenizer + { + Mode = KuromojiTokenizationMode.Extended, + DiscardPunctuation = true, + NBestExamples = Example, + NBestCost = 1000 + }; + + protected override FuncTokenizer Fluent => (n, t) => t.Kuromoji(n, e => e + .Mode(KuromojiTokenizationMode.Extended) + .DiscardPunctuation() + .NBestExamples(Example) + .NBestCost(1000) + ); + + protected override object Json => new + { + discard_punctuation = true, + mode = "extended", + nbest_cost = 1000, + nbest_examples = Example, + type = "kuromoji_tokenizer" + }; + } + + public class UaxTests : TokenizerAssertionBase + { + protected override string Name => "uax"; + protected override ITokenizer Initializer => new UaxEmailUrlTokenizer {MaxTokenLength = 12}; + + protected override FuncTokenizer Fluent => (n, t) => t.UaxEmailUrl(n, e => e + .MaxTokenLength(12) + ); + + protected override object Json => new + { + max_token_length = 12, + type = "uax_url_email" + }; + } + + public class PatternTests : TokenizerAssertionBase + { + protected override string Name => "pat"; + + protected override ITokenizer Initializer => new PatternTokenizer + { + Flags = "CASE_INSENSITIVE", + Group = 1, + Pattern = @"\W+" + }; + + protected override FuncTokenizer Fluent => (n, t) => t.Pattern(n, e => e + .Flags("CASE_INSENSITIVE") + .Group(1) + .Pattern(@"\W+") + ); + + protected override object Json => new + { + pattern = @"\W+", + flags = "CASE_INSENSITIVE", + group = 1, + type = "pattern" + }; + } + + public class WhitespaceTests : TokenizerAssertionBase + { + protected override string Name => "ws"; + protected override ITokenizer Initializer => new WhitespaceTokenizer(); + + protected override FuncTokenizer Fluent => (n, t) => t.Whitespace(n); + + protected override object Json => new {type = "whitespace"}; + } + + public class StandardTests : TokenizerAssertionBase + { + protected override string Name => "ws"; + protected override ITokenizer Initializer => new StandardTokenizer(); + + protected override FuncTokenizer Fluent => (n, t) => t.Standard(n); + + protected override object Json => new {type = "standard"}; + } } } diff --git a/src/Tests/Tests/Framework/EndpointTests/TestState/AsyncLazy.cs b/src/Tests/Tests/Framework/EndpointTests/TestState/AsyncLazy.cs index 753a954fa8..011b4971fd 100644 --- a/src/Tests/Tests/Framework/EndpointTests/TestState/AsyncLazy.cs +++ b/src/Tests/Tests/Framework/EndpointTests/TestState/AsyncLazy.cs @@ -1,9 +1,6 @@ using System; -using System.Collections.Generic; using System.Runtime.CompilerServices; using System.Threading.Tasks; -using Nest; -using Tests.Framework.Integration; namespace Tests.Framework { @@ -22,27 +19,18 @@ public class AsyncLazy /// Initializes a new instance of the class. /// /// The delegate that is invoked on a background thread to produce the value when it is needed. - public AsyncLazy(Func factory) - { - instance = new Lazy>(() => Task.Run(factory)); - } + public AsyncLazy(Func factory) => instance = new Lazy>(() => Task.Run(factory)); /// /// Initializes a new instance of the class. /// /// The asynchronous delegate that is invoked on a background thread to produce the value when it is needed. - public AsyncLazy(Func> factory) - { - instance = new Lazy>(() => Task.Run(factory)); - } + public AsyncLazy(Func> factory) => instance = new Lazy>(() => Task.Run(factory)); /// /// Asynchronous infrastructure support. This method permits instances of to be await'ed. /// - public TaskAwaiter GetAwaiter() - { - return instance.Value.GetAwaiter(); - } + public TaskAwaiter GetAwaiter() => instance.Value.GetAwaiter(); /// /// Starts the asynchronous initialization, if it has not already started. @@ -52,13 +40,4 @@ public void Start() var unused = instance.Value; } } - - public class LazyResponses : AsyncLazy> - { - public static LazyResponses Empty { get; } = new LazyResponses(() => new Dictionary()); - - public LazyResponses(Func> factory) : base(factory) { } - - public LazyResponses(Func>> factory) : base(factory) { } - } } diff --git a/src/Tests/Tests/Framework/EndpointTests/TestState/CallUniqueValues.cs b/src/Tests/Tests/Framework/EndpointTests/TestState/CallUniqueValues.cs index e702b23155..aeb55e5720 100644 --- a/src/Tests/Tests/Framework/EndpointTests/TestState/CallUniqueValues.cs +++ b/src/Tests/Tests/Framework/EndpointTests/TestState/CallUniqueValues.cs @@ -1,33 +1,45 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; +using Elasticsearch.Net; +using static Tests.Framework.Integration.ClientMethod; namespace Tests.Framework.Integration { public class CallUniqueValues : Dictionary { - private string UniqueValue => "nest-" + Guid.NewGuid().ToString("N").Substring(0, 8); + private readonly string _prefix; + private string UniqueValue => $"{this._prefix}-{ViewName}-{Guid.NewGuid().ToString("N").Substring(0, 8)}"; - private IDictionary> ExtendedValues { get; } - = new Dictionary>(); + private IDictionary> ExtendedValues { get; } + = new Dictionary>(); - public ClientMethod CurrentView { get; set; } = ClientMethod.Fluent; - public ClientMethod[] Views { get; } = new[] { ClientMethod.Fluent, ClientMethod.FluentAsync, ClientMethod.Initializer, ClientMethod.InitializerAsync }; + public ClientMethod CurrentView { get; set; } = Fluent; + public string ViewName => this.CurrentView.GetStringValue().ToLowerInvariant(); + + public ClientMethod[] Views { get; } = { Fluent, FluentAsync, Initializer, InitializerAsync }; public string Value => this[CurrentView]; public T ExtendedValue(string key) where T : class => this.ExtendedValues[CurrentView][key] as T; public void ExtendedValue(string key, T value) where T : class => this.ExtendedValues[CurrentView][key] = value; + public T ExtendedValue(string key, Func value) where T : class => + this.ExtendedValues[CurrentView].GetOrAdd(key, value) as T; - public CallUniqueValues() + public CallUniqueValues(string prefix = "nest") { - this.Add(ClientMethod.Fluent, this.UniqueValue); - this.Add(ClientMethod.FluentAsync, this.UniqueValue); - this.Add(ClientMethod.Initializer, this.UniqueValue); - this.Add(ClientMethod.InitializerAsync, this.UniqueValue); + this._prefix = prefix; + this.SetupClientMethod(Fluent); + this.SetupClientMethod(FluentAsync); + this.SetupClientMethod(Initializer); + this.SetupClientMethod(InitializerAsync); + this.CurrentView = Fluent; + } - this.ExtendedValues.Add(ClientMethod.Fluent, new Dictionary()); - this.ExtendedValues.Add(ClientMethod.FluentAsync, new Dictionary()); - this.ExtendedValues.Add(ClientMethod.Initializer, new Dictionary()); - this.ExtendedValues.Add(ClientMethod.InitializerAsync, new Dictionary()); + private void SetupClientMethod(ClientMethod method) + { + this.CurrentView = method; + this.Add(method, this.UniqueValue); + this.ExtendedValues.Add(method, new ConcurrentDictionary()); } } -} \ No newline at end of file +} diff --git a/src/Tests/Tests/Framework/EndpointTests/TestState/EndpointUsage.cs b/src/Tests/Tests/Framework/EndpointTests/TestState/EndpointUsage.cs index 8f2bff98e0..d4529d31e2 100644 --- a/src/Tests/Tests/Framework/EndpointTests/TestState/EndpointUsage.cs +++ b/src/Tests/Tests/Framework/EndpointTests/TestState/EndpointUsage.cs @@ -1,5 +1,13 @@ using System; using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Runtime.ExceptionServices; +using System.Threading.Tasks; +using Elasticsearch.Net; +using Nest; +using Tests.Core.Client; namespace Tests.Framework.Integration { @@ -13,22 +21,120 @@ public class EndpointUsage public bool CalledSetup { get; internal set; } public bool CalledTeardown { get; internal set; } - public EndpointUsage() - { - this.CallUniqueValues = new CallUniqueValues(); - } + public EndpointUsage() : this("nest") { } + + public EndpointUsage(string prefix) => this.CallUniqueValues = new CallUniqueValues(prefix); - public LazyResponses CallOnce(Func clientUsage, int? k = null) + public LazyResponses CallOnce(Func clientUsage, int k = 0) { - var key = k ?? clientUsage.GetHashCode(); - if (_usages.TryGetValue(key, out var lazyResponses)) return lazyResponses; + if (_usages.TryGetValue(k, out var lazyResponses)) return lazyResponses; lock (_lock) { - if (_usages.TryGetValue(key, out lazyResponses)) return lazyResponses; + if (_usages.TryGetValue(k, out lazyResponses)) return lazyResponses; var response = clientUsage(); - _usages.TryAdd(key, response); + _usages.TryAdd(k, response); return response; } } } + + public class SingleEndpointUsage : EndpointUsage + where TResponse : class, IResponse + { + public SingleEndpointUsage( + Func fluent, + Func> fluentAsync, + Func request, + Func> requestAsync, + string valuePrefix = null + ) : base(valuePrefix) + { + _fluent = fluent; + _fluentAsync = fluentAsync; + _request = request; + _requestAsync = requestAsync; + } + + private readonly Func _fluent; + private readonly Func> _fluentAsync; + private readonly Func _request; + private readonly Func> _requestAsync; + + public Action IntegrationSetup { get; set; } + public Action IntegrationTeardown { get; set; } + public Action OnBeforeCall { get; set; } + public Action OnAfterCall { get; set; } + + private LazyResponses Responses { get; set; } + + public void KickOffOnce(IElasticClient client) => this.Responses = this.CallOnce(()=> new LazyResponses(async () => + { + if (TestClient.Configuration.RunIntegrationTests) + { + this.IntegrationSetup?.Invoke(client, this.CallUniqueValues); + this.CalledSetup = true; + } + + var dict = new Dictionary(); + + this.Call(client, dict, ClientMethod.Fluent, v => _fluent(v, client)); + + await this.CallAsync(client, dict, ClientMethod.FluentAsync, v => _fluentAsync(v, client)); + + this.Call(client, dict, ClientMethod.Initializer, v => _request(v, client)); + + await this.CallAsync(client, dict, ClientMethod.InitializerAsync, v => _requestAsync(v, client)); + + if (TestClient.Configuration.RunIntegrationTests) + { + foreach(var v in this.CallUniqueValues.Values.SelectMany(d=> d)) + this.IntegrationTeardown?.Invoke(client, this.CallUniqueValues); + this.CalledTeardown = true; + } + + return dict; + })); + + private void Call(IElasticClient client, IDictionary dict, ClientMethod method, Func call) + { + this.CallUniqueValues.CurrentView = method; + this.OnBeforeCall?.Invoke(client); + dict.Add(method, call(this.CallUniqueValues.Value)); + this.OnAfterCall?.Invoke(client); + } + private async Task CallAsync(IElasticClient client, IDictionary dict, ClientMethod method, Func> call) + { + this.CallUniqueValues.CurrentView = method; + this.OnBeforeCall?.Invoke(client); + dict.Add(method, await call(this.CallUniqueValues.Value)); + this.OnAfterCall?.Invoke(client); + } + + public async Task AssertOnAllResponses(Action assert) + { + var responses = await this.Responses; + foreach (var kv in responses) + { + var r = kv.Value as TResponse; + + //this is to make sure any unexpected exceptions on the response are rethrown and shown during testing + if (TestClient.Configuration.RunIntegrationTests && !r.IsValid && r.ApiCall.OriginalException != null + && !(r.ApiCall.OriginalException is ElasticsearchClientException)) + { + var e = ExceptionDispatchInfo.Capture(r.ApiCall.OriginalException.Demystify()); + throw new ResponseAssertionException(e.SourceException, r); + } + + try + { + assert(r); + } + catch (Exception e) + { + throw new ResponseAssertionException(e, r); + } + } + } + } + } diff --git a/src/Tests/Tests/Framework/EndpointTests/TestState/LazyResponses.cs b/src/Tests/Tests/Framework/EndpointTests/TestState/LazyResponses.cs new file mode 100644 index 0000000000..24de1fb63f --- /dev/null +++ b/src/Tests/Tests/Framework/EndpointTests/TestState/LazyResponses.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Nest; +using Tests.Framework.Integration; + +namespace Tests.Framework +{ + public class LazyResponses : AsyncLazy> + { + public static LazyResponses Empty { get; } = new LazyResponses(() => new Dictionary()); + + public LazyResponses(Func> factory) : base(factory) { } + + public LazyResponses(Func>> factory) : base(factory) { } + } +} \ No newline at end of file diff --git a/src/Tests/Tests/Mapping/Types/Core/Keyword/KeywordPropertyTests.cs b/src/Tests/Tests/Mapping/Types/Core/Keyword/KeywordPropertyTests.cs index 72ebd1ac37..96f9d20785 100644 --- a/src/Tests/Tests/Mapping/Types/Core/Keyword/KeywordPropertyTests.cs +++ b/src/Tests/Tests/Mapping/Types/Core/Keyword/KeywordPropertyTests.cs @@ -2,6 +2,7 @@ using Elastic.Xunit.XunitPlumbing; using Elasticsearch.Net; using Nest; +using Tests.Analysis.Tokenizers; using Tests.Core.ManagedElasticsearch.Clusters; using Tests.Domain; using Tests.Framework; @@ -20,7 +21,7 @@ public class KeywordPropertyTests : PropertyTestsBase .Settings(s => s .Analysis(a => a .CharFilters(t => Promise(Analysis.CharFilters.CharFilterUsageTests.FluentExample(s).Value.Analysis.CharFilters)) - .TokenFilters(t => Promise(Analysis.TokenFilters.TokenFilterUsageTests.FluentExample(s).Value.Analysis.TokenFilters)) + .TokenFilters(t => Promise(AnalysisUsageTests.TokenFiltersFluent.Analysis.TokenFilters)) .Normalizers(t => Promise(Analysis.Normalizers.NormalizerUsageTests.FluentExample(s).Value.Analysis.Normalizers)) ) );