diff --git a/src/Nest/Analysis/Tokenizers/Tokenizers.cs b/src/Nest/Analysis/Tokenizers/Tokenizers.cs index eaf6fe86dca..78536994658 100644 --- a/src/Nest/Analysis/Tokenizers/Tokenizers.cs +++ b/src/Nest/Analysis/Tokenizers/Tokenizers.cs @@ -19,7 +19,7 @@ public Tokenizers(Dictionary container) public void Add(string name, ITokenizer analyzer) => BackingDictionary.Add(name, analyzer); } - public class TokenizersDescriptor :IsADictionaryDescriptorBase + public class TokenizersDescriptor : IsADictionaryDescriptorBase { public TokenizersDescriptor() : base(new Tokenizers()) { } diff --git a/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/ReadOnlyCluster.cs b/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/ReadOnlyCluster.cs index 4026cf2b61d..5e9be4db3d1 100644 --- a/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/ReadOnlyCluster.cs +++ b/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/ReadOnlyCluster.cs @@ -1,11 +1,11 @@ -using Elastic.Managed.Ephemeral.Plugins; -using Tests.Core.ManagedElasticsearch.NodeSeeders; +using Tests.Core.ManagedElasticsearch.NodeSeeders; +using static Elastic.Managed.Ephemeral.Plugins.ElasticsearchPlugin; namespace Tests.Core.ManagedElasticsearch.Clusters { public class ReadOnlyCluster : ClientTestClusterBase { - public ReadOnlyCluster() : base(ElasticsearchPlugin.MapperMurmur3) { } + public ReadOnlyCluster() : base(MapperMurmur3) { } protected override void SeedCluster() => new DefaultSeeder(this.Client).SeedNode(); } diff --git a/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/WritableCluster.cs b/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/WritableCluster.cs index 216eefc9d4b..33346801c75 100644 --- a/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/WritableCluster.cs +++ b/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/WritableCluster.cs @@ -1,5 +1,5 @@ -using Elastic.Managed.Ephemeral.Plugins; -using Tests.Core.ManagedElasticsearch.NodeSeeders; +using Tests.Core.ManagedElasticsearch.NodeSeeders; +using static Elastic.Managed.Ephemeral.Plugins.ElasticsearchPlugin; namespace Tests.Core.ManagedElasticsearch.Clusters { @@ -7,7 +7,12 @@ namespace Tests.Core.ManagedElasticsearch.Clusters public class WritableCluster : ClientTestClusterBase { public WritableCluster() : base(new ClientTestClusterConfiguration( - ElasticsearchPlugin.IngestGeoIp, ElasticsearchPlugin.IngestAttachment, ElasticsearchPlugin.AnalysisKuromoji, ElasticsearchPlugin.AnalysisIcu, ElasticsearchPlugin.AnalysisPhonetic, ElasticsearchPlugin.MapperMurmur3 + IngestGeoIp, + IngestAttachment, + AnalysisKuromoji, + AnalysisIcu, + AnalysisPhonetic, + MapperMurmur3 ) { MaxConcurrency = 4 diff --git a/src/Tests/Tests.Core/Tests.Core.csproj b/src/Tests/Tests.Core/Tests.Core.csproj index 50aa941ee60..107a57626a9 100644 --- a/src/Tests/Tests.Core/Tests.Core.csproj +++ b/src/Tests/Tests.Core/Tests.Core.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Tests/Tests.Domain/Tests.Domain.csproj b/src/Tests/Tests.Domain/Tests.Domain.csproj index 2fd21b9925a..5caa0651059 100644 --- a/src/Tests/Tests.Domain/Tests.Domain.csproj +++ b/src/Tests/Tests.Domain/Tests.Domain.csproj @@ -12,7 +12,7 @@ - + \ No newline at end of file diff --git a/src/Tests/Tests/Analysis/AnalysisComponentTestBase.cs b/src/Tests/Tests/Analysis/AnalysisComponentTestBase.cs new file mode 100644 index 00000000000..3683e45be95 --- /dev/null +++ b/src/Tests/Tests/Analysis/AnalysisComponentTestBase.cs @@ -0,0 +1,82 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Elastic.Xunit; +using Elastic.Xunit.XunitPlumbing; +using FluentAssertions; +using Nest; +using Tests.Core.Client; +using Tests.Core.ManagedElasticsearch.Clusters; +using Tests.Core.Serialization; +using Tests.Framework.Integration; + +namespace Tests.Analysis +{ + public interface IAnalysisAssertion + { + string Name { get; } + object Json { get; } + } + public interface IAnalysisAssertion : IAnalysisAssertion + where TContainer : class + { + TComponent Initializer { get; } + Func> Fluent { get; } + } + + [IntegrationTestCluster(typeof(WritableCluster))] + public abstract class AnalysisComponentTestBase + : IAnalysisAssertion + where TAssertion : AnalysisComponentTestBase, new() + where TContainer : class + { + private static readonly SingleEndpointUsage Usage = new SingleEndpointUsage + ( + fluent: (s, c) => c.CreateIndex(s, AssertionSetup.FluentCall), + fluentAsync: (s, c) => c.CreateIndexAsync(s, AssertionSetup.FluentCall), + request: (s, c) => c.CreateIndex(AssertionSetup.InitializerCall(s)), + requestAsync: (s, c) => c.CreateIndexAsync(AssertionSetup.InitializerCall(s)), + valuePrefix: $"test-{typeof(TAssertion).Name.ToLowerInvariant()}" + ) + { + OnAfterCall = c=> c.DeleteIndex(Usage.CallUniqueValues.Value) + }; + protected static TAssertion AssertionSetup { get; } = new TAssertion(); + + protected AnalysisComponentTestBase() + { + this.Client = (ElasticXunitRunner.CurrentCluster as INestTestCluster)?.Client ?? TestClient.DefaultInMemoryClient; + Usage.KickOffOnce(this.Client, oneRandomCall: true); + } + + private IElasticClient Client { get; } + + public abstract string Name { get; } + public abstract TComponent Initializer { get; } + public abstract Func> Fluent { get; } + public abstract object Json { get; } + + private Func FluentCall => i =>i.Settings(s => s.Analysis(this.FluentAnalysis)); + protected abstract IAnalysis FluentAnalysis(AnalysisDescriptor an); + + private CreateIndexRequest InitializerCall(string index) => new CreateIndexRequest(index) + { + Settings = new IndexSettings { Analysis = this.InitializerAnalysis() } + }; + protected abstract Nest.Analysis InitializerAnalysis(); + + [U] public virtual async Task TestPutSettingsRequest() => await Usage.AssertOnAllResponses(r => + { + var json = new { settings = new { analysis = this.AnalysisJson } }; + SerializationTestHelper.Expect(json).FromRequest(r); + }); + + protected abstract object AnalysisJson { get; } + + [I] public virtual async Task TestPutSettingsResponse() => await Usage.AssertOnAllResponses(r => + { + r.ApiCall.HttpStatusCode.Should().Be(200); + }); + + } +} diff --git a/src/Tests/Tests/Analysis/AnalysisCrudTests.cs b/src/Tests/Tests/Analysis/AnalysisCrudTests.cs index af05d17443b..4ca45a20628 100644 --- a/src/Tests/Tests/Analysis/AnalysisCrudTests.cs +++ b/src/Tests/Tests/Analysis/AnalysisCrudTests.cs @@ -1,19 +1,16 @@ using System.Linq; -using Elastic.Xunit.XunitPlumbing; using FluentAssertions; using Nest; +using Tests.Analysis.Tokenizers; using Tests.Core.Extensions; using Tests.Core.ManagedElasticsearch.Clusters; using Tests.Framework; using Tests.Framework.Integration; -using Tests.Framework.ManagedElasticsearch.Clusters; -using Xunit; using static Tests.Framework.Promisify; namespace Tests.Analysis { - [SkipVersion("<5.2.0", "This tests contains analyzers/tokenfilters not found in previous versions, need a clean way to seperate these out")] public class AnalysisCrudTests : CrudWithNoDeleteTestBase { @@ -46,10 +43,10 @@ protected virtual CreateIndexRequest CreateInitializer(string indexName) => new { Analysis = new Nest.Analysis { - Analyzers = Analyzers.AnalyzerUsageTests.InitializerExample.Analysis.Analyzers, - CharFilters = CharFilters.CharFilterUsageTests.InitializerExample.Analysis.CharFilters, - Tokenizers = Tokenizers.TokenizerUsageTests.InitializerExample.Analysis.Tokenizers, - TokenFilters = TokenFilters.TokenFilterUsageTests.InitializerExample.Analysis.TokenFilters, + Analyzers = AnalysisUsageTests.AnalyzersInitializer.Analysis.Analyzers, + CharFilters = AnalysisUsageTests.CharFiltersInitializer.Analysis.CharFilters, + Tokenizers = AnalysisUsageTests.TokenizersInitializer.Analysis.Tokenizers, + TokenFilters = AnalysisUsageTests.TokenFiltersInitializer.Analysis.TokenFilters, } } }; @@ -57,10 +54,10 @@ protected virtual CreateIndexRequest CreateInitializer(string indexName) => new protected virtual ICreateIndexRequest CreateFluent(string indexName, CreateIndexDescriptor c) => c.Settings(s => s .Analysis(a => a - .Analyzers(t => Promise(Analyzers.AnalyzerUsageTests.FluentExample(s).Value.Analysis.Analyzers)) - .CharFilters(t => Promise(CharFilters.CharFilterUsageTests.FluentExample(s).Value.Analysis.CharFilters)) - .Tokenizers(t => Promise(Tokenizers.TokenizerUsageTests.FluentExample(s).Value.Analysis.Tokenizers)) - .TokenFilters(t => Promise(TokenFilters.TokenFilterUsageTests.FluentExample(s).Value.Analysis.TokenFilters)) + .Analyzers(t => Promise(AnalysisUsageTests.AnalyzersFluent.Analysis.Analyzers)) + .CharFilters(t => Promise(AnalysisUsageTests.CharFiltersFluent.Analysis.CharFilters)) + .Tokenizers(t => Promise(AnalysisUsageTests.TokenizersFluent.Analysis.Tokenizers)) + .TokenFilters(t => Promise(AnalysisUsageTests.TokenFiltersFluent.Analysis.TokenFilters)) ) ); @@ -82,7 +79,7 @@ protected virtual CreateIndexRequest CreateInitializer(string indexName) => new /** * Here we assert over the response from `GetIndexSettings()` after the index creation to make sure our analysis chain did infact - * store our html char filter called `stripMe` + * store our html char filter called `htmls` */ protected override void ExpectAfterCreate(IGetIndexSettingsResponse response) { @@ -94,7 +91,7 @@ protected override void ExpectAfterCreate(IGetIndexSettingsResponse response) indexSettings.Analysis.Should().NotBeNull(); indexSettings.Analysis.CharFilters.Should().NotBeNull(); - var firstHtmlCharFilter = indexSettings.Analysis.CharFilters["stripMe"]; + var firstHtmlCharFilter = indexSettings.Analysis.CharFilters["htmls"]; firstHtmlCharFilter.Should().NotBeNull(); } diff --git a/src/Tests/Tests/Analysis/AnalysisUsageTests.cs b/src/Tests/Tests/Analysis/AnalysisUsageTests.cs new file mode 100644 index 00000000000..9205d7e5f77 --- /dev/null +++ b/src/Tests/Tests/Analysis/AnalysisUsageTests.cs @@ -0,0 +1,104 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using Elastic.Xunit.XunitPlumbing; +using FluentAssertions; +using Nest; +using Tests.Analysis.Analyzers; +using Tests.Analysis.CharFilters; +using Tests.Analysis.Normalizers; +using Tests.Analysis.TokenFilters; +using Tests.Analysis.Tokenizers; +using Tests.Core.Client; + +namespace Tests.Analysis +{ + public class AnalysisUsageTestsTests + { + [U] public static void CollectionsShouldNotBeEmpty() + { + var analyzers = AnalysisUsageTests.AnalyzersInitializer.Analysis.Analyzers; + var charFilters = AnalysisUsageTests.CharFiltersInitializer.Analysis.CharFilters; + var tokenizers = AnalysisUsageTests.TokenizersInitializer.Analysis.Tokenizers; + var tokenFilters = AnalysisUsageTests.TokenFiltersInitializer.Analysis.TokenFilters; + + analyzers.Should().NotBeNull().And.NotBeEmpty(); + charFilters.Should().NotBeNull().And.NotBeEmpty(); + tokenizers.Should().NotBeNull().And.NotBeEmpty(); + tokenFilters.Should().NotBeNull().And.NotBeEmpty(); + } + } + + public static class AnalysisUsageTests + { + + public static IndexSettings NormalizersFluent => Fluent(i => i.Fluent, (a, v) => a.Normalizers = v.Value); + + public static IndexSettings AnalyzersFluent => Fluent(i => i.Fluent, (a, v) => a.Analyzers = v.Value); + + public static IndexSettings TokenizersFluent => Fluent(i => i.Fluent, (a, v) => a.Tokenizers = v.Value); + + public static IndexSettings TokenFiltersFluent => Fluent(i => i.Fluent, (a, v) => a.TokenFilters = v.Value); + + public static IndexSettings CharFiltersFluent => Fluent(i => i.Fluent, (a, v) => a.CharFilters = v.Value); + + public static IndexSettings NormalizersInitializer => Init(i => i.Initializer, (a, v) => a.Normalizers = v); + + public static IndexSettings AnalyzersInitializer => Init(i => i.Initializer, (a, v) => a.Analyzers = v); + + public static IndexSettings TokenizersInitializer => Init(i => i.Initializer, (a, v) => a.Tokenizers = v); + + public static IndexSettings TokenFiltersInitializer => Init(i => i.Initializer, (a, v) => a.TokenFilters = v); + + public static IndexSettings CharFiltersInitializer => Init(i => i.Initializer, (a, v) => a.CharFilters = v); + + private static IndexSettings Fluent(Func>> fluent, Action> set) + where TAssertion : IAnalysisAssertion + where TContainer : IPromise, new() + where TValue : class => Wrap(an => set(an, Apply((t, a) => fluent(a)(a.Name, t)))); + + private static IndexSettings Init(Func value, Action set) + where TAssertion : IAnalysisAssertion + where TContainer : IDictionary, new() => Wrap(an => set(an, Apply((t, a) => t[a.Name] = value(a)))); + + private static TContainer Apply(Action act) + where TAssertion : IAnalysisAssertion + where TContainer : new() => All().Aggregate(new TContainer() , (t,a) => { act(t,a); return t; }, t=>t); + + private static IndexSettings Wrap(Action set) + { + var a = new Nest.Analysis(); + var s =new IndexSettings { Analysis = a }; + set(a); + return s; + } + + private static List All() + where TAssertion : IAnalysisAssertion + { + var assertions = typeof(TokenizerTests).GetNestedTypes() + .Union(typeof(TokenFilterTests).GetNestedTypes()) + .Union(typeof(NormalizerTests).GetNestedTypes()) + .Union(typeof(AnalyzerTests).GetNestedTypes()) + .Union(typeof(CharFilterTests).GetNestedTypes()) + .ToList(); + + var nestedTypes = assertions + .Where(t => typeof(TAssertion).IsAssignableFrom(t) && t.IsClass) + .ToList(); + + var types = nestedTypes + .Select(t => new + { + t, + a = t.GetCustomAttributes(typeof(SkipVersionAttribute)).FirstOrDefault() as SkipVersionAttribute + }) + .Where(@t1 => @t1.a == null || !@t1.a.Ranges.Any(r => r.IsSatisfied(TestClient.Configuration.ElasticsearchVersion))) + .Select(@t1 => (TAssertion) Activator.CreateInstance(@t1.t)); + return types.ToList(); + } + + + } +} diff --git a/src/Tests/Tests/Analysis/AnalysisWithNormalizerCrudTests.cs b/src/Tests/Tests/Analysis/AnalysisWithNormalizerCrudTests.cs new file mode 100644 index 00000000000..0832c9d1c2b --- /dev/null +++ b/src/Tests/Tests/Analysis/AnalysisWithNormalizerCrudTests.cs @@ -0,0 +1,39 @@ +using Nest; +using Tests.Analysis.Tokenizers; +using Tests.Core.ManagedElasticsearch.Clusters; +using Tests.Framework.Integration; +using static Tests.Framework.Promisify; + +namespace Tests.Analysis +{ + public class AnalysisWithNormalizerCrudTests : AnalysisCrudTests + { + public AnalysisWithNormalizerCrudTests(WritableCluster cluster, EndpointUsage usage) : base(cluster, usage) { } + + protected override CreateIndexRequest CreateInitializer(string indexName) => new CreateIndexRequest(indexName) + { + Settings = new IndexSettings + { + Analysis = new Nest.Analysis + { + Analyzers = AnalysisUsageTests.AnalyzersInitializer.Analysis.Analyzers, + CharFilters = AnalysisUsageTests.CharFiltersInitializer.Analysis.CharFilters, + Tokenizers = AnalysisUsageTests.TokenizersInitializer.Analysis.Tokenizers, + TokenFilters = AnalysisUsageTests.TokenFiltersInitializer.Analysis.TokenFilters, + Normalizers = AnalysisUsageTests.NormalizersInitializer.Analysis.Normalizers, + } + } + }; + + protected override ICreateIndexRequest CreateFluent(string indexName, CreateIndexDescriptor c) => + c.Settings(s => s + .Analysis(a => a + .Analyzers(t => Promise(AnalysisUsageTests.AnalyzersFluent.Analysis.Analyzers)) + .CharFilters(t => Promise(AnalysisUsageTests.CharFiltersFluent.Analysis.CharFilters)) + .Tokenizers(t => Promise(AnalysisUsageTests.TokenizersFluent.Analysis.Tokenizers)) + .TokenFilters(t => Promise(AnalysisUsageTests.TokenFiltersFluent.Analysis.TokenFilters)) + .Normalizers(t => Promise(AnalysisUsageTests.NormalizersFluent.Analysis.Normalizers)) + ) + ); + } +} diff --git a/src/Tests/Tests/Analysis/Analyzers/AnalyzerAssertionBase.cs b/src/Tests/Tests/Analysis/Analyzers/AnalyzerAssertionBase.cs new file mode 100644 index 00000000000..aeee879a7bb --- /dev/null +++ b/src/Tests/Tests/Analysis/Analyzers/AnalyzerAssertionBase.cs @@ -0,0 +1,29 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using Elastic.Xunit.XunitPlumbing; +using Nest; + +namespace Tests.Analysis.Analyzers +{ + public interface IAnalyzerAssertion : IAnalysisAssertion { } + + public abstract class AnalyzerAssertionBase + : AnalysisComponentTestBase + , IAnalyzerAssertion + where TAssertion : AnalyzerAssertionBase, new() + { + protected override IAnalysis FluentAnalysis(AnalysisDescriptor an) => + an.Analyzers(d => AssertionSetup.Fluent(AssertionSetup.Name, d)); + + protected override Nest.Analysis InitializerAnalysis() => + new Nest.Analysis {Analyzers = new Nest.Analyzers {{AssertionSetup.Name, AssertionSetup.Initializer}}}; + + protected override object AnalysisJson => new + { + analyzer = new Dictionary { {AssertionSetup.Name, AssertionSetup.Json} } + }; + // https://youtrack.jetbrains.com/issue/RIDER-19912 + [U] public override Task TestPutSettingsRequest() => base.TestPutSettingsRequest(); + [I] public override Task TestPutSettingsResponse() => base.TestPutSettingsResponse(); + } +} diff --git a/src/Tests/Tests/Analysis/Analyzers/AnalyzerTests.cs b/src/Tests/Tests/Analysis/Analyzers/AnalyzerTests.cs new file mode 100644 index 00000000000..0a663b5bc1f --- /dev/null +++ b/src/Tests/Tests/Analysis/Analyzers/AnalyzerTests.cs @@ -0,0 +1,201 @@ +using System; +using Nest; + +namespace Tests.Analysis.Analyzers +{ + using FuncTokenizer = Func>; + + public class AnalyzerTests + { + public class KeywordTests : AnalyzerAssertionBase + { + public override string Name => "myKeyword"; + + public override IAnalyzer Initializer => + new KeywordAnalyzer(); + + public override FuncTokenizer Fluent => (n, an) => an.Keyword("myKeyword"); + + public override object Json => new + { + type = "keyword" + }; + + } + + public class CustomTests : AnalyzerAssertionBase + { + public override string Name => "myCustom"; + + public override IAnalyzer Initializer => new CustomAnalyzer + { + CharFilter = new[] {"html_strip"}, + Tokenizer = "standard", + Filter = new []{"lowercase", "asciifolding" } + }; + + public override FuncTokenizer Fluent => (n, an) => an + .Custom("myCustom", a => a + .Filters("lowercase", "asciifolding") + .CharFilters("html_strip") + .Tokenizer("standard") + ); + + public override object Json => new + { + type = "custom", + tokenizer = "standard", + filter = new[] {"lowercase", "asciifolding"}, + char_filter = new[] {"html_strip"} + }; + + } + public class PatternTests : AnalyzerAssertionBase + { + public override string Name => "myPattern "; + + public override IAnalyzer Initializer => new PatternAnalyzer {Pattern = @"\w"}; + + public override FuncTokenizer Fluent => (n, an) => an.Pattern(n, a => a.Pattern(@"\w")); + + public override object Json => new { type = "pattern", pattern = "\\w" }; + + } + public class SimpleTests : AnalyzerAssertionBase + { + public override string Name => "mySimple"; + + public override IAnalyzer Initializer => new SimpleAnalyzer(); + + public override FuncTokenizer Fluent => (n, an) => an.Simple("mySimple"); + public override object Json => new {type = "simple"}; + + } + public class LanguageTests : AnalyzerAssertionBase + { + public override string Name => "myLanguage"; + + public override IAnalyzer Initializer => new LanguageAnalyzer {Language = Language.Dutch}; + + public override FuncTokenizer Fluent => (n, an) => an + .Language("myLanguage", a => a.Language(Language.Dutch)); + + public override object Json => new {type = "dutch"}; + + } + public class SnowballTests : AnalyzerAssertionBase + { + public override string Name => "mySnow"; + + public override IAnalyzer Initializer => new SnowballAnalyzer {Language = SnowballLanguage.Dutch}; + + public override FuncTokenizer Fluent => (n, an) => an + .Snowball("mySnow", a => a.Language(SnowballLanguage.Dutch)); + + public override object Json => new + { + type = "snowball", + language = "Dutch" + }; + + } + public class StandardTests : AnalyzerAssertionBase + { + public override string Name => "myStandard"; + + public override IAnalyzer Initializer => new StandardAnalyzer {MaxTokenLength = 2}; + + public override FuncTokenizer Fluent => (n, an) => an + .Standard("myStandard", a => a.MaxTokenLength(2)); + + public override object Json => new + { + type = "standard", + max_token_length = 2 + }; + + } + public class StopTests : AnalyzerAssertionBase + { + public override string Name => "myStop"; + + public override IAnalyzer Initializer => new StopAnalyzer {StopwordsPath = "analysis/stopwords.txt"}; + + public override FuncTokenizer Fluent => (n, an) => an + .Stop("myStop", a => a.StopwordsPath("analysis/stopwords.txt")); + + public override object Json => new + { + type = "stop", + stopwords_path = "analysis/stopwords.txt" + }; + + } + public class WhitespaceTests : AnalyzerAssertionBase + { + public override string Name => "myWhiteSpace"; + + public override IAnalyzer Initializer => new WhitespaceAnalyzer(); + + public override FuncTokenizer Fluent => (n, an) => an.Whitespace(n); + public override object Json => new {type = "whitespace"}; + + } + + public class FingerprintTests : AnalyzerAssertionBase + { + public override string Name => "myFingerprint"; + + public override IAnalyzer Initializer => + new FingerprintAnalyzer + { + PreserveOriginal = true, + Separator = ",", + MaxOutputSize = 100, + StopWords = new[] {"a", "he", "the"} + }; + + public override FuncTokenizer Fluent => (n, an) => an + .Fingerprint("myFingerprint", a => a + .PreserveOriginal() + .Separator(",") + .MaxOutputSize(100) + .StopWords("a", "he", "the") + ); + + public override object Json => new + { + type = "fingerprint", + preserve_original = true, + separator = ",", + max_output_size = 100, + stopwords = new[] {"a", "he", "the"} + }; + + } + + + public class KuromojuTests : AnalyzerAssertionBase + { + public override string Name => "kuro"; + + public override IAnalyzer Initializer => + new KuromojiAnalyzer + { + Mode = KuromojiTokenizationMode.Search + }; + + public override FuncTokenizer Fluent => (n, an) => an + .Kuromoji("kuro", a => a + .Mode(KuromojiTokenizationMode.Search) + ); + + public override object Json => new + { + type = "kuromoji", + mode = "search" + }; + } + + } +} diff --git a/src/Tests/Tests/Analysis/Analyzers/AnalyzerUsageTests.cs b/src/Tests/Tests/Analysis/Analyzers/AnalyzerUsageTests.cs deleted file mode 100644 index 5c9be365a8b..00000000000 --- a/src/Tests/Tests/Analysis/Analyzers/AnalyzerUsageTests.cs +++ /dev/null @@ -1,166 +0,0 @@ -using System; -using Nest; -using Tests.Framework; - -namespace Tests.Analysis.Analyzers -{ - /** - */ - - public class AnalyzerUsageTests : PromiseUsageTestBase - { - protected override object ExpectJson => new - { - analysis = new - { - analyzer = new - { - @default = new - { - type = "keyword" - }, - myCustom = new - { - type = "custom", - tokenizer = "ng", - filter = new[] {"myAscii", "kstem"}, - char_filter = new[] {"stripMe", "patterned"} - }, - myKeyword = new - { - type = "keyword" - }, - myPattern = new - { - type = "pattern", - pattern = "\\w" - }, - mySimple = new - { - type = "simple" - }, - myLanguage = new {type = "dutch"}, - mySnow = new - { - type = "snowball", - language = "Dutch" - }, - myStandard = new - { - type = "standard", - max_token_length = 2 - }, - myStop = new - { - type = "stop", - stopwords_path = "analysis/stopwords.txt" - }, - myWhiteSpace = new - { - type = "whitespace" - }, - myWhiteSpace2 = new - { - type = "whitespace" - }, - myFingerprint = new - { - type = "fingerprint", - preserve_original = true, - separator = ",", - max_output_size = 100, - stopwords = new[] {"a", "he", "the"} - }, - kuro = new - { - type = "kuromoji", - mode = "search" - } - } - } - }; - - /** - * - */ - protected override Func> Fluent => FluentExample; - - public static Func> FluentExample => s => s - .Analysis(analysis => analysis - .Analyzers(analyzers => analyzers - .Keyword("default") - .Custom("myCustom", a => a - .Filters("myAscii", "kstem") - .CharFilters("stripMe", "patterned") - .Tokenizer("ng") - ) - .Keyword("myKeyword") - .Pattern("myPattern", a => a.Pattern(@"\w")) - .Language("myLanguage", a => a.Language(Language.Dutch)) - .Simple("mySimple") - .Snowball("mySnow", a => a.Language(SnowballLanguage.Dutch)) - .Standard("myStandard", a => a.MaxTokenLength(2)) - .Stop("myStop", a => a.StopwordsPath("analysis/stopwords.txt")) - .Whitespace("myWhiteSpace") - .Whitespace("myWhiteSpace2") - .Fingerprint("myFingerprint", a => a - .PreserveOriginal() - .Separator(",") - .MaxOutputSize(100) - .StopWords("a", "he", "the") - ) - .Kuromoji("kuro", a => a - .Mode(KuromojiTokenizationMode.Search) - ) - ) - ); - - /** - */ - protected override IndexSettings Initializer => InitializerExample; - - public static IndexSettings InitializerExample => - new IndexSettings - { - Analysis = new Nest.Analysis - { - Analyzers = new Nest.Analyzers - { - {"default", new KeywordAnalyzer()}, - { - "myCustom", new CustomAnalyzer - { - CharFilter = new[] {"stripMe", "patterned"}, - Filter = new[] {"myAscii", "kstem"}, - Tokenizer = "ng" - } - }, - {"myKeyword", new KeywordAnalyzer()}, - {"myPattern", new PatternAnalyzer {Pattern = @"\w"}}, - {"myLanguage", new LanguageAnalyzer {Language = Language.Dutch}}, - {"mySimple", new SimpleAnalyzer()}, - {"mySnow", new SnowballAnalyzer {Language = SnowballLanguage.Dutch}}, - {"myStandard", new StandardAnalyzer {MaxTokenLength = 2}}, - {"myStop", new StopAnalyzer {StopwordsPath = "analysis/stopwords.txt"}}, - {"myWhiteSpace", new WhitespaceAnalyzer()}, - {"myWhiteSpace2", new WhitespaceAnalyzer()}, - { - "myFingerprint", new FingerprintAnalyzer - { - PreserveOriginal = true, - Separator = ",", - MaxOutputSize = 100, - StopWords = new[] {"a", "he", "the"} - } - }, - { - "kuro", new KuromojiAnalyzer - { - Mode = KuromojiTokenizationMode.Search - } - } - } - } - }; - } -} diff --git a/src/Tests/Tests/Analysis/CharFilters/CharFilterAssertionBase.cs b/src/Tests/Tests/Analysis/CharFilters/CharFilterAssertionBase.cs new file mode 100644 index 00000000000..ba1bafba1f6 --- /dev/null +++ b/src/Tests/Tests/Analysis/CharFilters/CharFilterAssertionBase.cs @@ -0,0 +1,29 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using Elastic.Xunit.XunitPlumbing; +using Nest; + +namespace Tests.Analysis.CharFilters +{ + public interface ICharFilterAssertion : IAnalysisAssertion { } + + public abstract class CharFilterAssertionBase + : AnalysisComponentTestBase + , ICharFilterAssertion + where TAssertion : CharFilterAssertionBase, new() + { + protected override IAnalysis FluentAnalysis(AnalysisDescriptor an) => + an.CharFilters(d => AssertionSetup.Fluent(AssertionSetup.Name, d)); + + protected override Nest.Analysis InitializerAnalysis() => + new Nest.Analysis {CharFilters = new Nest.CharFilters {{AssertionSetup.Name, AssertionSetup.Initializer}}}; + + protected override object AnalysisJson => new + { + char_filter = new Dictionary { {AssertionSetup.Name, AssertionSetup.Json} } + }; + // https://youtrack.jetbrains.com/issue/RIDER-19912 + [U] public override Task TestPutSettingsRequest() => base.TestPutSettingsRequest(); + [I] public override Task TestPutSettingsResponse() => base.TestPutSettingsResponse(); + } +} diff --git a/src/Tests/Tests/Analysis/CharFilters/CharFilterTests.cs b/src/Tests/Tests/Analysis/CharFilters/CharFilterTests.cs new file mode 100644 index 00000000000..0cbd28b0c81 --- /dev/null +++ b/src/Tests/Tests/Analysis/CharFilters/CharFilterTests.cs @@ -0,0 +1,74 @@ +using System; +using Nest; + +namespace Tests.Analysis.CharFilters +{ + using FuncTokenizer = Func>; + + public class CharFilterTests + { + public class MappingTests : CharFilterAssertionBase + { + public override string Name => "mapping"; + public override ICharFilter Initializer => new MappingCharFilter {Mappings = new[] {"a=>b"}}; + public override FuncTokenizer Fluent => (n, cf) => cf.Mapping(n, c => c.Mappings("a=>b")); + public override object Json => new { mappings = new[] {"a=>b"}, type = "mapping" }; + } + + public class PatternReplaceTests : CharFilterAssertionBase + { + public override string Name => "pr"; + public override ICharFilter Initializer => new PatternReplaceCharFilter {Pattern = "x", Replacement = "y"}; + public override FuncTokenizer Fluent => (n, cf) => cf.PatternReplace(n, c => c.Pattern("x").Replacement("y")); + public override object Json => new {pattern = "x", replacement = "y", type = "pattern_replace"}; + } + + public class IcuNormalizerTests : CharFilterAssertionBase + { + public override string Name => "icunorm"; + + public override ICharFilter Initializer => + new IcuNormalizationCharFilter + { + Mode = IcuNormalizationMode.Compose, + Name = IcuNormalizationType.CompatibilityCaseFold + }; + + public override FuncTokenizer Fluent => (n, cf) => cf + .IcuNormalization(n, c => c + .Mode(IcuNormalizationMode.Compose) + .Name(IcuNormalizationType.CompatibilityCaseFold) + ); + + public override object Json => new {mode = "compose", name = "nfkc_cf", type = "icu_normalizer"}; + + } + + public class KuromojiIterationMarkTests : CharFilterAssertionBase + { + public override string Name => "kmark"; + + public override ICharFilter Initializer => + new KuromojiIterationMarkCharFilter { NormalizeKana = true, NormalizeKanji = true }; + + public override FuncTokenizer Fluent => + (n, cf) => cf.KuromojiIterationMark("kmark", c => c.NormalizeKana().NormalizeKanji()); + + public override object Json => new + { + normalize_kanji = true, + normalize_kana = true, + type = "kuromoji_iteration_mark" + }; + } + + public class HtmlStripTests : CharFilterAssertionBase + { + public override string Name => "htmls"; + public override ICharFilter Initializer => new HtmlStripCharFilter { }; + public override FuncTokenizer Fluent => (n, cf) => cf.HtmlStrip(n); + public override object Json => new {type = "html_strip"}; + } + + } +} diff --git a/src/Tests/Tests/Analysis/CharFilters/CharFilterUsageTests.cs b/src/Tests/Tests/Analysis/CharFilters/CharFilterUsageTests.cs deleted file mode 100644 index a073a9cbe33..00000000000 --- a/src/Tests/Tests/Analysis/CharFilters/CharFilterUsageTests.cs +++ /dev/null @@ -1,88 +0,0 @@ -using System; -using Nest; -using Tests.Framework; - -namespace Tests.Analysis.CharFilters -{ - public class CharFilterUsageTests : PromiseUsageTestBase - { - protected override object ExpectJson => new - { - analysis = new - { - char_filter = new - { - icun = new { - mode = "compose", - name = "nfkc_cf", - type = "icu_normalizer" - }, - stripMe = new { type = "html_strip" }, - patterned = new - { - pattern = "x", - replacement = "y", - type = "pattern_replace" - }, - mapped = new - { - mappings = new[] { "a=>b" }, - type = "mapping" - }, - kmark = new - { - normalize_kanji = true, - normalize_kana = true, - type = "kuromoji_iteration_mark" - } - } - } - }; - - - /** - * - */ - protected override Func> Fluent => FluentExample; - public static Func> FluentExample => s => s - .Analysis(a => a - .CharFilters(charfilters => charfilters - .HtmlStrip("stripMe") - .PatternReplace("patterned", c => c.Pattern("x").Replacement("y")) - .Mapping("mapped", c => c.Mappings("a=>b")) - .KuromojiIterationMark("kmark", c => c.NormalizeKana().NormalizeKanji()) - .IcuNormalization("icun", c => c - .Mode(IcuNormalizationMode.Compose) - .Name(IcuNormalizationType.CompatibilityCaseFold) - ) - ) - ); - - /** - */ - protected override IndexSettings Initializer => InitializerExample; - public static IndexSettings InitializerExample => - new IndexSettings - { - Analysis = new Nest.Analysis - { - CharFilters = new Nest.CharFilters - { - { "stripMe", new HtmlStripCharFilter { } }, - { "patterned", new PatternReplaceCharFilter { Pattern = "x", Replacement = "y" } }, - { "mapped", new MappingCharFilter { Mappings = new [] { "a=>b"} } }, - { "kmark", new KuromojiIterationMarkCharFilter - { - NormalizeKana = true, - NormalizeKanji = true - } }, - { "icun", new IcuNormalizationCharFilter - { - Mode = IcuNormalizationMode.Compose, - Name = IcuNormalizationType.CompatibilityCaseFold - } } - } - } - }; - } -} diff --git a/src/Tests/Tests/Analysis/Normalizers/AnalysisWithNormalizerCrudTests.cs b/src/Tests/Tests/Analysis/Normalizers/AnalysisWithNormalizerCrudTests.cs deleted file mode 100644 index 5e8decd87dd..00000000000 --- a/src/Tests/Tests/Analysis/Normalizers/AnalysisWithNormalizerCrudTests.cs +++ /dev/null @@ -1,46 +0,0 @@ -using System.Linq; -using Elastic.Xunit.XunitPlumbing; -using FluentAssertions; -using Nest; -using Tests.Core.ManagedElasticsearch.Clusters; -using Tests.Framework; -using Tests.Framework.Integration; -using Tests.Framework.ManagedElasticsearch.Clusters; -using Xunit; -using static Tests.Framework.Promisify; - -namespace Tests.Analysis -{ - - [SkipVersion("<5.2.0", "Normalizers are a new 5.2.0 feature")] - public class AnalysisWithNormalizerCrudTests : AnalysisCrudTests - { - public AnalysisWithNormalizerCrudTests(WritableCluster cluster, EndpointUsage usage) : base(cluster, usage) { } - - protected override CreateIndexRequest CreateInitializer(string indexName) => new CreateIndexRequest(indexName) - { - Settings = new IndexSettings - { - Analysis = new Nest.Analysis - { - Analyzers = Analyzers.AnalyzerUsageTests.InitializerExample.Analysis.Analyzers, - CharFilters = CharFilters.CharFilterUsageTests.InitializerExample.Analysis.CharFilters, - Tokenizers = Tokenizers.TokenizerUsageTests.InitializerExample.Analysis.Tokenizers, - TokenFilters = TokenFilters.TokenFilterUsageTests.InitializerExample.Analysis.TokenFilters, - Normalizers = Normalizers.NormalizerUsageTests.InitializerExample.Analysis.Normalizers, - } - } - }; - - protected override ICreateIndexRequest CreateFluent(string indexName, CreateIndexDescriptor c) => - c.Settings(s => s - .Analysis(a => a - .Analyzers(t => Promise(Analyzers.AnalyzerUsageTests.FluentExample(s).Value.Analysis.Analyzers)) - .CharFilters(t => Promise(CharFilters.CharFilterUsageTests.FluentExample(s).Value.Analysis.CharFilters)) - .Tokenizers(t => Promise(Tokenizers.TokenizerUsageTests.FluentExample(s).Value.Analysis.Tokenizers)) - .TokenFilters(t => Promise(TokenFilters.TokenFilterUsageTests.FluentExample(s).Value.Analysis.TokenFilters)) - .Normalizers(t => Promise(Normalizers.NormalizerUsageTests.FluentExample(s).Value.Analysis.Normalizers)) - ) - ); - } -} diff --git a/src/Tests/Tests/Analysis/Normalizers/NormalizerAssertionBase.cs b/src/Tests/Tests/Analysis/Normalizers/NormalizerAssertionBase.cs new file mode 100644 index 00000000000..86ce05af883 --- /dev/null +++ b/src/Tests/Tests/Analysis/Normalizers/NormalizerAssertionBase.cs @@ -0,0 +1,30 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using Elastic.Xunit.XunitPlumbing; +using Nest; + +namespace Tests.Analysis.Normalizers +{ + public interface INormalizerAssertion : IAnalysisAssertion { } + + public abstract class NormalizerAssertionBase + : AnalysisComponentTestBase + , INormalizerAssertion + where TAssertion : NormalizerAssertionBase, new() + { + protected override IAnalysis FluentAnalysis(AnalysisDescriptor an) => + an.Normalizers(d => AssertionSetup.Fluent(AssertionSetup.Name, d)); + + protected override Nest.Analysis InitializerAnalysis() => + new Nest.Analysis {Normalizers = new Nest.Normalizers {{AssertionSetup.Name, AssertionSetup.Initializer}}}; + + protected override object AnalysisJson => new + { + normalizer = new Dictionary { {AssertionSetup.Name, AssertionSetup.Json} } + }; + + // https://youtrack.jetbrains.com/issue/RIDER-19912 + [U] public override Task TestPutSettingsRequest() => base.TestPutSettingsRequest(); + [I] public override Task TestPutSettingsResponse() => base.TestPutSettingsResponse(); + } +} diff --git a/src/Tests/Tests/Analysis/Normalizers/NormalizerTests.cs b/src/Tests/Tests/Analysis/Normalizers/NormalizerTests.cs new file mode 100644 index 00000000000..1265b37b7f6 --- /dev/null +++ b/src/Tests/Tests/Analysis/Normalizers/NormalizerTests.cs @@ -0,0 +1,32 @@ +using System; +using Nest; + +namespace Tests.Analysis.Normalizers +{ + using FuncTokenizer = Func>; + + public class NormalizerTests + { + public class CustomTests : NormalizerAssertionBase + { + public override string Name => "myCustom"; + + public override INormalizer Initializer => new CustomNormalizer + { + Filter = new[] {"lowercase", "asciifolding"}, + }; + + public override FuncTokenizer Fluent => (n, an) => an + .Custom("myCustom", a => a + .Filters("lowercase", "asciifolding") + ); + + public override object Json => new + { + type = "custom", + filter = new[] {"lowercase", "asciifolding"}, + }; + } + + } +} diff --git a/src/Tests/Tests/Analysis/Normalizers/NormalizerUsageTests.cs b/src/Tests/Tests/Analysis/Normalizers/NormalizerUsageTests.cs deleted file mode 100644 index e5c3a06e09c..00000000000 --- a/src/Tests/Tests/Analysis/Normalizers/NormalizerUsageTests.cs +++ /dev/null @@ -1,67 +0,0 @@ -using System; -using Elastic.Xunit.XunitPlumbing; -using Nest; -using Tests.Framework; - -namespace Tests.Analysis.Normalizers -{ - /** - */ - - [SkipVersion("<5.2.0", "Normalizers are a new 5.2.0 feature")] - public class NormalizerUsageTests : PromiseUsageTestBase - { - protected override object ExpectJson => new - { - analysis = new - { - normalizer = new - { - myCustom = new - { - type = "custom", - filter = new[] {"lowercase", "asciifolding"}, - char_filter = new[] {"mapped"} - } - } - } - }; - - /** - * - */ - protected override Func> Fluent => FluentExample; - - public static Func> FluentExample => s => s - .Analysis(analysis => analysis - .Normalizers(analyzers => analyzers - .Custom("myCustom", a => a - .Filters("lowercase", "asciifolding") - .CharFilters("mapped") - ) - ) - ); - - /** - */ - protected override IndexSettings Initializer => InitializerExample; - - public static IndexSettings InitializerExample => - new IndexSettings - { - Analysis = new Nest.Analysis - { - Normalizers = new Nest.Normalizers - { - { - "myCustom", new CustomNormalizer - { - CharFilter = new[] {"mapped"}, - Filter = new[] {"lowercase", "asciifolding"}, - } - } - } - } - }; - } -} diff --git a/src/Tests/Tests/Analysis/TokenFilters/TokenFilterAssertionBase.cs b/src/Tests/Tests/Analysis/TokenFilters/TokenFilterAssertionBase.cs new file mode 100644 index 00000000000..9687ac69b2c --- /dev/null +++ b/src/Tests/Tests/Analysis/TokenFilters/TokenFilterAssertionBase.cs @@ -0,0 +1,31 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using Elastic.Xunit.XunitPlumbing; +using Nest; + +namespace Tests.Analysis.TokenFilters +{ + public interface ITokenFilterAssertion : IAnalysisAssertion { } + + public abstract class TokenFilterAssertionBase + : AnalysisComponentTestBase + , ITokenFilterAssertion + where TAssertion : TokenFilterAssertionBase, new() + { + protected override IAnalysis FluentAnalysis(AnalysisDescriptor an) => + an.TokenFilters(d => AssertionSetup.Fluent(AssertionSetup.Name, d)); + + protected override Nest.Analysis InitializerAnalysis() => + new Nest.Analysis {TokenFilters = new Nest.TokenFilters {{AssertionSetup.Name, AssertionSetup.Initializer}}}; + + protected override object AnalysisJson => new + { + filter = new Dictionary { {AssertionSetup.Name, AssertionSetup.Json} } + }; + + // https://youtrack.jetbrains.com/issue/RIDER-19912 + [U] public override Task TestPutSettingsRequest() => base.TestPutSettingsRequest(); + [I] public override Task TestPutSettingsResponse() => base.TestPutSettingsResponse(); + } + +} diff --git a/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs b/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs new file mode 100644 index 00000000000..4c702cf10d1 --- /dev/null +++ b/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs @@ -0,0 +1,882 @@ +using System; +using Nest; +using Tests.Framework; + +namespace Tests.Analysis.TokenFilters +{ + using FuncTokenFilters = Func>; + + public static class TokenFilterTests + { + public class AsciiFoldingTests : TokenFilterAssertionBase + { + public override string Name => "ascii"; + public override ITokenFilter Initializer => new AsciiFoldingTokenFilter {PreserveOriginal = true}; + public override FuncTokenFilters Fluent => (n, tf) => tf.AsciiFolding(n, t => t.PreserveOriginal()); + public override object Json => new {type = "asciifolding", preserve_original = true}; + } + + public class CommonGramsTests : TokenFilterAssertionBase + { + public override string Name => "mycomgram"; + + public override ITokenFilter Initializer => + new CommonGramsTokenFilter {QueryMode = true, IgnoreCase = true, CommonWords = new[] {"x", "y", "z"}}; + + public override FuncTokenFilters Fluent => (n, tf) => tf + .CommonGrams(n, t => t.CommonWords("x", "y", "z").IgnoreCase().QueryMode()); + + public override object Json => new + { + type = "common_grams", + common_words = new[] {"x", "y", "z"}, + ignore_case = true, + query_mode = true + }; + } + + public class DelimitedPayloadFilterTests : TokenFilterAssertionBase + { + public override string Name => "mydp"; + + public override ITokenFilter Initializer => + new DelimitedPayloadTokenFilter {Delimiter = '-', Encoding = DelimitedPayloadEncoding.Identity}; + + public override FuncTokenFilters Fluent => (n, tf) => tf + .DelimitedPayload(n, t => t.Delimiter('-').Encoding(DelimitedPayloadEncoding.Identity)); + + public override object Json => new { type = "delimited_payload_filter", delimiter = "-", encoding = "identity" }; + } + + public class DictionaryDecompounderTests : TokenFilterAssertionBase + { + public override string Name => "dcc"; + + public override ITokenFilter Initializer => + new DictionaryDecompounderTokenFilter + { + MinWordSize = 2, + MinSubwordSize = 2, + MaxSubwordSize = 2, + OnlyLongestMatch = true, + WordList = new[] {"x", "y", "z"} + }; + + public override FuncTokenFilters Fluent => (n, tf) => tf + .DictionaryDecompounder(n, t => t + .MaxSubwordSize(2) + .MinSubwordSize(2) + .MinWordSize(2) + .OnlyLongestMatch() + .WordList("x", "y", "z") + ); + + public override object Json => new + { + type = "dictionary_decompounder", + word_list = new[] {"x", "y", "z"}, + min_word_size = 2, + min_subword_size = 2, + max_subword_size = 2, + only_longest_match = true + }; + + } + + public class EdgeNgramTests : TokenFilterAssertionBase + { + public override string Name => "etf"; + + public override ITokenFilter Initializer => new EdgeNGramTokenFilter {MaxGram = 2, MinGram = 1}; + + public override FuncTokenFilters Fluent => (n, tf) => tf + .EdgeNGram(n, t => t.MaxGram(2).MinGram(1)); + + public override object Json => new { type = "edge_ngram", min_gram = 1, max_gram = 2 }; + + } + + public class ElisionTests : TokenFilterAssertionBase + { + public override string Name => "el"; + + public override ITokenFilter Initializer => new ElisionTokenFilter {Articles = new[] {"a", "b", "c"}}; + + public override FuncTokenFilters Fluent => (n, tf) => tf.Elision(n, t => t.Articles("a", "b", "c")); + + public override object Json => new { type = "elision", articles = new[] {"a", "b", "c"} }; + } + + public class HunspellTests : TokenFilterAssertionBase + { + public override string Name => "huns"; + + public override ITokenFilter Initializer => + new HunspellTokenFilter + { + Dedup = true, + Dictionary = "path_to_dict", + Locale = "en_US", + LongestOnly = true + }; + + public override FuncTokenFilters Fluent => (n, tf) => tf + .Hunspell(n, t => t + .Dedup() + .Dictionary("path_to_dict") + .Locale("en_US") + .LongestOnly() + ); + + public override object Json => new + { + type = "hunspell", + locale = "en_US", + dictionary = "path_to_dict", + dedup = true, + longest_only = true + }; + + } + + public class HyphenationDecompounderTests : TokenFilterAssertionBase + { + public override string Name => "hyphdecomp"; + + public override ITokenFilter Initializer => + new HyphenationDecompounderTokenFilter + { + MaxSubwordSize = 2, + MinSubwordSize = 2, + MinWordSize = 2, + OnlyLongestMatch = true, + WordList = new[] {"x", "y", "z"}, + HyphenationPatternsPath = "analysis/fop.xml" + }; + + public override FuncTokenFilters Fluent => (n, tf) => tf + .HyphenationDecompounder(n, t => t + .MaxSubwordSize(2) + .MinSubwordSize(2) + .MinWordSize(2) + .OnlyLongestMatch() + .WordList("x", "y", "z") + .HyphenationPatternsPath("analysis/fop.xml") + ); + + public override object Json => new + { + type = "hyphenation_decompounder", + word_list = new[] {"x", "y", "z"}, + min_word_size = 2, + min_subword_size = 2, + max_subword_size = 2, + only_longest_match = true, + hyphenation_patterns_path = "analysis/fop.xml" + }; + + } + + public class KeepTypesTests : TokenFilterAssertionBase + { + public override string Name => "keeptypes"; + + public override ITokenFilter Initializer => + new KeepTypesTokenFilter {Types = new[] {"", ""}}; + + public override FuncTokenFilters Fluent => (n, tf) => tf + .KeepTypes(n, t => t + .Types("", "") + ); + + public override object Json => new + { + type = "keep_types", + types = new[] {"", ""} + }; + + } + + public class IcuCollationTests : TokenFilterAssertionBase + { + public override string Name => "icuc"; + + public override ITokenFilter Initializer => + new IcuCollationTokenFilter + { + Alternate = IcuCollationAlternate.NonIgnorable, + CaseFirst = IcuCollationCaseFirst.Lower, + HiraganaQuaternaryMode = true, + Decomposition = IcuCollationDecomposition.No, + Numeric = true, + CaseLevel = true, + Country = "DE", + Language = "de", + Strength = IcuCollationStrength.Tertiary, + Variant = "@collation=phonebook" + }; + + public override FuncTokenFilters Fluent => (n, tf) => tf + .IcuCollation(n, t => t + .Alternate(IcuCollationAlternate.NonIgnorable) + .CaseFirst(IcuCollationCaseFirst.Lower) + .HiraganaQuaternaryMode() + .Decomposition(IcuCollationDecomposition.No) + .Numeric() + .CaseLevel() + .Country("DE") + .Language("de") + .Strength(IcuCollationStrength.Tertiary) + .Variant("@collation=phonebook") + ); + + public override object Json => new + { + alternate = "non-ignorable", + caseFirst = "lower", + caseLevel = true, + country = "DE", + decomposition = "no", + hiraganaQuaternaryMode = true, + language = "de", + numeric = true, + strength = "tertiary", + type = "icu_collation", + variant = "@collation=phonebook" + }; + + } + + public class IcuFoldingTests : TokenFilterAssertionBase + { + public override string Name => "icuf"; + + public override ITokenFilter Initializer => + new IcuFoldingTokenFilter { UnicodeSetFilter = "[^åäöÅÄÖ]" }; + + public override FuncTokenFilters Fluent => (n, tf) => tf.IcuFolding(n, t => t.UnicodeSetFilter("[^åäöÅÄÖ]")); + + public override object Json => new + { + type = "icu_folding", + unicodeSetFilter = "[^åäöÅÄÖ]" + }; + + } + + public class IcuNormalizerTests : TokenFilterAssertionBase + { + public override string Name => "icun"; + + public override ITokenFilter Initializer => new IcuNormalizationTokenFilter { Name = IcuNormalizationType.Canonical }; + + public override FuncTokenFilters Fluent => (n, tf) => tf .IcuNormalization(n, t => t.Name(IcuNormalizationType.Canonical)); + + public override object Json => new + { + name = "nfc", + type = "icu_normalizer" + }; + + } + + public class IcuTransformTests : TokenFilterAssertionBase + { + public override string Name => "icut"; + + public override ITokenFilter Initializer => + new IcuTransformTokenFilter + { + Direction = IcuTransformDirection.Forward, + Id = "Any-Latin; NFD; [:Nonspacing Mark:] Remove; NFC" + }; + + public override FuncTokenFilters Fluent => (n, tf) => tf + .IcuTransform(n, t => t + .Direction(IcuTransformDirection.Forward) + .Id("Any-Latin; NFD; [:Nonspacing Mark:] Remove; NFC") + ); + + public override object Json => new + { + dir = "forward", + id = "Any-Latin; NFD; [:Nonspacing Mark:] Remove; NFC", + type = "icu_transform" + }; + + } + + public class KeepwordsTests : TokenFilterAssertionBase + { + public override string Name => "keepwords"; + + public override ITokenFilter Initializer => + new KeepWordsTokenFilter {KeepWordsCase = true, KeepWords = new[] {"a", "b", "c"}}; + + public override FuncTokenFilters Fluent => (n, tf) => tf + .KeepWords(n, t => t + .KeepWords("a", "b", "c") + .KeepWordsCase() + ); + + public override object Json => new + { + type = "keep", + keep_words = new[] {"a", "b", "c"}, + keep_words_case = true + }; + + } + + public class MarkerTests : TokenFilterAssertionBase + { + public override string Name => "marker"; + + public override ITokenFilter Initializer => new KeywordMarkerTokenFilter {IgnoreCase = true, Keywords = new[] {"a", "b"}}; + + public override FuncTokenFilters Fluent => (n, tf) => tf + .KeywordMarker("marker", t => t + .IgnoreCase() + .Keywords("a", "b") + ); + + public override object Json => new + { + type = "keyword_marker", + keywords = new[] {"a", "b"}, + ignore_case = true + }; + + + } + + public class KuromojiReadingFormTests : TokenFilterAssertionBase + { + public override string Name => "kfr"; + + public override ITokenFilter Initializer => new KuromojiReadingFormTokenFilter {UseRomaji = true}; + + public override FuncTokenFilters Fluent => (n, tf) => tf.KuromojiReadingForm(n, t => t.UseRomaji()); + + public override object Json => new + { + type = "kuromoji_readingform", + use_romaji = true + }; + + } + + public class KuromojiPartOfSpeechTests : TokenFilterAssertionBase + { + public override string Name => "kpos"; + + public override ITokenFilter Initializer => + new KuromojiPartOfSpeechTokenFilter {StopTags = new[] {"# verb-main:", "動詞-自立"}}; + + public override FuncTokenFilters Fluent => (n, tf) => tf + .KuromojiPartOfSpeech(n, t => t.StopTags("# verb-main:", "動詞-自立")); + + public override object Json => new + { + stoptags = new[] + { + "# verb-main:", + "動詞-自立" + }, + type = "kuromoji_part_of_speech" + }; + + } + + public class KuromojiStemmerTests : TokenFilterAssertionBase + { + public override string Name => "ks"; + + public override ITokenFilter Initializer => new KuromojiStemmerTokenFilter {MinimumLength = 4}; + + public override FuncTokenFilters Fluent => (n, tf) => tf.KuromojiStemmer(n, t => t.MinimumLength(4)); + + public override object Json => new + { + minimum_length = 4, + type = "kuromoji_stemmer" + }; + + } + + public class KStemTests : TokenFilterAssertionBase + { + public override string Name => "kstem"; + public override ITokenFilter Initializer => new KStemTokenFilter { }; + public override FuncTokenFilters Fluent => (n, tf) => tf.KStem(n); + public override object Json => new {type = "kstem"}; + } + + public class LengthTests : TokenFilterAssertionBase + { + public override string Name => "length"; + public override ITokenFilter Initializer => new LengthTokenFilter {Min = 10, Max = 200}; + + public override FuncTokenFilters Fluent => (n, tf) => tf.Length(n, t => t.Max(200).Min(10)); + public override object Json => new {type = "length", min = 10, max = 200}; + + } + + public class LimitTests : TokenFilterAssertionBase + { + public override string Name => "limit"; + + public override ITokenFilter Initializer => new LimitTokenCountTokenFilter {ConsumeAllTokens = true, MaxTokenCount = 12}; + + public override FuncTokenFilters Fluent => (n, tf) => tf.LimitTokenCount(n, t => t.ConsumeAllToken().MaxTokenCount(12)); + + public override object Json => new + { + type = "limit", + max_token_count = 12, + consume_all_tokens = true + }; + + } + + public class LowercaseTests : TokenFilterAssertionBase + { + public override string Name => "lc"; + + public override ITokenFilter Initializer => new LowercaseTokenFilter(); + + public override FuncTokenFilters Fluent => (n, tf) => tf.Lowercase(n); + + public override object Json => new {type = "lowercase"}; + + } + + public class NGramTests : TokenFilterAssertionBase + { + public override string Name => "ngram"; + + public override ITokenFilter Initializer => new NGramTokenFilter {MinGram = 3, MaxGram = 4}; + + public override FuncTokenFilters Fluent => (n, tf) => tf.NGram(n, t => t.MinGram(3).MaxGram(4)); + + public override object Json => new {type = "ngram", min_gram = 3, max_gram = 4}; + + } + + public class PatternCaptureTests : TokenFilterAssertionBase + { + public override string Name => "pc"; + + public override ITokenFilter Initializer => + new PatternCaptureTokenFilter {Patterns = new[] {@"\d", @"\w"}, PreserveOriginal = true}; + + public override FuncTokenFilters Fluent => (n, tf) => tf + .PatternCapture(n, t => t.Patterns(@"\d", @"\w").PreserveOriginal()); + + public override object Json => new + { + type = "pattern_capture", + patterns = new[] {"\\d", "\\w"}, + preserve_original = true + }; + } + + public class PatternReplaceTests : TokenFilterAssertionBase + { + public override string Name => "pr"; + + public override ITokenFilter Initializer => + new PatternReplaceTokenFilter {Pattern = @"(\d|\w)", Replacement = "replacement"}; + + public override FuncTokenFilters Fluent => (n, tf) => tf + .PatternReplace(n, t => t + .Pattern(@"(\d|\w)") + .Replacement("replacement") + ); + + public override object Json => new + { + type = "pattern_replace", + pattern = "(\\d|\\w)", + replacement = "replacement" + }; + + } + + public class PorterStemTests : TokenFilterAssertionBase + { + public override string Name => "porter"; + public override ITokenFilter Initializer => new PorterStemTokenFilter(); + public override FuncTokenFilters Fluent => (n, tf) => tf.PorterStem(n); + public override object Json => new { type = "porter_stem" }; + } + + public class ReverseTests : TokenFilterAssertionBase + { + public override string Name => "rev"; + public override ITokenFilter Initializer => new ReverseTokenFilter(); + public override FuncTokenFilters Fluent => (n, tf) => tf.Reverse(n); + public override object Json => new {type = "reverse"}; + + } + + public class ShingleTests : TokenFilterAssertionBase + { + public override string Name => "shing"; + + public override ITokenFilter Initializer => new ShingleTokenFilter + { + FillerToken = "x", + MaxShingleSize = 10, + MinShingleSize = 8, + OutputUnigrams = true, + OutputUnigramsIfNoShingles = true, + TokenSeparator = "|" + }; + + public override FuncTokenFilters Fluent => (n, tf) => tf + .Shingle(n, t => t + .FillerToken("x") + .MaxShingleSize(10) + .MinShingleSize(8) + .OutputUnigrams() + .OutputUnigramsIfNoShingles() + .TokenSeparator("|") + ); + + public override object Json => new + { + type = "shingle", + min_shingle_size = 8, + max_shingle_size = 10, + output_unigrams = true, + output_unigrams_if_no_shingles = true, + token_separator = "|", + filler_token = "x" + }; + + } + + public class SnowballTests : TokenFilterAssertionBase + { + public override string Name => "snow"; + + public override ITokenFilter Initializer => new SnowballTokenFilter {Language = SnowballLanguage.Dutch}; + + public override FuncTokenFilters Fluent => (n, tf) => tf.Snowball(n, t => t.Language(SnowballLanguage.Dutch)); + + public override object Json => new + { + type = "snowball", + language = "Dutch" + }; + + } + + public class StandardTests : TokenFilterAssertionBase + { + public override string Name => "standard"; + + public override ITokenFilter Initializer => new StandardTokenFilter(); + + public override FuncTokenFilters Fluent => (n, tf) => tf.Standard(n); + + public override object Json => new { type = "standard" }; + + } + + public class StemmerTests : TokenFilterAssertionBase + { + public override string Name => "stem"; + + public override ITokenFilter Initializer => new StemmerTokenFilter {Language = "arabic"}; + + public override FuncTokenFilters Fluent => (n, tf) => tf.Stemmer(n, t => t.Language("arabic")); + + public override object Json => new + { + type = "stemmer", + language = "arabic" + }; + + } + + public class StemmerOverrideTests : TokenFilterAssertionBase + { + public override string Name => "stemo"; + + public override ITokenFilter Initializer => new StemmerOverrideTokenFilter {RulesPath = "analysis/custom_stems.txt"}; + + public override FuncTokenFilters Fluent => (n, tf) => tf.StemmerOverride(n, t => t.RulesPath("analysis/custom_stems.txt")); + + public override object Json => new + { + type = "stemmer_override", + rules_path = "analysis/custom_stems.txt" + }; + + } + + public class StopTests : TokenFilterAssertionBase + { + public override string Name => "stop"; + + public override ITokenFilter Initializer => + new StopTokenFilter {IgnoreCase = true, RemoveTrailing = true, StopWords = new[] {"x", "y", "z"}}; + + public override FuncTokenFilters Fluent => (n, tf) => tf + .Stop(n, t => t + .IgnoreCase() + .RemoveTrailing() + .StopWords("x", "y", "z") + ); + + public override object Json => new + { + type = "stop", + stopwords = new[] {"x", "y", "z"}, + ignore_case = true, + remove_trailing = true + }; + + } + + public class SynonymTests : TokenFilterAssertionBase + { + public override string Name => "syn"; + + public override ITokenFilter Initializer => + new SynonymTokenFilter + { + Expand = true, + Format = SynonymFormat.WordNet, + SynonymsPath = "analysis/stopwords.txt", + Synonyms = new[] {"x=>y", "z=>s"}, + Tokenizer = "whitespace" + }; + + public override FuncTokenFilters Fluent => (n, tf) => tf + .Synonym(n, t => t + .Expand() + .Format(SynonymFormat.WordNet) + .SynonymsPath("analysis/stopwords.txt") + .Synonyms("x=>y", "z=>s") + .Tokenizer("whitespace") + ); + + public override object Json => new + { + type = "synonym", + synonyms_path = "analysis/stopwords.txt", + format = "wordnet", + synonyms = new[] {"x=>y", "z=>s"}, + expand = true, + tokenizer = "whitespace" + }; + + } + + public class SynonymGraphTests : TokenFilterAssertionBase + { + public override string Name => "syn_graph"; + + public override ITokenFilter Initializer => + new SynonymGraphTokenFilter + { + Expand = true, + Format = SynonymFormat.WordNet, + SynonymsPath = "analysis/stopwords.txt", + Synonyms = new[] {"x=>y", "z=>s"}, + Tokenizer = "whitespace" + }; + + public override FuncTokenFilters Fluent => (n, tf) => tf + .SynonymGraph(n, t => t + .Expand() + .Format(SynonymFormat.WordNet) + .SynonymsPath("analysis/stopwords.txt") + .Synonyms("x=>y", "z=>s") + .Tokenizer("whitespace") + ); + + public override object Json => new + { + type = "synonym_graph", + synonyms_path = "analysis/stopwords.txt", + format = "wordnet", + synonyms = new[] {"x=>y", "z=>s"}, + expand = true, + tokenizer = "whitespace" + }; + + } + + public class TrimTests : TokenFilterAssertionBase + { + public override string Name => "trimmer"; + public override ITokenFilter Initializer => new TrimTokenFilter(); + public override FuncTokenFilters Fluent => (n, tf) => tf.Trim(n); + public override object Json => new {type = "trim"}; + } + + public class TruncateTests : TokenFilterAssertionBase + { + public override string Name => "truncer"; + public override ITokenFilter Initializer => new TruncateTokenFilter {Length = 100}; + public override FuncTokenFilters Fluent => (n, tf) => tf.Truncate(n, t => t.Length(100)); + public override object Json => new {type = "truncate", length = 100}; + } + + public class UniqueTests : TokenFilterAssertionBase + { + public override string Name => "uq"; + public override ITokenFilter Initializer => new UniqueTokenFilter {OnlyOnSamePosition = true,}; + public override FuncTokenFilters Fluent => (n, tf) => tf.Unique(n, t => t.OnlyOnSamePosition()); + public override object Json => new {type = "unique", only_on_same_position = true}; + + } + public class UppercaseTests : TokenFilterAssertionBase + { + public override string Name => "upper"; + public override ITokenFilter Initializer => new UppercaseTokenFilter(); + public override FuncTokenFilters Fluent => (n, tf) => tf.Uppercase(n); + public override object Json => new {type = "uppercase"}; + + } + public class WordDelimiterTests : TokenFilterAssertionBase + { + public override string Name => "wd"; + + public override ITokenFilter Initializer => + new WordDelimiterTokenFilter + { + CatenateAll = true, + CatenateNumbers = true, + CatenateWords = true, + GenerateNumberParts = true, + GenerateWordParts = true, + PreserveOriginal = true, + ProtectedWords = new[] {"x", "y", "z"}, + SplitOnCaseChange = true, + SplitOnNumerics = true, + StemEnglishPossessive = true + }; + + public override FuncTokenFilters Fluent => (n, tf) => tf + .WordDelimiter(n, t => t + .CatenateAll() + .CatenateNumbers() + .CatenateWords() + .GenerateNumberParts() + .GenerateWordParts() + .PreserveOriginal() + .ProtectedWords("x", "y", "z") + .SplitOnCaseChange() + .SplitOnNumerics() + .StemEnglishPossessive() + ); + + public override object Json => new + { + type = "word_delimiter", + generate_word_parts = true, + generate_number_parts = true, + catenate_words = true, + catenate_numbers = true, + catenate_all = true, + split_on_case_change = true, + preserve_original = true, + split_on_numerics = true, + stem_english_possessive = true, + protected_words = new[] {"x", "y", "z"} + }; + + } + + public class WordDelimiterGraphTests : TokenFilterAssertionBase + { + public override string Name => "wdg"; + + public override ITokenFilter Initializer => + new WordDelimiterGraphTokenFilter + { + CatenateAll = true, + CatenateNumbers = true, + CatenateWords = true, + GenerateNumberParts = true, + GenerateWordParts = true, + PreserveOriginal = true, + ProtectedWords = new[] {"x", "y", "z"}, + SplitOnCaseChange = true, + SplitOnNumerics = true, + StemEnglishPossessive = true + }; + + public override FuncTokenFilters Fluent => (n, tf) => tf + .WordDelimiterGraph(n, t => t + .CatenateAll() + .CatenateNumbers() + .CatenateWords() + .GenerateNumberParts() + .GenerateWordParts() + .PreserveOriginal() + .ProtectedWords("x", "y", "z") + .SplitOnCaseChange() + .SplitOnNumerics() + .StemEnglishPossessive() + ); + + public override object Json => new + { + type = "word_delimiter_graph", + generate_word_parts = true, + generate_number_parts = true, + catenate_words = true, + catenate_numbers = true, + catenate_all = true, + split_on_case_change = true, + preserve_original = true, + split_on_numerics = true, + stem_english_possessive = true, + protected_words = new[] {"x", "y", "z"} + }; + + } + + public class PhoneticTests : TokenFilterAssertionBase + { + public override string Name => "phonetic"; + + public override ITokenFilter Initializer => + new PhoneticTokenFilter + { + Encoder = PhoneticEncoder.Beidermorse, + RuleType = PhoneticRuleType.Exact, + NameType = PhoneticNameType.Sephardic, + LanguageSet = new[] {PhoneticLanguage.Cyrillic, PhoneticLanguage.English, PhoneticLanguage.Hebrew} + }; + + public override FuncTokenFilters Fluent => (n, tf) => tf + .Phonetic(n, t => t + .Encoder(PhoneticEncoder.Beidermorse) + .RuleType(PhoneticRuleType.Exact) + .NameType(PhoneticNameType.Sephardic) + .LanguageSet( + PhoneticLanguage.Cyrillic, + PhoneticLanguage.English, + PhoneticLanguage.Hebrew + ) + ); + + public override object Json => new + { + type = "phonetic", + encoder = "beider_morse", + rule_type = "exact", + name_type = "sephardic", + languageset = new[] {"cyrillic", "english", "hebrew"} + }; + + } + } +} diff --git a/src/Tests/Tests/Analysis/TokenFilters/TokenFilterUsageTests.cs b/src/Tests/Tests/Analysis/TokenFilters/TokenFilterUsageTests.cs deleted file mode 100644 index 2f647f54907..00000000000 --- a/src/Tests/Tests/Analysis/TokenFilters/TokenFilterUsageTests.cs +++ /dev/null @@ -1,641 +0,0 @@ -using System; -using Nest; -using Tests.Framework; - -namespace Tests.Analysis.TokenFilters -{ - /** - */ - - public class TokenFilterUsageTests : PromiseUsageTestBase - { - protected override object ExpectJson => new - { - analysis = new - { - filter = new - { - myAscii = new - { - type = "asciifolding", - preserve_original = true - }, - myCommonGrams = new - { - type = "common_grams", - common_words = new[] {"x", "y", "z"}, - ignore_case = true, - query_mode = true - }, - mydp = new - { - type = "delimited_payload_filter", - delimiter = "-", - encoding = "identity" - }, - dcc = new - { - type = "dictionary_decompounder", - word_list = new[] {"x", "y", "z"}, - min_word_size = 2, - min_subword_size = 2, - max_subword_size = 2, - only_longest_match = true - }, - etf = new - { - type = "edge_ngram", - min_gram = 1, - max_gram = 2 - }, - elision = new - { - type = "elision", - articles = new[] {"a", "b", "c"} - }, - hunspell = new - { - type = "hunspell", - locale = "en_US", - dictionary = "path_to_dict", - dedup = true, - longest_only = true - }, - hypdecomp = new - { - type = "hyphenation_decompounder", - word_list = new[] {"x", "y", "z"}, - min_word_size = 2, - min_subword_size = 2, - max_subword_size = 2, - only_longest_match = true, - hyphenation_patterns_path = "analysis/fop.xml" - }, - keeptypes = new - { - type = "keep_types", - types = new[] {"", ""} - }, - icuc = new { - alternate = "non-ignorable", - caseFirst = "lower", - caseLevel = true, - country = "DE", - decomposition = "no", - hiraganaQuaternaryMode = true, - language = "de", - numeric = true, - strength = "tertiary", - type = "icu_collation", - variant = "@collation=phonebook" - }, - icuf = new { - type = "icu_folding", - unicodeSetFilter = "[^åäöÅÄÖ]" - }, - icun = new { - name = "nfc", - type = "icu_normalizer" - }, - icut = new { - dir = "forward", - id = "Any-Latin; NFD; [:Nonspacing Mark:] Remove; NFC", - type = "icu_transform" - }, - keepwords = new - { - type = "keep", - keep_words = new[] {"a", "b", "c"}, - keep_words_case = true - }, - marker = new - { - type = "keyword_marker", - keywords = new[] {"a", "b"}, - ignore_case = true - }, - kfr = new - { - type = "kuromoji_readingform", - use_romaji = true - }, - kpos = new - { - stoptags = new[] - { - "# verb-main:", - "動詞-自立" - }, - type = "kuromoji_part_of_speech" - }, - ks = new - { - minimum_length = 4, - type = "kuromoji_stemmer" - }, - kstem = new - { - type = "kstem" - }, - length = new - { - type = "length", - min = 10, - max = 200 - }, - limit = new - { - type = "limit", - max_token_count = 12, - consume_all_tokens = true - }, - lc = new - { - type = "lowercase" - }, - ngram = new - { - type = "ngram", - min_gram = 3, - max_gram = 4 - }, - pc = new - { - type = "pattern_capture", - patterns = new[] {"\\d", "\\w"}, - preserve_original = true - }, - pr = new - { - type = "pattern_replace", - pattern = "(\\d|\\w)", - replacement = "replacement" - }, - porter = new - { - type = "porter_stem" - }, - rev = new - { - type = "reverse" - }, - shing = new - { - type = "shingle", - min_shingle_size = 8, - max_shingle_size = 10, - output_unigrams = true, - output_unigrams_if_no_shingles = true, - token_separator = "|", - filler_token = "x" - }, - snow = new - { - type = "snowball", - language = "Dutch" - }, - standard = new - { - type = "standard" - }, - stem = new - { - type = "stemmer", - language = "arabic" - }, - stemo = new - { - type = "stemmer_override", - rules_path = "analysis/custom_stems.txt" - }, - stop = new - { - type = "stop", - stopwords = new[] {"x", "y", "z"}, - ignore_case = true, - remove_trailing = true - }, - syn = new - { - type = "synonym", - synonyms_path = "analysis/stopwords.txt", - format = "wordnet", - synonyms = new[] {"x=>y", "z=>s"}, - expand = true, - tokenizer = "whitespace" - }, - syn_graph = new - { - type = "synonym_graph", - synonyms_path = "analysis/stopwords.txt", - format = "wordnet", - synonyms = new[] {"x=>y", "z=>s"}, - expand = true, - tokenizer = "whitespace" - }, - trimmer = new - { - type = "trim" - }, - truncer = new - { - type = "truncate", - length = 100 - }, - uq = new - { - type = "unique", - only_on_same_position = true - }, - upper = new - { - type = "uppercase" - }, - wd = new - { - type = "word_delimiter", - generate_word_parts = true, - generate_number_parts = true, - catenate_words = true, - catenate_numbers = true, - catenate_all = true, - split_on_case_change = true, - preserve_original = true, - split_on_numerics = true, - stem_english_possessive = true, - protected_words = new[] {"x", "y", "z"} - }, - wdg = new - { - type = "word_delimiter_graph", - generate_word_parts = true, - generate_number_parts = true, - catenate_words = true, - catenate_numbers = true, - catenate_all = true, - split_on_case_change = true, - preserve_original = true, - split_on_numerics = true, - stem_english_possessive = true, - protected_words = new[] {"x", "y", "z"} - }, - phonetic = new - { - type = "phonetic", - encoder = "beider_morse", - rule_type = "exact", - name_type = "sephardic", - languageset = new [] { "cyrillic", "english", "hebrew" } - } - } - } - }; - - /** - * - */ - protected override Func> Fluent => FluentExample; - - public static Func> FluentExample => s => s - .Analysis(analysis => analysis - .TokenFilters(tf => tf - .AsciiFolding("myAscii", t => t.PreserveOriginal()) - .CommonGrams("myCommonGrams", t => t - .CommonWords("x", "y", "z") - .IgnoreCase() - .QueryMode() - ) - .DelimitedPayload("mydp", t => t - .Delimiter('-') - .Encoding(DelimitedPayloadEncoding.Identity) - ) - .DictionaryDecompounder("dcc", t => t - .MaxSubwordSize(2) - .MinSubwordSize(2) - .MinWordSize(2) - .OnlyLongestMatch() - .WordList("x", "y", "z") - ) - .EdgeNGram("etf", t => t - .MaxGram(2) - .MinGram(1) - ) - .Elision("elision", t => t - .Articles("a", "b", "c") - ) - .Hunspell("hunspell", t => t - .Dedup() - .Dictionary("path_to_dict") - .Locale("en_US") - .LongestOnly() - ) - .HyphenationDecompounder("hypdecomp", t => t - .MaxSubwordSize(2) - .MinSubwordSize(2) - .MinWordSize(2) - .OnlyLongestMatch() - .WordList("x", "y", "z") - .HyphenationPatternsPath("analysis/fop.xml") - ) - .KeepTypes("keeptypes", t => t - .Types("", "") - ) - .KeepWords("keepwords", t => t - .KeepWords("a", "b", "c") - .KeepWordsCase() - ) - .KeywordMarker("marker", t => t - .IgnoreCase() - .Keywords("a", "b") - ) - .KStem("kstem") - .Length("length", t => t - .Max(200) - .Min(10) - ) - .LimitTokenCount("limit", t => t - .ConsumeAllToken() - .MaxTokenCount(12) - ) - .Lowercase("lc") - .NGram("ngram", t => t - .MinGram(3) - .MaxGram(4) - ) - .PatternCapture("pc", t => t - .Patterns(@"\d", @"\w") - .PreserveOriginal() - ) - .PatternReplace("pr", t => t - .Pattern(@"(\d|\w)") - .Replacement("replacement") - ) - .PorterStem("porter") - .Reverse("rev") - .Shingle("shing", t => t - .FillerToken("x") - .MaxShingleSize(10) - .MinShingleSize(8) - .OutputUnigrams() - .OutputUnigramsIfNoShingles() - .TokenSeparator("|") - ) - .Snowball("snow", t => t.Language(SnowballLanguage.Dutch)) - .Standard("standard") - .Stemmer("stem", t => t.Language("arabic")) - .StemmerOverride("stemo", t => t.RulesPath("analysis/custom_stems.txt")) - .Stop("stop", t => t - .IgnoreCase() - .RemoveTrailing() - .StopWords("x", "y", "z") - ) - .Synonym("syn", t => t - .Expand() - .Format(SynonymFormat.WordNet) - .SynonymsPath("analysis/stopwords.txt") - .Synonyms("x=>y", "z=>s") - .Tokenizer("whitespace") - ) - .SynonymGraph("syn_graph", t => t - .Expand() - .Format(SynonymFormat.WordNet) - .SynonymsPath("analysis/stopwords.txt") - .Synonyms("x=>y", "z=>s") - .Tokenizer("whitespace") - ) - .Trim("trimmer") - .Truncate("truncer", t => t.Length(100)) - .Unique("uq", t => t.OnlyOnSamePosition()) - .Uppercase("upper") - .WordDelimiter("wd", t => t - .CatenateAll() - .CatenateNumbers() - .CatenateWords() - .GenerateNumberParts() - .GenerateWordParts() - .PreserveOriginal() - .ProtectedWords("x", "y", "z") - .SplitOnCaseChange() - .SplitOnNumerics() - .StemEnglishPossessive() - ) - .WordDelimiterGraph("wdg", t => t - .CatenateAll() - .CatenateNumbers() - .CatenateWords() - .GenerateNumberParts() - .GenerateWordParts() - .PreserveOriginal() - .ProtectedWords("x", "y", "z") - .SplitOnCaseChange() - .SplitOnNumerics() - .StemEnglishPossessive() - ) - .KuromojiPartOfSpeech("kpos", t => t - .StopTags("# verb-main:", "動詞-自立") - ) - .KuromojiReadingForm("kfr", t => t - .UseRomaji() - ) - .KuromojiStemmer("ks", t => t - .MinimumLength(4) - ) - .IcuCollation("icuc", t => t - .Alternate(IcuCollationAlternate.NonIgnorable) - .CaseFirst(IcuCollationCaseFirst.Lower) - .HiraganaQuaternaryMode() - .Decomposition(IcuCollationDecomposition.No) - .Numeric() - .CaseLevel() - .Country("DE") - .Language("de") - .Strength(IcuCollationStrength.Tertiary) - .Variant("@collation=phonebook") - ) - .IcuFolding("icuf", t=>t.UnicodeSetFilter("[^åäöÅÄÖ]")) - .IcuNormalization("icun", t=>t.Name(IcuNormalizationType.Canonical)) - .IcuTransform("icut", t=>t - .Direction(IcuTransformDirection.Forward) - .Id("Any-Latin; NFD; [:Nonspacing Mark:] Remove; NFC") - ) - .Phonetic("phonetic", t => t - .Encoder(PhoneticEncoder.Beidermorse) - .RuleType(PhoneticRuleType.Exact) - .NameType(PhoneticNameType.Sephardic) - .LanguageSet(PhoneticLanguage.Cyrillic, PhoneticLanguage.English, PhoneticLanguage.Hebrew) - ) - ) - ); - - /** - */ - protected override IndexSettings Initializer => InitializerExample; - - public static IndexSettings InitializerExample => - new IndexSettings - { - Analysis = new Nest.Analysis - { - TokenFilters = new Nest.TokenFilters - { - {"myAscii", new AsciiFoldingTokenFilter {PreserveOriginal = true}}, - {"myCommonGrams", new CommonGramsTokenFilter {QueryMode = true, IgnoreCase = true, CommonWords = new[] {"x", "y", "z"}}}, - {"mydp", new DelimitedPayloadTokenFilter {Delimiter = '-', Encoding = DelimitedPayloadEncoding.Identity}}, - { - "dcc", new DictionaryDecompounderTokenFilter - { - MinWordSize = 2, - MinSubwordSize = 2, - MaxSubwordSize = 2, - OnlyLongestMatch = true, - WordList = new[] {"x", "y", "z"} - } - }, - {"etf", new EdgeNGramTokenFilter {MaxGram = 2, MinGram = 1}}, - {"elision", new ElisionTokenFilter {Articles = new[] {"a", "b", "c"}}}, - { - "hunspell", new HunspellTokenFilter - { - Dedup = true, - Dictionary = "path_to_dict", - Locale = "en_US", - LongestOnly = true - } - }, - { - "hypdecomp", new HyphenationDecompounderTokenFilter - { - MaxSubwordSize = 2, - MinSubwordSize = 2, - MinWordSize = 2, - OnlyLongestMatch = true, - WordList = new[] {"x", "y", "z"}, - HyphenationPatternsPath = "analysis/fop.xml" - } - }, - {"keeptypes", new KeepTypesTokenFilter {Types = new[] {"", ""}}}, - {"keepwords", new KeepWordsTokenFilter {KeepWordsCase = true, KeepWords = new[] {"a", "b", "c"}}}, - {"marker", new KeywordMarkerTokenFilter {IgnoreCase = true, Keywords = new[] {"a", "b"}}}, - {"kstem", new KStemTokenFilter { }}, - {"length", new LengthTokenFilter {Min = 10, Max = 200}}, - {"limit", new LimitTokenCountTokenFilter {ConsumeAllTokens = true, MaxTokenCount = 12}}, - {"lc", new LowercaseTokenFilter()}, - {"ngram", new NGramTokenFilter {MinGram = 3, MaxGram = 4}}, - {"pc", new PatternCaptureTokenFilter {Patterns = new[] {@"\d", @"\w"}, PreserveOriginal = true}}, - {"pr", new PatternReplaceTokenFilter {Pattern = @"(\d|\w)", Replacement = "replacement"}}, - {"porter", new PorterStemTokenFilter()}, - {"rev", new ReverseTokenFilter()}, - { - "shing", new ShingleTokenFilter - { - FillerToken = "x", - MaxShingleSize = 10, - MinShingleSize = 8, - OutputUnigrams = true, - OutputUnigramsIfNoShingles = true, - TokenSeparator = "|" - } - }, - {"snow", new SnowballTokenFilter {Language = SnowballLanguage.Dutch}}, - {"standard", new StandardTokenFilter()}, - {"stem", new StemmerTokenFilter {Language = "arabic"}}, - {"stemo", new StemmerOverrideTokenFilter {RulesPath = "analysis/custom_stems.txt"}}, - {"stop", new StopTokenFilter {IgnoreCase = true, RemoveTrailing = true, StopWords = new[] {"x", "y", "z"}}}, - { - "syn", new SynonymTokenFilter - { - Expand = true, - Format = SynonymFormat.WordNet, - SynonymsPath = "analysis/stopwords.txt", - Synonyms = new[] {"x=>y", "z=>s"}, - Tokenizer = "whitespace" - } - }, - { - "syn_graph", new SynonymGraphTokenFilter - { - Expand = true, - Format = SynonymFormat.WordNet, - SynonymsPath = "analysis/stopwords.txt", - Synonyms = new[] {"x=>y", "z=>s"}, - Tokenizer = "whitespace" - } - }, - {"trimmer", new TrimTokenFilter()}, - {"truncer", new TruncateTokenFilter {Length = 100}}, - {"uq", new UniqueTokenFilter {OnlyOnSamePosition = true,}}, - {"upper", new UppercaseTokenFilter()}, - { - "wd", new WordDelimiterTokenFilter - { - CatenateAll = true, - CatenateNumbers = true, - CatenateWords = true, - GenerateNumberParts = true, - GenerateWordParts = true, - PreserveOriginal = true, - ProtectedWords = new[] {"x", "y", "z"}, - SplitOnCaseChange = true, - SplitOnNumerics = true, - StemEnglishPossessive = true - } - }, - { - "wdg", new WordDelimiterGraphTokenFilter - { - CatenateAll = true, - CatenateNumbers = true, - CatenateWords = true, - GenerateNumberParts = true, - GenerateWordParts = true, - PreserveOriginal = true, - ProtectedWords = new[] {"x", "y", "z"}, - SplitOnCaseChange = true, - SplitOnNumerics = true, - StemEnglishPossessive = true - } - }, - {"kpos", new KuromojiPartOfSpeechTokenFilter {StopTags = new[] {"# verb-main:", "動詞-自立"}}}, - {"kfr", new KuromojiReadingFormTokenFilter {UseRomaji = true}}, - {"ks", new KuromojiStemmerTokenFilter {MinimumLength = 4}}, - - { - "icuc", new IcuCollationTokenFilter - { - Alternate = IcuCollationAlternate.NonIgnorable, - CaseFirst = IcuCollationCaseFirst.Lower, - HiraganaQuaternaryMode = true, - Decomposition = IcuCollationDecomposition.No, - Numeric = true, - CaseLevel = true, - Country = "DE", - Language = "de", - Strength = IcuCollationStrength.Tertiary, - Variant = "@collation=phonebook" - } - }, - {"icuf", new IcuFoldingTokenFilter - { - UnicodeSetFilter = "[^åäöÅÄÖ]" - }}, - {"icun", new IcuNormalizationTokenFilter - { - Name = IcuNormalizationType.Canonical - }}, - {"icut", new IcuTransformTokenFilter - { - Direction = IcuTransformDirection.Forward, - Id = "Any-Latin; NFD; [:Nonspacing Mark:] Remove; NFC" - }}, - { "phonetic", new PhoneticTokenFilter - { - Encoder = PhoneticEncoder.Beidermorse, - RuleType = PhoneticRuleType.Exact, - NameType = PhoneticNameType.Sephardic, - LanguageSet = new [] { PhoneticLanguage.Cyrillic, PhoneticLanguage.English, PhoneticLanguage.Hebrew } - }}, - } - } - }; - } -} diff --git a/src/Tests/Tests/Analysis/Tokenizers/TokenizerAssertionBase.cs b/src/Tests/Tests/Analysis/Tokenizers/TokenizerAssertionBase.cs new file mode 100644 index 00000000000..063d4d3c5e9 --- /dev/null +++ b/src/Tests/Tests/Analysis/Tokenizers/TokenizerAssertionBase.cs @@ -0,0 +1,31 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using Elastic.Xunit.XunitPlumbing; +using Nest; + +namespace Tests.Analysis.Tokenizers +{ + public interface ITokenizerAssertion : IAnalysisAssertion { } + + public abstract class TokenizerAssertionBase + : AnalysisComponentTestBase + , ITokenizerAssertion + where TAssertion : TokenizerAssertionBase, new() + { + protected override IAnalysis FluentAnalysis(AnalysisDescriptor an) => + an.Tokenizers(d => AssertionSetup.Fluent(AssertionSetup.Name, d)); + + protected override Nest.Analysis InitializerAnalysis() => + new Nest.Analysis {Tokenizers = new Nest.Tokenizers {{AssertionSetup.Name, AssertionSetup.Initializer}}}; + + protected override object AnalysisJson => new + { + tokenizer = new Dictionary { {AssertionSetup.Name, AssertionSetup.Json} } + }; + + // https://youtrack.jetbrains.com/issue/RIDER-19912 + [U] public override Task TestPutSettingsRequest() => base.TestPutSettingsRequest(); + [I] public override Task TestPutSettingsResponse() => base.TestPutSettingsResponse(); + } + +} diff --git a/src/Tests/Tests/Analysis/Tokenizers/TokenizerTests.cs b/src/Tests/Tests/Analysis/Tokenizers/TokenizerTests.cs new file mode 100644 index 00000000000..a36399ad2a3 --- /dev/null +++ b/src/Tests/Tests/Analysis/Tokenizers/TokenizerTests.cs @@ -0,0 +1,207 @@ +using System; +using Nest; + +namespace Tests.Analysis.Tokenizers +{ + using FuncTokenizer = Func>; + + public static class TokenizerTests + { + public class EdgeNGramTests : TokenizerAssertionBase + { + public override string Name => "endgen"; + + public override ITokenizer Initializer => new EdgeNGramTokenizer + { + MaxGram = 2, + MinGram = 1, + TokenChars = new[] {TokenChar.Digit, TokenChar.Letter} + }; + + public override FuncTokenizer Fluent => (n, t) => t.EdgeNGram(n, e => e + .MaxGram(2) + .MinGram(1) + .TokenChars(TokenChar.Digit, TokenChar.Letter) + ); + + public override object Json => new + { + min_gram = 1, + max_gram = 2, + token_chars = new[] {"digit", "letter"}, + type = "edge_ngram" + }; + } + + public class NGramTests : TokenizerAssertionBase + { + public override string Name => "ng"; + + public override ITokenizer Initializer => new NGramTokenizer + { + MaxGram = 2, + MinGram = 1, + TokenChars = new[] {TokenChar.Digit, TokenChar.Letter} + }; + + public override FuncTokenizer Fluent => (n, t) => t.NGram(n, e => e + .MaxGram(2) + .MinGram(1) + .TokenChars(TokenChar.Digit, TokenChar.Letter) + ); + + public override object Json => new + { + min_gram = 1, + max_gram = 2, + token_chars = new[] {"digit", "letter"}, + type = "ngram" + }; + } + + public class PathHierarchyTests : TokenizerAssertionBase + { + public override string Name => "path"; + + public override ITokenizer Initializer => new PathHierarchyTokenizer + { + BufferSize = 2048, + Delimiter = '|', + Replacement = '-', + Reverse = true, + Skip = 1 + }; + + public override FuncTokenizer Fluent => (n, t) => t.PathHierarchy(n, e => e + .BufferSize(2048) + .Delimiter('|') + .Replacement('-') + .Reverse() + .Skip(1) + ); + + public override object Json => new + { + delimiter = "|", + replacement = "-", + buffer_size = 2048, + reverse = true, + skip = 1, + type = "path_hierarchy" + }; + } + + public class IcuTests : TokenizerAssertionBase + { + public override string Name => "icu"; + private const string RuleFiles = "Latn:icu-files/KeywordTokenizer.rbbi"; + + public override ITokenizer Initializer => new IcuTokenizer + { + RuleFiles = RuleFiles, + }; + + public override FuncTokenizer Fluent => (n, t) => t.Icu(n, e => e + .RuleFiles(RuleFiles) + ); + + public override object Json => new + { + rule_files = RuleFiles, + type = "icu_tokenizer" + }; + } + + public class KuromojiTests : TokenizerAssertionBase + { + public override string Name => "kuro"; + private const string Example = "/箱根山-箱根/成田空港-成田/"; + + public override ITokenizer Initializer => new KuromojiTokenizer + { + Mode = KuromojiTokenizationMode.Extended, + DiscardPunctuation = true, + NBestExamples = Example, + NBestCost = 1000 + }; + + public override FuncTokenizer Fluent => (n, t) => t.Kuromoji(n, e => e + .Mode(KuromojiTokenizationMode.Extended) + .DiscardPunctuation() + .NBestExamples(Example) + .NBestCost(1000) + ); + + public override object Json => new + { + discard_punctuation = true, + mode = "extended", + nbest_cost = 1000, + nbest_examples = Example, + type = "kuromoji_tokenizer" + }; + } + + public class UaxTests : TokenizerAssertionBase + { + public override string Name => "uax"; + public override ITokenizer Initializer => new UaxEmailUrlTokenizer {MaxTokenLength = 12}; + + public override FuncTokenizer Fluent => (n, t) => t.UaxEmailUrl(n, e => e + .MaxTokenLength(12) + ); + + public override object Json => new + { + max_token_length = 12, + type = "uax_url_email" + }; + } + + public class PatternTests : TokenizerAssertionBase + { + public override string Name => "pat"; + + public override ITokenizer Initializer => new PatternTokenizer + { + Flags = "CASE_INSENSITIVE", + Group = 1, + Pattern = @"\W+" + }; + + public override FuncTokenizer Fluent => (n, t) => t.Pattern(n, e => e + .Flags("CASE_INSENSITIVE") + .Group(1) + .Pattern(@"\W+") + ); + + public override object Json => new + { + pattern = @"\W+", + flags = "CASE_INSENSITIVE", + group = 1, + type = "pattern" + }; + } + + public class WhitespaceTests : TokenizerAssertionBase + { + public override string Name => "ws"; + public override ITokenizer Initializer => new WhitespaceTokenizer(); + + public override FuncTokenizer Fluent => (n, t) => t.Whitespace(n); + + public override object Json => new {type = "whitespace"}; + } + + public class StandardTests : TokenizerAssertionBase + { + public override string Name => "stan"; + public override ITokenizer Initializer => new StandardTokenizer(); + + public override FuncTokenizer Fluent => (n, t) => t.Standard(n); + + public override object Json => new {type = "standard"}; + } + } +} diff --git a/src/Tests/Tests/Analysis/Tokenizers/TokenizerUsageTests.cs b/src/Tests/Tests/Analysis/Tokenizers/TokenizerUsageTests.cs deleted file mode 100644 index d89658685e7..00000000000 --- a/src/Tests/Tests/Analysis/Tokenizers/TokenizerUsageTests.cs +++ /dev/null @@ -1,186 +0,0 @@ -using System; -using Nest; -using Tests.Framework; - -namespace Tests.Analysis.Tokenizers -{ - /** - */ - - public class TokenizerUsageTests : PromiseUsageTestBase - { - protected override object ExpectJson => new - { - analysis = new - { - tokenizer = new - { - endgen = new - { - min_gram = 1, - max_gram = 2, - token_chars = new[] {"digit", "letter"}, - type = "edge_ngram" - }, - icu = new - { - rule_files = "Latn:icu-files/KeywordTokenizer.rbbi", - type = "icu_tokenizer" - }, - kuromoji = new - { - discard_punctuation = true, - mode = "extended", - nbest_cost = 1000, - nbest_examples = "/箱根山-箱根/成田空港-成田/", - type = "kuromoji_tokenizer" - }, - ng = new - { - min_gram = 1, - max_gram = 2, - token_chars = new[] {"digit", "letter"}, - type = "ngram" - }, - path = new - { - delimiter = "|", - replacement = "-", - buffer_size = 2048, - reverse = true, - skip = 1, - type = "path_hierarchy" - }, - pattern = new - { - pattern = @"\W+", - flags = "CASE_INSENSITIVE", - group = 1, - type = "pattern" - }, - standard = new - { - type = "standard" - }, - uax = new - { - max_token_length = 12, - type = "uax_url_email" - }, - whitespace = new - { - type = "whitespace" - } - } - } - }; - - /** - * - */ - protected override Func> Fluent => FluentExample; - - public static Func> FluentExample => s => s - .Analysis(analysis => analysis - .Tokenizers(tokenizer => tokenizer - .EdgeNGram("endgen", t => t - .MaxGram(2) - .MinGram(1) - .TokenChars(TokenChar.Digit, TokenChar.Letter) - ) - .NGram("ng", t => t - .MaxGram(2) - .MinGram(1) - .TokenChars(TokenChar.Digit, TokenChar.Letter) - ) - .PathHierarchy("path", t => t - .BufferSize(2048) - .Delimiter('|') - .Replacement('-') - .Reverse() - .Skip(1) - ) - .Pattern("pattern", t => t - .Flags("CASE_INSENSITIVE") - .Group(1) - .Pattern(@"\W+") - ) - .Standard("standard") - .UaxEmailUrl("uax", t => t.MaxTokenLength(12)) - .Whitespace("whitespace") - .Kuromoji("kuromoji", t => t - .Mode(KuromojiTokenizationMode.Extended) - .DiscardPunctuation() - .NBestExamples("/箱根山-箱根/成田空港-成田/") - .NBestCost(1000) - ) - .Icu("icu", t => t.RuleFiles("Latn:icu-files/KeywordTokenizer.rbbi")) - ) - ); - - /** - */ - protected override IndexSettings Initializer => InitializerExample; - - public static IndexSettings InitializerExample => - new IndexSettings - { - Analysis = new Nest.Analysis - { - Tokenizers = new Nest.Tokenizers - { - { - "endgen", new EdgeNGramTokenizer - { - MaxGram = 2, - MinGram = 1, - TokenChars = new[] {TokenChar.Digit, TokenChar.Letter} - } - }, - { - "ng", new NGramTokenizer - { - MaxGram = 2, - MinGram = 1, - TokenChars = new[] {TokenChar.Digit, TokenChar.Letter} - } - }, - { - "path", new PathHierarchyTokenizer - { - BufferSize = 2048, - Delimiter = '|', - Replacement = '-', - Reverse = true, - Skip = 1 - } - }, - { - "pattern", new PatternTokenizer - { - Flags = "CASE_INSENSITIVE", - Group = 1, - Pattern = @"\W+" - } - }, - {"standard", new StandardTokenizer()}, - {"uax", new UaxEmailUrlTokenizer {MaxTokenLength = 12}}, - {"icu", new IcuTokenizer - { - RuleFiles = "Latn:icu-files/KeywordTokenizer.rbbi", - }}, - {"whitespace", new WhitespaceTokenizer()}, - { - "kuromoji", new KuromojiTokenizer - { - Mode = KuromojiTokenizationMode.Extended, - DiscardPunctuation = true, - NBestExamples = "/箱根山-箱根/成田空港-成田/", - NBestCost = 1000 - } - }, - } - } - }; - } -} diff --git a/src/Tests/Tests/Framework/EndpointTests/CrudTestBase.cs b/src/Tests/Tests/Framework/EndpointTests/CrudTestBase.cs index 57a04adbb57..f6b63a02aee 100644 --- a/src/Tests/Tests/Framework/EndpointTests/CrudTestBase.cs +++ b/src/Tests/Tests/Framework/EndpointTests/CrudTestBase.cs @@ -3,17 +3,13 @@ using System.Diagnostics.CodeAnalysis; using System.Threading.Tasks; using Elastic.Managed.Ephemeral; -using Elastic.Xunit.Sdk; using Elastic.Xunit.XunitPlumbing; -using Elasticsearch.Net; using FluentAssertions; using Nest; using Tests.Core.Client; using Tests.Core.Extensions; using Tests.Core.ManagedElasticsearch.Clusters; using Tests.Framework.Integration; -using Tests.Framework.ManagedElasticsearch; -using Tests.Framework.ManagedElasticsearch.Clusters; using Xunit; namespace Tests.Framework @@ -27,6 +23,17 @@ public abstract class CrudWithNoDeleteTestBase false; protected override bool SupportsExists => false; + + // https://youtrack.jetbrains.com/issue/RIDER-19912 + [I] protected override Task CreateCallIsValid() => base.CreateCallIsValid(); + [I] protected override Task GetAfterCreateIsValid() => base.GetAfterCreateIsValid(); + [I] protected override Task ExistsAfterCreateIsValid() => base.ExistsAfterCreateIsValid(); + [I] protected override Task UpdateCallIsValid() => base.UpdateCallIsValid(); + [I] protected override Task GetAfterUpdateIsValid() => base.GetAfterUpdateIsValid(); + [I] protected override Task DeleteCallIsValid() => base.DeleteCallIsValid(); + [I] protected override Task GetAfterDeleteIsValid() => base.GetAfterDeleteIsValid(); + [I] protected override Task ExistsAfterDeleteIsValid() => base.ExistsAfterDeleteIsValid(); + [I] protected override Task DeleteNotFoundIsNotValid() => base.DeleteNotFoundIsNotValid(); } public abstract class CrudTestBase @@ -38,6 +45,16 @@ public abstract class CrudTestBase false; + // https://youtrack.jetbrains.com/issue/RIDER-19912 + [I] protected override Task CreateCallIsValid() => base.CreateCallIsValid(); + [I] protected override Task GetAfterCreateIsValid() => base.GetAfterCreateIsValid(); + [I] protected override Task ExistsAfterCreateIsValid() => base.ExistsAfterCreateIsValid(); + [I] protected override Task UpdateCallIsValid() => base.UpdateCallIsValid(); + [I] protected override Task GetAfterUpdateIsValid() => base.GetAfterUpdateIsValid(); + [I] protected override Task DeleteCallIsValid() => base.DeleteCallIsValid(); + [I] protected override Task GetAfterDeleteIsValid() => base.GetAfterDeleteIsValid(); + [I] protected override Task ExistsAfterDeleteIsValid() => base.ExistsAfterDeleteIsValid(); + [I] protected override Task DeleteNotFoundIsNotValid() => base.DeleteNotFoundIsNotValid(); } public abstract class CrudTestBase : CrudTestBase @@ -49,6 +66,16 @@ public abstract class CrudTestBase false; + // https://youtrack.jetbrains.com/issue/RIDER-19912 + [I] protected override Task CreateCallIsValid() => base.CreateCallIsValid(); + [I] protected override Task GetAfterCreateIsValid() => base.GetAfterCreateIsValid(); + [I] protected override Task ExistsAfterCreateIsValid() => base.ExistsAfterCreateIsValid(); + [I] protected override Task UpdateCallIsValid() => base.UpdateCallIsValid(); + [I] protected override Task GetAfterUpdateIsValid() => base.GetAfterUpdateIsValid(); + [I] protected override Task DeleteCallIsValid() => base.DeleteCallIsValid(); + [I] protected override Task GetAfterDeleteIsValid() => base.GetAfterDeleteIsValid(); + [I] protected override Task ExistsAfterDeleteIsValid() => base.ExistsAfterDeleteIsValid(); + [I] protected override Task DeleteNotFoundIsNotValid() => base.DeleteNotFoundIsNotValid(); } public abstract class CrudTestBase diff --git a/src/Tests/Tests/Framework/EndpointTests/TestState/AsyncLazy.cs b/src/Tests/Tests/Framework/EndpointTests/TestState/AsyncLazy.cs index 753a954fa80..011b4971fda 100644 --- a/src/Tests/Tests/Framework/EndpointTests/TestState/AsyncLazy.cs +++ b/src/Tests/Tests/Framework/EndpointTests/TestState/AsyncLazy.cs @@ -1,9 +1,6 @@ using System; -using System.Collections.Generic; using System.Runtime.CompilerServices; using System.Threading.Tasks; -using Nest; -using Tests.Framework.Integration; namespace Tests.Framework { @@ -22,27 +19,18 @@ public class AsyncLazy /// Initializes a new instance of the class. /// /// The delegate that is invoked on a background thread to produce the value when it is needed. - public AsyncLazy(Func factory) - { - instance = new Lazy>(() => Task.Run(factory)); - } + public AsyncLazy(Func factory) => instance = new Lazy>(() => Task.Run(factory)); /// /// Initializes a new instance of the class. /// /// The asynchronous delegate that is invoked on a background thread to produce the value when it is needed. - public AsyncLazy(Func> factory) - { - instance = new Lazy>(() => Task.Run(factory)); - } + public AsyncLazy(Func> factory) => instance = new Lazy>(() => Task.Run(factory)); /// /// Asynchronous infrastructure support. This method permits instances of to be await'ed. /// - public TaskAwaiter GetAwaiter() - { - return instance.Value.GetAwaiter(); - } + public TaskAwaiter GetAwaiter() => instance.Value.GetAwaiter(); /// /// Starts the asynchronous initialization, if it has not already started. @@ -52,13 +40,4 @@ public void Start() var unused = instance.Value; } } - - public class LazyResponses : AsyncLazy> - { - public static LazyResponses Empty { get; } = new LazyResponses(() => new Dictionary()); - - public LazyResponses(Func> factory) : base(factory) { } - - public LazyResponses(Func>> factory) : base(factory) { } - } } diff --git a/src/Tests/Tests/Framework/EndpointTests/TestState/CallUniqueValues.cs b/src/Tests/Tests/Framework/EndpointTests/TestState/CallUniqueValues.cs index e702b231553..aeb55e57205 100644 --- a/src/Tests/Tests/Framework/EndpointTests/TestState/CallUniqueValues.cs +++ b/src/Tests/Tests/Framework/EndpointTests/TestState/CallUniqueValues.cs @@ -1,33 +1,45 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; +using Elasticsearch.Net; +using static Tests.Framework.Integration.ClientMethod; namespace Tests.Framework.Integration { public class CallUniqueValues : Dictionary { - private string UniqueValue => "nest-" + Guid.NewGuid().ToString("N").Substring(0, 8); + private readonly string _prefix; + private string UniqueValue => $"{this._prefix}-{ViewName}-{Guid.NewGuid().ToString("N").Substring(0, 8)}"; - private IDictionary> ExtendedValues { get; } - = new Dictionary>(); + private IDictionary> ExtendedValues { get; } + = new Dictionary>(); - public ClientMethod CurrentView { get; set; } = ClientMethod.Fluent; - public ClientMethod[] Views { get; } = new[] { ClientMethod.Fluent, ClientMethod.FluentAsync, ClientMethod.Initializer, ClientMethod.InitializerAsync }; + public ClientMethod CurrentView { get; set; } = Fluent; + public string ViewName => this.CurrentView.GetStringValue().ToLowerInvariant(); + + public ClientMethod[] Views { get; } = { Fluent, FluentAsync, Initializer, InitializerAsync }; public string Value => this[CurrentView]; public T ExtendedValue(string key) where T : class => this.ExtendedValues[CurrentView][key] as T; public void ExtendedValue(string key, T value) where T : class => this.ExtendedValues[CurrentView][key] = value; + public T ExtendedValue(string key, Func value) where T : class => + this.ExtendedValues[CurrentView].GetOrAdd(key, value) as T; - public CallUniqueValues() + public CallUniqueValues(string prefix = "nest") { - this.Add(ClientMethod.Fluent, this.UniqueValue); - this.Add(ClientMethod.FluentAsync, this.UniqueValue); - this.Add(ClientMethod.Initializer, this.UniqueValue); - this.Add(ClientMethod.InitializerAsync, this.UniqueValue); + this._prefix = prefix; + this.SetupClientMethod(Fluent); + this.SetupClientMethod(FluentAsync); + this.SetupClientMethod(Initializer); + this.SetupClientMethod(InitializerAsync); + this.CurrentView = Fluent; + } - this.ExtendedValues.Add(ClientMethod.Fluent, new Dictionary()); - this.ExtendedValues.Add(ClientMethod.FluentAsync, new Dictionary()); - this.ExtendedValues.Add(ClientMethod.Initializer, new Dictionary()); - this.ExtendedValues.Add(ClientMethod.InitializerAsync, new Dictionary()); + private void SetupClientMethod(ClientMethod method) + { + this.CurrentView = method; + this.Add(method, this.UniqueValue); + this.ExtendedValues.Add(method, new ConcurrentDictionary()); } } -} \ No newline at end of file +} diff --git a/src/Tests/Tests/Framework/EndpointTests/TestState/EndpointUsage.cs b/src/Tests/Tests/Framework/EndpointTests/TestState/EndpointUsage.cs index 8f2bff98e04..226772332a2 100644 --- a/src/Tests/Tests/Framework/EndpointTests/TestState/EndpointUsage.cs +++ b/src/Tests/Tests/Framework/EndpointTests/TestState/EndpointUsage.cs @@ -1,5 +1,15 @@ using System; using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Runtime.ExceptionServices; +using System.Threading.Tasks; +using Bogus; +using Elasticsearch.Net; +using Nest; +using Tests.Configuration; +using Tests.Core.Client; namespace Tests.Framework.Integration { @@ -13,22 +23,129 @@ public class EndpointUsage public bool CalledSetup { get; internal set; } public bool CalledTeardown { get; internal set; } - public EndpointUsage() - { - this.CallUniqueValues = new CallUniqueValues(); - } + public EndpointUsage() : this("nest") { } + + protected EndpointUsage(string prefix) => this.CallUniqueValues = new CallUniqueValues(prefix); - public LazyResponses CallOnce(Func clientUsage, int? k = null) + public LazyResponses CallOnce(Func clientUsage, int k = 0) { - var key = k ?? clientUsage.GetHashCode(); - if (_usages.TryGetValue(key, out var lazyResponses)) return lazyResponses; + if (_usages.TryGetValue(k, out var lazyResponses)) return lazyResponses; lock (_lock) { - if (_usages.TryGetValue(key, out lazyResponses)) return lazyResponses; + if (_usages.TryGetValue(k, out lazyResponses)) return lazyResponses; var response = clientUsage(); - _usages.TryAdd(key, response); + _usages.TryAdd(k, response); return response; } } } + + public class SingleEndpointUsage : EndpointUsage + where TResponse : class, IResponse + { + public SingleEndpointUsage( + Func fluent, + Func> fluentAsync, + Func request, + Func> requestAsync, + string valuePrefix = null + ) : base(valuePrefix) + { + _fluent = fluent; + _fluentAsync = fluentAsync; + _request = request; + _requestAsync = requestAsync; + } + + private readonly Func _fluent; + private readonly Func> _fluentAsync; + private readonly Func _request; + private readonly Func> _requestAsync; + + public Action IntegrationSetup { get; set; } + public Action IntegrationTeardown { get; set; } + public Action OnBeforeCall { get; set; } + public Action OnAfterCall { get; set; } + + private LazyResponses Responses { get; set; } + + public static Randomizer Random { get; } = new Randomizer(TestConfiguration.Instance.Seed); + + public void KickOffOnce(IElasticClient client, bool oneRandomCall = false) => + this.Responses = this.CallOnce(()=> new LazyResponses(async () => + { + if (TestClient.Configuration.RunIntegrationTests) + { + this.IntegrationSetup?.Invoke(client, this.CallUniqueValues); + this.CalledSetup = true; + } + + var randomCall = Random.Number(0, 3); + + var dict = new Dictionary(); + + if (!oneRandomCall || randomCall == 0) + this.Call(client, dict, ClientMethod.Fluent, v => _fluent(v, client)); + + if (!oneRandomCall || randomCall == 1) + await this.CallAsync(client, dict, ClientMethod.FluentAsync, v => _fluentAsync(v, client)); + + if (!oneRandomCall || randomCall == 2) + this.Call(client, dict, ClientMethod.Initializer, v => _request(v, client)); + + if (!oneRandomCall || randomCall == 3) + await this.CallAsync(client, dict, ClientMethod.InitializerAsync, v => _requestAsync(v, client)); + + if (TestClient.Configuration.RunIntegrationTests) + { + foreach (var v in this.CallUniqueValues.Values.SelectMany(d => d)) + this.IntegrationTeardown?.Invoke(client, this.CallUniqueValues); + this.CalledTeardown = true; + } + + return dict; + })); + + private void Call(IElasticClient client, IDictionary dict, ClientMethod method, Func call) + { + this.CallUniqueValues.CurrentView = method; + this.OnBeforeCall?.Invoke(client); + dict.Add(method, call(this.CallUniqueValues.Value)); + this.OnAfterCall?.Invoke(client); + } + private async Task CallAsync(IElasticClient client, IDictionary dict, ClientMethod method, Func> call) + { + this.CallUniqueValues.CurrentView = method; + this.OnBeforeCall?.Invoke(client); + dict.Add(method, await call(this.CallUniqueValues.Value)); + this.OnAfterCall?.Invoke(client); + } + + public async Task AssertOnAllResponses(Action assert) + { + var responses = await this.Responses; + foreach (var kv in responses) + { + var r = kv.Value as TResponse; + + //this is to make sure any unexpected exceptions on the response are rethrown and shown during testing + if (TestClient.Configuration.RunIntegrationTests && !r.IsValid && r.ApiCall.OriginalException != null + && !(r.ApiCall.OriginalException is ElasticsearchClientException)) + { + var e = ExceptionDispatchInfo.Capture(r.ApiCall.OriginalException.Demystify()); + throw new ResponseAssertionException(e.SourceException, r); + } + + try + { + assert(r); + } + catch (Exception e) + { + throw new ResponseAssertionException(e, r); + } + } + } + } + } diff --git a/src/Tests/Tests/Framework/EndpointTests/TestState/LazyResponses.cs b/src/Tests/Tests/Framework/EndpointTests/TestState/LazyResponses.cs new file mode 100644 index 00000000000..24de1fb63f1 --- /dev/null +++ b/src/Tests/Tests/Framework/EndpointTests/TestState/LazyResponses.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Nest; +using Tests.Framework.Integration; + +namespace Tests.Framework +{ + public class LazyResponses : AsyncLazy> + { + public static LazyResponses Empty { get; } = new LazyResponses(() => new Dictionary()); + + public LazyResponses(Func> factory) : base(factory) { } + + public LazyResponses(Func>> factory) : base(factory) { } + } +} \ No newline at end of file diff --git a/src/Tests/Tests/Mapping/Types/Core/Keyword/KeywordPropertyTests.cs b/src/Tests/Tests/Mapping/Types/Core/Keyword/KeywordPropertyTests.cs index 72ebd1ac37e..56864394f1e 100644 --- a/src/Tests/Tests/Mapping/Types/Core/Keyword/KeywordPropertyTests.cs +++ b/src/Tests/Tests/Mapping/Types/Core/Keyword/KeywordPropertyTests.cs @@ -2,6 +2,8 @@ using Elastic.Xunit.XunitPlumbing; using Elasticsearch.Net; using Nest; +using Tests.Analysis; +using Tests.Analysis.Tokenizers; using Tests.Core.ManagedElasticsearch.Clusters; using Tests.Domain; using Tests.Framework; @@ -19,9 +21,9 @@ public class KeywordPropertyTests : PropertyTestsBase protected override ICreateIndexRequest CreateIndexSettings(CreateIndexDescriptor create) => create .Settings(s => s .Analysis(a => a - .CharFilters(t => Promise(Analysis.CharFilters.CharFilterUsageTests.FluentExample(s).Value.Analysis.CharFilters)) - .TokenFilters(t => Promise(Analysis.TokenFilters.TokenFilterUsageTests.FluentExample(s).Value.Analysis.TokenFilters)) - .Normalizers(t => Promise(Analysis.Normalizers.NormalizerUsageTests.FluentExample(s).Value.Analysis.Normalizers)) + .CharFilters(t => Promise(AnalysisUsageTests.CharFiltersFluent.Analysis.CharFilters)) + .TokenFilters(t => Promise(AnalysisUsageTests.TokenFiltersFluent.Analysis.TokenFilters)) + .Normalizers(t => Promise(AnalysisUsageTests.NormalizersInitializer.Analysis.Normalizers)) ) );