From c1b0706f4f3f437e48a60cd755d24391cf2eaf17 Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Mon, 15 Jan 2018 11:32:12 +1100 Subject: [PATCH 1/3] ignore .vs files --- .gitignore | 3 +- .vs/config/applicationhost.config | 1030 ----------------------------- 2 files changed, 2 insertions(+), 1031 deletions(-) delete mode 100644 .vs/config/applicationhost.config diff --git a/.gitignore b/.gitignore index b3140a14fe6..8970d6c10d8 100644 --- a/.gitignore +++ b/.gitignore @@ -75,7 +75,8 @@ launchSettings.json # https://github.com/elastic/elasticsearch-net/pull/1822#issuecomment-183722698 *.project.lock.json project.lock.json +/src/.vs +/src/.vs/* .idea/ *.sln.iml -/src/.vs/restore.dg diff --git a/.vs/config/applicationhost.config b/.vs/config/applicationhost.config deleted file mode 100644 index c2abfb4801d..00000000000 --- a/.vs/config/applicationhost.config +++ /dev/null @@ -1,1030 +0,0 @@ - - - - - - - - -
-
-
-
-
-
-
-
- - - -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
- -
-
-
-
-
-
- -
-
-
-
-
- -
-
-
- -
-
- -
-
- -
-
-
- - -
-
-
-
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From 5d9843596d8e4031e51cce22d547fcb5ce298e5e Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Mon, 18 Dec 2017 18:10:27 +1100 Subject: [PATCH 2/3] Include similarity in index settings (#2924) Non binary breaking back port of 648c3cf9d016839bc6aa4dbd60da28d90db984b3 Similarity settings are nested under settings>index Add ClassicSimilarity Add DFISimilarity Add similarity settings to create index integration tests Add XML comments for each similarity Refactor SimilarityJsonConverter to not look up the NEST ISimilarity type with Type.GetType() but to infer it from the type property value directly. Add Similarity to the updatable index settings Closes #2890 (cherry picked from commit bf78582ab961895ac8d2ac0e9c42696586a12aa4) --- .../writing-aggregations.asciidoc | 2 + .../working-with-certificates.asciidoc | 3 + .../exceptions/unexpected-exceptions.asciidoc | 7 + .../unrecoverable-exceptions.asciidoc | 5 + .../disable-sniff-ping-per-request.asciidoc | 5 + .../round-robin/skip-dead-nodes.asciidoc | 3 + .../analysis/writing-analyzers.asciidoc | 2 + .../high-level/getting-started.asciidoc | 1 + .../inference/field-inference.asciidoc | 4 + .../inference/indices-paths.asciidoc | 5 + .../high-level/mapping/auto-map.asciidoc | 1 + .../mapping/fluent-mapping.asciidoc | 6 + .../high-level/mapping/multi-fields.asciidoc | 3 + .../mapping/visitor-pattern-mapping.asciidoc | 1 + .../low-level/connecting.asciidoc | 9 + .../low-level/getting-started.asciidoc | 4 + ...logging-with-on-request-completed.asciidoc | 12 + docs/code-standards/elastic-client.asciidoc | 6 +- docs/search/returned-fields.asciidoc | 2 + docs/search/writing-queries.asciidoc | 3 + .../DictionaryLike/IsADictionaryBase.cs | 36 +- .../IndexModules/IndexSettings/IndexState.cs | 18 +- .../Settings/DynamicIndexSettings.cs | 55 ++- .../Settings/IndexSettingsConverter.cs | 437 +++++++++--------- .../Settings/UpdatableIndexSettings.cs | 2 + .../IndexModules/Similarity/BM25Similarity.cs | 6 +- .../Similarity/CustomSimilarity.cs | 19 +- .../Similarity/DFI/DFIIndependenceMeasure.cs | 22 + .../Similarity/DFI/DFISimilarity.cs | 37 ++ .../Similarity/DFR/DFRSimilarity.cs | 19 + .../Similarity/DefaultSimilarity.cs | 16 +- .../Similarity/IB/IBSimilarity.cs | 31 +- .../Similarity/LMDirichletSimilarity.cs | 14 +- .../Similarity/LMJelinekMercerSimilarity.cs | 13 +- .../IndexModules/Similarity/Similarities.cs | 37 +- .../IndexModules/Similarity/Similarity.cs | 6 + .../Similarity/SimilarityJsonConverter.cs | 39 +- .../CreateIndex/CreateIndexRequest.cs | 38 +- .../InstallationTasks/InstallationTaskBase.cs | 8 +- .../Similarity/SimilaritySettings.cs | 91 ++-- .../CreateIndex/CreateIndexApiTests.cs | 147 ++++++ 41 files changed, 830 insertions(+), 345 deletions(-) create mode 100644 src/Nest/IndexModules/Similarity/DFI/DFIIndependenceMeasure.cs create mode 100644 src/Nest/IndexModules/Similarity/DFI/DFISimilarity.cs diff --git a/docs/aggregations/writing-aggregations.asciidoc b/docs/aggregations/writing-aggregations.asciidoc index 3c4327c4afe..5e1c43c35e3 100644 --- a/docs/aggregations/writing-aggregations.asciidoc +++ b/docs/aggregations/writing-aggregations.asciidoc @@ -172,6 +172,7 @@ return s => s ); ---- <1> a list of aggregation functions to apply + <2> Using LINQ's `Aggregate()` function to accumulate/apply all of the aggregation functions Combining multiple `AggregationDescriptor` is also possible using the bitwise `&&` operator @@ -237,5 +238,6 @@ var maxPerChild = childAggregation.Max("max_per_child"); maxPerChild.Should().NotBeNull(); <2> ---- <1> Do something with the average per child. Here we just assert it's not null + <2> Do something with the max per child. Here we just assert it's not null diff --git a/docs/client-concepts/certificates/working-with-certificates.asciidoc b/docs/client-concepts/certificates/working-with-certificates.asciidoc index 29edd81616c..8b82d82eea0 100644 --- a/docs/client-concepts/certificates/working-with-certificates.asciidoc +++ b/docs/client-concepts/certificates/working-with-certificates.asciidoc @@ -144,8 +144,11 @@ public class PkiCluster : CertgenCaCluster } ---- <1> Set the client certificate on `ConnectionSettings` + <2> The path to the `.cer` file + <3> The path to the `.key` file + <4> The password for the private key Or per request on `RequestConfiguration` which will take precedence over the ones defined on `ConnectionConfiguration` diff --git a/docs/client-concepts/connection-pooling/exceptions/unexpected-exceptions.asciidoc b/docs/client-concepts/connection-pooling/exceptions/unexpected-exceptions.asciidoc index 84d54d1f276..d84f33033eb 100644 --- a/docs/client-concepts/connection-pooling/exceptions/unexpected-exceptions.asciidoc +++ b/docs/client-concepts/connection-pooling/exceptions/unexpected-exceptions.asciidoc @@ -60,8 +60,11 @@ audit = await audit.TraceUnexpectedException( ); ---- <1> set up a cluster with 10 nodes + <2> where node 2 on port 9201 always throws an exception + <3> The first call to 9200 returns a healthy response + <4> ...but the second call, to 9201, returns a bad response Sometimes, an unexpected exception happens further down in the pipeline. In this scenario, we @@ -100,7 +103,9 @@ audit = await audit.TraceUnexpectedException( ); ---- <1> calls on 9200 set up to throw a `WebException` + <2> calls on 9201 set up to throw an `Exception` + <3> Assert that the audit trail for the client call includes the bad response from 9200 and 9201 An unexpected hard exception on ping and sniff is something we *do* try to recover from and failover to retrying on the next node. @@ -145,6 +150,8 @@ audit = await audit.TraceUnexpectedException( ); ---- <1> `InnerException` is the exception that brought the request down + <2> The hard exception that happened on ping is still available though + <3> An exception can be hard to relate back to a point in time, so the exception is also available on the audit trail diff --git a/docs/client-concepts/connection-pooling/exceptions/unrecoverable-exceptions.asciidoc b/docs/client-concepts/connection-pooling/exceptions/unrecoverable-exceptions.asciidoc index 46a7a8dfd87..18d6565e8c9 100644 --- a/docs/client-concepts/connection-pooling/exceptions/unrecoverable-exceptions.asciidoc +++ b/docs/client-concepts/connection-pooling/exceptions/unrecoverable-exceptions.asciidoc @@ -68,6 +68,7 @@ var audit = new Auditor(() => Framework.Cluster ); ---- <1> Always succeed on ping + <2> ...but always fail on calls with a 401 Bad Authentication response Now, let's make a client call. We'll see that the first audit event is a successful ping @@ -88,7 +89,9 @@ audit = await audit.TraceElasticsearchException( ); ---- <1> First call results in a successful ping + <2> Second call results in a bad response + <3> The reason for the bad response is Bad Authentication When a bad authentication response occurs, the client does not attempt to deserialize the response body returned; @@ -122,6 +125,7 @@ audit = await audit.TraceElasticsearchException( ); ---- <1> Always return a 401 bad response with a HTML response on client calls + <2> Assert that the response body bytes are null Now in this example, by turning on `DisableDirectStreaming()` on `ConnectionSettings`, we see the same behaviour exhibited @@ -154,5 +158,6 @@ audit = await audit.TraceElasticsearchException( ); ---- <1> Response bytes are set on the response + <2> Assert that the response contains `"nginx/"` diff --git a/docs/client-concepts/connection-pooling/request-overrides/disable-sniff-ping-per-request.asciidoc b/docs/client-concepts/connection-pooling/request-overrides/disable-sniff-ping-per-request.asciidoc index d30752053b2..3e3fb4f9af7 100644 --- a/docs/client-concepts/connection-pooling/request-overrides/disable-sniff-ping-per-request.asciidoc +++ b/docs/client-concepts/connection-pooling/request-overrides/disable-sniff-ping-per-request.asciidoc @@ -67,8 +67,11 @@ audit = await audit.TraceCalls( ); ---- <1> disable sniffing + <2> first call is a successful ping + <3> sniff on startup call happens here, on the second call + <4> No sniff on startup again Now, let's disable pinging on the request @@ -92,6 +95,7 @@ audit = await audit.TraceCall( ); ---- <1> disable ping + <2> No ping after sniffing Finally, let's demonstrate disabling both sniff and ping on the request @@ -113,5 +117,6 @@ audit = await audit.TraceCall( ); ---- <1> diable ping and sniff + <2> no ping or sniff before the call diff --git a/docs/client-concepts/connection-pooling/round-robin/skip-dead-nodes.asciidoc b/docs/client-concepts/connection-pooling/round-robin/skip-dead-nodes.asciidoc index cf83a1e8fd2..d79270b7321 100644 --- a/docs/client-concepts/connection-pooling/round-robin/skip-dead-nodes.asciidoc +++ b/docs/client-concepts/connection-pooling/round-robin/skip-dead-nodes.asciidoc @@ -142,7 +142,9 @@ await audit.TraceCalls( ); ---- <1> The first call goes to 9200 which succeeds + <2> The 2nd call does a ping on 9201 because its used for the first time. It fails so we wrap over to node 9202 + <3> The next call goes to 9203 which fails so we should wrap over A cluster with 2 nodes where the second node fails on ping @@ -192,5 +194,6 @@ await audit.TraceCalls( ); ---- <1> All the calls fail + <2> After all our registered nodes are marked dead we want to sample a single dead node each time to quickly see if the cluster is back up. We do not want to retry all 4 nodes diff --git a/docs/client-concepts/high-level/analysis/writing-analyzers.asciidoc b/docs/client-concepts/high-level/analysis/writing-analyzers.asciidoc index 486b090d039..91d5a699824 100644 --- a/docs/client-concepts/high-level/analysis/writing-analyzers.asciidoc +++ b/docs/client-concepts/high-level/analysis/writing-analyzers.asciidoc @@ -102,6 +102,7 @@ var createIndexResponse = client.CreateIndex("my-index", c => c ); ---- <1> Pre-defined list of English stopwords within Elasticsearch + <2> Use the `standard_english` analyzer configured [source,javascript] @@ -267,6 +268,7 @@ var createIndexResponse = client.CreateIndex("questions", c => c ); ---- <1> Use an analyzer at index time that strips HTML tags + <2> Use an analyzer at search time that does not strip HTML tags With this in place, the text of a question body will be analyzed with the `index_question` analyzer diff --git a/docs/client-concepts/high-level/getting-started.asciidoc b/docs/client-concepts/high-level/getting-started.asciidoc index 36e41d7707e..6150fd3dc33 100644 --- a/docs/client-concepts/high-level/getting-started.asciidoc +++ b/docs/client-concepts/high-level/getting-started.asciidoc @@ -109,6 +109,7 @@ var indexResponse = client.Index(person); <1> var asyncIndexResponse = await client.IndexAsync(person); <2> ---- <1> synchronous method that returns an `IIndexResponse` + <2> asynchronous method that returns a `Task` that can be awaited NOTE: All methods available within NEST are exposed as both synchronous and asynchronous versions, diff --git a/docs/client-concepts/high-level/inference/field-inference.asciidoc b/docs/client-concepts/high-level/inference/field-inference.asciidoc index 867467dd8d5..c7a24d33ca7 100644 --- a/docs/client-concepts/high-level/inference/field-inference.asciidoc +++ b/docs/client-concepts/high-level/inference/field-inference.asciidoc @@ -473,9 +473,13 @@ class Precedence } ---- <1> Even though this property has a NEST property mapping _and_ a `JsonProperty` attribute, We are going to provide a hard rename for it on ConnectionSettings later that should win. + <2> This property has both a NEST attribute and a `JsonProperty`, NEST should win. + <3> We should take the json property into account by itself + <4> This property we are going to special case in our custom serializer to resolve to ask + <5> We are going to register a DefaultFieldNameInferrer on ConnectionSettings that will uppercase all properties. Here we create a custom serializer that renames any property named `AskSerializer` to `ask` diff --git a/docs/client-concepts/high-level/inference/indices-paths.asciidoc b/docs/client-concepts/high-level/inference/indices-paths.asciidoc index 2b7fe31b5cd..21227fd346a 100644 --- a/docs/client-concepts/high-level/inference/indices-paths.asciidoc +++ b/docs/client-concepts/high-level/inference/indices-paths.asciidoc @@ -90,6 +90,7 @@ singleIndexFromIndexName.Match( ); ---- <1> `_all` will override any specific index names here + <2> The `Project` type has been mapped to a specific index name using <`>> [[nest-indices]] @@ -120,7 +121,9 @@ ISearchRequest singleTypedRequest = new SearchDescriptor().Index(single var invalidSingleString = Nest.Indices.Index("name1, name2"); <3> ---- <1> specifying a single index using a string + <2> specifying a single index using a type + <3> an **invalid** single index name ===== Multiple indices @@ -145,7 +148,9 @@ manyStringRequest = new SearchDescriptor().Type(new[] { "name1", "name2 ((IUrlParameter)manyStringRequest.Type).GetString(client.ConnectionSettings).Should().Be("name1,name2"); ---- <1> specifying multiple indices using strings + <2> specifying multiple indices using types + <3> The index names here come from the Connection Settings passed to `TestClient`. See the documentation on <> for more details. ===== All Indices diff --git a/docs/client-concepts/high-level/mapping/auto-map.asciidoc b/docs/client-concepts/high-level/mapping/auto-map.asciidoc index db661474b52..9a5cb1530f8 100644 --- a/docs/client-concepts/high-level/mapping/auto-map.asciidoc +++ b/docs/client-concepts/high-level/mapping/auto-map.asciidoc @@ -58,6 +58,7 @@ var descriptor = new CreateIndexDescriptor("myindex") ); ---- <1> Auto map `Company` + <2> Auto map `Employee` [source,javascript] diff --git a/docs/client-concepts/high-level/mapping/fluent-mapping.asciidoc b/docs/client-concepts/high-level/mapping/fluent-mapping.asciidoc index 1f8e87be255..0e5b365058f 100644 --- a/docs/client-concepts/high-level/mapping/fluent-mapping.asciidoc +++ b/docs/client-concepts/high-level/mapping/fluent-mapping.asciidoc @@ -277,8 +277,11 @@ var descriptor = new CreateIndexDescriptor("myindex") ); ---- <1> Automap company + <2> Override company inferred mappings + <3> Auto map employee + <4> Override employee inferred mappings [source,javascript] @@ -401,8 +404,11 @@ var descriptor = new CreateIndexDescriptor("myindex") ); ---- <1> Automap `Company` + <2> Override specific `Company` mappings + <3> Automap `Employees` property + <4> Override specific `Employee` properties [source,javascript] diff --git a/docs/client-concepts/high-level/mapping/multi-fields.asciidoc b/docs/client-concepts/high-level/mapping/multi-fields.asciidoc index c23e0393877..91abaa792b5 100644 --- a/docs/client-concepts/high-level/mapping/multi-fields.asciidoc +++ b/docs/client-concepts/high-level/mapping/multi-fields.asciidoc @@ -65,7 +65,9 @@ var descriptor = new CreateIndexDescriptor("myindex") ); ---- <1> Use the stop analyzer on this sub field + <2> Use a custom analyzer named "named_shingles" that is configured in the index + <3> Index as not analyzed [source,javascript] @@ -135,6 +137,7 @@ var searchResponse = client.Search(s => s ); ---- <1> Use the shingles subfield on `Name` + <2> Use the keyword subfield on `Name` [source,javascript] diff --git a/docs/client-concepts/high-level/mapping/visitor-pattern-mapping.asciidoc b/docs/client-concepts/high-level/mapping/visitor-pattern-mapping.asciidoc index 02f352dafb8..b797938b62f 100644 --- a/docs/client-concepts/high-level/mapping/visitor-pattern-mapping.asciidoc +++ b/docs/client-concepts/high-level/mapping/visitor-pattern-mapping.asciidoc @@ -73,6 +73,7 @@ public class DisableDocValuesPropertyVisitor : NoopPropertyVisitor } ---- <1> Override the `Visit` method on `INumberProperty` and set `DocValues = false` + <2> Similarily, override the `Visit` method on `IBooleanProperty` and set `DocValues = false` Now we can pass an instance of our custom visitor to `.AutoMap()` diff --git a/docs/client-concepts/low-level/connecting.asciidoc b/docs/client-concepts/low-level/connecting.asciidoc index 0e212e37fec..404f870fc44 100644 --- a/docs/client-concepts/low-level/connecting.asciidoc +++ b/docs/client-concepts/low-level/connecting.asciidoc @@ -86,7 +86,9 @@ var client = new ElasticLowLevelClient(config); var result = client.Search>(new { size = 12 }); ---- <1> Disable automatic proxy detection. When called, defaults to `true`. + <2> Enable compressed request and responses from Elasticsearch (Note that nodes need to be configured to allow this. See the {ref_current}/modules-http.html[http module settings] for more info). + <3> By default responses are deserialized directly from the response stream to the object you tell it to. For debugging purposes, it can be very useful to keep a copy of the raw response on the result object, which is what calling this method will do. `.ResponseBodyInBytes` will only have a value if the client configuration has `DisableDirectStreaming` set @@ -124,10 +126,15 @@ config = config .BasicAuthentication("username", "password"); <6> ---- <1> Allows you to set querystring parameters that have to be added to every request. For instance, if you use a hosted elasticserch provider, and you need need to pass an `apiKey` parameter onto every request. + <2> Sets proxy information on the connection. + <3> [[request-timeout]] Sets the global maximum time a connection may take. Please note that this is the request timeout, the builtin .NET `WebRequest` has no way to set connection timeouts (see http://msdn.microsoft.com/en-us/library/system.net.httpwebrequest.timeout(v=vs.110).aspx[the MSDN documentation on `HttpWebRequest.Timeout` Property]). + <4> As an alternative to the C/go like error checking on `response.IsValid`, you can instead tell the client to <>. + <5> forces all serialization to be indented and appends `pretty=true` to all the requests so that the responses are indented as well + <6> sets the HTTP basic authentication credentials to specify with all requests. NOTE: Basic authentication credentials can alternatively be specified on the node URI directly: @@ -309,7 +316,9 @@ public class MyJsonNetSerializer : JsonNetSerializer } ---- <1> Call this constructor if you only need access to `JsonSerializerSettings` without local state + <2> Call OverwriteDefaultSerializers if you need access to `JsonSerializerSettings` with local state + <3> You can inject contract resolved converters by implementing the ContractConverters property. This can be much faster then registering them on `JsonSerializerSettings.Converters` You can then register a factory on `ConnectionSettings` to create an instance of your subclass instead. diff --git a/docs/client-concepts/low-level/getting-started.asciidoc b/docs/client-concepts/low-level/getting-started.asciidoc index c23bd167bd4..de6ccf1a725 100644 --- a/docs/client-concepts/low-level/getting-started.asciidoc +++ b/docs/client-concepts/low-level/getting-started.asciidoc @@ -108,6 +108,7 @@ var asyncIndexResponse = await lowlevelClient.IndexAsync("people", "pers string responseString = asyncIndexResponse.Body; ---- <1> synchronous method that returns an `IIndexResponse` + <2> asynchronous method that returns a `Task` that can be awaited NOTE: All available methods within Elasticsearch.Net are exposed as both synchronous and asynchronous versions, @@ -243,8 +244,11 @@ var serverError = searchResponse.ServerError; <3> var exception = searchResponse.OriginalException; <4> ---- <1> Response is in the 200 range, or an expected response for the given request + <2> Response is successful, or has a response code between 400-599 that indicates the request cannot be retried. + <3> Details of any error returned from Elasticsearch + <4> If the response is unsuccessful, will hold the original exception. Using these details, it is possible to make decisions around what should be done in your application. diff --git a/docs/client-concepts/troubleshooting/logging-with-on-request-completed.asciidoc b/docs/client-concepts/troubleshooting/logging-with-on-request-completed.asciidoc index e91d5a94823..b2d9c6a4347 100644 --- a/docs/client-concepts/troubleshooting/logging-with-on-request-completed.asciidoc +++ b/docs/client-concepts/troubleshooting/logging-with-on-request-completed.asciidoc @@ -39,7 +39,9 @@ await client.RootNodeInfoAsync(); <3> counter.Should().Be(2); ---- <1> Construct a client + <2> Make a synchronous call and assert the counter is incremented + <3> Make an asynchronous call and assert the counter is incremented `OnRequestCompleted` is called even when an exception is thrown, so it can be used even if the client is @@ -63,7 +65,9 @@ await Assert.ThrowsAsync(async () => await client. counter.Should().Be(2); ---- <1> Configure a client with a connection that **always returns a HTTP 500 response + <2> Always throw exceptions when a call results in an exception + <3> Assert an exception is thrown and the counter is incremented Here's an example using `OnRequestCompleted()` for more complex logging @@ -144,10 +148,15 @@ list.ShouldAllBeEquivalentTo(new[] <6> }); ---- <1> Here we use `InMemoryConnection` but in a real application, you'd use an `IConnection` that _actually_ sends the request, such as `HttpConnection` + <2> Disable direct streaming so we can capture the request and response bytes + <3> Perform some action when a request completes. Here, we're just adding to a list, but in your application you may be logging to a file. + <4> Make a synchronous call + <5> Make an asynchronous call + <6> Assert the list contains the contents written in the delegate passed to `OnRequestCompleted` When running an application in production, you probably don't want to disable direct streaming for _all_ @@ -227,7 +236,10 @@ list.ShouldAllBeEquivalentTo(new[] }); ---- <1> Make a synchronous call where the request and response bytes will not be buffered + <2> Make an asynchronous call where `DisableDirectStreaming()` is enabled + <3> Only the method and url for the first request is captured + <4> the body of the second request is captured diff --git a/docs/code-standards/elastic-client.asciidoc b/docs/code-standards/elastic-client.asciidoc index 5e538a1c105..0a9c81491af 100644 --- a/docs/code-standards/elastic-client.asciidoc +++ b/docs/code-standards/elastic-client.asciidoc @@ -105,15 +105,15 @@ private class MethodWithRequestParameter { public string Name { get; } - public MethodInfo MethodInfo { get; } + public MethodInfo MethodInfo { get; } public bool IsAsync { get; } public ClientMethodType MethodType { get; } - public ParameterInfo Parameter { get; } + public ParameterInfo Parameter { get; } - public MethodWithRequestParameter(MethodInfo methodInfo) + public MethodWithRequestParameter(MethodInfo methodInfo) { Name = methodInfo.Name.EndsWith("Async") ? methodInfo.Name.Substring(0, methodInfo.Name.Length - "Async".Length) diff --git a/docs/search/returned-fields.asciidoc b/docs/search/returned-fields.asciidoc index b7196ea9a54..63300584cae 100644 --- a/docs/search/returned-fields.asciidoc +++ b/docs/search/returned-fields.asciidoc @@ -113,7 +113,9 @@ var searchResponse = client.Search(s => s ); ---- <1> **Include** the following fields + <2> **Exclude** the following fields + <3> Fields can be included or excluded through patterns With source filtering specified on the request, `.Documents` will diff --git a/docs/search/writing-queries.asciidoc b/docs/search/writing-queries.asciidoc index c29fc7e6978..f3900cbd7bd 100644 --- a/docs/search/writing-queries.asciidoc +++ b/docs/search/writing-queries.asciidoc @@ -279,7 +279,9 @@ var searchResponse = client.Search(s => s ); ---- <1> match documents where lead developer first name contains Russ + <2> ...and where the lead developer last name contains Cam + <3> ...and where the project started in 2017 which yields the following query JSON @@ -354,6 +356,7 @@ searchResponse = client.Search(s => s ); ---- <1> combine queries using the binary `&&` operator + <2> wrap a query in a `bool` query filter clause using the unary `+` operator and combine using the binary `&&` operator Take a look at the dedicated section on <> for more detail diff --git a/src/Nest/CommonAbstractions/DictionaryLike/IsADictionaryBase.cs b/src/Nest/CommonAbstractions/DictionaryLike/IsADictionaryBase.cs index 79fc358491f..707e9a74721 100644 --- a/src/Nest/CommonAbstractions/DictionaryLike/IsADictionaryBase.cs +++ b/src/Nest/CommonAbstractions/DictionaryLike/IsADictionaryBase.cs @@ -11,7 +11,7 @@ public abstract class IsADictionaryBase : IIsADictionary BackingDictionary { get; } private IDictionary Self => BackingDictionary; - protected IsADictionaryBase() { this.BackingDictionary = new Dictionary(); } + protected IsADictionaryBase() => this.BackingDictionary = new Dictionary(); //protected IsADictionaryBase(Dictionary backingDictionary) { this.BackingDictionary = backingDictionary; } protected IsADictionaryBase(IDictionary backingDictionary) { @@ -22,9 +22,9 @@ protected IsADictionaryBase(IDictionary backingDictionary) ? new Dictionary(backingDictionary) : new Dictionary(); } + IDictionaryEnumerator IDictionary.GetEnumerator() => Self?.GetEnumerator(); void IDictionary.Clear() => this.BackingDictionary.Clear(); - IDictionaryEnumerator IDictionary.GetEnumerator() => Self?.GetEnumerator(); void IDictionary.Remove(object key) => Self?.Remove(key); object IDictionary.this[object key] { @@ -36,27 +36,27 @@ object IDictionary.this[object key] void IDictionary.Add(object key, object value) => Self?.Add(key, value); bool IDictionary.IsReadOnly=> (Self?.IsReadOnly).GetValueOrDefault(false); bool IDictionary.IsFixedSize=> (Self?.IsFixedSize).GetValueOrDefault(false); - ICollection IDictionary.Keys => Self?.Keys; + bool ICollection>.Remove(KeyValuePair item) => ((ICollection>)this.BackingDictionary).Remove(item); IEnumerator IEnumerable.GetEnumerator() => this.BackingDictionary.GetEnumerator(); + void ICollection>.Add(KeyValuePair item) + { + ValidateKey(item.Key); + ((ICollection>)this.BackingDictionary).Add(item); + } IEnumerator> IEnumerable>.GetEnumerator() => this.BackingDictionary.GetEnumerator(); //public IEnumerator> GetEnumerator() => this.BackingDictionary.GetEnumerator(); void ICollection>.Clear() => this.BackingDictionary.Clear(); + void ICollection.CopyTo(Array array, int index) => Self?.CopyTo(array, index); [EditorBrowsable(EditorBrowsableState.Never)] bool ICollection>.Contains(KeyValuePair item) => Self?.Contains(item) ?? false; + bool ICollection>.IsReadOnly => Self?.IsReadOnly ?? false; void ICollection>.CopyTo( KeyValuePair[] array, int arrayIndex) => Self?.CopyTo(array, arrayIndex); - void ICollection.CopyTo(Array array, int index) => Self?.CopyTo(array, index); + ICollection IDictionary.Keys => Self?.Keys; int ICollection>.Count => this.BackingDictionary.Count; - bool ICollection>.IsReadOnly => Self?.IsReadOnly ?? false; - bool ICollection>.Remove(KeyValuePair item) => ((ICollection>)this.BackingDictionary).Remove(item); - void ICollection>.Add(KeyValuePair item) - { - ValidateKey(item.Key); - ((ICollection>)this.BackingDictionary).Add(item); - } int ICollection.Count => (Self?.Count).GetValueOrDefault(0); object ICollection.SyncRoot => Self?.SyncRoot; @@ -67,26 +67,22 @@ void ICollection>.Add(KeyValuePair item bool IDictionary.ContainsKey(TKey key) => this.BackingDictionary.ContainsKey(key); void IDictionary.Add(TKey key, TValue value) => this.BackingDictionary.Add(ValidateKey(key), value); bool IDictionary.Remove(TKey key) => this.BackingDictionary.Remove(key); - bool IDictionary.TryGetValue(TKey key, out TValue value) => this.BackingDictionary.TryGetValue(key, out value); ICollection IDictionary.Keys => this.BackingDictionary.Keys; ICollection IDictionary.Values => this.BackingDictionary.Values; + bool IDictionary.TryGetValue(TKey key, out TValue value) => this.BackingDictionary.TryGetValue(key, out value); public virtual TKey ValidateKey(TKey key) => key; TValue IDictionary.this[TKey key] { - get { return this.BackingDictionary[key]; } - set { this.BackingDictionary[ValidateKey(key)] = value; } + get => this.BackingDictionary[key]; + set => this.BackingDictionary[ValidateKey(key)] = value; } public TValue this[TKey key] { - get { return this.BackingDictionary[key]; } - set { this.BackingDictionary[ValidateKey(key)] = value; } + get => this.BackingDictionary[key]; + set => this.BackingDictionary[ValidateKey(key)] = value; } - - } - - } diff --git a/src/Nest/IndexModules/IndexSettings/IndexState.cs b/src/Nest/IndexModules/IndexSettings/IndexState.cs index 7e745d09815..e9bcbf33fe8 100644 --- a/src/Nest/IndexModules/IndexSettings/IndexState.cs +++ b/src/Nest/IndexModules/IndexSettings/IndexState.cs @@ -1,4 +1,5 @@ -using Newtonsoft.Json; +using System; +using Newtonsoft.Json; namespace Nest { @@ -17,24 +18,21 @@ public interface IIndexState [JsonProperty("mappings")] IMappings Mappings { get; set; } - [JsonProperty("similarity")] + [JsonIgnore] + [Obsolete("Use Similarity within Settings. Removed in NEST 6.x")] ISimilarities Similarity { get; set; } } public class IndexState : IIndexState { public IIndexSettings Settings { get; set; } - + public IMappings Mappings { get; set; } public IAliases Aliases { get; set; } - + public IWarmers Warmers { get; set; } + [Obsolete("Use Similarity within Settings. Removed in NEST 6.x")] public ISimilarities Similarity { get; set; } } - - - - - -} \ No newline at end of file +} diff --git a/src/Nest/IndexModules/IndexSettings/Settings/DynamicIndexSettings.cs b/src/Nest/IndexModules/IndexSettings/Settings/DynamicIndexSettings.cs index 6c4f728c127..43cc77a9995 100644 --- a/src/Nest/IndexModules/IndexSettings/Settings/DynamicIndexSettings.cs +++ b/src/Nest/IndexModules/IndexSettings/Settings/DynamicIndexSettings.cs @@ -8,13 +8,13 @@ namespace Nest public interface IDynamicIndexSettings : IIsADictionary { /// - ///The number of replicas each primary shard has. Defaults to 1. + ///The number of replicas each primary shard has. Defaults to 1. /// int? NumberOfReplicas { get; set; } /// - ///Auto-expand the number of replicas based on the number of available nodes. - /// Set to a dash delimited lower and upper bound (e.g. 0-5) or use all for the upper bound (e.g. 0-all). Defaults to false (i.e. disabled). + ///Auto-expand the number of replicas based on the number of available nodes. + /// Set to a dash delimited lower and upper bound (e.g. 0-5) or use all for the upper bound (e.g. 0-all). Defaults to false (i.e. disabled). /// //TODO SPECIAL TYPE FOR THIS INSTEAD OF JUST STRING string AutoExpandReplicas { get; set; } @@ -51,17 +51,17 @@ public interface IDynamicIndexSettings : IIsADictionary int? Priority { get; set; } /// - /// Index warmup can be disabled by setting index.warmer.enabled to false. This can be handy when - /// doing initial bulk indexing: disable pre registered warmers to make indexing faster + /// Index warmup can be disabled by setting index.warmer.enabled to false. This can be handy when + /// doing initial bulk indexing: disable pre registered warmers to make indexing faster /// and less expensive and then enable it. /// bool? WarmersEnabled { get; set; } /// /// When a search request is run against an index or against many indices, each involved shard executes the search locally and - /// returns its local results to the coordinating node, which combines these shard-level results into a global result set. + /// returns its local results to the coordinating node, which combines these shard-level results into a “global” result set. /// - /// The shard-level request cache module caches the local results on each shard.This allows frequently used + /// The shard-level request cache module caches the local results on each shard.This allows frequently used /// (and potentially heavy) search requests to return results almost instantly. /// bool? RequestCacheEnabled { get; set; } @@ -73,7 +73,7 @@ public interface IDynamicIndexSettings : IIsADictionary Union RecoveryInitialShards { get; set; } /// - /// The allocation of replica shards which become unassigned because a node has left can be + /// The allocation of replica shards which become unassigned because a node has left can be /// delayed with this dynamic setting, which defaults to 1m. /// Time UnassignedNodeLeftDelayedTimeout { get; set; } @@ -99,6 +99,11 @@ public interface IDynamicIndexSettings : IIsADictionary ITranslogSettings Translog { get; set; } IAnalysis Analysis { get; set; } + + /// + /// Configure similarity + /// + ISimilarities Similarity { get; set; } } public class DynamicIndexSettings : IsADictionaryBase, IDynamicIndexSettings @@ -120,46 +125,49 @@ public DynamicIndexSettings(Dictionary container) /// public bool? BlocksReadOnly { get; set; } - + /// public bool? BlocksWrite { get; set; } - + /// public int? Priority { get; set; } - + /// public bool? WarmersEnabled { get; set; } - + /// public bool? RequestCacheEnabled { get; set; } - + /// public IMergeSettings Merge { get; set; } - + /// public int? NumberOfReplicas { get; set; } - + /// public Union RecoveryInitialShards { get; set; } - + /// public Time RefreshInterval { get; set; } - + /// public int? RoutingAllocationTotalShardsPerNode { get; set; } - + /// public ISlowLog SlowLog { get; set; } - + /// public ITranslogSettings Translog { get; set; } - + /// public Time UnassignedNodeLeftDelayedTimeout { get; set; } /// public IAnalysis Analysis { get; set; } + /// + public ISimilarities Similarity { get; set; } + /// /// Add any setting to the index /// @@ -242,8 +250,13 @@ public TDescriptor Translog(Func public TDescriptor UnassignedNodeLeftDelayedTimeout(Time time) => Assign(a => a.UnassignedNodeLeftDelayedTimeout = time); + /// public TDescriptor Analysis(Func selector) => Assign(a => a.Analysis = selector?.Invoke(new AnalysisDescriptor())); + + /// + public TDescriptor Similarity(Func> selector) => + Assign(a => a.Similarity = selector?.Invoke(new SimilaritiesDescriptor())?.Value); } -} \ No newline at end of file +} diff --git a/src/Nest/IndexModules/IndexSettings/Settings/IndexSettingsConverter.cs b/src/Nest/IndexModules/IndexSettings/Settings/IndexSettingsConverter.cs index 2cbc02e01eb..b58d840e7d2 100644 --- a/src/Nest/IndexModules/IndexSettings/Settings/IndexSettingsConverter.cs +++ b/src/Nest/IndexModules/IndexSettings/Settings/IndexSettingsConverter.cs @@ -1,221 +1,226 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.Linq; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; - -namespace Nest -{ - internal class IndexSettingsConverter : VerbatimDictionaryKeysJsonConverter - { - public override bool CanRead => true; - public override bool CanWrite => true; - public override bool CanConvert(Type objectType) => true; - - public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) - { - var ds = value as IDynamicIndexSettings ?? (value as IUpdateIndexSettingsRequest)?.IndexSettings; - ; - if (ds == null) return; - - IDictionary d = ds; - - d[UpdatableIndexSettings.NumberOfReplicas] = ds.NumberOfReplicas; - d[UpdatableIndexSettings.AutoExpandReplicas] = ds.AutoExpandReplicas; - d[UpdatableIndexSettings.RefreshInterval] = ds.RefreshInterval; - d[UpdatableIndexSettings.BlocksReadOnly] = ds.BlocksReadOnly; - d[UpdatableIndexSettings.BlocksRead] = ds.BlocksRead; - d[UpdatableIndexSettings.BlocksWrite] = ds.BlocksWrite; - d[UpdatableIndexSettings.BlocksMetadata] = ds.BlocksMetadata; - d[UpdatableIndexSettings.Priority] = ds.Priority; - d[UpdatableIndexSettings.WarmersEnabled] = ds.WarmersEnabled; - d[UpdatableIndexSettings.RequestCacheEnable] = ds.RequestCacheEnabled; - d[UpdatableIndexSettings.RecoveryInitialShards] = ds.RecoveryInitialShards; - d[UpdatableIndexSettings.RoutingAllocationTotalShardsPerNode] = - ds.RoutingAllocationTotalShardsPerNode; - d[UpdatableIndexSettings.UnassignedNodeLeftDelayedTimeout] = ds.UnassignedNodeLeftDelayedTimeout; - - var translog = ds.Translog; - d[UpdatableIndexSettings.TranslogSyncInterval] = translog?.SyncInterval; - d[UpdatableIndexSettings.TranslogDurability] = translog?.Durability; - d[UpdatableIndexSettings.TranslogFsType] = translog?.FileSystemType; - - var flush = ds.Translog?.Flush; - d[UpdatableIndexSettings.TranslogFlushThresholdSize] = flush?.ThresholdSize; - d[UpdatableIndexSettings.TranslogFlushTreshHoldOps] = flush?.ThresholdOps; - d[UpdatableIndexSettings.TranslogFlushThresholdPeriod] = flush?.ThresholdPeriod; - d[UpdatableIndexSettings.TranslogInterval] = flush?.Interval; - - d[UpdatableIndexSettings.MergePolicyExpungeDeletesAllowed] = ds.Merge?.Policy?.ExpungeDeletesAllowed; - d[UpdatableIndexSettings.MergePolicyFloorSegment] = ds.Merge?.Policy?.FloorSegment; - d[UpdatableIndexSettings.MergePolicyMaxMergeAtOnce] = ds.Merge?.Policy?.MaxMergeAtOnce; - d[UpdatableIndexSettings.MergePolicyMaxMergeAtOnceExplicit] = ds.Merge?.Policy?.MaxMergeAtOnceExplicit; - d[UpdatableIndexSettings.MergePolicyMaxMergedSegment] = ds.Merge?.Policy?.MaxMergedSegment; - d[UpdatableIndexSettings.MergePolicySegmentsPerTier] = ds.Merge?.Policy?.SegmentsPerTier; - d[UpdatableIndexSettings.MergePolicyReclaimDeletesWeight] = ds.Merge?.Policy?.ReclaimDeletesWeight; - - d[UpdatableIndexSettings.MergeSchedulerMaxThreadCount] = ds.Merge?.Scheduler?.MaxThreadCount; - d[UpdatableIndexSettings.MergeSchedulerAutoThrottle] = ds.Merge?.Scheduler?.AutoThrottle; - - var log = ds.SlowLog; - var search = log?.Search; - var indexing = log?.Indexing; - - d[UpdatableIndexSettings.SlowlogSearchThresholdQueryWarn] = search?.Query?.ThresholdWarn; - d[UpdatableIndexSettings.SlowlogSearchThresholdQueryInfo] = search?.Query?.ThresholdInfo; - d[UpdatableIndexSettings.SlowlogSearchThresholdQueryDebug] = search?.Query?.ThresholdDebug; - d[UpdatableIndexSettings.SlowlogSearchThresholdQueryTrace] = search?.Query?.ThresholdTrace; - - d[UpdatableIndexSettings.SlowlogSearchThresholdFetchWarn] = search?.Fetch?.ThresholdWarn; - d[UpdatableIndexSettings.SlowlogSearchThresholdFetchInfo] = search?.Fetch?.ThresholdInfo; - d[UpdatableIndexSettings.SlowlogSearchThresholdFetchDebug] = search?.Fetch?.ThresholdDebug; - d[UpdatableIndexSettings.SlowlogSearchThresholdFetchTrace] = search?.Fetch?.ThresholdTrace; - d[UpdatableIndexSettings.SlowlogSearchLevel] = search?.LogLevel; - - d[UpdatableIndexSettings.SlowlogIndexingThresholdFetchWarn] = indexing?.ThresholdWarn; - d[UpdatableIndexSettings.SlowlogIndexingThresholdFetchInfo] = indexing?.ThresholdInfo; - d[UpdatableIndexSettings.SlowlogIndexingThresholdFetchDebug] = indexing?.ThresholdDebug; - d[UpdatableIndexSettings.SlowlogIndexingThresholdFetchTrace] = indexing?.ThresholdTrace; - d[UpdatableIndexSettings.SlowlogIndexingLevel] = indexing?.LogLevel; - d[UpdatableIndexSettings.SlowlogIndexingSource] = indexing?.Source; - - - var indexSettings = value as IIndexSettings; - d["index.number_of_shards"] = indexSettings?.NumberOfShards; - d["index.store.type"] = indexSettings?.FileSystemStorageImplementation; - - d["analysis"] = ds.Analysis; - base.WriteJson(writer, d, serializer); - } - - - public JObject Flatten(JObject original, string prefix = "", JObject newObject = null) - { - newObject = newObject ?? new JObject(); - foreach (var property in original.Properties()) - { - if (property.Value is JObject && property.Name != "analysis") - Flatten(property.Value.Value(), prefix + property.Name + ".", newObject); - else newObject.Add(prefix + property.Name, property.Value); - } - return newObject; - } - - public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) - { - var s = new IndexSettings(); - SetKnownIndexSettings(reader, serializer, s); - if (!typeof (IUpdateIndexSettingsRequest).IsAssignableFrom(objectType)) return s; - - var request = new UpdateIndexSettingsRequest() { IndexSettings = s}; - return request; - } - - private void SetKnownIndexSettings(JsonReader reader, JsonSerializer serializer, IIndexSettings s) - { - var settings = Flatten(JObject.Load(reader)).Properties().ToDictionary(kv => kv.Name); - - Set(s, settings, UpdatableIndexSettings.NumberOfReplicas, v => s.NumberOfReplicas = v); - Set(s, settings, UpdatableIndexSettings.AutoExpandReplicas, v => s.AutoExpandReplicas = v); - Set