Fix/doc gen 5.3

docs/aggregations/bucket/adjacency-matrix/adjacency-matrix-usage.asciidoc

@@ -0,0 +1,107 @@
+:ref_current: https://www.elastic.co/guide/en/elasticsearch/reference/5.0
+
+:github: https://github.com/elastic/elasticsearch-net
+
+:nuget: https://www.nuget.org/packages
+
+////
+IMPORTANT NOTE
+==============
+This file has been generated from https://github.com/elastic/elasticsearch-net/tree/5.x/src/Tests/Aggregations/Bucket/AdjacencyMatrix/AdjacencyMatrixUsageTests.cs. 
+If you wish to submit a PR for any spelling mistakes, typos or grammatical errors for this file,
+please modify the original csharp file found at the link and submit the PR with that change. Thanks!
+////
+
+[[adjacency-matrix-usage]]
+== Adjacency Matrix Usage
+
+=== Object Initializer Syntax Example
+
+[source,csharp]
+----
+new SearchRequest<Project>
+{
+    Size =  0,
+    Aggregations = new AdjacencyMatrixAggregation("interactions")
+    {
+        Filters = new NamedFiltersContainer
+        {
+            { "grpA", new TermQuery { Field = "state", Value = StateOfBeing.BellyUp } },
+            { "grpB", new TermQuery { Field = "state", Value = StateOfBeing.Stable } },
+            { "grpC", new TermQuery { Field = "state", Value = StateOfBeing.VeryActive } },
+        }
+    }
+}
+----
+
+[source,javascript]
+.Example json output
+----
+{
+  "size": 0,
+  "aggs": {
+    "interactions": {
+      "adjacency_matrix": {
+        "filters": {
+          "grpA": {
+            "term": {
+              "state": {
+                "value": "BellyUp"
+              }
+            }
+          },
+          "grpB": {
+            "term": {
+              "state": {
+                "value": "Stable"
+              }
+            }
+          },
+          "grpC": {
+            "term": {
+              "state": {
+                "value": "VeryActive"
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+----
+
+=== Fluent DSL Example
+
+[source,csharp]
+----
+s => s
+.Size(0)
+.Aggregations(aggs => aggs
+    .AdjacencyMatrix("interactions", am => am
+        .Filters(fs => fs
+            .Filter("grpA", f => f.Term(p => p.State, StateOfBeing.BellyUp))
+            .Filter("grpB", f => f.Term(p => p.State, StateOfBeing.Stable))
+            .Filter("grpC", f => f.Term(p => p.State, StateOfBeing.VeryActive))
+        )
+    )
+)
+----
+
+=== Handling Responses
+
+[source,csharp]
+----
+response.ShouldBeValid();
+var interactions = response.Aggs.AdjacencyMatrix("interactions");
+interactions.Should().NotBeNull();
+var buckets = interactions.Buckets;
+buckets.Should().NotBeNullOrEmpty();
+
+foreach (var bucket in buckets)
+{
+    bucket.Key.Should().NotBeNullOrEmpty();
+    bucket.DocCount.Should().BeGreaterThan(0);
+}
+----
+

docs/client-concepts/certificates/working-with-certificates.asciidoc

@@ -0,0 +1,89 @@
+:ref_current: https://www.elastic.co/guide/en/elasticsearch/reference/5.0
+
+:github: https://github.com/elastic/elasticsearch-net
+
+:nuget: https://www.nuget.org/packages
+
+////
+IMPORTANT NOTE
+==============
+This file has been generated from https://github.com/elastic/elasticsearch-net/tree/5.x/src/Tests/ClientConcepts/Certificates/WorkingWithCertificates.doc.cs. 
+If you wish to submit a PR for any spelling mistakes, typos or grammatical errors for this file,
+please modify the original csharp file found at the link and submit the PR with that change. Thanks!
+////
+
+[[working-with-certificates]]
+== Working with certificates
+
+=== Server Certificates
+
+If you've enabled SSL on elasticsearch with x-pack or through a proxy in front of elasticsearch and the Certificate Authority (CA)
+That generated the certificate is trusted by the machine running the client code there should be nothing you'll have to do to to talk
+to over https with the client. If you are using your own CA which is not trusted .NET won't allow you to make https calls to that endpoint.
+
+.NET allows you to preempt this though through a custom validation through the the global static `ServicePointManager.ServerCertificateValidationCallback`.
+Most examples you will find on the .NET will simply return `true` from this delegate and call it quits. This is not advisable as this will allow any HTTPS
+traffic in the current AppDomain and not run any validations. Imagine you deploy a web app that talks to Elasticsearch over HTTPS but also some third party
+SOAP/WSDL endpoint setting `ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, errors) => true;` will skip validation of BOTH
+Elasticsearch and that external web service.
+
+.NET also allows you to set that callback per service endpoint and Elasticsearch.NET/NEST exposes this through connection settings.
+You can do your own validation in that handler or simply assign baked in handler that we ship with out of the box on the static
+class `CertificateValidations`.
+
+The two most basic ones are `AllowAll` and `DenyAll` which does accept or deny any ssl trafic to our nodes`:
+
+If your client application however has access to the public CA certificate locally Elasticsearch.NET/NEST ships with handy helpers that assert
+that the certificate that the server presented was one that came from our local CA certificate. If you use x-pack's `certgen` tool to
+[generate SSL certificates] https://www.elastic.co/guide/en/x-pack/current/ssl-tls.html)[] the generated node certificate does not include the CA in the
+certificate chain. This to cut back on SSL handshake size. In those case you can use `CertificateValidations.AuthorityIsRoot` and pass it your local copy
+of the CA public key to assert that the certificate the server presented was generated off that.
+
+If your local copy does not match the servers CA Elasticsearch.NET/NEST will fail to connect
+
+If you go for a vendor generated SSL certificate its common practice for them to include the CA and any intermediary CA's in the certificate chain
+in those case use `CertificateValidations.AuthorityPartOfChain` which validates that the local CA certificate is part of that chain and was used to
+generate the servers key.
+
+=== Client Certificates
+
+X-Pack also allows you to configure a [PKI realm] https://www.elastic.co/guide/en/x-pack/current/pki-realm.html)[] to enable user authentication
+through client certificates. The `certgen` tool included with X-Pack allows you to
+[generate client certificates as well] https://www.elastic.co/guide/en/x-pack/current/ssl-tls.html#CO13-4)[] and assign the distinguished name (DN) of the
+certificate as a user with a certain role.
+
+certgen by default only generates a public certificate `.cer`) and a private key `.key`. To authenticate with client certificates you need to present both
+as one certificate. The easiest way to do this is to generate a `pfx` or `p12` file from the two and present that to `new X509Certificate(pathToPfx)`.
+
+If you do not have a way to run `openssl` or `Pvk2Pfx` to do so as part of your deployments the clients ships with a handy helper to generate one
+on the fly in code based of `.cer`  and `.key` files that `certgen` outputs. Sadly this is not available on .NET core because we can no longer set `PublicKey`
+crypto service provider.
+
+You can set Client Certificates to use on all connections on `ConnectionSettings`
+
+Or per request on `RequestConfiguration` which will take precedence over the ones defined on `ConnectionConfiguration`
+
+=== Object Initializer Syntax Example
+
+[source,csharp]
+----
+new RootNodeInfoRequest
+{
+    RequestConfiguration = new RequestConfiguration
+    {
+        ClientCertificates = new X509Certificate2Collection { new X509Certificate2(this.BadCertificate) }
+    }
+}
+----
+
+=== Fluent DSL Example
+
+[source,csharp]
+----
+s => s
+.RequestConfiguration(r => r
+    .ClientCertificate(this.BadCertificate)
+
+)
+----
+

docs/search/search/collapsing/field-collapse-usage.asciidoc

@@ -0,0 +1,86 @@
+:ref_current: https://www.elastic.co/guide/en/elasticsearch/reference/5.0
+
+:github: https://github.com/elastic/elasticsearch-net
+
+:nuget: https://www.nuget.org/packages
+
+////
+IMPORTANT NOTE
+==============
+This file has been generated from https://github.com/elastic/elasticsearch-net/tree/5.x/src/Tests/Search/Search/Collapsing/FieldCollapseUsageTests.cs. 
+If you wish to submit a PR for any spelling mistakes, typos or grammatical errors for this file,
+please modify the original csharp file found at the link and submit the PR with that change. Thanks!
+////
+
+[[field-collapse-usage]]
+== Field Collapse Usage
+
+=== Fluent DSL Example
+
+[source,csharp]
+----
+s => s
+.Collapse(c => c
+    .Field(f => f.State)
+    .MaxConcurrentGroupSearches(1000)
+    .InnerHits(i => i
+        .Name(nameof(StateOfBeing).ToLowerInvariant())
+        .Size(5)
+        .From(1)
+    )
+)
+----
+
+=== Object Initializer Syntax Example
+
+[source,csharp]
+----
+new SearchRequest<Project>
+{
+    Collapse = new FieldCollapse
+    {
+        Field = Field<Project>(p => p.State),
+        MaxConcurrentGroupSearches = 1000,
+        InnerHits = new InnerHits
+        {
+            Name = nameof(StateOfBeing).ToLowerInvariant(),
+            Size = 5,
+            From = 1
+        }
+    }
+}
+----
+
+[source,javascript]
+.Example json output
+----
+{
+  "collapse": {
+    "field": "state",
+    "max_concurrent_group_searches": 1000,
+    "inner_hits": {
+      "from": 1,
+      "name": "stateofbeing",
+      "size": 5
+    }
+  }
+}
+----
+
+=== Handling Responses
+
+[source,csharp]
+----
+var numberOfStates = Enum.GetValues(typeof(StateOfBeing)).Length;
+response.HitsMetaData.Total.Should().BeGreaterThan(numberOfStates);
+response.Hits.Count.Should().Be(numberOfStates);
+
+foreach (var hit in response.Hits)
+{
+    var name = nameof(StateOfBeing).ToLowerInvariant();
+    hit.InnerHits.Should().NotBeNull().And.ContainKey(name);
+    var innherHits = hit.InnerHits[name];
+    innherHits.Hits.Total.Should().BeGreaterThan(0);
+}
+----
+

src/CodeGeneration/ApiGenerator/ApiGenerator.cs

@@ -56,8 +56,8 @@ private static RestApiSpec CreateRestApiSpecModel(string downloadBranch, string[
 					{
 						foreach (var file in jsonFiles)
 						{
-							if (file.EndsWith("_common.json"))
-								RestApiSpec.CommonApiQueryParameters = CreateCommonApiQueryParameters(file);
+							if (file.EndsWith("_common.json")) RestApiSpec.CommonApiQueryParameters = CreateCommonApiQueryParameters(file);
+							else if (file.EndsWith(".obsolete.json")) continue;
 							else
 							{
 								var endpoint = CreateApiEndpoint(file);
@@ -86,9 +86,22 @@ private static KeyValuePair<string, ApiEndpoint> CreateApiEndpoint(string jsonFi
 			var json = File.ReadAllText(jsonFile);
 			var endpoint = JsonConvert.DeserializeObject<Dictionary<string, ApiEndpoint>>(json).First();
 			endpoint.Value.CsharpMethodName = CreateMethodName(endpoint.Key);
+			PatchObsoleteValues(jsonFile, endpoint.Value);
 			return endpoint;
 		}
 
+		private static void PatchObsoleteValues(string jsonFile, ApiEndpoint endpoint)
+		{
+			var directory = Path.GetDirectoryName(jsonFile);
+			var obsoleteFile = Path.Combine(directory, Path.GetFileNameWithoutExtension(jsonFile)) + ".obsolete.json";
+			if (!File.Exists(obsoleteFile)) return;
+
+			var json = File.ReadAllText(obsoleteFile);
+			var endpointOverride = JsonConvert.DeserializeObject<Dictionary<string, ApiEndpoint>>(json).First();
+			endpoint.ObsoleteQueryParameters = endpointOverride.Value?.Url?.Params ?? new Dictionary<string, ApiQueryParameters>();
+			endpoint.RemovedMethods = endpointOverride.Value?.RemovedMethods ?? new Dictionary<string, string>();
+		}
+
 		private static Dictionary<string, ApiQueryParameters> CreateCommonApiQueryParameters(string jsonFile)
 		{
 			var json = File.ReadAllText(jsonFile);

src/CodeGeneration/ApiGenerator/ApiGenerator.csproj

@@ -39,7 +39,7 @@
       <Paket>True</Paket>
     </Content>
     <Content Include="RestSpecification\Core\root.html" />
-    <Content Include="RestSpecification\DeleteByQuery\root.html" />
+    <Content Include="RestSpecification\Core\_common.json" />
     <Reference Include="System" />
     <Reference Include="System.Core" />
     <Reference Include="System.Xml.Linq" />
@@ -104,6 +104,7 @@
     <None Include="RestSpecification\Core\cat.tasks.json" />
     <None Include="RestSpecification\Core\cat.templates.json" />
     <None Include="RestSpecification\Core\cat.thread_pool.json" />
+    <None Include="RestSpecification\Core\cat.thread_pool.obsolete.json" />
     <None Include="RestSpecification\Core\clear_scroll.json" />
     <None Include="RestSpecification\Core\cluster.allocation_explain.json" />
     <None Include="RestSpecification\Core\cluster.get_settings.json" />
@@ -114,14 +115,17 @@
     <None Include="RestSpecification\Core\cluster.state.json" />
     <None Include="RestSpecification\Core\cluster.stats.json" />
     <None Include="RestSpecification\Core\count.json" />
+    <None Include="RestSpecification\Core\count.obsolete.json" />
     <None Include="RestSpecification\Core\count_percolate.json" />
     <None Include="RestSpecification\Core\create.json" />
     <None Include="RestSpecification\Core\delete.json" />
     <None Include="RestSpecification\Core\delete_by_query.json" />
+    <None Include="RestSpecification\Core\delete_by_query.obsolete.json" />
     <None Include="RestSpecification\Core\delete_script.json" />
     <None Include="RestSpecification\Core\delete_template.json" />
     <None Include="RestSpecification\Core\exists.json" />
     <None Include="RestSpecification\Core\explain.json" />
+    <None Include="RestSpecification\Core\explain.obsolete.json" />
     <None Include="RestSpecification\Core\field_stats.json" />
     <None Include="RestSpecification\Core\get.json" />
     <None Include="RestSpecification\Core\get_script.json" />
@@ -137,6 +141,7 @@
     <None Include="RestSpecification\Core\indices.delete_template.json" />
     <None Include="RestSpecification\Core\indices.exists.json" />
     <None Include="RestSpecification\Core\indices.exists_alias.json" />
+    <None Include="RestSpecification\Core\indices.exists_alias.obsolete.json" />
     <None Include="RestSpecification\Core\indices.exists_template.json" />
     <None Include="RestSpecification\Core\indices.exists_type.json" />
     <None Include="RestSpecification\Core\indices.flush.json" />
@@ -164,6 +169,7 @@
     <None Include="RestSpecification\Core\indices.update_aliases.json" />
     <None Include="RestSpecification\Core\indices.upgrade.json" />
     <None Include="RestSpecification\Core\indices.validate_query.json" />
+    <None Include="RestSpecification\Core\indices.validate_query.obsolete.json" />
     <None Include="RestSpecification\Core\info.json" />
     <None Include="RestSpecification\Core\ingest.delete_pipeline.json" />
     <None Include="RestSpecification\Core\ingest.get_pipeline.json" />
@@ -186,6 +192,7 @@
     <None Include="RestSpecification\Core\render_search_template.json" />
     <None Include="RestSpecification\Core\scroll.json" />
     <None Include="RestSpecification\Core\search.json" />
+    <None Include="RestSpecification\Core\search.obsolete.json" />
     <None Include="RestSpecification\Core\search_shards.json" />
     <None Include="RestSpecification\Core\search_template.json" />
     <None Include="RestSpecification\Core\snapshot.create.json" />
@@ -198,13 +205,14 @@
     <None Include="RestSpecification\Core\snapshot.status.json" />
     <None Include="RestSpecification\Core\snapshot.verify_repository.json" />
     <None Include="RestSpecification\Core\suggest.json" />
+    <None Include="RestSpecification\Core\suggest.obsolete.json" />
     <None Include="RestSpecification\Core\tasks.cancel.json" />
     <None Include="RestSpecification\Core\tasks.get.json" />
     <None Include="RestSpecification\Core\tasks.list.json" />
     <None Include="RestSpecification\Core\termvectors.json" />
     <None Include="RestSpecification\Core\update.json" />
     <None Include="RestSpecification\Core\update_by_query.json" />
-    <None Include="RestSpecification\DeleteByQuery\delete_by_query.json" />
+    <None Include="RestSpecification\Core\update_by_query.obsolete.json" />
     <None Include="RestSpecification\XPack\Graph\xpack.graph.explore.json" />
     <None Include="RestSpecification\XPack\License\xpack.license.delete.json" />
     <None Include="RestSpecification\XPack\License\xpack.license.get.json" />