From 37f9710180e9692be74a6362934593dcac880bab Mon Sep 17 00:00:00 2001 From: Mpdreamz Date: Wed, 15 Aug 2018 11:48:20 +0200 Subject: [PATCH 01/63] 6.4 updated x-pack API's --- .../XPack/Watcher/xpack.watcher.put_watch.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Watcher/xpack.watcher.put_watch.json b/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Watcher/xpack.watcher.put_watch.json index d41f8ea221e..7e29aeaaf43 100644 --- a/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Watcher/xpack.watcher.put_watch.json +++ b/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Watcher/xpack.watcher.put_watch.json @@ -29,7 +29,7 @@ }, "body": { "description" : "The watch", - "required" : true + "required" : false } } } From 52c96a0723582cb6542fa1bc92b1869b7b15d23e Mon Sep 17 00:00:00 2001 From: Mpdreamz Date: Wed, 15 Aug 2018 11:48:43 +0200 Subject: [PATCH 02/63] 6.4 new x-pack API's --- .../xpack.ml.update_filter.json | 20 +++++++++++ .../xpack.rollup.get_rollup_index_caps.json | 17 ++++++++++ .../xpack.security.delete_privileges.json | 30 +++++++++++++++++ .../xpack.security.get_privileges.json | 24 ++++++++++++++ .../xpack.security.has_privileges.json | 22 +++++++++++++ .../xpack.security.put_privilege.json | 33 +++++++++++++++++++ .../xpack.security.put_privileges.json | 27 +++++++++++++++ 7 files changed, 173 insertions(+) create mode 100644 src/CodeGeneration/ApiGenerator/RestSpecification/XPack/MachineLearning/xpack.ml.update_filter.json create mode 100644 src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Rollup/xpack.rollup.get_rollup_index_caps.json create mode 100644 src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Security/xpack.security.delete_privileges.json create mode 100644 src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Security/xpack.security.get_privileges.json create mode 100644 src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Security/xpack.security.has_privileges.json create mode 100644 src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Security/xpack.security.put_privilege.json create mode 100644 src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Security/xpack.security.put_privileges.json diff --git a/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/MachineLearning/xpack.ml.update_filter.json b/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/MachineLearning/xpack.ml.update_filter.json new file mode 100644 index 00000000000..06aceea4c12 --- /dev/null +++ b/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/MachineLearning/xpack.ml.update_filter.json @@ -0,0 +1,20 @@ +{ + "xpack.ml.update_filter": { + "methods": [ "POST" ], + "url": { + "path": "/_xpack/ml/filters/{filter_id}/_update", + "paths": [ "/_xpack/ml/filters/{filter_id}/_update" ], + "parts": { + "filter_id": { + "type": "string", + "required": true, + "description": "The ID of the filter to update" + } + } + }, + "body": { + "description" : "The filter update", + "required" : true + } + } +} diff --git a/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Rollup/xpack.rollup.get_rollup_index_caps.json b/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Rollup/xpack.rollup.get_rollup_index_caps.json new file mode 100644 index 00000000000..458311417d4 --- /dev/null +++ b/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Rollup/xpack.rollup.get_rollup_index_caps.json @@ -0,0 +1,17 @@ +{ + "xpack.rollup.get_rollup_index_caps": { + "documentation": "", + "methods": [ "GET" ], + "url": { + "path": "/{index}/_xpack/rollup/data", + "paths": [ "/{index}/_xpack/rollup/data" ], + "parts": { + "index": { + "type": "string", + "required": true, + "description": "The rollup index or index pattern to obtain rollup capabilities from." + } + } + } + } +} diff --git a/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Security/xpack.security.delete_privileges.json b/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Security/xpack.security.delete_privileges.json new file mode 100644 index 00000000000..6086e46eade --- /dev/null +++ b/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Security/xpack.security.delete_privileges.json @@ -0,0 +1,30 @@ +{ + "xpack.security.delete_privileges": { + "documentation": "TODO", + "methods": [ "DELETE" ], + "url": { + "path": "/_xpack/security/privilege/{application}/{name}", + "paths": [ "/_xpack/security/privilege/{application}/{name}" ], + "parts": { + "application": { + "type" : "string", + "description" : "Application name", + "required" : true + }, + "name": { + "type" : "string", + "description" : "Privilege name", + "required" : true + } + }, + "params": { + "refresh": { + "type" : "enum", + "options": ["true", "false", "wait_for"], + "description" : "If `true` (the default) then refresh the affected shards to make this operation visible to search, if `wait_for` then wait for a refresh to make this operation visible to search, if `false` then do nothing with refreshes." + } + } + }, + "body": null + } +} diff --git a/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Security/xpack.security.get_privileges.json b/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Security/xpack.security.get_privileges.json new file mode 100644 index 00000000000..4286ffa954b --- /dev/null +++ b/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Security/xpack.security.get_privileges.json @@ -0,0 +1,24 @@ +{ + "xpack.security.get_privileges": { + "documentation": "TODO", + "methods": [ "GET" ], + "url": { + "path": "/_xpack/security/privilege/{application}/{name}", + "paths": [ "/_xpack/security/privilege/{application}/{name}" ], + "parts": { + "application": { + "type" : "string", + "description" : "Application name", + "required" : false + }, + "name": { + "type" : "string", + "description" : "Privilege name", + "required" : false + } + }, + "params": {} + }, + "body": null + } +} diff --git a/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Security/xpack.security.has_privileges.json b/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Security/xpack.security.has_privileges.json new file mode 100644 index 00000000000..64b15ae9c02 --- /dev/null +++ b/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Security/xpack.security.has_privileges.json @@ -0,0 +1,22 @@ +{ + "xpack.security.has_privileges": { + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-privileges.html", + "methods": [ "GET", "POST" ], + "url": { + "path": "/_xpack/security/user/_has_privileges", + "paths": [ "/_xpack/security/user/_has_privileges", "/_xpack/security/user/{user}/_has_privileges" ], + "parts": { + "user": { + "type" : "string", + "description" : "Username", + "required" : false + } + }, + "params": {} + }, + "body": { + "description" : "The privileges to test", + "required" : true + } + } +} diff --git a/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Security/xpack.security.put_privilege.json b/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Security/xpack.security.put_privilege.json new file mode 100644 index 00000000000..3d453682c64 --- /dev/null +++ b/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Security/xpack.security.put_privilege.json @@ -0,0 +1,33 @@ +{ + "xpack.security.put_privilege": { + "documentation": "TODO", + "methods": [ "POST", "PUT" ], + "url": { + "path": "/_xpack/security/privilege/{application}/{name}", + "paths": [ "/_xpack/security/privilege/{application}/{name}" ], + "parts": { + "application": { + "type" : "string", + "description" : "Application name", + "required" : true + }, + "name": { + "type" : "string", + "description" : "Privilege name", + "required" : true + } + }, + "params": { + "refresh": { + "type" : "enum", + "options": ["true", "false", "wait_for"], + "description" : "If `true` (the default) then refresh the affected shards to make this operation visible to search, if `wait_for` then wait for a refresh to make this operation visible to search, if `false` then do nothing with refreshes." + } + } + }, + "body": { + "description" : "The privilege to add", + "required" : true + } + } +} diff --git a/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Security/xpack.security.put_privileges.json b/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Security/xpack.security.put_privileges.json new file mode 100644 index 00000000000..07eb5417158 --- /dev/null +++ b/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Security/xpack.security.put_privileges.json @@ -0,0 +1,27 @@ +{ + "xpack.security.put_privileges": { + "documentation": "TODO", + "methods": [ "POST" ], + "url": { + "path": "/_xpack/security/privilege/", + "paths": [ + "/_xpack/security/privilege/" + ], + "params": { + "refresh": { + "type": "enum", + "options": [ + "true", + "false", + "wait_for" + ], + "description": "If `true` (the default) then refresh the affected shards to make this operation visible to search, if `wait_for` then wait for a refresh to make this operation visible to search, if `false` then do nothing with refreshes." + } + } + }, + "body": { + "description" : "The privilege(s) to add", + "required" : true + } + } +} From 8e1e8262c3ffdaca75a9c97659a364538703af0e Mon Sep 17 00:00:00 2001 From: Mpdreamz Date: Wed, 15 Aug 2018 11:51:06 +0200 Subject: [PATCH 03/63] 6.4 updated rest spec files --- .../RestSpecification/Core/get_script.json | 4 ++++ .../RestSpecification/Core/indices.analyze.json | 10 ---------- .../RestSpecification/Core/indices.get.json | 4 ++++ .../RestSpecification/Core/indices.get_mapping.json | 4 ++++ .../RestSpecification/Core/indices.get_settings.json | 4 ++++ .../RestSpecification/Core/indices.shrink.json | 4 ++++ .../RestSpecification/Core/indices.split.json | 4 ++++ 7 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/CodeGeneration/ApiGenerator/RestSpecification/Core/get_script.json b/src/CodeGeneration/ApiGenerator/RestSpecification/Core/get_script.json index 2240f0e1a0b..0b2d6c5a5b9 100644 --- a/src/CodeGeneration/ApiGenerator/RestSpecification/Core/get_script.json +++ b/src/CodeGeneration/ApiGenerator/RestSpecification/Core/get_script.json @@ -13,6 +13,10 @@ } }, "params" : { + "master_timeout": { + "type" : "time", + "description" : "Specify timeout for connection to master" + } } }, "body": null diff --git a/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.analyze.json b/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.analyze.json index 93965388916..c340ccd2f37 100644 --- a/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.analyze.json +++ b/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.analyze.json @@ -15,16 +15,6 @@ "index": { "type" : "string", "description" : "The name of the index to scope the operation" - }, - "prefer_local": { - "type" : "boolean", - "description" : "With `true`, specify that a local shard should be used if available, with `false`, use a random shard (default: true)" - }, - "format": { - "type": "enum", - "options" : ["detailed","text"], - "default": "detailed", - "description": "Format of the output" } } }, diff --git a/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.get.json b/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.get.json index f615718c7d4..6474b8acf52 100644 --- a/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.get.json +++ b/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.get.json @@ -39,6 +39,10 @@ "type": "boolean", "description": "Whether to return all default setting for each of the indices.", "default": false + }, + "master_timeout": { + "type" : "time", + "description" : "Specify timeout for connection to master" } } }, diff --git a/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.get_mapping.json b/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.get_mapping.json index c3c0622844b..e29ed43b3e4 100644 --- a/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.get_mapping.json +++ b/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.get_mapping.json @@ -30,6 +30,10 @@ "default" : "open", "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both." }, + "master_timeout": { + "type" : "time", + "description" : "Specify timeout for connection to master" + }, "local": { "type": "boolean", "description": "Return local information, do not retrieve the state from master node (default: false)" diff --git a/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.get_settings.json b/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.get_settings.json index 706cce5277a..ed22cc837d6 100644 --- a/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.get_settings.json +++ b/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.get_settings.json @@ -16,6 +16,10 @@ } }, "params": { + "master_timeout": { + "type": "time", + "description": "Specify timeout for connection to master" + }, "ignore_unavailable": { "type" : "boolean", "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)" diff --git a/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.shrink.json b/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.shrink.json index 5ef943eacba..f92421b79ae 100644 --- a/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.shrink.json +++ b/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.shrink.json @@ -18,6 +18,10 @@ } }, "params": { + "copy_settings": { + "type" : "boolean", + "description" : "whether or not to copy settings from the source index (defaults to false)" + }, "timeout": { "type" : "time", "description" : "Explicit operation timeout" diff --git a/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.split.json b/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.split.json index a79fa7b7082..2c14fced28c 100644 --- a/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.split.json +++ b/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.split.json @@ -18,6 +18,10 @@ } }, "params": { + "copy_settings": { + "type" : "boolean", + "description" : "whether or not to copy settings from the source index (defaults to false)" + }, "timeout": { "type" : "time", "description" : "Explicit operation timeout" From a037e28f0fad6b5a32a3e2620fa2b1a364f869ab Mon Sep 17 00:00:00 2001 From: Mpdreamz Date: Wed, 15 Aug 2018 11:58:36 +0200 Subject: [PATCH 04/63] add patch file for indices.analyze.json in order to maintain backwards compat --- .../Core/indices.analyze.patch.json | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.analyze.patch.json diff --git a/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.analyze.patch.json b/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.analyze.patch.json new file mode 100644 index 00000000000..7b2cd2c4713 --- /dev/null +++ b/src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.analyze.patch.json @@ -0,0 +1,21 @@ +{ + "indices.analyze": { + "url": { + "params": { + "prefer_local": { + "type": "boolean", + "description": "With `true`, specify that a local shard should be used if available, with `false`, use a random shard (default: true)", + "obsolete": "Erroneously documented as a valid option, no longer document from 6.4 onwards as per https://github.com/elastic/elasticsearch/pull/31795" + + }, + "format": { + "type": "enum", + "options": [ "detailed", "text" ], + "default": "detailed", + "description": "Format of the output", + "obsolete": "Erroneously documented as a valid option, no longer document from 6.4 onwards as per https://github.com/elastic/elasticsearch/pull/31795" + } + } + } + } +} From eda0b3b073d7844667649a43b704aa61f38d6b71 Mon Sep 17 00:00:00 2001 From: Mpdreamz Date: Wed, 15 Aug 2018 13:20:37 +0200 Subject: [PATCH 05/63] Ran codegen against the updated 6.4 json rest spec --- src/CodeGeneration/ApiGenerator/ApiGenerator.cs | 12 ++++++++++++ .../RequestParameters.Generated.cs | 14 ++++++++++++++ src/Nest/_Generated/_Descriptors.generated.cs | 14 ++++++++++++++ src/Nest/_Generated/_Requests.generated.cs | 14 ++++++++++++++ 4 files changed, 54 insertions(+) diff --git a/src/CodeGeneration/ApiGenerator/ApiGenerator.cs b/src/CodeGeneration/ApiGenerator/ApiGenerator.cs index c969cc3ba01..de4ebde8e53 100644 --- a/src/CodeGeneration/ApiGenerator/ApiGenerator.cs +++ b/src/CodeGeneration/ApiGenerator/ApiGenerator.cs @@ -84,6 +84,18 @@ public static void Generate(string downloadBranch, params string[] folders) "xpack.sql.query.json", "xpack.sql.translate.json", "xpack.ssl.certificates.json", + + "scripts_painless_execute.json", + + // 6.4 new API's + + "xpack.ml.update_filter.json", + "xpack.rollup.get_rollup_index_caps.json", + "xpack.security.delete_privileges.json", + "xpack.security.get_privileges.json", + "xpack.security.has_privileges.json", + "xpack.security.put_privilege.json", + "xpack.security.put_privileges.json", }; private static RestApiSpec CreateRestApiSpecModel(string downloadBranch, string[] folders) diff --git a/src/Elasticsearch.Net/Domain/RequestParameters/RequestParameters.Generated.cs b/src/Elasticsearch.Net/Domain/RequestParameters/RequestParameters.Generated.cs index 62b67eae6c3..3f86f39ccb6 100644 --- a/src/Elasticsearch.Net/Domain/RequestParameters/RequestParameters.Generated.cs +++ b/src/Elasticsearch.Net/Domain/RequestParameters/RequestParameters.Generated.cs @@ -873,6 +873,8 @@ public partial class GetRequestParameters : RequestParameters { public override HttpMethod DefaultHttpMethod => HttpMethod.GET; + ///Specify timeout for connection to master + public TimeSpan MasterTimeout { get => Q("master_timeout"); set => Q("master_timeout", value); } } ///Request options for GetSource
http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-get.html
 public partial class SourceRequestParameters : RequestParameters @@ -938,8 +940,10 @@ public partial class AnalyzeRequestParameters : RequestParametersThe name of the index to scope the operation public string IndexQueryString { get => Q("index"); set => Q("index", value); } ///With `true`, specify that a local shard should be used if available, with `false`, use a random shard (default: true) + [Obsolete("Scheduled to be removed in 7.0, Erroneously documented as a valid option, no longer document from 6.4 onwards as per https://github.com/elastic/elasticsearch/pull/31795")] public bool? PreferLocal { get => Q("prefer_local"); set => Q("prefer_local", value); } ///Format of the output + [Obsolete("Scheduled to be removed in 7.0, Erroneously documented as a valid option, no longer document from 6.4 onwards as per https://github.com/elastic/elasticsearch/pull/31795")] public Format? Format { get => Q("format"); set => Q("format", value); } } ///Request options for IndicesClearCacheForAll
http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-clearcache.html
 @@ -1170,6 +1174,8 @@ public partial class GetIndexRequestParameters : RequestParameters Q("flat_settings"); set => Q("flat_settings", value); } ///Whether to return all default setting for each of the indices. public bool? IncludeDefaults { get => Q("include_defaults"); set => Q("include_defaults", value); } + ///Specify timeout for connection to master + public TimeSpan MasterTimeout { get => Q("master_timeout"); set => Q("master_timeout", value); } } ///Request options for IndicesGetAliasForAll
http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-aliases.html
 public partial class GetAliasRequestParameters : RequestParameters @@ -1218,6 +1224,8 @@ public partial class GetMappingRequestParameters : RequestParameters Q("allow_no_indices"); set => Q("allow_no_indices", value); } ///Whether to expand wildcard expression to concrete indices that are open, closed or both. public ExpandWildcards? ExpandWildcards { get => Q("expand_wildcards"); set => Q("expand_wildcards", value); } + ///Specify timeout for connection to master + public TimeSpan MasterTimeout { get => Q("master_timeout"); set => Q("master_timeout", value); } ///Return local information, do not retrieve the state from master node (default: false) public bool? Local { get => Q("local"); set => Q("local", value); } } @@ -1225,6 +1233,8 @@ public partial class GetMappingRequestParameters : RequestParameters { public override HttpMethod DefaultHttpMethod => HttpMethod.GET; + ///Specify timeout for connection to master + public TimeSpan MasterTimeout { get => Q("master_timeout"); set => Q("master_timeout", value); } ///Whether specified concrete indices should be ignored when unavailable (missing or closed) public bool? IgnoreUnavailable { get => Q("ignore_unavailable"); set => Q("ignore_unavailable", value); } /// @@ -1423,6 +1433,8 @@ public partial class IndicesShardStoresRequestParameters : RequestParameters { public override HttpMethod DefaultHttpMethod => HttpMethod.PUT; + ///whether or not to copy settings from the source index (defaults to false) + public bool? CopySettings { get => Q("copy_settings"); set => Q("copy_settings", value); } ///Explicit operation timeout public TimeSpan Timeout { get => Q("timeout"); set => Q("timeout", value); } ///Specify timeout for connection to master @@ -1434,6 +1446,8 @@ public partial class ShrinkIndexRequestParameters : RequestParameters { public override HttpMethod DefaultHttpMethod => HttpMethod.PUT; + ///whether or not to copy settings from the source index (defaults to false) + public bool? CopySettings { get => Q("copy_settings"); set => Q("copy_settings", value); } ///Explicit operation timeout public TimeSpan Timeout { get => Q("timeout"); set => Q("timeout", value); } ///Specify timeout for connection to master diff --git a/src/Nest/_Generated/_Descriptors.generated.cs b/src/Nest/_Generated/_Descriptors.generated.cs index aef3329e885..b91790e96be 100644 --- a/src/Nest/_Generated/_Descriptors.generated.cs +++ b/src/Nest/_Generated/_Descriptors.generated.cs @@ -1438,6 +1438,8 @@ public GetScriptDescriptor(Id id) : base(r=>r.Required("id", id)){} // Request parameters + ///Specify timeout for connection to master + public GetScriptDescriptor MasterTimeout(Time masterTimeout) => Qs("master_timeout", masterTimeout); } ///descriptor for GetSource 
http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-get.html
 public partial class SourceDescriptor : RequestDescriptorBase,SourceRequestParameters, ISourceRequest>, ISourceRequest @@ -1584,8 +1586,10 @@ public AnalyzeDescriptor() : base(){} // Request parameters ///With `true`, specify that a local shard should be used if available, with `false`, use a random shard (default: true) + [Obsolete("Scheduled to be removed in 7.0, Erroneously documented as a valid option, no longer document from 6.4 onwards as per https://github.com/elastic/elasticsearch/pull/31795")] public AnalyzeDescriptor PreferLocal(bool? preferLocal = true) => Qs("prefer_local", preferLocal); ///Format of the output + [Obsolete("Scheduled to be removed in 7.0, Erroneously documented as a valid option, no longer document from 6.4 onwards as per https://github.com/elastic/elasticsearch/pull/31795")] public AnalyzeDescriptor Format(Format? format) => Qs("format", format); } ///descriptor for IndicesClearCacheForAll 
http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-clearcache.html
 @@ -2003,6 +2007,8 @@ public GetIndexDescriptor(Indices index) : base(r=>r.Required("index", index)){} public GetIndexDescriptor FlatSettings(bool? flatSettings = true) => Qs("flat_settings", flatSettings); ///Whether to return all default setting for each of the indices. public GetIndexDescriptor IncludeDefaults(bool? includeDefaults = true) => Qs("include_defaults", includeDefaults); + ///Specify timeout for connection to master + public GetIndexDescriptor MasterTimeout(Time masterTimeout) => Qs("master_timeout", masterTimeout); } ///descriptor for IndicesGetAliasForAll 
http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-aliases.html
 public partial class GetAliasDescriptor : RequestDescriptorBase, IGetAliasRequest @@ -2113,6 +2119,8 @@ public GetMappingDescriptor() : base(r=> r.Required("index", (Indices)typeof(T)) public GetMappingDescriptor AllowNoIndices(bool? allowNoIndices = true) => Qs("allow_no_indices", allowNoIndices); ///Whether to expand wildcard expression to concrete indices that are open, closed or both. public GetMappingDescriptor ExpandWildcards(ExpandWildcards? expandWildcards) => Qs("expand_wildcards", expandWildcards); + ///Specify timeout for connection to master + public GetMappingDescriptor MasterTimeout(Time masterTimeout) => Qs("master_timeout", masterTimeout); ///Return local information, do not retrieve the state from master node (default: false) public GetMappingDescriptor Local(bool? local = true) => Qs("local", local); } @@ -2139,6 +2147,8 @@ public GetIndexSettingsDescriptor() : base(){} // Request parameters + ///Specify timeout for connection to master + public GetIndexSettingsDescriptor MasterTimeout(Time masterTimeout) => Qs("master_timeout", masterTimeout); ///Whether specified concrete indices should be ignored when unavailable (missing or closed) public GetIndexSettingsDescriptor IgnoreUnavailable(bool? ignoreUnavailable = true) => Qs("ignore_unavailable", ignoreUnavailable); ///Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified) @@ -2502,6 +2512,8 @@ public ShrinkIndexDescriptor(IndexName index, IndexName target) : base(r=>r.Requ // Request parameters + ///whether or not to copy settings from the source index (defaults to false) + public ShrinkIndexDescriptor CopySettings(bool? copySettings = true) => Qs("copy_settings", copySettings); ///Explicit operation timeout public ShrinkIndexDescriptor Timeout(Time timeout) => Qs("timeout", timeout); ///Specify timeout for connection to master @@ -2528,6 +2540,8 @@ public SplitIndexDescriptor(IndexName index, IndexName target) : base(r=>r.Requi // Request parameters + ///whether or not to copy settings from the source index (defaults to false) + public SplitIndexDescriptor CopySettings(bool? copySettings = true) => Qs("copy_settings", copySettings); ///Explicit operation timeout public SplitIndexDescriptor Timeout(Time timeout) => Qs("timeout", timeout); ///Specify timeout for connection to master diff --git a/src/Nest/_Generated/_Requests.generated.cs b/src/Nest/_Generated/_Requests.generated.cs index cac7ff292b9..76723a042f2 100644 --- a/src/Nest/_Generated/_Requests.generated.cs +++ b/src/Nest/_Generated/_Requests.generated.cs @@ -126,8 +126,10 @@ public AnalyzeRequest(IndexName index) : base(r=>r.Optional("index", index)){} // Request parameters ///With `true`, specify that a local shard should be used if available, with `false`, use a random shard (default: true) + [Obsolete("Scheduled to be removed in 7.0, Erroneously documented as a valid option, no longer document from 6.4 onwards as per https://github.com/elastic/elasticsearch/pull/31795")] public bool? PreferLocal { get => Q("prefer_local"); set => Q("prefer_local", value); } ///Format of the output + [Obsolete("Scheduled to be removed in 7.0, Erroneously documented as a valid option, no longer document from 6.4 onwards as per https://github.com/elastic/elasticsearch/pull/31795")] public Format? Format { get => Q("format"); set => Q("format", value); } } [JsonObject(MemberSerialization = MemberSerialization.OptIn)] @@ -2699,6 +2701,8 @@ public GetIndexRequest(Indices index) : base(r=>r.Required("index", index)){} public bool? FlatSettings { get => Q("flat_settings"); set => Q("flat_settings", value); } ///Whether to return all default setting for each of the indices. public bool? IncludeDefaults { get => Q("include_defaults"); set => Q("include_defaults", value); } + ///Specify timeout for connection to master + public Time MasterTimeout { get => Q public SearchDescriptor Collapse(Func, IFieldCollapse> collapseSelector) => diff --git a/src/Tests/Tests/Search/Search/Collapsing/FieldCollapseUsageTests.cs b/src/Tests/Tests/Search/Search/Collapsing/FieldCollapseUsageTests.cs index ff101ef5835..76e48bf1818 100644 --- a/src/Tests/Tests/Search/Search/Collapsing/FieldCollapseUsageTests.cs +++ b/src/Tests/Tests/Search/Search/Collapsing/FieldCollapseUsageTests.cs @@ -1,4 +1,5 @@ using System; +using Elastic.Xunit.XunitPlumbing; using FluentAssertions; using Nest; using Tests.Core.Extensions; @@ -6,14 +7,10 @@ using Tests.Core.ManagedElasticsearch.NodeSeeders; using Tests.Domain; using Tests.Framework.Integration; -using Tests.Framework.ManagedElasticsearch.Clusters; -using Tests.Framework.ManagedElasticsearch.NodeSeeders; using static Nest.Infer; namespace Tests.Search.Search.Collapsing { - /** - */ public class FieldCollapseUsageTests : SearchUsageTestBase { protected override string UrlPath => $"/{DefaultSeeder.ProjectsAliasFilter}/doc/_search"; @@ -77,4 +74,93 @@ protected override void ExpectResponse(ISearchResponse response) } } } + + [SkipVersion("<6.4.0", "2nd level collapsing is a new feature in 6.4.0")] + public class FieldCollapseSecondLevelUsageTests : SearchUsageTestBase + { + protected override string UrlPath => $"/{DefaultSeeder.ProjectsAliasFilter}/doc/_search"; + + public FieldCollapseSecondLevelUsageTests(ReadOnlyCluster cluster, EndpointUsage usage) : base(cluster, usage) { } + + protected override object ExpectJson => new + { + _source = new { excludes = new [] { "*" } }, + collapse = new { + field = "state", + inner_hits = new { + _source = new { + excludes = new [] { "*" } + }, + collapse = new { + field = "name" + }, + from = 1, + name = "stateofbeing", + size = 5 + }, + max_concurrent_group_searches = 1000 + } + }; + + protected override Func, ISearchRequest> Fluent => s => s + .Source(source=>source.ExcludeAll()) + .Index(DefaultSeeder.ProjectsAliasFilter) + .Collapse(c => c + .Field(f => f.State) + .MaxConcurrentGroupSearches(1000) + .InnerHits(i => i + .Source(source=>source.ExcludeAll()) + .Name(nameof(StateOfBeing).ToLowerInvariant()) + .Size(5) + .From(1) + .Collapse(c2 => c2 + .Field(p=>p.Name) + ) + ) + ); + + protected override SearchRequest Initializer => new SearchRequest(DefaultSeeder.ProjectsAliasFilter) + { + Source = SourceFilter.ExcludeAll, + Collapse = new FieldCollapse + { + Field = Field(p => p.State), + MaxConcurrentGroupSearches = 1000, + InnerHits = new InnerHits + { + Source = SourceFilter.ExcludeAll, + Name = nameof(StateOfBeing).ToLowerInvariant(), + Size = 5, + From = 1, + Collapse = new FieldCollapse + { + Field = Field(p=>p.Name) + } + } + } + }; + + protected override void ExpectResponse(ISearchResponse response) + { + var numberOfStates = Enum.GetValues(typeof(StateOfBeing)).Length; + response.HitsMetadata.Total.Should().BeGreaterThan(numberOfStates); + response.Hits.Count.Should().Be(numberOfStates); + foreach (var hit in response.Hits) + { + var name = nameof(StateOfBeing).ToLowerInvariant(); + hit.InnerHits.Should().NotBeNull().And.ContainKey(name); + var innerHits = hit.InnerHits[name]; + innerHits.Hits.Total.Should().BeGreaterThan(0); + var i = 0; + foreach (var innerHit in innerHits.Hits.Hits) + { + i++; + innerHit.Fields.Should().NotBeEmpty() + .And.ContainKey("name"); + } + + i.Should().NotBe(0, "we expect to inspect 2nd level collapsed fields"); + } + } + } } From 98a3943cf968de7a289b2e4c141aa5e8725254c6 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 17 Oct 2018 06:38:46 +0200 Subject: [PATCH 36/63] Feature/alias is write index (#3447) * Add support for is_write_index on put alias and add alias operation * extended tests so that we assert the property is available when getting the alias as well * OnBeforeCall and OnAfterCall should only be called in integration mode, unit test mode was failing for AliasIsWriteIndexApiTests * Make sure IsWriteIndex() on descriptor is nullable, ty CodeStandards tests --- .../Response/DictionaryResponseBase.cs | 4 +- .../AliasManagement/Alias/Actions/AliasAdd.cs | 9 ++- .../Alias/Actions/AliasAddOperation.cs | 9 ++- .../AliasManagement/AliasDefinition.cs | 4 ++ .../PutAlias/PutAliasRequest.cs | 29 +++++++- .../RequestResponseApiTestBase.cs | 21 +++--- .../AliasManagement/Alias/AliasApiTests.cs | 69 ++++++++++++++++++- .../PutAlias/PutAliasApiTests.cs | 31 +++++++++ 8 files changed, 160 insertions(+), 16 deletions(-) diff --git a/src/Nest/CommonAbstractions/Response/DictionaryResponseBase.cs b/src/Nest/CommonAbstractions/Response/DictionaryResponseBase.cs index 25c2ed12b95..b8af6a47120 100644 --- a/src/Nest/CommonAbstractions/Response/DictionaryResponseBase.cs +++ b/src/Nest/CommonAbstractions/Response/DictionaryResponseBase.cs @@ -14,7 +14,9 @@ public interface IDictionaryResponse : IResponse public abstract class DictionaryResponseBase : ResponseBase, IDictionaryResponse { protected IDictionaryResponse Self => this; - IReadOnlyDictionary IDictionaryResponse.BackingDictionary { get; set; } + + IReadOnlyDictionary IDictionaryResponse.BackingDictionary { get; set; } = + EmptyReadOnly.Dictionary; } internal class DictionaryResponseJsonConverterHelpers diff --git a/src/Nest/Indices/AliasManagement/Alias/Actions/AliasAdd.cs b/src/Nest/Indices/AliasManagement/Alias/Actions/AliasAdd.cs index 811ca7cb459..36c2e8d32ba 100644 --- a/src/Nest/Indices/AliasManagement/Alias/Actions/AliasAdd.cs +++ b/src/Nest/Indices/AliasManagement/Alias/Actions/AliasAdd.cs @@ -3,7 +3,7 @@ namespace Nest { - + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] public interface IAliasAddAction : IAliasAction { @@ -60,6 +60,11 @@ public AliasAddDescriptor SearchRouting(string searchRouting) Self.Add.SearchRouting = searchRouting; return this; } + public AliasAddDescriptor IsWriteIndex(bool? isWriteIndex = true) + { + Self.Add.IsWriteIndex = isWriteIndex; + return this; + } public AliasAddDescriptor Filter(Func, QueryContainer> filterSelector) where T : class { @@ -67,4 +72,4 @@ public AliasAddDescriptor Filter(Func, QueryConta return this; } } -} \ No newline at end of file +} diff --git a/src/Nest/Indices/AliasManagement/Alias/Actions/AliasAddOperation.cs b/src/Nest/Indices/AliasManagement/Alias/Actions/AliasAddOperation.cs index b20576fc600..bdf93c836dd 100644 --- a/src/Nest/Indices/AliasManagement/Alias/Actions/AliasAddOperation.cs +++ b/src/Nest/Indices/AliasManagement/Alias/Actions/AliasAddOperation.cs @@ -21,5 +21,12 @@ public class AliasAddOperation [JsonProperty("search_routing")] public string SearchRouting { get; set; } + + /// + /// If an alias points to multiple indices elasticsearch will reject the write operations + /// unless one is explicitly marked with as the write alias using this property. + /// + [JsonProperty("is_write_index")] + public bool? IsWriteIndex { get; set; } } -} \ No newline at end of file +} diff --git a/src/Nest/Indices/AliasManagement/AliasDefinition.cs b/src/Nest/Indices/AliasManagement/AliasDefinition.cs index 94cb10fe525..8b342b77b9e 100644 --- a/src/Nest/Indices/AliasManagement/AliasDefinition.cs +++ b/src/Nest/Indices/AliasManagement/AliasDefinition.cs @@ -15,5 +15,9 @@ public class AliasDefinition [JsonProperty("search_routing")] public string SearchRouting { get; internal set; } + + [JsonProperty("is_write_index")] + public bool? IsWriteIndex { get; internal set; } + } } diff --git a/src/Nest/Indices/AliasManagement/PutAlias/PutAliasRequest.cs b/src/Nest/Indices/AliasManagement/PutAlias/PutAliasRequest.cs index fb9b1433e3a..533ae6e2ef6 100644 --- a/src/Nest/Indices/AliasManagement/PutAlias/PutAliasRequest.cs +++ b/src/Nest/Indices/AliasManagement/PutAlias/PutAliasRequest.cs @@ -11,13 +11,27 @@ public partial interface IPutAliasRequest [JsonProperty("filter")] QueryContainer Filter { get; set; } + + [JsonProperty("index_routing")] + Routing IndexRouting { get; set; } + + [JsonProperty("search_routing")] + Routing SearchRouting { get; set; } + + /// + [JsonProperty("is_write_index")] + bool? IsWriteIndex { get; set; } } public partial class PutAliasRequest { + public QueryContainer Filter { get; set; } public Routing Routing { get; set; } + public Routing IndexRouting { get; set; } + public Routing SearchRouting { get; set; } + /// + public bool? IsWriteIndex { get; set; } - public QueryContainer Filter { get; set; } } [DescriptorFor("IndicesPutAlias")] @@ -25,11 +39,20 @@ public partial class PutAliasDescriptor { Routing IPutAliasRequest.Routing { get; set; } QueryContainer IPutAliasRequest.Filter { get; set; } + Routing IPutAliasRequest.IndexRouting { get; set; } + Routing IPutAliasRequest.SearchRouting { get; set; } + bool? IPutAliasRequest.IsWriteIndex { get; set; } public PutAliasDescriptor Routing(Routing routing) => Assign(a => a.Routing = routing); - public PutAliasDescriptor Filter(Func, QueryContainer> filterSelector) - where T : class => + public PutAliasDescriptor IndexRouting(Routing routing) => Assign(a => a.IndexRouting = routing); + + public PutAliasDescriptor SearchRouting(Routing routing) => Assign(a => a.SearchRouting = routing); + + /// + public PutAliasDescriptor IsWriteIndex(bool? isWriteIndex = true) => Assign(a => a.IsWriteIndex = isWriteIndex); + + public PutAliasDescriptor Filter(Func, QueryContainer> filterSelector) where T : class => Assign(a => a.Filter = filterSelector?.Invoke(new QueryContainerDescriptor())); } } diff --git a/src/Tests/Tests/Framework/EndpointTests/RequestResponseApiTestBase.cs b/src/Tests/Tests/Framework/EndpointTests/RequestResponseApiTestBase.cs index f5c0237d687..0f0cc0ae84b 100644 --- a/src/Tests/Tests/Framework/EndpointTests/RequestResponseApiTestBase.cs +++ b/src/Tests/Tests/Framework/EndpointTests/RequestResponseApiTestBase.cs @@ -71,6 +71,11 @@ Func> requestAsync return new LazyResponses(async () => { var client = this.Client; + void IntegrateOnly(Action act) + { + if (!TestClient.Configuration.RunIntegrationTests) return; + act(client); + } if (TestClient.Configuration.RunIntegrationTests) { this.IntegrationSetup(client, UniqueValues); @@ -80,24 +85,24 @@ Func> requestAsync var dict = new Dictionary(); UniqueValues.CurrentView = ClientMethod.Fluent; - OnBeforeCall(client); + IntegrateOnly(this.OnBeforeCall); dict.Add(ClientMethod.Fluent, fluent(client, this.Fluent)); - OnAfterCall(client); + IntegrateOnly(this.OnAfterCall); UniqueValues.CurrentView = ClientMethod.FluentAsync; - OnBeforeCall(client); + IntegrateOnly(this.OnBeforeCall); dict.Add(ClientMethod.FluentAsync, await fluentAsync(client, this.Fluent)); - OnAfterCall(client); + IntegrateOnly(this.OnAfterCall); UniqueValues.CurrentView = ClientMethod.Initializer; - OnBeforeCall(client); + IntegrateOnly(this.OnBeforeCall); dict.Add(ClientMethod.Initializer, request(client, this.Initializer)); - OnAfterCall(client); + IntegrateOnly(this.OnAfterCall); UniqueValues.CurrentView = ClientMethod.InitializerAsync; - OnBeforeCall(client); + IntegrateOnly(this.OnBeforeCall); dict.Add(ClientMethod.InitializerAsync, await requestAsync(client, this.Initializer)); - OnAfterCall(client); + IntegrateOnly(this.OnAfterCall); if (TestClient.Configuration.RunIntegrationTests) { diff --git a/src/Tests/Tests/Indices/AliasManagement/Alias/AliasApiTests.cs b/src/Tests/Tests/Indices/AliasManagement/Alias/AliasApiTests.cs index 7c2be44063b..0d9ff78d926 100644 --- a/src/Tests/Tests/Indices/AliasManagement/Alias/AliasApiTests.cs +++ b/src/Tests/Tests/Indices/AliasManagement/Alias/AliasApiTests.cs @@ -1,7 +1,10 @@ using System; using System.Collections.Generic; +using Elastic.Xunit.XunitPlumbing; using Elasticsearch.Net; +using FluentAssertions; using Nest; +using Tests.Core.Extensions; using Tests.Core.ManagedElasticsearch.Clusters; using Tests.Framework; using Tests.Framework.Integration; @@ -47,7 +50,71 @@ protected override LazyResponses ClientUsage() => Calls( Actions = new List { new AliasAddAction { Add = new AliasAddOperation {Alias = "alias", Index = CallIsolatedValue, IndexRouting = "x", SearchRouting = "y"} }, - new AliasRemoveAction {Remove = new AliasRemoveOperation {Alias = "alias", Index = Infer.Index(CallIsolatedValue) }}, + new AliasRemoveAction {Remove = new AliasRemoveOperation {Alias = "alias", Index = CallIsolatedValue }}, + } + }; + } + + [SkipVersion("<6.4.0", "is_write_index is a new feature")] + public class AliasIsWriteIndexApiTests : ApiIntegrationAgainstNewIndexTestBase + { + public AliasIsWriteIndexApiTests(WritableCluster cluster, EndpointUsage usage) : base(cluster, usage) { } + + protected override LazyResponses ClientUsage() => Calls( + fluent: (client, f) => client.Alias(f), + fluentAsync: (client, f) => client.AliasAsync(f), + request: (client, r) => client.Alias(r), + requestAsync: (client, r) => client.AliasAsync(r) + ); + + protected override bool ExpectIsValid => true; + protected override int ExpectStatusCode => 200; + protected override HttpMethod HttpMethod => HttpMethod.POST; + protected override string UrlPath => $"/_aliases"; + private string Index => this.CallIsolatedValue; + private string Alias(int i) => $"alias-{this.CallIsolatedValue}-{i}"; + + protected override void OnAfterCall(IElasticClient client) + { + var secondAlias = this.Alias(2); + var aliasResponse = this.Client.GetAlias(a => a.Name(secondAlias)); + aliasResponse.ShouldBeValid(); + aliasResponse.Indices.Should().NotBeEmpty().And.ContainKey(this.Index); + var indexAliases = aliasResponse.Indices[this.Index].Aliases; + + indexAliases.Should().NotBeEmpty().And.ContainKey(secondAlias); + var alias = indexAliases[secondAlias]; + alias.IsWriteIndex.Should().HaveValue().And + .BeTrue($"{secondAlias} was stored is is_write_index, so we need to be able to read it too"); + + } + + protected override bool SupportsDeserialization => false; + + protected override object ExpectJson => new + { + actions = new object[] + { + new Dictionary { { "add", + new { alias = this.Alias(1), index = this.Index, index_routing="x", search_routing="y", is_write_index = false} } }, + new Dictionary { { "add", + new { alias = this.Alias(2), index = this.Index, index_routing="x", search_routing="y", is_write_index = true} } }, + } + }; + + protected override Func Fluent => d => d + .Add(a=>a.Alias(this.Alias(1)).Index(this.Index).IndexRouting("x").SearchRouting("y").IsWriteIndex(false)) + .Add(a=>a.Alias(this.Alias(2)).Index(this.Index).IndexRouting("x").SearchRouting("y").IsWriteIndex()) + ; + + protected override BulkAliasRequest Initializer => new BulkAliasRequest + { + Actions = new List + { + new AliasAddAction { Add = + new AliasAddOperation {Alias = this.Alias(1), Index = this.Index, IndexRouting = "x", SearchRouting = "y", IsWriteIndex = false} }, + new AliasAddAction { Add = + new AliasAddOperation {Alias = this.Alias(2), Index = this.Index, IndexRouting = "x", SearchRouting = "y", IsWriteIndex = true} }, } }; } diff --git a/src/Tests/Tests/Indices/AliasManagement/PutAlias/PutAliasApiTests.cs b/src/Tests/Tests/Indices/AliasManagement/PutAlias/PutAliasApiTests.cs index e45567eb80a..5ef06c7edae 100644 --- a/src/Tests/Tests/Indices/AliasManagement/PutAlias/PutAliasApiTests.cs +++ b/src/Tests/Tests/Indices/AliasManagement/PutAlias/PutAliasApiTests.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using Elastic.Xunit.XunitPlumbing; using Elasticsearch.Net; using Nest; using Tests.Core.ManagedElasticsearch.Clusters; @@ -30,4 +31,34 @@ protected override LazyResponses ClientUsage() => Calls( protected override Func Fluent => null; protected override PutAliasRequest Initializer => new PutAliasRequest(CallIsolatedValue, CallIsolatedValue + "-alias"); } + + [SkipVersion("<6.4.0", "is_write_index is a new feature")] + public class PutAliasIsWriteIndexApiTests : ApiIntegrationAgainstNewIndexTestBase + { + public PutAliasIsWriteIndexApiTests(WritableCluster cluster, EndpointUsage usage) : base(cluster, usage) { } + protected override LazyResponses ClientUsage() => Calls( + fluent: (client, f) => client.PutAlias(CallIsolatedValue, CallIsolatedValue + "-alias"), + fluentAsync: (client, f) => client.PutAliasAsync(CallIsolatedValue, CallIsolatedValue + "-alias"), + request: (client, r) => client.PutAlias(r), + requestAsync: (client, r) => client.PutAliasAsync(r) + ); + + protected override bool ExpectIsValid => true; + protected override int ExpectStatusCode => 200; + protected override HttpMethod HttpMethod => HttpMethod.PUT; + protected override string UrlPath => $"/{CallIsolatedValue}/_alias/{CallIsolatedValue + "-alias"}"; + + protected override bool SupportsDeserialization => false; + + protected override object ExpectJson => new {is_write_index = true}; + + protected override PutAliasDescriptor NewDescriptor() => new PutAliasDescriptor(CallIsolatedValue, CallIsolatedValue + "-alias"); + + protected override Func Fluent => f => f.IsWriteIndex(); + + protected override PutAliasRequest Initializer => new PutAliasRequest(CallIsolatedValue, CallIsolatedValue + "-alias") + { + IsWriteIndex = true + }; + } } From 657481c57d2901e61ee367cd68bc7580823a867e Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 17 Oct 2018 06:40:00 +0200 Subject: [PATCH 37/63] Allow Processors tested and guarded in isolation with a SkipVersion (#3438) --- .../SerializationTesterAssertionExtensions.cs | 11 +- .../DeletePipeline/DeletePipelineApiTests.cs | 3 +- .../Ingest/GetPipeline/GetPipelineApiTests.cs | 1 + .../GrokProcessorPatternsApiTests.cs | 0 .../GrokProcessorPatternsTests.cs | 0 .../GrokProcessorPatternsUnitTests.cs | 0 src/Tests/Tests/Ingest/ProcessorAssertions.cs | 345 +++++++++++++++++ .../Ingest/PutPipeline/PutPipelineApiTests.cs | 348 +----------------- 8 files changed, 364 insertions(+), 344 deletions(-) rename src/Tests/Tests/Ingest/{Processor => GrokProcessorPatterns}/GrokProcessorPatternsApiTests.cs (100%) rename src/Tests/Tests/Ingest/{Processor => GrokProcessorPatterns}/GrokProcessorPatternsTests.cs (100%) rename src/Tests/Tests/Ingest/{Processor => GrokProcessorPatterns}/GrokProcessorPatternsUnitTests.cs (100%) create mode 100644 src/Tests/Tests/Ingest/ProcessorAssertions.cs diff --git a/src/Tests/Tests.Core/Extensions/SerializationTesterAssertionExtensions.cs b/src/Tests/Tests.Core/Extensions/SerializationTesterAssertionExtensions.cs index 541be627f20..1c2af9eef38 100644 --- a/src/Tests/Tests.Core/Extensions/SerializationTesterAssertionExtensions.cs +++ b/src/Tests/Tests.Core/Extensions/SerializationTesterAssertionExtensions.cs @@ -1,12 +1,17 @@ -using FluentAssertions; +using System; using Tests.Core.Serialization; namespace Tests.Core.Extensions { public static class SerializationTesterAssertionExtensions { - public static void ShouldBeValid(this SerializationResult result, string message = null) => - result.Success.Should().BeTrue("{0}", message + result); + public static void ShouldBeValid(this SerializationResult result, string message = null) + { + if (result.Success) return; + throw new Exception($@"Expected serialization to succeed but failed. +{(message ?? string.Empty) + result} +"); + } public static T AssertRoundTrip(this SerializationTester tester, T @object, string message = null, bool preserveNullInExpected = false) { diff --git a/src/Tests/Tests/Ingest/DeletePipeline/DeletePipelineApiTests.cs b/src/Tests/Tests/Ingest/DeletePipeline/DeletePipelineApiTests.cs index 568878bdb02..45fb17d903a 100644 --- a/src/Tests/Tests/Ingest/DeletePipeline/DeletePipelineApiTests.cs +++ b/src/Tests/Tests/Ingest/DeletePipeline/DeletePipelineApiTests.cs @@ -4,11 +4,10 @@ using Tests.Core.ManagedElasticsearch.Clusters; using Tests.Framework; using Tests.Framework.Integration; -using Tests.Framework.ManagedElasticsearch.Clusters; -using Xunit; namespace Tests.Ingest.DeletePipeline { + //integration test is part of PipelineCrudTests public class DeletePipelineApiTests : ApiTestBase { public DeletePipelineApiTests(ReadOnlyCluster cluster, EndpointUsage usage) : base(cluster, usage) { } diff --git a/src/Tests/Tests/Ingest/GetPipeline/GetPipelineApiTests.cs b/src/Tests/Tests/Ingest/GetPipeline/GetPipelineApiTests.cs index 8e7d02137c7..5661952b237 100644 --- a/src/Tests/Tests/Ingest/GetPipeline/GetPipelineApiTests.cs +++ b/src/Tests/Tests/Ingest/GetPipeline/GetPipelineApiTests.cs @@ -9,6 +9,7 @@ namespace Tests.Ingest.GetPipeline { + //integration test is part of PipelineCrudTests public class GetPipelineApiTests : ApiTestBase { public GetPipelineApiTests(ReadOnlyCluster cluster, EndpointUsage usage) : base(cluster, usage) { } diff --git a/src/Tests/Tests/Ingest/Processor/GrokProcessorPatternsApiTests.cs b/src/Tests/Tests/Ingest/GrokProcessorPatterns/GrokProcessorPatternsApiTests.cs similarity index 100% rename from src/Tests/Tests/Ingest/Processor/GrokProcessorPatternsApiTests.cs rename to src/Tests/Tests/Ingest/GrokProcessorPatterns/GrokProcessorPatternsApiTests.cs diff --git a/src/Tests/Tests/Ingest/Processor/GrokProcessorPatternsTests.cs b/src/Tests/Tests/Ingest/GrokProcessorPatterns/GrokProcessorPatternsTests.cs similarity index 100% rename from src/Tests/Tests/Ingest/Processor/GrokProcessorPatternsTests.cs rename to src/Tests/Tests/Ingest/GrokProcessorPatterns/GrokProcessorPatternsTests.cs diff --git a/src/Tests/Tests/Ingest/Processor/GrokProcessorPatternsUnitTests.cs b/src/Tests/Tests/Ingest/GrokProcessorPatterns/GrokProcessorPatternsUnitTests.cs similarity index 100% rename from src/Tests/Tests/Ingest/Processor/GrokProcessorPatternsUnitTests.cs rename to src/Tests/Tests/Ingest/GrokProcessorPatterns/GrokProcessorPatternsUnitTests.cs diff --git a/src/Tests/Tests/Ingest/ProcessorAssertions.cs b/src/Tests/Tests/Ingest/ProcessorAssertions.cs new file mode 100644 index 00000000000..52e1d6c32c3 --- /dev/null +++ b/src/Tests/Tests/Ingest/ProcessorAssertions.cs @@ -0,0 +1,345 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using Elastic.Xunit.XunitPlumbing; +using Nest; +using Tests.Core.Client; +using Tests.Domain; + +namespace Tests.Ingest +{ + using ProcFunc = Func>>; + public interface IProcessorAssertion + { + object Json { get; } + string Key { get; } + IProcessor Initializer { get; } + ProcFunc Fluent { get; } + } + public abstract class ProcessorAssertion : IProcessorAssertion + { + public abstract string Key { get; } + public abstract IProcessor Initializer { get; } + public abstract ProcFunc Fluent { get; } + public abstract object Json { get; } + } + + + public static class ProcessorAssertions + { + public static IEnumerable All => + from t in typeof(ProcessorAssertions).GetNestedTypes() + where typeof(IProcessorAssertion).IsAssignableFrom(t) && t.IsClass + let a = t.GetCustomAttributes(typeof(SkipVersionAttribute)).FirstOrDefault() as SkipVersionAttribute + where a == null || !a.Ranges.Any(r => r.IsSatisfied(TestClient.Configuration.ElasticsearchVersion)) + select (IProcessorAssertion) (Activator.CreateInstance(t)); + + public static Dictionary[] Json => + All.Select(a => new Dictionary + { + {a.Key, a.Json} + }).ToArray(); + + public static IPromise> Fluent(ProcessorsDescriptor d) + { + foreach (var a in All) a.Fluent(d); + return d; + } + + public static IProcessor[] Initializer => All.Select(a => a.Initializer).ToArray(); + + public class Append : ProcessorAssertion + { + public override string Key => "append"; + public override object Json => new {field = "state", value = new[] {"Stable", "VeryActive"}}; + + public override IProcessor Initializer => new AppendProcessor + { + Field = "state", + Value = new object[] {StateOfBeing.Stable, StateOfBeing.VeryActive} + }; + public override Func>> Fluent => d => d + .Append(a => a + .Field(p => p.State) + .Value(StateOfBeing.Stable, StateOfBeing.VeryActive) + ); + + } + + public class Convert : ProcessorAssertion + { + public override string Key => "convert"; + public override object Json => new + { + field = "numberOfCommits", + target_field = "targetField", + type = "string" + }; + + public override IProcessor Initializer => new ConvertProcessor + { + Field = "numberOfCommits", + TargetField = "targetField", + Type = ConvertProcessorType.String + }; + public override Func>> Fluent => d => d + .Convert(c => c + .Field(p => p.NumberOfCommits) + .TargetField("targetField") + .Type(ConvertProcessorType.String) + ); + } + + public class Date : ProcessorAssertion + { + public override string Key => "date"; + public override object Json => new + { + field = "startedOn", + target_field = "timestamp", + formats = new[] {"dd/MM/yyyy hh:mm:ss"}, + timezone = "Europe/Amsterdam" + }; + + public override IProcessor Initializer => new DateProcessor + { + Field = "startedOn", + TargetField = "timestamp", + Formats = new string[] {"dd/MM/yyyy hh:mm:ss"}, + Timezone = "Europe/Amsterdam" + }; + public override Func>> Fluent => d => d + .Date(dt => dt + .Field(p => p.StartedOn) + .TargetField("timestamp") + .Formats("dd/MM/yyyy hh:mm:ss") + .Timezone("Europe/Amsterdam") + ); + } + + public class Fail : ProcessorAssertion + { + public override string Key => "fail"; + + public override object Json => new { message = "an error message" }; + + public override IProcessor Initializer => new FailProcessor {Message = "an error message"}; + public override Func>> Fluent => d => d.Fail(f => f.Message("an error message")); + } + + public class Foreach : ProcessorAssertion + { + public override string Key => "foreach"; + + public override object Json => new + { + field = "tags", + processor = new + { + uppercase = new + { + field = "_value.name" + } + } + }; + + public override IProcessor Initializer => new ForeachProcessor + { + Field = Infer.Field(p => p.Tags), + Processor = new UppercaseProcessor + { + Field = "_value.name" + } + }; + public override Func>> Fluent => d => d + .Foreach(fe => fe + .Field(p => p.Tags) + .Processor(pps => pps + .Uppercase(uc => uc + .Field("_value.name") + ) + ) + ); + } + + public class Grok : ProcessorAssertion + { + public override string Key => "grok"; + + public override object Json => new + { + field = "description", + patterns = new[] {"my %{FAVORITE_DOG:dog} is colored %{RGB:color}"}, + pattern_definitions = new Dictionary + { + {"FAVORITE_DOG", "border collie"}, + {"RGB", "RED|BLUE|GREEN"}, + } + }; + + public override IProcessor Initializer => new GrokProcessor + { + Field = "description", + Patterns = new[] {"my %{FAVORITE_DOG:dog} is colored %{RGB:color}"}, + PatternDefinitions = new Dictionary + { + {"FAVORITE_DOG", "border collie"}, + {"RGB", "RED|BLUE|GREEN"}, + } + }; + public override Func>> Fluent => d => d + .Grok(gk => gk + .Field(p => p.Description) + .Patterns("my %{FAVORITE_DOG:dog} is colored %{RGB:color}") + .PatternDefinitions(pds => pds + .Add("FAVORITE_DOG", "border collie") + .Add("RGB", "RED|BLUE|GREEN") + ) + ); + } + + public class Gsub : ProcessorAssertion + { + public override string Key => "gsub"; + + public override object Json => new { field = "name", pattern = "-", replacement = "_" }; + + public override IProcessor Initializer => new GsubProcessor + { + Field = "name", + Pattern = "-", + Replacement = "_" + }; + public override Func>> Fluent => d => d + .Gsub(gs => gs + .Field(p => p.Name) + .Pattern("-") + .Replacement("_") + ); + } + + public class Join : ProcessorAssertion + { + public override string Key => "join"; + + public override object Json => new { field = "branches", separator = "," }; + + public override IProcessor Initializer => new JoinProcessor + { + Field = "branches", + Separator = "," + }; + public override Func>> Fluent => d => d.Join(j => j.Field(p => p.Branches).Separator(",")); + } + + public class Lowercase : ProcessorAssertion + { + public override string Key => "lowercase"; + + public override object Json => new { field = "name" }; + + public override IProcessor Initializer => new LowercaseProcessor { Field = "name" }; + public override Func>> Fluent => d => d.Lowercase(l => l.Field(p => p.Name)); + } + + public class Remove : ProcessorAssertion + { + public override string Key => "remove"; + + public override object Json => new { field = "suggest" }; + + public override IProcessor Initializer => new RemoveProcessor {Field = "suggest"}; + public override Func>> Fluent => d => d.Remove(r => r.Field(p => p.Suggest)); + } + + public class Rename : ProcessorAssertion + { + public override string Key => "rename"; + + public override object Json => new { field = "leadDeveloper", target_field = "projectLead" }; + + public override IProcessor Initializer => new RenameProcessor + { + Field = "leadDeveloper", + TargetField = "projectLead" + }; + public override Func>> Fluent => d => d.Rename(rn => rn.Field(p => p.LeadDeveloper).TargetField("projectLead")); + } + + public class Set : ProcessorAssertion + { + public override string Key => "set"; + + public override object Json => new { field = "name", value = "foo" }; + + public override IProcessor Initializer => new SetProcessor { Field = Infer.Field(p => p.Name), Value = "foo" }; + public override Func>> Fluent => d => d.Set(s => s.Field(p => p.Name).Value("foo")); + } + + public class Split : ProcessorAssertion + { + public override string Key => "split"; + + public override object Json => new { field = "description", separator = "." }; + + public override IProcessor Initializer => new SplitProcessor {Field = "description", Separator = "."}; + public override Func>> Fluent => d => d.Split(sp => sp.Field(p => p.Description).Separator(".")); + } + + public class Trim : ProcessorAssertion + { + public override string Key => "trim"; + + public override object Json => new { field = "name" }; + + public override IProcessor Initializer => new TrimProcessor {Field = "name"}; + public override Func>> Fluent => d => d.Trim(t => t.Field(p => p.Name)); + } + + public class Uppercase : ProcessorAssertion + { + public override string Key => "uppercase"; + + public override object Json => new { field = "name" }; + + public override IProcessor Initializer => new UppercaseProcessor {Field = "name"}; + public override Func>> Fluent => d => d.Uppercase(u => u.Field(p => p.Name)); + } + + public class DotExpander : ProcessorAssertion + { + public override string Key => "dot_expander"; + + public override object Json => new { field = "field.withDots" }; + + public override IProcessor Initializer => new DotExpanderProcessor {Field = "field.withDots"}; + public override Func>> Fluent => d => d.DotExpander(de => de.Field("field.withDots")); + } + + public class Script : ProcessorAssertion + { + public override string Key => "script"; + + public override object Json => new { source = "ctx.numberOfCommits++" }; + + public override IProcessor Initializer => new ScriptProcessor {Source = "ctx.numberOfCommits++"}; + public override Func>> Fluent => d => d.Script(s => s.Source("ctx.numberOfCommits++")); + } + + [SkipVersion("<6.1.0", "uses url decode which was introduced in 6.1.0")] + public class UrlDecode : ProcessorAssertion + { + public override string Key => "urldecode"; + + public override object Json => new { field = "description", ignore_missing = true }; + + public override IProcessor Initializer => new UrlDecodeProcessor { Field = "description", IgnoreMissing = true }; + public override Func>> Fluent => d => d + .UrlDecode(ud => ud + .Field(p => p.Description) + .IgnoreMissing() + ); + } + } +} diff --git a/src/Tests/Tests/Ingest/PutPipeline/PutPipelineApiTests.cs b/src/Tests/Tests/Ingest/PutPipeline/PutPipelineApiTests.cs index ea2fa3c8bef..adb3b55c1fe 100644 --- a/src/Tests/Tests/Ingest/PutPipeline/PutPipelineApiTests.cs +++ b/src/Tests/Tests/Ingest/PutPipeline/PutPipelineApiTests.cs @@ -1,21 +1,18 @@ using System; using Elasticsearch.Net; +using FluentAssertions; using Nest; +using Tests.Core.Extensions; using Tests.Framework; using Tests.Framework.Integration; -using Xunit; -using System.Collections.Generic; -using Elastic.Xunit.XunitPlumbing; using Tests.Core.ManagedElasticsearch.Clusters; -using Tests.Domain; namespace Tests.Ingest.PutPipeline { - [SkipVersion("<6.1.0", "uses url decode which was introduced in 6.1.0")] public class PutPipelineApiTests - : ApiIntegrationTestBase + : ApiIntegrationTestBase { - public PutPipelineApiTests(ReadOnlyCluster cluster, EndpointUsage usage) : base(cluster, usage) { } + public PutPipelineApiTests(WritableCluster cluster, EndpointUsage usage) : base(cluster, usage) { } private static readonly string _id = "pipeline-1"; @@ -35,348 +32,21 @@ protected override LazyResponses ClientUsage() => Calls( protected override object ExpectJson { get; } = new { description = "My test pipeline", - processors = new object[] - { - new - { - append = new - { - field = "state", - value = new [] { "Stable", "VeryActive" } - } - }, - new - { - convert = new - { - field = "numberOfCommits", - target_field = "targetField", - type = "string" - } - }, - new - { - date = new - { - field = "startedOn", - target_field = "timestamp", - formats = new [] { "dd/MM/yyyy hh:mm:ss" }, - timezone = "Europe/Amsterdam" - } - }, - new - { - fail = new - { - message = "an error message" - } - }, - new - { - @foreach = new - { - field = "tags", - processor = new - { - uppercase = new - { - field = "_value.name" - } - } - } - }, - new - { - grok = new - { - field = "description", - patterns = new[] { "my %{FAVORITE_DOG:dog} is colored %{RGB:color}" }, - pattern_definitions = new Dictionary - { - { "FAVORITE_DOG", "border collie" }, - { "RGB", "RED|BLUE|GREEN" }, - } - } - }, - new - { - gsub = new - { - field = "name", - pattern = "-", - replacement = "_" - } - }, - new - { - join = new - { - field = "branches", - separator = "," - } - }, - new - { - lowercase = new - { - field = "name" - } - }, - new - { - remove = new - { - field = "suggest" - } - }, - new - { - rename = new - { - field = "leadDeveloper", - target_field = "projectLead" - } - }, - new - { - set = new - { - field = "name", - value = "foo" - } - }, - new - { - split = new - { - field = "description", - separator = "." - } - }, - new - { - trim = new - { - field = "name" - } - }, - new - { - uppercase = new - { - field = "name" - } - }, - new - { - dot_expander = new - { - field = "field.withDots" - } - }, - new - { - script = new - { - source = "ctx.numberOfCommits++" - } - }, - new - { - urldecode = new - { - field = "description", - ignore_missing = true - } - } - } + processors = ProcessorAssertions.Json }; protected override PutPipelineDescriptor NewDescriptor() => new PutPipelineDescriptor(_id); protected override Func Fluent => d => d .Description("My test pipeline") - .Processors(ps => ps - .Append(a => a - .Field(p => p.State) - .Value(StateOfBeing.Stable, StateOfBeing.VeryActive) - ) - .Convert(c => c - .Field(p => p.NumberOfCommits) - .TargetField("targetField") - .Type(ConvertProcessorType.String) - ) - .Date(dt => dt - .Field(p => p.StartedOn) - .TargetField("timestamp") - .Formats("dd/MM/yyyy hh:mm:ss") - .Timezone("Europe/Amsterdam") - ) - .Fail(f => f - .Message("an error message") - ) - .Foreach(fe => fe - .Field(p => p.Tags) - .Processor(pps => pps - .Uppercase(uc => uc - .Field("_value.name") - ) - ) - ) - .Grok(gk => gk - .Field(p => p.Description) - .Patterns("my %{FAVORITE_DOG:dog} is colored %{RGB:color}") - .PatternDefinitions(pds => pds - .Add("FAVORITE_DOG", "border collie") - .Add("RGB", "RED|BLUE|GREEN") - ) - ) - .Gsub(gs => gs - .Field(p => p.Name) - .Pattern("-") - .Replacement("_") - ) - .Join(j => j - .Field(p => p.Branches) - .Separator(",") - ) - .Lowercase(l => l - .Field(p => p.Name) - ) - .Remove(r => r - .Field(p => p.Suggest) - ) - .Rename(rn => rn - .Field(p => p.LeadDeveloper) - .TargetField("projectLead") - ) - .Set(s => s - .Field(p => p.Name) - .Value("foo") - ) - .Split(sp => sp - .Field(p => p.Description) - .Separator(".") - ) - .Trim(t => t - .Field(p => p.Name) - ) - .Uppercase(u => u - .Field(p => p.Name) - ) - .DotExpander(de => de - .Field("field.withDots") - ) - .Script(s => s - .Source("ctx.numberOfCommits++") - ) - .UrlDecode(ud => ud - .Field(p => p.Description) - .IgnoreMissing() - ) - ); + .Processors(ProcessorAssertions.Fluent); protected override PutPipelineRequest Initializer => new PutPipelineRequest(_id) { Description = "My test pipeline", - Processors = new IProcessor[] - { - new AppendProcessor - { - Field = "state", - Value = new object [] { StateOfBeing.Stable, StateOfBeing.VeryActive } - }, - new ConvertProcessor - { - Field = "numberOfCommits", - TargetField = "targetField", - Type = ConvertProcessorType.String - }, - new DateProcessor - { - Field = "startedOn", - TargetField = "timestamp", - Formats = new string [] { "dd/MM/yyyy hh:mm:ss"}, - Timezone = "Europe/Amsterdam" - }, - new FailProcessor - { - Message = "an error message" - }, - new ForeachProcessor - { - Field = Infer.Field(p => p.Tags), - Processor = new UppercaseProcessor - { - Field = "_value.name" - } - }, - new GrokProcessor - { - Field = "description", - Patterns = new [] { "my %{FAVORITE_DOG:dog} is colored %{RGB:color}" }, - PatternDefinitions = new Dictionary - { - { "FAVORITE_DOG", "border collie" }, - { "RGB", "RED|BLUE|GREEN" }, - } - }, - new GsubProcessor - { - Field = "name", - Pattern = "-", - Replacement = "_" - }, - new JoinProcessor - { - Field = "branches", - Separator = "," - }, - new LowercaseProcessor - { - Field = "name" - }, - new RemoveProcessor - { - Field = "suggest" - }, - new RenameProcessor - { - Field = "leadDeveloper", - TargetField = "projectLead" - }, - new SetProcessor - { - Field = Infer.Field(p => p.Name), - Value = "foo" - }, - new SplitProcessor - { - Field = "description", - Separator = "." - }, - new TrimProcessor - { - Field = "name" - }, - new UppercaseProcessor - { - Field = "name" - }, - new DotExpanderProcessor - { - Field = "field.withDots" - }, - new ScriptProcessor - { - Source = "ctx.numberOfCommits++" - }, - new UrlDecodeProcessor - { - Field = "description", - IgnoreMissing = true - } - } + Processors = ProcessorAssertions.Initializer }; + + protected override void ExpectResponse(IPutPipelineResponse response) => response.Acknowledged.Should().BeTrue(); } } From 31bb3c698b4c604aec469626bd9c547d7d4743fd Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 17 Oct 2018 07:21:25 +0200 Subject: [PATCH 38/63] add mode to keep_types token filter (#3429) --- .../TokenFilters/KeepTypesTokenFilter.cs | 41 ++++++++++++++----- .../Analysis/TokenFilters/TokenFilterTests.cs | 31 ++++++++++---- .../Analysis/Tokenizers/TokenizerTests.cs | 2 +- .../Core/Keyword/KeywordPropertyTests.cs | 1 + 4 files changed, 55 insertions(+), 20 deletions(-) diff --git a/src/Nest/Analysis/TokenFilters/KeepTypesTokenFilter.cs b/src/Nest/Analysis/TokenFilters/KeepTypesTokenFilter.cs index 33d2e9a11e6..70b88d679f6 100644 --- a/src/Nest/Analysis/TokenFilters/KeepTypesTokenFilter.cs +++ b/src/Nest/Analysis/TokenFilters/KeepTypesTokenFilter.cs @@ -1,42 +1,63 @@ using System.Collections.Generic; +using System.Runtime.Serialization; using Newtonsoft.Json; +using Newtonsoft.Json.Converters; namespace Nest { + [JsonConverter(typeof(StringEnumConverter))] + public enum KeepTypesMode + { + [EnumMember(Value="include")] + Include, + [EnumMember(Value = "exclude")] + Exclude + } + /// /// A token filter of type keep that only keeps tokens with text contained in a predefined set of words. /// public interface IKeepTypesTokenFilter : ITokenFilter { - /// - /// A list of types to keep. - /// + /// A list of types to keep. [JsonProperty("types")] IEnumerable Types { get; set; } + + /// Whether to include or exclude the types provided on + [JsonProperty("mode")] + KeepTypesMode? Mode { get; set; } } - /// + /// public class KeepTypesTokenFilter : TokenFilterBase { public KeepTypesTokenFilter() : base("keep_types") { } - /// + /// public IEnumerable Types { get; set; } + /// + public KeepTypesMode? Mode { get; set; } + } - /// - public class KeepTypesTokenFilterDescriptor + /// + public class KeepTypesTokenFilterDescriptor : TokenFilterDescriptorBase, IKeepTypesTokenFilter { protected override string Type => "keep_types"; IEnumerable IKeepTypesTokenFilter.Types { get; set; } + KeepTypesMode? IKeepTypesTokenFilter.Mode { get; set; } - /// + /// public KeepTypesTokenFilterDescriptor Types(IEnumerable types) => Assign(a => a.Types = types); - /// + /// public KeepTypesTokenFilterDescriptor Types(params string[] types) => Assign(a => a.Types = types); + /// + public KeepTypesTokenFilterDescriptor Mode(KeepTypesMode? mode) => Assign(a => a.Mode = mode); + + } -} \ No newline at end of file +} diff --git a/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs b/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs index bfe98a4dbef..ef0105a0a16 100644 --- a/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs +++ b/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs @@ -182,20 +182,33 @@ public class KeepTypesTests : TokenFilterAssertionBase { public override string Name => "keeptypes"; - public override ITokenFilter Initializer => - new KeepTypesTokenFilter {Types = new[] {"", ""}}; + private readonly string[] _types = {"", ""}; - public override FuncTokenFilters Fluent => (n, tf) => tf - .KeepTypes(n, t => t - .Types("", "") - ); + public override ITokenFilter Initializer => new KeepTypesTokenFilter {Types = _types}; - public override object Json => new + public override FuncTokenFilters Fluent => (n, tf) => tf.KeepTypes(n, t => t.Types(_types)); + + public override object Json => new { type = "keep_types", types = _types }; + } + + [SkipVersion("<6.4.0", "The mode option was introduced in https://github.com/elastic/elasticsearch/pull/32012")] + public class KeepTypesModeTests : TokenFilterAssertionBase + { + public override string Name => "keeptypes"; + private readonly string[] _types = {"", ""}; + + public override ITokenFilter Initializer => new KeepTypesTokenFilter { - type = "keep_types", - types = new[] {"", ""} + Mode = KeepTypesMode.Exclude, + Types = _types }; + public override FuncTokenFilters Fluent => (n, tf) => tf.KeepTypes(n, t => t + .Mode(KeepTypesMode.Exclude) + .Types(_types) + ); + + public override object Json => new { type = "keep_types", types = _types, mode = "exclude" }; } public class IcuCollationTests : TokenFilterAssertionBase diff --git a/src/Tests/Tests/Analysis/Tokenizers/TokenizerTests.cs b/src/Tests/Tests/Analysis/Tokenizers/TokenizerTests.cs index 8d27bff379d..e5341c46d31 100644 --- a/src/Tests/Tests/Analysis/Tokenizers/TokenizerTests.cs +++ b/src/Tests/Tests/Analysis/Tokenizers/TokenizerTests.cs @@ -203,7 +203,7 @@ public class StandardTests : TokenizerAssertionBase public override object Json => new {type = "standard"}; } - + public class CharGroupTests : TokenizerAssertionBase { private readonly string[] _chars = {"whitespace", "-", "\n"}; diff --git a/src/Tests/Tests/Mapping/Types/Core/Keyword/KeywordPropertyTests.cs b/src/Tests/Tests/Mapping/Types/Core/Keyword/KeywordPropertyTests.cs index b4400eba367..81f9d9c770f 100644 --- a/src/Tests/Tests/Mapping/Types/Core/Keyword/KeywordPropertyTests.cs +++ b/src/Tests/Tests/Mapping/Types/Core/Keyword/KeywordPropertyTests.cs @@ -2,6 +2,7 @@ using Elastic.Xunit.XunitPlumbing; using Nest; using Tests.Analysis; +using Tests.Analysis.Tokenizers; using Tests.Core.ManagedElasticsearch.Clusters; using Tests.Domain; using Tests.Framework.Integration; From ae5ae5b0a5c6894bd49161595dd30c249699413c Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 17 Oct 2018 07:34:07 +0200 Subject: [PATCH 39/63] Allow context to be specified on the painless execute API (#3435) --- .../ExecutePainlessScriptRequest.cs | 27 ++++++ .../PainlessContextSetup.cs | 55 ++++++++++++ .../TestState/CallUniqueValues.cs | 6 ++ .../ExecutePainlessScriptApiTests.cs | 89 ++++++++++++++++++- 4 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 src/Nest/Modules/Scripting/ExecutePainlessScript/PainlessContextSetup.cs diff --git a/src/Nest/Modules/Scripting/ExecutePainlessScript/ExecutePainlessScriptRequest.cs b/src/Nest/Modules/Scripting/ExecutePainlessScript/ExecutePainlessScriptRequest.cs index c9bf286cdb6..87fb5356a98 100644 --- a/src/Nest/Modules/Scripting/ExecutePainlessScript/ExecutePainlessScriptRequest.cs +++ b/src/Nest/Modules/Scripting/ExecutePainlessScript/ExecutePainlessScriptRequest.cs @@ -3,23 +3,50 @@ namespace Nest { + /// The Painless execute API allows an arbitrary script to be executed and a result to be returned. public partial interface IExecutePainlessScriptRequest { + /// The script to execute [JsonProperty("script")] IInlineScript Script { get; set; } + + /// The context the script should be executed in + [JsonProperty("context")] + string Context { get; set; } + + /// + [JsonProperty("context_setup")] + IPainlessContextSetup ContextSetup { get; set; } } + /// public partial class ExecutePainlessScriptRequest { + /// public IInlineScript Script { get; set; } + /// + public string Context { get; set; } + /// + public IPainlessContextSetup ContextSetup { get; set; } } + /// [DescriptorFor("ScriptsPainlessExecute")] public partial class ExecutePainlessScriptDescriptor { IInlineScript IExecutePainlessScriptRequest.Script { get; set; } + string IExecutePainlessScriptRequest.Context { get; set; } + IPainlessContextSetup IExecutePainlessScriptRequest.ContextSetup { get; set; } + /// public ExecutePainlessScriptDescriptor Script(Func selector) => Assign(a => a.Script = selector?.Invoke(new InlineScriptDescriptor())); + + /// + public ExecutePainlessScriptDescriptor ContextSetup(Func selector) => + Assign(a => a.ContextSetup = selector?.Invoke(new PainlessContextSetupDescriptor())); + + /// + public ExecutePainlessScriptDescriptor Context(string context) => Assign(a => a.Context = context); } } diff --git a/src/Nest/Modules/Scripting/ExecutePainlessScript/PainlessContextSetup.cs b/src/Nest/Modules/Scripting/ExecutePainlessScript/PainlessContextSetup.cs new file mode 100644 index 00000000000..95fd4c4274d --- /dev/null +++ b/src/Nest/Modules/Scripting/ExecutePainlessScript/PainlessContextSetup.cs @@ -0,0 +1,55 @@ +using System; +using Newtonsoft.Json; + +namespace Nest +{ + /// + /// Sets up contextual scope for the painless script the execute under. + /// + [JsonConverter(typeof(ReadAsTypeJsonConverter))] + public interface IPainlessContextSetup + { + /// + /// Contains the document that will be temporarily indexed in-memory and is accessible from the script. + /// + [JsonProperty("document")] + object Document { get; set; } + /// + /// The name of an index containing a mapping that is compatible with the document being indexed. + /// + [JsonProperty("index")] + IndexName Index { get; set; } + /// + /// If _score is used in the script then a query can specified that will be used to compute a score. + /// + [JsonProperty("query")] + QueryContainer Query { get; set; } + } + + /// + public class PainlessContextSetup : IPainlessContextSetup + { + /// + public object Document { get; set; } + /// + public IndexName Index { get; set; } + /// + public QueryContainer Query { get; set; } + } + + /// + public class PainlessContextSetupDescriptor : DescriptorBase, IPainlessContextSetup + { + object IPainlessContextSetup.Document { get; set; } + IndexName IPainlessContextSetup.Index { get; set; } + QueryContainer IPainlessContextSetup.Query { get; set; } + + /// + public PainlessContextSetupDescriptor Document(T document) => Assign(a => a.Document = document); + /// + public PainlessContextSetupDescriptor Index(IndexName index) => Assign(a => a.Index = index); + /// + public PainlessContextSetupDescriptor Query(Func, QueryContainer> querySelector) where T : class => + Assign(a => a.Query = querySelector?.Invoke(new QueryContainerDescriptor())); + } +} diff --git a/src/Tests/Tests/Framework/EndpointTests/TestState/CallUniqueValues.cs b/src/Tests/Tests/Framework/EndpointTests/TestState/CallUniqueValues.cs index aeb55e57205..3156d0f8455 100644 --- a/src/Tests/Tests/Framework/EndpointTests/TestState/CallUniqueValues.cs +++ b/src/Tests/Tests/Framework/EndpointTests/TestState/CallUniqueValues.cs @@ -6,10 +6,15 @@ namespace Tests.Framework.Integration { + /// + /// Holds unique values for the the two DSL's and the exposed sync and async methods we expose + /// + /// public class CallUniqueValues : Dictionary { private readonly string _prefix; private string UniqueValue => $"{this._prefix}-{ViewName}-{Guid.NewGuid().ToString("N").Substring(0, 8)}"; + public string FixedForAllCallsValue { get; } private IDictionary> ExtendedValues { get; } = new Dictionary>(); @@ -28,6 +33,7 @@ public T ExtendedValue(string key, Func value) where T : class => public CallUniqueValues(string prefix = "nest") { this._prefix = prefix; + this.FixedForAllCallsValue = this.UniqueValue; this.SetupClientMethod(Fluent); this.SetupClientMethod(FluentAsync); this.SetupClientMethod(Initializer); diff --git a/src/Tests/Tests/Modules/Scripting/ExecutePainlessScript/ExecutePainlessScriptApiTests.cs b/src/Tests/Tests/Modules/Scripting/ExecutePainlessScript/ExecutePainlessScriptApiTests.cs index 31b4447e729..9084437e1a7 100644 --- a/src/Tests/Tests/Modules/Scripting/ExecutePainlessScript/ExecutePainlessScriptApiTests.cs +++ b/src/Tests/Tests/Modules/Scripting/ExecutePainlessScript/ExecutePainlessScriptApiTests.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Threading.Tasks; using Elastic.Xunit.XunitPlumbing; using Elasticsearch.Net; using FluentAssertions; @@ -9,8 +8,6 @@ using Tests.Core.ManagedElasticsearch.Clusters; using Tests.Framework; using Tests.Framework.Integration; -using Tests.Framework.ManagedElasticsearch.Clusters; -using Xunit; namespace Tests.Modules.Scripting.ExecutePainlessScript { @@ -69,4 +66,90 @@ protected override void ExpectResponse(IExecutePainlessScriptResponse re response.Result.Should().NotBeNullOrWhiteSpace(); } } + + [SkipVersion("<6.4.0", "Context only tested on 6.4.0 when they were introduced")] + public class ExecutePainlessScriptContextApiTests + : ApiIntegrationTestBase, IExecutePainlessScriptRequest, ExecutePainlessScriptDescriptor, ExecutePainlessScriptRequest> + { + public ExecutePainlessScriptContextApiTests(WritableCluster cluster, EndpointUsage usage) : base(cluster, usage) { } + + private static readonly string _painlessScript = "doc['rank'].value / params.max_rank"; + + protected override LazyResponses ClientUsage() => Calls( + fluent: (client, f) => client.ExecutePainlessScript(f), + fluentAsync: (client, f) => client.ExecutePainlessScriptAsync(f), + request: (client, r) => client.ExecutePainlessScript(r), + requestAsync: (client, r) => client.ExecutePainlessScriptAsync(r) + ); + + protected override HttpMethod HttpMethod => HttpMethod.POST; + protected override string UrlPath => "/_scripts/painless/_execute"; + protected override int ExpectStatusCode => 200; + protected override bool ExpectIsValid => true; + protected override bool SupportsDeserialization => false; + + private class ScriptDocument + { + public string Field { get; set; } + public long Rank { get; set; } + } + + protected override void IntegrationSetup(IElasticClient client, CallUniqueValues values) + { + var create =client.CreateIndex(values.FixedForAllCallsValue, c => c.Mappings(map => map.Map(m => m.AutoMap()))); + create.ShouldBeValid(); + } + + protected override object ExpectJson => new + { + context = "score", + context_setup = new + { + document = new { rank = 4 }, + index = this.UniqueValues.FixedForAllCallsValue, + query = new { match_all = new {} } + }, + script = new + { + source = _painlessScript, + @params = new { max_rank = 5.0 } + }, + }; + + protected override Func Fluent => d => d + .ContextSetup(cs=>cs + .Index(this.UniqueValues.FixedForAllCallsValue) + .Document(new ScriptDocument { Rank = 4 }) + .Query(q=>q.MatchAll()) + ) + .Context("score") + .Script(s=>s + .Source(_painlessScript) + .Params(p => p.Add("max_rank", 5.0)) + ); + + protected override ExecutePainlessScriptRequest Initializer => new ExecutePainlessScriptRequest + { + ContextSetup = new PainlessContextSetup + { + Index = this.UniqueValues.FixedForAllCallsValue, + Document = new ScriptDocument { Rank = 4 }, + Query = new MatchAllQuery() + }, + Context = "score", + Script = new InlineScript(_painlessScript) + { + Params = new Dictionary + { + { "max_rank", 5.0 }, + } + } + }; + + protected override void ExpectResponse(IExecutePainlessScriptResponse response) + { + response.ShouldBeValid(); + response.Result.Should().NotBeNullOrWhiteSpace(); + } + } } From ee4096e7ac8e8419f16e9ec6413f99708849374e Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 17 Oct 2018 07:34:59 +0200 Subject: [PATCH 40/63] Add ignore_missing to the remove processor (#3441) --- src/Nest/Ingest/Processors/RemoveProcessor.cs | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/Nest/Ingest/Processors/RemoveProcessor.cs b/src/Nest/Ingest/Processors/RemoveProcessor.cs index 1c77bbadb29..bbb35948fa9 100644 --- a/src/Nest/Ingest/Processors/RemoveProcessor.cs +++ b/src/Nest/Ingest/Processors/RemoveProcessor.cs @@ -1,10 +1,6 @@ using Newtonsoft.Json; using System; -using System.Collections.Generic; -using System.Linq; using System.Linq.Expressions; -using System.Text; -using System.Threading.Tasks; namespace Nest { @@ -14,14 +10,26 @@ public interface IRemoveProcessor : IProcessor { [JsonProperty("field")] Field Field { get; set; } + + /// + /// If true and does not exist or is null, + /// the processor quietly exits without modifying the document. Default is false + /// + [JsonProperty("ignore_missing")] + bool? IgnoreMissing { get; set; } } + /// public class RemoveProcessor : ProcessorBase, IRemoveProcessor { protected override string Name => "remove"; + /// public Field Field { get; set; } + /// + public bool? IgnoreMissing { get; set; } } + /// public class RemoveProcessorDescriptor : ProcessorDescriptorBase, IRemoveProcessor>, IRemoveProcessor where T : class @@ -29,10 +37,15 @@ public class RemoveProcessorDescriptor protected override string Name => "remove"; Field IRemoveProcessor.Field { get; set; } + bool? IRemoveProcessor.IgnoreMissing { get; set; } + /// public RemoveProcessorDescriptor Field(Field field) => Assign(a => a.Field = field); - public RemoveProcessorDescriptor Field(Expression> objectPath) => - Assign(a => a.Field = objectPath); + /// + public RemoveProcessorDescriptor Field(Expression> objectPath) => Assign(a => a.Field = objectPath); + + /// + public RemoveProcessorDescriptor IgnoreMissing(bool? ignoreMissing = true) => Assign(a => a.IgnoreMissing = ignoreMissing); } } From cdfcc2ca670cfa50fb22fc852f084d84b68a4ddd Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 17 Oct 2018 07:40:29 +0200 Subject: [PATCH 41/63] Add uuid to indices stats (#3437) As per https://github.com/elastic/elasticsearch/pull/31871 --- src/Nest/Indices/Monitoring/IndicesStats/IndicesStats.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Nest/Indices/Monitoring/IndicesStats/IndicesStats.cs b/src/Nest/Indices/Monitoring/IndicesStats/IndicesStats.cs index cecd026b98f..ca84e2e3bd4 100644 --- a/src/Nest/Indices/Monitoring/IndicesStats/IndicesStats.cs +++ b/src/Nest/Indices/Monitoring/IndicesStats/IndicesStats.cs @@ -6,6 +6,15 @@ namespace Nest [JsonObject] public class IndicesStats { + /// + /// Universal Unique Identifier for the index + /// + /// + /// Introduced in Elasticsearch 6.4.0 + /// + [JsonProperty("uuid")] + public string UUID { get; } + [JsonProperty("primaries")] public IndexStats Primaries { get; internal set; } From 2130ddfd7671d631eb7e2e9b331509d3e6291ac1 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 17 Oct 2018 07:44:16 +0200 Subject: [PATCH 42/63] Add cluster_uuid to root of the cluster_state response (#3443) --- src/Nest/Cluster/ClusterState/ClusterStateResponse.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Nest/Cluster/ClusterState/ClusterStateResponse.cs b/src/Nest/Cluster/ClusterState/ClusterStateResponse.cs index 1eb593da67d..ea640c12623 100644 --- a/src/Nest/Cluster/ClusterState/ClusterStateResponse.cs +++ b/src/Nest/Cluster/ClusterState/ClusterStateResponse.cs @@ -14,6 +14,11 @@ public interface IClusterStateResponse : IResponse [JsonProperty("state_uuid")] string StateUUID { get; } + /// The Universally Unique Identifier for the cluster. + /// While the cluster is still forming, it is possible for the `cluster_uuid` to be `_na_`. + [JsonProperty("cluster_uuid")] + string ClusterUUID { get; } + [JsonProperty("version")] long Version { get; } @@ -42,6 +47,9 @@ public class ClusterStateResponse : ResponseBase, IClusterStateResponse public string StateUUID { get; internal set; } + /// + public string ClusterUUID { get; internal set; } + public long Version { get; internal set; } public IReadOnlyDictionary Nodes { get; internal set; } = EmptyReadOnly.Dictionary; From c7e51cb4705f4d7de1afa2e6428116641faff093 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 17 Oct 2018 07:44:55 +0200 Subject: [PATCH 43/63] Add support for nori plugin (#3430) * add analysis-nori plugin to writable cluster and added nori_tokenizer * add nori_part_of_speech token filter * add nori analyzer * add Nori() to AnalyzeTokenDescriptor, CodeStandard tests caught this --- .../Analyzers/AnalyzerJsonConverter.cs | 1 + src/Nest/Analysis/Analyzers/Analyzers.cs | 4 ++ src/Nest/Analysis/Analyzers/NoriAnalyzer.cs | 65 +++++++++++++++++++ .../NoriPartOfSpeechTokenFilter.cs | 38 +++++++++++ .../Analysis/TokenFilters/TokenFilters.cs | 4 ++ src/Nest/Analysis/Tokenizers/NoriTokenizer.cs | 63 ++++++++++++++++++ src/Nest/Analysis/Tokenizers/Tokenizers.cs | 4 ++ .../Indices/Analyze/AnalyzeTokenFilters.cs | 4 ++ .../Analyze/AnalyzeTokenizersDescriptor.cs | 4 ++ .../Clusters/WritableCluster.cs | 7 +- .../Tests/Analysis/Analyzers/AnalyzerTests.cs | 25 +++++++ .../Analysis/TokenFilters/TokenFilterTests.cs | 13 ++++ .../Analysis/Tokenizers/TokenizerTests.cs | 19 +++++- 13 files changed, 248 insertions(+), 3 deletions(-) create mode 100644 src/Nest/Analysis/Analyzers/NoriAnalyzer.cs create mode 100644 src/Nest/Analysis/TokenFilters/NoriPartOfSpeechTokenFilter.cs create mode 100644 src/Nest/Analysis/Tokenizers/NoriTokenizer.cs diff --git a/src/Nest/Analysis/Analyzers/AnalyzerJsonConverter.cs b/src/Nest/Analysis/Analyzers/AnalyzerJsonConverter.cs index e0709d282e0..d47253dfda4 100644 --- a/src/Nest/Analysis/Analyzers/AnalyzerJsonConverter.cs +++ b/src/Nest/Analysis/Analyzers/AnalyzerJsonConverter.cs @@ -29,6 +29,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist case "simple": return o.ToObject(ElasticContractResolver.Empty); case "fingerprint": return o.ToObject(ElasticContractResolver.Empty); case "kuromoji": return o.ToObject(ElasticContractResolver.Empty); + case "nori": return o.ToObject(ElasticContractResolver.Empty); default: if (o.Property("tokenizer") != null) return o.ToObject(ElasticContractResolver.Empty); diff --git a/src/Nest/Analysis/Analyzers/Analyzers.cs b/src/Nest/Analysis/Analyzers/Analyzers.cs index 990d9882417..4f78b5e4741 100644 --- a/src/Nest/Analysis/Analyzers/Analyzers.cs +++ b/src/Nest/Analysis/Analyzers/Analyzers.cs @@ -97,5 +97,9 @@ public AnalyzersDescriptor Fingerprint(string name, Func public AnalyzersDescriptor Kuromoji(string name, Func selector = null) => Assign(name, selector.InvokeOrDefault(new KuromojiAnalyzerDescriptor())); + + /// + public AnalyzersDescriptor Nori(string name, Func selector) => + Assign(name, selector?.Invoke(new NoriAnalyzerDescriptor())); } } diff --git a/src/Nest/Analysis/Analyzers/NoriAnalyzer.cs b/src/Nest/Analysis/Analyzers/NoriAnalyzer.cs new file mode 100644 index 00000000000..010caad0020 --- /dev/null +++ b/src/Nest/Analysis/Analyzers/NoriAnalyzer.cs @@ -0,0 +1,65 @@ +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace Nest +{ + /// + ///The nori analyzer consists of the following tokenizer and token filters: + /// - nori_tokenizer + /// - nori_part_of_speech token filter + /// - nori_readingform token filter + /// - lowercase token filter + /// + public interface INoriAnalyzer : IAnalyzer + { + /// + [JsonProperty("decompound_mode")] + NoriDecompoundMode? DecompoundMode { get; set; } + + /// + [JsonProperty("user_dictionary")] + string UserDictionary { get; set; } + + /// + [JsonProperty("stoptags")] + IEnumerable StopTags { get; set; } + } + + /// + public class NoriAnalyzer : AnalyzerBase, INoriAnalyzer + { + public NoriAnalyzer() : base("nori") {} + + /// + public NoriDecompoundMode? DecompoundMode { get; set; } + + /// + public string UserDictionary { get; set; } + + /// + public IEnumerable StopTags { get; set; } + } + + /// + public class NoriAnalyzerDescriptor : AnalyzerDescriptorBase, INoriAnalyzer + { + protected override string Type => "nori"; + + NoriDecompoundMode? INoriAnalyzer.DecompoundMode { get; set; } + string INoriAnalyzer.UserDictionary { get; set; } + IEnumerable INoriAnalyzer.StopTags { get; set; } + + /// + public NoriAnalyzerDescriptor DecompoundMode(NoriDecompoundMode? mode) => Assign(a => a.DecompoundMode = mode); + + /// + public NoriAnalyzerDescriptor UserDictionary(string path) => Assign(a => a.UserDictionary = path); + + /// + public NoriAnalyzerDescriptor StopTags(IEnumerable stopTags) => Assign(a => a.StopTags = stopTags); + + /// + public NoriAnalyzerDescriptor StopTags(params string[] stopTags) => Assign(a => a.StopTags = stopTags); + + } +} diff --git a/src/Nest/Analysis/TokenFilters/NoriPartOfSpeechTokenFilter.cs b/src/Nest/Analysis/TokenFilters/NoriPartOfSpeechTokenFilter.cs new file mode 100644 index 00000000000..93870344550 --- /dev/null +++ b/src/Nest/Analysis/TokenFilters/NoriPartOfSpeechTokenFilter.cs @@ -0,0 +1,38 @@ +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace Nest +{ + /// The nori_part_of_speech token filter removes tokens that match a set of part-of-speech tags. + public interface INoriPartOfSpeechTokenFilter : ITokenFilter + { + /// An array of part-of-speech tags that should be removed. + [JsonProperty("stoptags")] + IEnumerable StopTags { get; set; } + } + + /// + public class NoriPartOfSpeechTokenFilter : TokenFilterBase, INoriPartOfSpeechTokenFilter + { + public NoriPartOfSpeechTokenFilter() : base("nori_part_of_speech") { } + + /// + public IEnumerable StopTags { get; set; } + } + /// + public class NoriPartOfSpeechTokenFilterDescriptor + : TokenFilterDescriptorBase, INoriPartOfSpeechTokenFilter + { + protected override string Type => "nori_part_of_speech"; + + IEnumerable INoriPartOfSpeechTokenFilter.StopTags { get; set; } + + /// + public NoriPartOfSpeechTokenFilterDescriptor StopTags(IEnumerable stopTags) => Assign(a => a.StopTags = stopTags); + + /// + public NoriPartOfSpeechTokenFilterDescriptor StopTags(params string[] stopTags) => Assign(a => a.StopTags = stopTags); + + } + +} diff --git a/src/Nest/Analysis/TokenFilters/TokenFilters.cs b/src/Nest/Analysis/TokenFilters/TokenFilters.cs index 71fb54bc842..a0bc22e0c02 100644 --- a/src/Nest/Analysis/TokenFilters/TokenFilters.cs +++ b/src/Nest/Analysis/TokenFilters/TokenFilters.cs @@ -318,6 +318,10 @@ public TokenFiltersDescriptor IcuNormalization(string name, Func selector) => Assign(name, selector.Invoke(new IcuTransformTokenFilterDescriptor())); + /// + public TokenFiltersDescriptor NoriPartOfSpeech(string name, Func selector) => + Assign(name, selector.Invoke(new NoriPartOfSpeechTokenFilterDescriptor())); + /// /// A token filter of type multiplexer will emit multiple tokens at the same position, each version of the token /// having been run through a different filter. Identical output tokens at the same position will be removed. diff --git a/src/Nest/Analysis/Tokenizers/NoriTokenizer.cs b/src/Nest/Analysis/Tokenizers/NoriTokenizer.cs new file mode 100644 index 00000000000..435d24f7117 --- /dev/null +++ b/src/Nest/Analysis/Tokenizers/NoriTokenizer.cs @@ -0,0 +1,63 @@ +using System.Runtime.Serialization; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; + +namespace Nest +{ + /// The decompound mode determines how the tokenizer handles compound tokens. + [JsonConverter(typeof(StringEnumConverter))] + public enum NoriDecompoundMode + { + /// Decomposes compounds and discards the original form (default). + [EnumMember(Value="discard")] + Discard, + /// No decomposition for compounds + [EnumMember(Value="none")] + None, + /// Decomposes compounds and keeps the original form + [EnumMember(Value="mixed")] + Mixed + } + + /// Tokenizer that ships with the analysis-nori plugin + public interface INoriTokenizer : ITokenizer + { + /// + /// The regular expression pattern, defaults to \W+. + /// + [JsonProperty("decompound_mode")] + NoriDecompoundMode? DecompoundMode { get; set; } + + /// + /// The Nori tokenizer uses the mecab-ko-dic dictionary by default. A user_dictionary with custom nouns (NNG) may be appended to + /// the default dictionary. This property allows you to specify this file on disk + /// + [JsonProperty("user_dictionary")] + string UserDictionary { get; set; } + } + + /// + public class NoriTokenizer : TokenizerBase, INoriTokenizer + { + public NoriTokenizer() => this.Type = "nori_tokenizer"; + /// + public NoriDecompoundMode? DecompoundMode { get; set; } + /// + public string UserDictionary { get; set; } + } + /// + public class NoriTokenizerDescriptor + : TokenizerDescriptorBase, INoriTokenizer + { + protected override string Type => "nori_tokenizer"; + + NoriDecompoundMode? INoriTokenizer.DecompoundMode { get; set; } + string INoriTokenizer.UserDictionary { get; set; } + + /// + public NoriTokenizerDescriptor DecompoundMode(NoriDecompoundMode? mode) => Assign(a => a.DecompoundMode = mode); + + /// + public NoriTokenizerDescriptor UserDictionary(string path) => Assign(a => a.UserDictionary = path); + } +} diff --git a/src/Nest/Analysis/Tokenizers/Tokenizers.cs b/src/Nest/Analysis/Tokenizers/Tokenizers.cs index bc23e2d5f4c..330162c49b4 100644 --- a/src/Nest/Analysis/Tokenizers/Tokenizers.cs +++ b/src/Nest/Analysis/Tokenizers/Tokenizers.cs @@ -113,6 +113,10 @@ public TokenizersDescriptor Kuromoji(string name, Func selector) => Assign(name, selector?.Invoke(new IcuTokenizerDescriptor())); + /// + public TokenizersDescriptor Nori(string name, Func selector) => + Assign(name, selector?.Invoke(new NoriTokenizerDescriptor())); + /// > public TokenizersDescriptor CharGroup(string name, Func selector) => Assign(name, selector?.Invoke(new CharGroupTokenizerDescriptor())); diff --git a/src/Nest/Indices/Analyze/AnalyzeTokenFilters.cs b/src/Nest/Indices/Analyze/AnalyzeTokenFilters.cs index b0ce11eeb87..440ab4fccb8 100644 --- a/src/Nest/Indices/Analyze/AnalyzeTokenFilters.cs +++ b/src/Nest/Indices/Analyze/AnalyzeTokenFilters.cs @@ -324,6 +324,10 @@ public AnalyzeTokenFiltersDescriptor IcuNormalization(Func selector) => AssignIfNotNull(selector.Invoke(new IcuTransformTokenFilterDescriptor())); + /// + public AnalyzeTokenFiltersDescriptor NoriPartOfSpeech(Func selector) => + AssignIfNotNull(selector.Invoke(new NoriPartOfSpeechTokenFilterDescriptor())); + /// public AnalyzeTokenFiltersDescriptor Multiplexer(Func selector) => AssignIfNotNull(selector.Invoke(new MultiplexerTokenFilterDescriptor())); diff --git a/src/Nest/Indices/Analyze/AnalyzeTokenizersDescriptor.cs b/src/Nest/Indices/Analyze/AnalyzeTokenizersDescriptor.cs index edb9133e86c..1335f271811 100644 --- a/src/Nest/Indices/Analyze/AnalyzeTokenizersDescriptor.cs +++ b/src/Nest/Indices/Analyze/AnalyzeTokenizersDescriptor.cs @@ -92,6 +92,10 @@ public ITokenizer Kuromoji(Func public ITokenizer Icu(Func selector) => (selector?.Invoke(new IcuTokenizerDescriptor())); + /// + public ITokenizer Nori(Func selector) => + selector.Invoke(new NoriTokenizerDescriptor()); + /// > public ITokenizer CharGroup(Func selector) => selector?.Invoke(new CharGroupTokenizerDescriptor()); } diff --git a/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/WritableCluster.cs b/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/WritableCluster.cs index 33346801c75..a0f53a9c0b5 100644 --- a/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/WritableCluster.cs +++ b/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/WritableCluster.cs @@ -1,4 +1,5 @@ -using Tests.Core.ManagedElasticsearch.NodeSeeders; +using Elastic.Managed.Ephemeral.Plugins; +using Tests.Core.ManagedElasticsearch.NodeSeeders; using static Elastic.Managed.Ephemeral.Plugins.ElasticsearchPlugin; namespace Tests.Core.ManagedElasticsearch.Clusters @@ -12,7 +13,9 @@ public WritableCluster() : base(new ClientTestClusterConfiguration( AnalysisKuromoji, AnalysisIcu, AnalysisPhonetic, - MapperMurmur3 + MapperMurmur3, + //TODO move this to elasticsearch-net abstractions + new ElasticsearchPlugin("analysis-nori", v => v >= "6.4.0") ) { MaxConcurrency = 4 diff --git a/src/Tests/Tests/Analysis/Analyzers/AnalyzerTests.cs b/src/Tests/Tests/Analysis/Analyzers/AnalyzerTests.cs index 0a663b5bc1f..b18c38cbe2c 100644 --- a/src/Tests/Tests/Analysis/Analyzers/AnalyzerTests.cs +++ b/src/Tests/Tests/Analysis/Analyzers/AnalyzerTests.cs @@ -1,5 +1,7 @@ using System; +using Elastic.Xunit.XunitPlumbing; using Nest; +using Tests.Analysis.TokenFilters; namespace Tests.Analysis.Analyzers { @@ -197,5 +199,28 @@ public class KuromojuTests : AnalyzerAssertionBase }; } + [SkipVersion("<6.4.0", "analysis-nori plugin introduced in 6.4.0")] + public class NoriTests : AnalyzerAssertionBase + { + public override string Name => "nori"; + private readonly string[] _stopTags = {"NR", "SP"}; + public override IAnalyzer Initializer => new NoriAnalyzer + { + StopTags = _stopTags, + DecompoundMode = NoriDecompoundMode.Mixed + }; + + public override FuncTokenizer Fluent => (n, t) => t.Nori(n, e => e + .StopTags(_stopTags) + .DecompoundMode(NoriDecompoundMode.Mixed) + ); + + public override object Json => new + { + type = "nori", + decompound_mode = "mixed", + stoptags =_stopTags + }; + } } } diff --git a/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs b/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs index ef0105a0a16..cc1ac2ab15b 100644 --- a/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs +++ b/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs @@ -919,6 +919,19 @@ public class PhoneticTests : TokenFilterAssertionBase } + [SkipVersion("<6.4.0", "analysis-nori plugin introduced in 6.4.0")] + public class NoriPartOfSpeechTests : TokenFilterAssertionBase + { + public override string Name => "nori_pos"; + private readonly string[] _stopTags = {"NR", "SP"}; + + public override ITokenFilter Initializer => new NoriPartOfSpeechTokenFilter {StopTags = _stopTags}; + + public override FuncTokenFilters Fluent => (n, tf) => tf.NoriPartOfSpeech(n, t => t.StopTags(_stopTags)); + + public override object Json => new { type = "nori_part_of_speech", stoptags = _stopTags }; + } + [SkipVersion("<6.4.0", "Introduced in 6.4.0")] public class MultiplexerTests : TokenFilterAssertionBase { diff --git a/src/Tests/Tests/Analysis/Tokenizers/TokenizerTests.cs b/src/Tests/Tests/Analysis/Tokenizers/TokenizerTests.cs index e5341c46d31..dadb8e36553 100644 --- a/src/Tests/Tests/Analysis/Tokenizers/TokenizerTests.cs +++ b/src/Tests/Tests/Analysis/Tokenizers/TokenizerTests.cs @@ -1,4 +1,5 @@ -using System; +using System; +using Elastic.Xunit.XunitPlumbing; using Nest; namespace Tests.Analysis.Tokenizers @@ -204,6 +205,22 @@ public class StandardTests : TokenizerAssertionBase public override object Json => new {type = "standard"}; } + [SkipVersion("<6.4.0", "analysis-nori plugin introduced in 6.4.0")] + public class NoriTests : TokenizerAssertionBase + { + public override string Name => "nori"; + public override ITokenizer Initializer => new NoriTokenizer + { + DecompoundMode = NoriDecompoundMode.Mixed + }; + + public override FuncTokenizer Fluent => (n, t) => t.Nori(n, e => e + .DecompoundMode(NoriDecompoundMode.Mixed) + ); + + public override object Json => new {type = "nori_tokenizer", decompound_mode = "mixed"}; + } + public class CharGroupTests : TokenizerAssertionBase { private readonly string[] _chars = {"whitespace", "-", "\n"}; From b3e286f73ff8cc1219b9a0b1512c4efb2b0c8248 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 17 Oct 2018 08:25:12 +0200 Subject: [PATCH 44/63] Add KeyValueProcessor tests and add support for trimming properties (#3439) --- .../Ingest/Processors/KeyValueProcessor.cs | 75 ++++++++++++------- src/Nest/Ingest/ProcessorsDescriptor.cs | 5 ++ src/Tests/Tests/Ingest/ProcessorAssertions.cs | 67 +++++++++++++++++ 3 files changed, 121 insertions(+), 26 deletions(-) diff --git a/src/Nest/Ingest/Processors/KeyValueProcessor.cs b/src/Nest/Ingest/Processors/KeyValueProcessor.cs index c72895ad2a8..adac1a88493 100644 --- a/src/Nest/Ingest/Processors/KeyValueProcessor.cs +++ b/src/Nest/Ingest/Processors/KeyValueProcessor.cs @@ -1,7 +1,5 @@ using System; -using System.Collections; using System.Collections.Generic; -using System.Linq; using System.Linq.Expressions; using Newtonsoft.Json; @@ -11,33 +9,23 @@ namespace Nest [JsonConverter(typeof(ProcessorJsonConverter))] public interface IKeyValueProcessor : IProcessor { - /// - /// The field to be parsed - /// + /// The field to be parsed [JsonProperty("field")] Field Field { get; set; } - /// - /// The field to insert the extracted keys into. Defaults to the root of the document - /// + /// The field to insert the extracted keys into. Defaults to the root of the document [JsonProperty("target_field")] Field TargetField { get; set; } - /// - /// Regex pattern to use for splitting key-value pairs - /// + /// Regex pattern to use for splitting key-value pairs [JsonProperty("field_split")] string FieldSplit { get; set; } - /// - /// Regex pattern to use for splitting the key from the value within a key-value pair - /// + /// Regex pattern to use for splitting the key from the value within a key-value pair [JsonProperty("value_split")] string ValueSplit { get; set; } - /// - /// List of keys to filter and insert into document. Defaults to including all keys - /// + /// List of keys to filter and insert into document. Defaults to including all keys [JsonProperty("include_keys")] IEnumerable IncludeKeys { get; set; } @@ -46,8 +34,21 @@ public interface IKeyValueProcessor : IProcessor /// [JsonProperty("ignore_missing")] bool? IgnoreMissing { get; set; } + + /// String of characters to trim from extracted keys + [JsonProperty("trim_key")] + string TrimKey { get; set; } + + /// String of characters to trim from extracted values + [JsonProperty("trim_value")] + string TrimValue { get; set; } + + /// If true strip brackets (), <>, [] as well as quotes ' and " from extracted values + [JsonProperty("strip_brackets")] + bool? StripBrackets { get; set; } } + /// public class KeyValueProcessor : ProcessorBase, IKeyValueProcessor { protected override string Name => "kv"; @@ -69,8 +70,18 @@ public class KeyValueProcessor : ProcessorBase, IKeyValueProcessor /// public bool? IgnoreMissing { get; set; } + + /// + public string TrimKey { get; set; } + + /// + public string TrimValue { get; set; } + + /// + public bool? StripBrackets { get; set; } } + /// public class KeyValueProcessorDescriptor : ProcessorDescriptorBase, IKeyValueProcessor>, IKeyValueProcessor where T : class { @@ -82,33 +93,45 @@ public class KeyValueProcessorDescriptor : ProcessorDescriptorBase IKeyValueProcessor.IncludeKeys { get; set; } bool? IKeyValueProcessor.IgnoreMissing { get; set; } + string IKeyValueProcessor.TrimKey { get; set; } + string IKeyValueProcessor.TrimValue { get; set; } + bool? IKeyValueProcessor.StripBrackets { get; set; } - /// + /// public KeyValueProcessorDescriptor Field(Field field) => Assign(a => a.Field = field); - /// + /// public KeyValueProcessorDescriptor Field(Expression> objectPath) => Assign(a => a.Field = objectPath); - /// + /// public KeyValueProcessorDescriptor TargetField(Field field) => Assign(a => a.TargetField = field); - /// + /// public KeyValueProcessorDescriptor TargetField(Expression> objectPath) => Assign(a => a.TargetField = objectPath); - /// + /// public KeyValueProcessorDescriptor FieldSplit(string split) => Assign(a => a.FieldSplit = split); - /// + /// public KeyValueProcessorDescriptor ValueSplit(string split) => Assign(a => a.ValueSplit = split); - /// + /// public KeyValueProcessorDescriptor IgnoreMissing(bool? ignoreMissing = true) => Assign(a => a.IgnoreMissing = ignoreMissing); - /// + /// public KeyValueProcessorDescriptor IncludeKeys(IEnumerable includeKeys) => Assign(a => a.IncludeKeys = includeKeys); - /// + /// public KeyValueProcessorDescriptor IncludeKeys(params string[] includeKeys) => Assign(a => a.IncludeKeys = includeKeys); + + /// + public KeyValueProcessorDescriptor TrimKey(string trimKeys) => Assign(a => a.TrimKey = trimKeys); + + /// + public KeyValueProcessorDescriptor TrimValue(string trimValues) => Assign(a => a.TrimValue = trimValues); + + /// + public KeyValueProcessorDescriptor StripBrackets(bool? skip = true) => Assign(a => a.StripBrackets = skip); } } diff --git a/src/Nest/Ingest/ProcessorsDescriptor.cs b/src/Nest/Ingest/ProcessorsDescriptor.cs index e186ba4636d..ce380b4c902 100644 --- a/src/Nest/Ingest/ProcessorsDescriptor.cs +++ b/src/Nest/Ingest/ProcessorsDescriptor.cs @@ -124,9 +124,14 @@ public ProcessorsDescriptor Json(Func, IJsonProces public ProcessorsDescriptor UserAgent(Func, IUserAgentProcessor> selector) where T : class => Assign(a => a.AddIfNotNull(selector?.Invoke(new UserAgentProcessorDescriptor()))); + [Obsolete("This method takes the wrong descriptor please use Kv")] public ProcessorsDescriptor KeyValue(Func, IUserAgentProcessor> selector) where T : class => Assign(a => a.AddIfNotNull(selector?.Invoke(new UserAgentProcessorDescriptor()))); + /// + public ProcessorsDescriptor Kv(Func, IKeyValueProcessor> selector) where T : class => + Assign(a => a.AddIfNotNull(selector?.Invoke(new KeyValueProcessorDescriptor()))); + /// /// URL-decodes a string /// diff --git a/src/Tests/Tests/Ingest/ProcessorAssertions.cs b/src/Tests/Tests/Ingest/ProcessorAssertions.cs index 52e1d6c32c3..9c9f2cbe5da 100644 --- a/src/Tests/Tests/Ingest/ProcessorAssertions.cs +++ b/src/Tests/Tests/Ingest/ProcessorAssertions.cs @@ -341,5 +341,72 @@ public class UrlDecode : ProcessorAssertion .IgnoreMissing() ); } + + public class KeyValue : ProcessorAssertion + { + public override string Key => "kv"; + + public override object Json => new + { + field = "description", + ignore_missing = true, + field_split = "_", + value_split = " " + }; + + public override IProcessor Initializer => new KeyValueProcessor + { + Field = "description", + FieldSplit = "_", + ValueSplit = " ", + IgnoreMissing = true + }; + + public override Func>> Fluent => d => d + .Kv(ud => ud + .Field(p => p.Description) + .FieldSplit("_") + .ValueSplit(" ") + .IgnoreMissing() + ); + } + [SkipVersion("<6.4.0", "trimming options were introduced later")] + public class KeyValueTrimming : ProcessorAssertion + { + public override string Key => "kv"; + + public override object Json => new + { + field = "description", + ignore_missing = true, + field_split = "_", + value_split = " ", + trim_key = "xyz", + trim_value = "abc", + strip_brackets = true + }; + + public override IProcessor Initializer => new KeyValueProcessor + { + Field = "description", + FieldSplit = "_", + ValueSplit = " ", + TrimKey = "xyz", + TrimValue = "abc", + StripBrackets = true, + IgnoreMissing = true + }; + + public override Func>> Fluent => d => d + .Kv(ud => ud + .Field(p => p.Description) + .FieldSplit("_") + .ValueSplit(" ") + .TrimKey("xyz") + .TrimValue("abc") + .StripBrackets() + .IgnoreMissing() + ); + } } } From 577888a428207ac5c7d6245f188bae6abf5cb78d Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 17 Oct 2018 08:54:18 +0200 Subject: [PATCH 45/63] add support for the bytes processor (#3440) --- src/Nest/Ingest/PipelineJsonConverter.cs | 3 + src/Nest/Ingest/ProcessorJsonConverter.cs | 4 - src/Nest/Ingest/Processors/BytesProcessor.cs | 76 +++++++++++++++++++ src/Nest/Ingest/ProcessorsDescriptor.cs | 4 + src/Tests/Tests/Ingest/ProcessorAssertions.cs | 16 ++++ 5 files changed, 99 insertions(+), 4 deletions(-) create mode 100644 src/Nest/Ingest/Processors/BytesProcessor.cs diff --git a/src/Nest/Ingest/PipelineJsonConverter.cs b/src/Nest/Ingest/PipelineJsonConverter.cs index f4cc9be92aa..5bd6ffba4c9 100644 --- a/src/Nest/Ingest/PipelineJsonConverter.cs +++ b/src/Nest/Ingest/PipelineJsonConverter.cs @@ -108,6 +108,9 @@ private List GetProcessors(JToken jsonProcessors, JsonSerializer ser case "urldecode": processors.Add(jsonProcessor.ToObject(serializer)); break; + case "bytes": + processors.Add(jsonProcessor.ToObject(serializer)); + break; } } return processors; diff --git a/src/Nest/Ingest/ProcessorJsonConverter.cs b/src/Nest/Ingest/ProcessorJsonConverter.cs index f462b2d8950..e013710c422 100644 --- a/src/Nest/Ingest/ProcessorJsonConverter.cs +++ b/src/Nest/Ingest/ProcessorJsonConverter.cs @@ -1,9 +1,5 @@ using Newtonsoft.Json; using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace Nest { diff --git a/src/Nest/Ingest/Processors/BytesProcessor.cs b/src/Nest/Ingest/Processors/BytesProcessor.cs new file mode 100644 index 00000000000..dfeada41d2c --- /dev/null +++ b/src/Nest/Ingest/Processors/BytesProcessor.cs @@ -0,0 +1,76 @@ +using Newtonsoft.Json; +using System; +using System.Linq.Expressions; + +namespace Nest +{ + /// + /// Converts a human readable byte value (e.g. 1kb) to its value in bytes (e.g. 1024). + /// Supported human readable units are "b", "kb", "mb", "gb", "tb", "pb" case insensitive. + /// An error will occur if the field is not a supported format or resultant value exceeds 2^63. + /// + [JsonObject(MemberSerialization.OptIn)] + [JsonConverter(typeof(ProcessorJsonConverter))] + public interface IBytesProcessor : IProcessor + { + /// The field to convert bytes from + [JsonProperty("field")] + Field Field { get; set; } + + /// The field to assign the converted value to, by default is updated in-place + [JsonProperty("target_field")] + Field TargetField { get; set; } + + /// + /// If true and does not exist or is null, + /// the processor quietly exits without modifying the document. Default is false + /// + [JsonProperty("ignore_missing")] + bool? IgnoreMissing { get; set; } + } + + /// + public class BytesProcessor : ProcessorBase, IBytesProcessor + { + protected override string Name => "bytes"; + + /// + [JsonProperty("field")] + public Field Field { get; set; } + + /// + [JsonProperty("target_field")] + public Field TargetField { get; set; } + + /// + [JsonProperty("ignore_missing")] + public bool? IgnoreMissing { get; set; } + } + + /// + public class BytesProcessorDescriptor + : ProcessorDescriptorBase, IBytesProcessor>, IBytesProcessor + where T : class + { + protected override string Name => "bytes"; + + Field IBytesProcessor.Field { get; set; } + Field IBytesProcessor.TargetField { get; set; } + bool? IBytesProcessor.IgnoreMissing { get; set; } + + /// + public BytesProcessorDescriptor Field(Field field) => Assign(a => a.Field = field); + + /// + public BytesProcessorDescriptor Field(Expression> objectPath) => Assign(a => a.Field = objectPath); + + /// + public BytesProcessorDescriptor TargetField(Field field) => Assign(a => a.TargetField = field); + + /// + public BytesProcessorDescriptor TargetField(Expression> objectPath) => Assign(a => a.TargetField = objectPath); + + /// + public BytesProcessorDescriptor IgnoreMissing(bool? ignoreMissing = true) => Assign(a => a.IgnoreMissing = ignoreMissing); + } +} diff --git a/src/Nest/Ingest/ProcessorsDescriptor.cs b/src/Nest/Ingest/ProcessorsDescriptor.cs index ce380b4c902..ebdc135acf9 100644 --- a/src/Nest/Ingest/ProcessorsDescriptor.cs +++ b/src/Nest/Ingest/ProcessorsDescriptor.cs @@ -137,5 +137,9 @@ public ProcessorsDescriptor Kv(Func, IKeyValue /// public ProcessorsDescriptor UrlDecode(Func, IUrlDecodeProcessor> selector) where T : class => Assign(a => a.AddIfNotNull(selector?.Invoke(new UrlDecodeProcessorDescriptor()))); + + /// + public ProcessorsDescriptor Bytes(Func, IBytesProcessor> selector) where T : class => + Assign(a => a.AddIfNotNull(selector?.Invoke(new BytesProcessorDescriptor()))); } } diff --git a/src/Tests/Tests/Ingest/ProcessorAssertions.cs b/src/Tests/Tests/Ingest/ProcessorAssertions.cs index 9c9f2cbe5da..12b77b11d69 100644 --- a/src/Tests/Tests/Ingest/ProcessorAssertions.cs +++ b/src/Tests/Tests/Ingest/ProcessorAssertions.cs @@ -342,6 +342,22 @@ public class UrlDecode : ProcessorAssertion ); } + + [SkipVersion("<6.4.0", "")] + public class Bytes : ProcessorAssertion + { + public override string Key => "bytes"; + + public override object Json => new { field = "description", ignore_missing = true }; + + public override IProcessor Initializer => new BytesProcessor { Field = "description", IgnoreMissing = true }; + + public override Func>> Fluent => d => d + .Bytes(ud => ud + .Field(p => p.Description) + ); + } + public class KeyValue : ProcessorAssertion { public override string Key => "kv"; From d48dc4ee2080215a91e0df889565f4814a647701 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 17 Oct 2018 09:37:41 +0200 Subject: [PATCH 46/63] Add support for indexed_chars_field on the AttachmentProcessor (#3442) --- .../Processors/Plugins/AttachmentProcessor.cs | 103 ++++++------------ src/Tests/Tests/Ingest/ProcessorAssertions.cs | 37 ++++++- 2 files changed, 69 insertions(+), 71 deletions(-) diff --git a/src/Nest/Ingest/Processors/Plugins/AttachmentProcessor.cs b/src/Nest/Ingest/Processors/Plugins/AttachmentProcessor.cs index ff2960ce449..eed3e5fbaf7 100644 --- a/src/Nest/Ingest/Processors/Plugins/AttachmentProcessor.cs +++ b/src/Nest/Ingest/Processors/Plugins/AttachmentProcessor.cs @@ -17,15 +17,11 @@ namespace Nest [JsonConverter(typeof(ProcessorJsonConverter))] public interface IAttachmentProcessor : IProcessor { - /// - /// The field to get the base64 encoded field from - /// + /// The field to get the base64 encoded field from [JsonProperty("field")] Field Field { get; set; } - /// - /// The field that will hold the attachment information - /// + /// The field that will hold the attachment information [JsonProperty("target_field")] Field TargetField { get; set; } @@ -43,59 +39,42 @@ public interface IAttachmentProcessor : IProcessor [JsonProperty("indexed_chars")] long? IndexedCharacters { get; set; } - /// - /// If `true` and `field` does not exist, the processor quietly exits without modifying the document - /// + /// Field name from which you can overwrite the number of chars being used for extraction. + [JsonProperty("indexed_chars_field")] + Field IndexedCharactersField { get; set; } + + + /// If `true` and `field` does not exist, the processor quietly exits without modifying the document [JsonProperty("ignore_missing")] bool? IgnoreMissing { get; set; } } - /// - /// The ingest attachment plugin lets Elasticsearch extract file attachments in common formats - /// (such as PPT, XLS, and PDF) by using the Apache text extraction library Tika. - /// You can use the ingest attachment plugin as a replacement for the mapper attachment plugin. - /// - /// - /// Requires the Ingest Attachment Processor Plugin to be installed on the cluster. - /// + /// public class AttachmentProcessor : ProcessorBase, IAttachmentProcessor { protected override string Name => "attachment"; - /// - /// The field to get the base64 encoded field from - /// + /// public Field Field { get; set; } - /// - /// The field that will hold the attachment information - /// + /// public Field TargetField { get; set; } - /// - /// Properties to select to be stored. Can be content, title, name, author, - /// keywords, date, content_type, content_length, language. Defaults to all - /// + /// public IEnumerable Properties { get; set; } - /// - /// The number of chars being used for extraction to prevent huge fields. Use -1 for no limit. - /// Defaults to 100000. - /// + /// public long? IndexedCharacters { get; set; } + /// + public Field IndexedCharactersField { get; set; } + /// + /// public bool? IgnoreMissing { get; set; } } - /// - /// The ingest attachment plugin lets Elasticsearch extract file attachments in common formats - /// (such as PPT, XLS, and PDF) by using the Apache text extraction library Tika. - /// You can use the ingest attachment plugin as a replacement for the mapper attachment plugin. - /// - /// - /// Requires the Ingest Attachment Processor Plugin to be installed on the cluster. - /// + /// public class AttachmentProcessorDescriptor : ProcessorDescriptorBase, IAttachmentProcessor>, IAttachmentProcessor where T : class @@ -107,48 +86,36 @@ public class AttachmentProcessorDescriptor IEnumerable IAttachmentProcessor.Properties { get; set; } long? IAttachmentProcessor.IndexedCharacters { get; set; } bool? IAttachmentProcessor.IgnoreMissing { get; set; } + Field IAttachmentProcessor.IndexedCharactersField { get; set; } - /// - /// The field to get the base64 encoded field from - /// + /// public AttachmentProcessorDescriptor Field(Field field) => Assign(a => a.Field = field); - /// - /// The field to get the base64 encoded field from - /// - public AttachmentProcessorDescriptor Field(Expression> objectPath) => - Assign(a => a.Field = objectPath); + /// + public AttachmentProcessorDescriptor Field(Expression> objectPath) => Assign(a => a.Field = objectPath); - /// - /// The field that will hold the attachment information - /// + /// public AttachmentProcessorDescriptor TargetField(Field field) => Assign(a => a.TargetField = field); - /// - /// The field that will hold the attachment information - /// - public AttachmentProcessorDescriptor TargetField(Expression> objectPath) => - Assign(a => a.TargetField = objectPath); + /// + public AttachmentProcessorDescriptor TargetField(Expression> objectPath) => Assign(a => a.TargetField = objectPath); - /// - /// The number of chars being used for extraction to prevent huge fields. Use -1 for no limit. - /// Defaults to 100000. - /// + /// public AttachmentProcessorDescriptor IndexedCharacters(long? indexedCharacters) => Assign(a => a.IndexedCharacters = indexedCharacters); - /// + /// + public AttachmentProcessorDescriptor IndexedCharactersField(Field field) => Assign(a => a.IndexedCharactersField = field); + + /// + public AttachmentProcessorDescriptor IndexedCharactersField(Expression> objectPath) => Assign(a => a.IndexedCharactersField = objectPath); + + /// public AttachmentProcessorDescriptor IgnoreMissing(bool? ignoreMissing = true) => Assign(a => a.IgnoreMissing = ignoreMissing); - /// - /// Properties to select to be stored. Can be content, title, name, author, - /// keywords, date, content_type, content_length, language. Defaults to all - /// + /// public AttachmentProcessorDescriptor Properties(IEnumerable properties) => Assign(a => a.Properties = properties); - /// - /// Properties to select to be stored. Can be content, title, name, author, - /// keywords, date, content_type, content_length, language. Defaults to all - /// + /// public AttachmentProcessorDescriptor Properties(params string[] properties) => Assign(a => a.Properties = properties); } } diff --git a/src/Tests/Tests/Ingest/ProcessorAssertions.cs b/src/Tests/Tests/Ingest/ProcessorAssertions.cs index 12b77b11d69..70f35b889f9 100644 --- a/src/Tests/Tests/Ingest/ProcessorAssertions.cs +++ b/src/Tests/Tests/Ingest/ProcessorAssertions.cs @@ -342,6 +342,36 @@ public class UrlDecode : ProcessorAssertion ); } + [SkipVersion("<6.4.0", "")] + public class Attachment : ProcessorAssertion + { + public override string Key => "attachment"; + + public override object Json => new + { + field = "description", + ignore_missing = true, + properties = new [] {"title", "author"}, + indexed_chars = 100_000, + }; + + public override IProcessor Initializer => new AttachmentProcessor + { + Field = "description", + Properties = new [] {"title", "author"}, + IndexedCharacters = 100_000, + IgnoreMissing = true + + }; + public override Func>> Fluent => d => d + .Attachment(ud => ud + .Field(p => p.Description) + .IndexedCharacters(100_000) + .Properties("title", "author") + .IgnoreMissing() + ); + } + [SkipVersion("<6.4.0", "")] public class Bytes : ProcessorAssertion @@ -351,12 +381,12 @@ public class Bytes : ProcessorAssertion public override object Json => new { field = "description", ignore_missing = true }; public override IProcessor Initializer => new BytesProcessor { Field = "description", IgnoreMissing = true }; - + public override Func>> Fluent => d => d .Bytes(ud => ud .Field(p => p.Description) - ); - } + ); + } public class KeyValue : ProcessorAssertion { @@ -386,6 +416,7 @@ public class KeyValue : ProcessorAssertion .IgnoreMissing() ); } + [SkipVersion("<6.4.0", "trimming options were introduced later")] public class KeyValueTrimming : ProcessorAssertion { From f7712ff804afba998aafa9458d90028c006b7f1e Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Thu, 18 Oct 2018 02:27:20 +0200 Subject: [PATCH 47/63] add support for missing_bucket on Composite Aggregation (#3420) per https://github.com/elastic/elasticsearch/pull/29465 --- .../Composite/CompositeAggregationSource.cs | 14 ++ src/Tests/Tests.Domain/Project.cs | 2 +- .../CompositeAggregationUsageTests.cs | 130 +++++++++++++++++- 3 files changed, 140 insertions(+), 6 deletions(-) diff --git a/src/Nest/Aggregations/Bucket/Composite/CompositeAggregationSource.cs b/src/Nest/Aggregations/Bucket/Composite/CompositeAggregationSource.cs index 23754272e33..cb736b1f704 100644 --- a/src/Nest/Aggregations/Bucket/Composite/CompositeAggregationSource.cs +++ b/src/Nest/Aggregations/Bucket/Composite/CompositeAggregationSource.cs @@ -37,6 +37,13 @@ public interface ICompositeAggregationSource /// [JsonProperty("order")] SortOrder? Order { get; set; } + + /// + /// By default documents without a value for a given source are ignored. It is possible to include + /// them in the response as null by setting this to true + /// + [JsonProperty("missing_bucket")] + bool? MissingBucket { get; set; } } /// @@ -59,6 +66,9 @@ protected CompositeAggregationSourceBase(string name) => /// public SortOrder? Order { get; set; } + + /// + public bool? MissingBucket { get; set; } } /// @@ -93,6 +103,7 @@ public abstract class CompositeAggregationSourceDescriptorBase _sourceType; Field ICompositeAggregationSource.Field { get; set; } SortOrder? ICompositeAggregationSource.Order { get; set; } + bool? ICompositeAggregationSource.MissingBucket { get; set; } protected CompositeAggregationSourceDescriptorBase(string name, string sourceType) { @@ -108,6 +119,9 @@ protected CompositeAggregationSourceDescriptorBase(string name, string sourceTyp /// public TDescriptor Order(SortOrder? order) => Assign(a => a.Order = order); + + /// + public TDescriptor MissingBucket(bool? includeMissing = true) => Assign(a => a.MissingBucket = includeMissing); } internal class CompositeAggregationSourceConverter : ReserializeJsonConverter diff --git a/src/Tests/Tests.Domain/Project.cs b/src/Tests/Tests.Domain/Project.cs index 233a9248712..62e20b06cc9 100644 --- a/src/Tests/Tests.Domain/Project.cs +++ b/src/Tests/Tests.Domain/Project.cs @@ -57,7 +57,7 @@ public class Project .RuleFor(p => p.NumberOfCommits, f => Gimme.Random.Number(1, 1000)) .RuleFor(p => p.NumberOfContributors, f => Gimme.Random.Number(1, 200)) .RuleFor(p => p.Ranges, f => Ranges.Generator.Generate()) - .RuleFor(p => p.Branches, f => Gimme.Random.ListItems(new List { "master", "dev", "release", "qa", "test" }, 2)) + .RuleFor(p => p.Branches, f => Gimme.Random.ListItems(new List { "master", "dev", "release", "qa", "test" })) .RuleFor(p => p.SourceOnly, f => TestConfiguration.Instance.Random.SourceSerializer ? new SourceOnlyObject() : null ) diff --git a/src/Tests/Tests/Aggregations/Bucket/Composite/CompositeAggregationUsageTests.cs b/src/Tests/Tests/Aggregations/Bucket/Composite/CompositeAggregationUsageTests.cs index 0f22d0ebcf7..de33f936b45 100644 --- a/src/Tests/Tests/Aggregations/Bucket/Composite/CompositeAggregationUsageTests.cs +++ b/src/Tests/Tests/Aggregations/Bucket/Composite/CompositeAggregationUsageTests.cs @@ -155,7 +155,7 @@ public CompositeAggregationUsageTests(ReadOnlyCluster i, EndpointUsage usage) : }; /**==== Handling Responses - * Each Composite aggregation bucket key is an `CompositeKey`, a specialized + * Each Composite aggregation bucket key is a `CompositeKey` type, a specialized * `IReadOnlyDictionary` type with methods to convert values to supported types */ protected override void ExpectResponse(ISearchResponse response) @@ -196,6 +196,130 @@ protected override void ExpectResponse(ISearchResponse response) } } + /**[float] + * == Missing buckets + * By default documents without a value for a given source are ignored. + * It is possible to include them in the response by setting missing_bucket to `true` (defaults to `false`): + * + * NOTE: Only available in Elasticsearch 6.4.0+ + */ + [SkipVersion("<6.4.0", "Missing buckets added to Composite Aggregation Elasticsearch 6.4.0+")] + public class CompositeAggregationMissingBucketUsageTests : ProjectsOnlyAggregationUsageTestBase + { + public CompositeAggregationMissingBucketUsageTests(ReadOnlyCluster i, EndpointUsage usage) : base(i, usage) { } + + protected override object AggregationJson => new + { + my_buckets = new + { + composite = new + { + sources = new object[] + { + new + { + branches = new + { + terms = new + { + field = "branches.keyword", + order = "asc", + missing_bucket = true + } + } + }, + } + }, + aggs = new + { + project_tags = new + { + nested = new { path = "tags" }, + aggs = new + { + tags = new { terms = new {field = "tags.name"} } + } + } + } + } + }; + + protected override Func, IAggregationContainer> FluentAggs => a => a + .Composite("my_buckets", date => date + .Sources(s => s + .Terms("branches", t => t + .Field(f => f.Branches.Suffix("keyword")) + .MissingBucket() + .Order(SortOrder.Ascending) + ) + ) + .Aggregations(childAggs => childAggs + .Nested("project_tags", n => n + .Path(p => p.Tags) + .Aggregations(nestedAggs => nestedAggs + .Terms("tags", avg => avg.Field(p => p.Tags.First().Name)) + ) + ) + ) + ); + + protected override AggregationDictionary InitializerAggs => + new CompositeAggregation("my_buckets") + { + Sources = new List + { + new TermsCompositeAggregationSource("branches") + { + Field = Infer.Field(f => f.Branches.Suffix("keyword")), + MissingBucket = true, + Order = SortOrder.Ascending + } + }, + Aggregations = new NestedAggregation("project_tags") + { + Path = Field(p => p.Tags), + Aggregations = new TermsAggregation("tags") + { + Field = Field(p => p.Tags.First().Name) + } + } + }; + + /**==== Handling Responses + * Each Composite aggregation bucket key is an `CompositeKey`, a specialized + * `IReadOnlyDictionary` type with methods to convert values to supported types + */ + protected override void ExpectResponse(ISearchResponse response) + { + response.ShouldBeValid(); + + var composite = response.Aggregations.Composite("my_buckets"); + composite.Should().NotBeNull(); + composite.Buckets.Should().NotBeNullOrEmpty(); + composite.AfterKey.Should().NotBeNull(); + + if (TestConfiguration.Instance.InRange(">=6.3.0")) + composite.AfterKey.Should().HaveCount(1).And.ContainKeys("branches"); + + var i = 0; + foreach (var item in composite.Buckets) + { + var key = item.Key; + key.Should().NotBeNull(); + + key.TryGetValue("branches", out string branches).Should().BeTrue("expected to find 'branches' in composite bucket"); + if (i == 0) branches.Should().BeNull("First key should be null as we expect to have some projects with no branches"); + else branches.Should().NotBeNullOrEmpty(); + + var nested = item.Nested("project_tags"); + nested.Should().NotBeNull(); + + var nestedTerms = nested.Terms("tags"); + nestedTerms.Buckets.Count.Should().BeGreaterThan(0); + i++; + } + } + } //hide [SkipVersion("<6.3.0", "Date histogram source only supports format starting from Elasticsearch 6.3.0+")] @@ -286,10 +410,6 @@ public DateFormatCompositeAggregationUsageTests(ReadOnlyCluster i, EndpointUsage } }; - /**==== Handling Responses - * Each Composite aggregation bucket key is an `CompositeKey`, a specialized - * `IReadOnlyDictionary` type with methods to convert values to supported types - */ protected override void ExpectResponse(ISearchResponse response) { response.ShouldBeValid(); From 4cb05c0cda0d75bfc19cdfd51f7c64a0a631d0af Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Thu, 18 Oct 2018 04:01:01 +0200 Subject: [PATCH 48/63] Add support for Rollup APIs (#3448) This commit adds support for the Rollup APIs: - CreateRollupJob - GetRollupJob - DeleteRollupJob - StartRollupJob - StopRollupJob - GetRollupCapabilities - RollupSearch --- .../ApiGenerator/ApiGenerator.cs | 22 +- .../ApiGenerator/Domain/ApiUrl.cs | 14 +- src/CodeGeneration/ApiGenerator/Program.cs | 3 +- .../xpack.rollup.get_rollup_caps.patch.json | 16 + .../xpack.rollup.get_rollup_index_caps.json | 17 -- .../xpack.rollup.rollup_search.patch.json | 13 + .../RequestParameters.Generated.cs | 35 +++ .../ElasticLowLevelClient.Generated.cs | 126 ++++++++ .../IElasticLowLevelClient.Generated.cs | 102 +++++++ .../Multiple/BulkAll/BulkAllObservable.cs | 6 +- .../Multiple/BulkAll/BulkAllRequest.cs | 14 +- src/Nest/Nest.csproj | 1 + src/Nest/Search/Search/SearchRequest.cs | 4 +- .../CreateRollupJob/CreateRollupJobRequest.cs | 90 ++++++ .../CreateRollupJobResponse.cs | 9 + .../ElasticClient-CreateRollupJob.cs | 55 ++++ .../DeleteRollupJob/DeleteRollupJobRequest.cs | 15 + .../DeleteRollupJobResponse.cs | 9 + .../ElasticClient-DeleteRollupJob.cs | 54 ++++ .../ElasticClient-GetRollupCapabilities.cs | 62 ++++ .../GetRollupCapabilitiesRequest.cs | 15 + .../GetRollupCapabilitiesResponse.cs | 62 ++++ .../ElasticClient-GetRollupJob.cs | 56 ++++ .../GetRollupJob/GetRollupJobRequest.cs | 15 + .../GetRollupJob/GetRollupJobResponse.cs | 17 ++ .../RollUp/GetRollupJob/IndexingJobState.cs | 25 ++ .../GetRollupJob/RollupJobConfiguration.cs | 39 +++ .../GetRollupJob/RollupJobInformation.cs | 16 + .../RollUp/GetRollupJob/RollupJobStats.cs | 19 ++ .../RollUp/GetRollupJob/RollupJobStatus.cs | 18 ++ .../DateHistogramRollupGrouping.cs | 82 +++++ .../HistogramRollupGrouping.cs | 52 ++++ .../RollupConfiguration/IRollupGroupings.cs | 58 ++++ .../RollupConfiguration/RollupFieldMetric.cs | 40 +++ .../RollupConfiguration/RollupMetric.cs | 16 + .../TermsRollupGrouping.cs | 43 +++ .../ElasticClient-RollupSearch.cs | 86 ++++++ .../RollupSearch/RollupSearchRequest.cs | 53 ++++ .../RollupSearch/RollupSearchResponse.cs | 13 + .../ElasticClient-StartRollupJob.cs | 53 ++++ .../StartRollupJob/StartRollupJobRequest.cs | 15 + .../StartRollupJob/StartRollupJobResponse.cs | 16 + .../ElasticClient-StopRollupJob.cs | 54 ++++ .../StopRollupJob/StopRollupJobRequest.cs | 18 ++ .../StopRollupJob/StopRollupJobResponse.cs | 16 + src/Nest/_Generated/_Descriptors.generated.cs | 110 +++++++ .../_Generated/_LowLevelDispatch.generated.cs | 164 ++++++++++ src/Nest/_Generated/_Requests.generated.cs | 129 ++++++++ .../Clusters/BenchmarkCluster.cs | 5 - .../Clusters/TimeSeriesCluster.cs | 58 ++++ .../Clusters/WatcherStateCluster.cs | 10 +- src/Tests/Tests.Core/Xunit/Generators.cs | 2 +- src/Tests/Tests.Domain/Log.cs | 80 +++++ src/Tests/Tests.Domain/Message.cs | 41 --- .../Framework/EndpointTests/CrudTestBase.cs | 169 +++++++--- .../CreateRollupJob/CreateRollupUrlTests.cs | 22 ++ .../DeleteRollupJob/DeleteRollupUrlTests.cs | 21 ++ .../GetRollupCapabilitiesTests.cs | 27 ++ .../Rollup/GetRollupJob/GetRollupUrlTests.cs | 27 ++ .../Tests/XPack/Rollup/RollupJobCrudTests.cs | 289 ++++++++++++++++++ .../RollupSearch/RollupSearchUrlTests.cs | 28 ++ .../StartRollupJob/StartRollupUrlTests.cs | 21 ++ .../StopRollupJob/StopRollupUrlTests.cs | 21 ++ 63 files changed, 2643 insertions(+), 145 deletions(-) create mode 100644 src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Rollup/xpack.rollup.get_rollup_caps.patch.json delete mode 100644 src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Rollup/xpack.rollup.get_rollup_index_caps.json create mode 100644 src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Rollup/xpack.rollup.rollup_search.patch.json create mode 100644 src/Nest/XPack/RollUp/CreateRollupJob/CreateRollupJobRequest.cs create mode 100644 src/Nest/XPack/RollUp/CreateRollupJob/CreateRollupJobResponse.cs create mode 100644 src/Nest/XPack/RollUp/CreateRollupJob/ElasticClient-CreateRollupJob.cs create mode 100644 src/Nest/XPack/RollUp/DeleteRollupJob/DeleteRollupJobRequest.cs create mode 100644 src/Nest/XPack/RollUp/DeleteRollupJob/DeleteRollupJobResponse.cs create mode 100644 src/Nest/XPack/RollUp/DeleteRollupJob/ElasticClient-DeleteRollupJob.cs create mode 100644 src/Nest/XPack/RollUp/GetRollupCapabilities/ElasticClient-GetRollupCapabilities.cs create mode 100644 src/Nest/XPack/RollUp/GetRollupCapabilities/GetRollupCapabilitiesRequest.cs create mode 100644 src/Nest/XPack/RollUp/GetRollupCapabilities/GetRollupCapabilitiesResponse.cs create mode 100644 src/Nest/XPack/RollUp/GetRollupJob/ElasticClient-GetRollupJob.cs create mode 100644 src/Nest/XPack/RollUp/GetRollupJob/GetRollupJobRequest.cs create mode 100644 src/Nest/XPack/RollUp/GetRollupJob/GetRollupJobResponse.cs create mode 100644 src/Nest/XPack/RollUp/GetRollupJob/IndexingJobState.cs create mode 100644 src/Nest/XPack/RollUp/GetRollupJob/RollupJobConfiguration.cs create mode 100644 src/Nest/XPack/RollUp/GetRollupJob/RollupJobInformation.cs create mode 100644 src/Nest/XPack/RollUp/GetRollupJob/RollupJobStats.cs create mode 100644 src/Nest/XPack/RollUp/GetRollupJob/RollupJobStatus.cs create mode 100644 src/Nest/XPack/RollUp/RollupConfiguration/DateHistogramRollupGrouping.cs create mode 100644 src/Nest/XPack/RollUp/RollupConfiguration/HistogramRollupGrouping.cs create mode 100644 src/Nest/XPack/RollUp/RollupConfiguration/IRollupGroupings.cs create mode 100644 src/Nest/XPack/RollUp/RollupConfiguration/RollupFieldMetric.cs create mode 100644 src/Nest/XPack/RollUp/RollupConfiguration/RollupMetric.cs create mode 100644 src/Nest/XPack/RollUp/RollupConfiguration/TermsRollupGrouping.cs create mode 100644 src/Nest/XPack/RollUp/RollupSearch/ElasticClient-RollupSearch.cs create mode 100644 src/Nest/XPack/RollUp/RollupSearch/RollupSearchRequest.cs create mode 100644 src/Nest/XPack/RollUp/RollupSearch/RollupSearchResponse.cs create mode 100644 src/Nest/XPack/RollUp/StartRollupJob/ElasticClient-StartRollupJob.cs create mode 100644 src/Nest/XPack/RollUp/StartRollupJob/StartRollupJobRequest.cs create mode 100644 src/Nest/XPack/RollUp/StartRollupJob/StartRollupJobResponse.cs create mode 100644 src/Nest/XPack/RollUp/StopRollupJob/ElasticClient-StopRollupJob.cs create mode 100644 src/Nest/XPack/RollUp/StopRollupJob/StopRollupJobRequest.cs create mode 100644 src/Nest/XPack/RollUp/StopRollupJob/StopRollupJobResponse.cs delete mode 100644 src/Tests/Tests.Core/ManagedElasticsearch/Clusters/BenchmarkCluster.cs create mode 100644 src/Tests/Tests.Core/ManagedElasticsearch/Clusters/TimeSeriesCluster.cs create mode 100644 src/Tests/Tests.Domain/Log.cs delete mode 100644 src/Tests/Tests.Domain/Message.cs create mode 100644 src/Tests/Tests/XPack/Rollup/CreateRollupJob/CreateRollupUrlTests.cs create mode 100644 src/Tests/Tests/XPack/Rollup/DeleteRollupJob/DeleteRollupUrlTests.cs create mode 100644 src/Tests/Tests/XPack/Rollup/GetRollupCapabilities/GetRollupCapabilitiesTests.cs create mode 100644 src/Tests/Tests/XPack/Rollup/GetRollupJob/GetRollupUrlTests.cs create mode 100644 src/Tests/Tests/XPack/Rollup/RollupJobCrudTests.cs create mode 100644 src/Tests/Tests/XPack/Rollup/RollupSearch/RollupSearchUrlTests.cs create mode 100644 src/Tests/Tests/XPack/Rollup/StartRollupJob/StartRollupUrlTests.cs create mode 100644 src/Tests/Tests/XPack/Rollup/StopRollupJob/StopRollupUrlTests.cs diff --git a/src/CodeGeneration/ApiGenerator/ApiGenerator.cs b/src/CodeGeneration/ApiGenerator/ApiGenerator.cs index 6ec6adbfd35..c71340f3b11 100644 --- a/src/CodeGeneration/ApiGenerator/ApiGenerator.cs +++ b/src/CodeGeneration/ApiGenerator/ApiGenerator.cs @@ -73,20 +73,14 @@ public static void Generate(string downloadBranch, params string[] folders) "xpack.ml.put_calendar_job.json", "xpack.ml.get_calendar_job.json", - "xpack.rollup.delete_job.json", - "xpack.rollup.get_jobs.json", - "xpack.rollup.get_rollup_caps.json", - "xpack.rollup.put_job.json", - "xpack.rollup.rollup_search.json", - "xpack.rollup.start_job.json", - "xpack.rollup.stop_job.json", - + "xpack.sql.clear_cursor.json", + "xpack.sql.query.json", + "xpack.sql.translate.json", "xpack.ssl.certificates.json", // 6.4 new API's "xpack.ml.update_filter.json", - "xpack.rollup.get_rollup_index_caps.json", "xpack.security.delete_privileges.json", "xpack.security.get_privileges.json", "xpack.security.has_privileges.json", @@ -171,10 +165,20 @@ private static void PatchOfficialSpec(JObject original, string jsonFile) var patchedJson = JObject.Parse(File.ReadAllText(patchFile)); + var pathsOverride = patchedJson.SelectToken("*.url.paths"); + original.Merge(patchedJson, new JsonMergeSettings { MergeArrayHandling = MergeArrayHandling.Union }); + + if (pathsOverride != null) + { + original.SelectToken("*.url.paths").Replace(pathsOverride); + } + + + } private static Dictionary CreateCommonApiQueryParameters(string jsonFile) diff --git a/src/CodeGeneration/ApiGenerator/Domain/ApiUrl.cs b/src/CodeGeneration/ApiGenerator/Domain/ApiUrl.cs index 34bdcdb63ca..787b0adaa4a 100644 --- a/src/CodeGeneration/ApiGenerator/Domain/ApiUrl.cs +++ b/src/CodeGeneration/ApiGenerator/Domain/ApiUrl.cs @@ -15,16 +15,16 @@ public class ApiUrl public IEnumerable Paths { - get - { - return _paths?.Where(p => !BlackListRouteValues.Any(p.Contains)) - .ToList() ?? _paths; - } - set { _paths = value; } + get => _paths?.Where(p => !BlackListRouteValues.Any(p.Contains)) + .ToList() ?? _paths; + set => _paths = this.EnsureBackslash(value); } + public IList EnsureBackslash(IEnumerable paths) => + paths?.Select(p => p.StartsWith("/") ? p : $"/{p}").ToList(); + public IDictionary Parts { get; set; } public IDictionary Params { get; set; } } -} \ No newline at end of file +} diff --git a/src/CodeGeneration/ApiGenerator/Program.cs b/src/CodeGeneration/ApiGenerator/Program.cs index 9bff7d9db07..d45640a127e 100644 --- a/src/CodeGeneration/ApiGenerator/Program.cs +++ b/src/CodeGeneration/ApiGenerator/Program.cs @@ -41,8 +41,7 @@ static void Main(string[] args) if (redownloadCoreSpecification) RestSpecDownloader.Download(downloadBranch); - ApiGenerator.Generate(downloadBranch, "Core", "Graph", "License", "Security", "Watcher", "Info", - "MachineLearning", "Migration", "Sql"); + ApiGenerator.Generate(downloadBranch, "Core", "Graph", "License", "Security", "Watcher", "Info", "MachineLearning", "Migration", "Sql", "Rollup"); } } } diff --git a/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Rollup/xpack.rollup.get_rollup_caps.patch.json b/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Rollup/xpack.rollup.get_rollup_caps.patch.json new file mode 100644 index 00000000000..eefa6f7ed26 --- /dev/null +++ b/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Rollup/xpack.rollup.get_rollup_caps.patch.json @@ -0,0 +1,16 @@ +{ + "xpack.rollup.get_rollup_caps": { + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/rollup-get-rollup-caps.html", + "url": { + "path": "/_xpack/rollup/data/{index}", + "paths": [ "/_xpack/rollup/data/{index}", "/_xpack/rollup/data/" ], + "parts": { + "index": { + "type": "list", + "required": false, + "description": " Index, indices or index-pattern to return rollup capabilities for. _all may be used to fetch rollup capabilities from all job" + } + } + } + } +} diff --git a/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Rollup/xpack.rollup.get_rollup_index_caps.json b/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Rollup/xpack.rollup.get_rollup_index_caps.json deleted file mode 100644 index 458311417d4..00000000000 --- a/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Rollup/xpack.rollup.get_rollup_index_caps.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "xpack.rollup.get_rollup_index_caps": { - "documentation": "", - "methods": [ "GET" ], - "url": { - "path": "/{index}/_xpack/rollup/data", - "paths": [ "/{index}/_xpack/rollup/data" ], - "parts": { - "index": { - "type": "string", - "required": true, - "description": "The rollup index or index pattern to obtain rollup capabilities from." - } - } - } - } -} diff --git a/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Rollup/xpack.rollup.rollup_search.patch.json b/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Rollup/xpack.rollup.rollup_search.patch.json new file mode 100644 index 00000000000..e1835befa1f --- /dev/null +++ b/src/CodeGeneration/ApiGenerator/RestSpecification/XPack/Rollup/xpack.rollup.rollup_search.patch.json @@ -0,0 +1,13 @@ +{ + "xpack.rollup.rollup_search": { + "url": { + "parts": { + "index": { + "type": "list", + "required": true, + "description": "The index or index-pattern (containing rollup or regular data) that should be searched" + } + } + } + } +} diff --git a/src/Elasticsearch.Net/Domain/RequestParameters/RequestParameters.Generated.cs b/src/Elasticsearch.Net/Domain/RequestParameters/RequestParameters.Generated.cs index f8165cb15e6..fbcff7ef926 100644 --- a/src/Elasticsearch.Net/Domain/RequestParameters/RequestParameters.Generated.cs +++ b/src/Elasticsearch.Net/Domain/RequestParameters/RequestParameters.Generated.cs @@ -2433,6 +2433,41 @@ public partial class MigrationUpgradeRequestParameters : RequestParametersShould the request block until the upgrade operation is completed public bool? WaitForCompletion { get => Q("wait_for_completion"); set => Q("wait_for_completion", value); } } + ///Request options for XpackRollupDeleteJob

+	public partial class DeleteRollupJobRequestParameters : RequestParameters 
+	{
+		public override HttpMethod DefaultHttpMethod => HttpMethod.DELETE;
+	}
+	///Request options for XpackRollupGetJobs

+	public partial class GetRollupJobRequestParameters : RequestParameters 
+	{
+		public override HttpMethod DefaultHttpMethod => HttpMethod.GET;
+	}
+	///Request options for XpackRollupGetRollupCaps
https://www.elastic.co/guide/en/elasticsearch/reference/current/rollup-get-rollup-caps.html

+	public partial class GetRollupCapabilitiesRequestParameters : RequestParameters 
+	{
+		public override HttpMethod DefaultHttpMethod => HttpMethod.GET;
+	}
+	///Request options for XpackRollupPutJob

+	public partial class CreateRollupJobRequestParameters : RequestParameters 
+	{
+		public override HttpMethod DefaultHttpMethod => HttpMethod.PUT;
+	}
+	///Request options for XpackRollupRollupSearch

+	public partial class RollupSearchRequestParameters : RequestParameters 
+	{
+		public override HttpMethod DefaultHttpMethod => HttpMethod.POST;
+	}
+	///Request options for XpackRollupStartJob

+	public partial class StartRollupJobRequestParameters : RequestParameters 
+	{
+		public override HttpMethod DefaultHttpMethod => HttpMethod.POST;
+	}
+	///Request options for XpackRollupStopJob

+	public partial class StopRollupJobRequestParameters : RequestParameters 
+	{
+		public override HttpMethod DefaultHttpMethod => HttpMethod.POST;
+	}
 	///Request options for XpackSecurityAuthenticate
https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-authenticate.html

 	public partial class AuthenticateRequestParameters : RequestParameters 
 	{
diff --git a/src/Elasticsearch.Net/ElasticLowLevelClient.Generated.cs b/src/Elasticsearch.Net/ElasticLowLevelClient.Generated.cs
index bf4a41b3772..dde0703a3c5 100644
--- a/src/Elasticsearch.Net/ElasticLowLevelClient.Generated.cs
+++ b/src/Elasticsearch.Net/ElasticLowLevelClient.Generated.cs
@@ -3542,6 +3542,132 @@ public TResponse XpackMigrationUpgrade(string index, MigrationUpgrade
 		///A func that allows you to describe the querystring parameters & request specific connection settings.
 		public Task XpackMigrationUpgradeAsync(string index, MigrationUpgradeRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken))
 			where TResponse : class, IElasticsearchResponse, new() => this.DoRequestAsync(POST, Url($"_xpack/migration/upgrade/{index.NotNull("index")}"), ctx, null, _params(requestParameters));
+		///DELETE on /_xpack/rollup/job/{id} 
+		///The ID of the job to delete
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		public TResponse XpackRollupDeleteJob(string id, DeleteRollupJobRequestParameters requestParameters = null)
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequest(DELETE, Url($"_xpack/rollup/job/{id.NotNull("id")}"), null, _params(requestParameters));
+		///DELETE on /_xpack/rollup/job/{id} 
+		///The ID of the job to delete
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		public Task XpackRollupDeleteJobAsync(string id, DeleteRollupJobRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken))
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequestAsync(DELETE, Url($"_xpack/rollup/job/{id.NotNull("id")}"), ctx, null, _params(requestParameters));
+		///GET on /_xpack/rollup/job/{id} 
+		///The ID of the job(s) to fetch. Accepts glob patterns, or left blank for all jobs
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		public TResponse XpackRollupGetJobs(string id, GetRollupJobRequestParameters requestParameters = null)
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequest(GET, Url($"_xpack/rollup/job/{id.NotNull("id")}"), null, _params(requestParameters));
+		///GET on /_xpack/rollup/job/{id} 
+		///The ID of the job(s) to fetch. Accepts glob patterns, or left blank for all jobs
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		public Task XpackRollupGetJobsAsync(string id, GetRollupJobRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken))
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequestAsync(GET, Url($"_xpack/rollup/job/{id.NotNull("id")}"), ctx, null, _params(requestParameters));
+		///GET on /_xpack/rollup/job/ 
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		public TResponse XpackRollupGetJobs(GetRollupJobRequestParameters requestParameters = null)
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequest(GET, Url($"_xpack/rollup/job/"), null, _params(requestParameters));
+		///GET on /_xpack/rollup/job/ 
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		public Task XpackRollupGetJobsAsync(GetRollupJobRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken))
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequestAsync(GET, Url($"_xpack/rollup/job/"), ctx, null, _params(requestParameters));
+		///GET on /_xpack/rollup/data/{index} https://www.elastic.co/guide/en/elasticsearch/reference/current/rollup-get-rollup-caps.html
+		/// Index, indices or index-pattern to return rollup capabilities for. _all may be used to fetch rollup capabilities from all job
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		public TResponse XpackRollupGetRollupCaps(string index, GetRollupCapabilitiesRequestParameters requestParameters = null)
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequest(GET, Url($"_xpack/rollup/data/{index.NotNull("index")}"), null, _params(requestParameters));
+		///GET on /_xpack/rollup/data/{index} https://www.elastic.co/guide/en/elasticsearch/reference/current/rollup-get-rollup-caps.html
+		/// Index, indices or index-pattern to return rollup capabilities for. _all may be used to fetch rollup capabilities from all job
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		public Task XpackRollupGetRollupCapsAsync(string index, GetRollupCapabilitiesRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken))
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequestAsync(GET, Url($"_xpack/rollup/data/{index.NotNull("index")}"), ctx, null, _params(requestParameters));
+		///GET on /_xpack/rollup/data/ https://www.elastic.co/guide/en/elasticsearch/reference/current/rollup-get-rollup-caps.html
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		public TResponse XpackRollupGetRollupCaps(GetRollupCapabilitiesRequestParameters requestParameters = null)
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequest(GET, Url($"_xpack/rollup/data/"), null, _params(requestParameters));
+		///GET on /_xpack/rollup/data/ https://www.elastic.co/guide/en/elasticsearch/reference/current/rollup-get-rollup-caps.html
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		public Task XpackRollupGetRollupCapsAsync(GetRollupCapabilitiesRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken))
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequestAsync(GET, Url($"_xpack/rollup/data/"), ctx, null, _params(requestParameters));
+		///PUT on /_xpack/rollup/job/{id} 
+		///The ID of the job to create
+		///The job configuration
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		public TResponse XpackRollupPutJob(string id, PostData body, CreateRollupJobRequestParameters requestParameters = null)
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequest(PUT, Url($"_xpack/rollup/job/{id.NotNull("id")}"), body, _params(requestParameters));
+		///PUT on /_xpack/rollup/job/{id} 
+		///The ID of the job to create
+		///The job configuration
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		public Task XpackRollupPutJobAsync(string id, PostData body, CreateRollupJobRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken))
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequestAsync(PUT, Url($"_xpack/rollup/job/{id.NotNull("id")}"), ctx, body, _params(requestParameters));
+		///GET on /{index}/_rollup_search 
+		///The index or index-pattern (containing rollup or regular data) that should be searched
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		public TResponse XpackRollupRollupSearchGet(string index, RollupSearchRequestParameters requestParameters = null)
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequest(GET, Url($"{index.NotNull("index")}/_rollup_search"), null, _params(requestParameters));
+		///GET on /{index}/_rollup_search 
+		///The index or index-pattern (containing rollup or regular data) that should be searched
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		public Task XpackRollupRollupSearchGetAsync(string index, RollupSearchRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken))
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequestAsync(GET, Url($"{index.NotNull("index")}/_rollup_search"), ctx, null, _params(requestParameters));
+		///GET on /{index}/{type}/_rollup_search 
+		///The index or index-pattern (containing rollup or regular data) that should be searched
+		///The doc type inside the index
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		public TResponse XpackRollupRollupSearchGet(string index, string type, RollupSearchRequestParameters requestParameters = null)
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequest(GET, Url($"{index.NotNull("index")}/{type.NotNull("type")}/_rollup_search"), null, _params(requestParameters));
+		///GET on /{index}/{type}/_rollup_search 
+		///The index or index-pattern (containing rollup or regular data) that should be searched
+		///The doc type inside the index
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		public Task XpackRollupRollupSearchGetAsync(string index, string type, RollupSearchRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken))
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequestAsync(GET, Url($"{index.NotNull("index")}/{type.NotNull("type")}/_rollup_search"), ctx, null, _params(requestParameters));
+		///POST on /{index}/_rollup_search 
+		///The index or index-pattern (containing rollup or regular data) that should be searched
+		///The search request body
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		public TResponse XpackRollupRollupSearch(string index, PostData body, RollupSearchRequestParameters requestParameters = null)
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequest(POST, Url($"{index.NotNull("index")}/_rollup_search"), body, _params(requestParameters));
+		///POST on /{index}/_rollup_search 
+		///The index or index-pattern (containing rollup or regular data) that should be searched
+		///The search request body
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		public Task XpackRollupRollupSearchAsync(string index, PostData body, RollupSearchRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken))
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequestAsync(POST, Url($"{index.NotNull("index")}/_rollup_search"), ctx, body, _params(requestParameters));
+		///POST on /{index}/{type}/_rollup_search 
+		///The index or index-pattern (containing rollup or regular data) that should be searched
+		///The doc type inside the index
+		///The search request body
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		public TResponse XpackRollupRollupSearch(string index, string type, PostData body, RollupSearchRequestParameters requestParameters = null)
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequest(POST, Url($"{index.NotNull("index")}/{type.NotNull("type")}/_rollup_search"), body, _params(requestParameters));
+		///POST on /{index}/{type}/_rollup_search 
+		///The index or index-pattern (containing rollup or regular data) that should be searched
+		///The doc type inside the index
+		///The search request body
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		public Task XpackRollupRollupSearchAsync(string index, string type, PostData body, RollupSearchRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken))
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequestAsync(POST, Url($"{index.NotNull("index")}/{type.NotNull("type")}/_rollup_search"), ctx, body, _params(requestParameters));
+		///POST on /_xpack/rollup/job/{id}/_start 
+		///The ID of the job to start
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		public TResponse XpackRollupStartJob(string id, StartRollupJobRequestParameters requestParameters = null)
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequest(POST, Url($"_xpack/rollup/job/{id.NotNull("id")}/_start"), null, _params(requestParameters));
+		///POST on /_xpack/rollup/job/{id}/_start 
+		///The ID of the job to start
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		public Task XpackRollupStartJobAsync(string id, StartRollupJobRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken))
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequestAsync(POST, Url($"_xpack/rollup/job/{id.NotNull("id")}/_start"), ctx, null, _params(requestParameters));
+		///POST on /_xpack/rollup/job/{id}/_stop 
+		///The ID of the job to stop
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		public TResponse XpackRollupStopJob(string id, StopRollupJobRequestParameters requestParameters = null)
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequest(POST, Url($"_xpack/rollup/job/{id.NotNull("id")}/_stop"), null, _params(requestParameters));
+		///POST on /_xpack/rollup/job/{id}/_stop 
+		///The ID of the job to stop
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		public Task XpackRollupStopJobAsync(string id, StopRollupJobRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken))
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequestAsync(POST, Url($"_xpack/rollup/job/{id.NotNull("id")}/_stop"), ctx, null, _params(requestParameters));
 		///GET on /_xpack/security/_authenticate https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-authenticate.html
 		///A func that allows you to describe the querystring parameters & request specific connection settings.
 		public TResponse XpackSecurityAuthenticate(AuthenticateRequestParameters requestParameters = null)
diff --git a/src/Elasticsearch.Net/IElasticLowLevelClient.Generated.cs b/src/Elasticsearch.Net/IElasticLowLevelClient.Generated.cs
index e8d6de76aaf..e8fce529fa3 100644
--- a/src/Elasticsearch.Net/IElasticLowLevelClient.Generated.cs
+++ b/src/Elasticsearch.Net/IElasticLowLevelClient.Generated.cs
@@ -2872,6 +2872,108 @@ public partial interface IElasticLowLevelClient
 		///The name of the index
 		///A func that allows you to describe the querystring parameters & request specific connection settings.
 		Task XpackMigrationUpgradeAsync(string index, MigrationUpgradeRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken)) where TResponse : class, IElasticsearchResponse, new();
+		///DELETE on /_xpack/rollup/job/{id} 
+		///The ID of the job to delete
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		TResponse XpackRollupDeleteJob(string id, DeleteRollupJobRequestParameters requestParameters = null) where TResponse : class, IElasticsearchResponse, new();
+		///DELETE on /_xpack/rollup/job/{id} 
+		///The ID of the job to delete
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		Task XpackRollupDeleteJobAsync(string id, DeleteRollupJobRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken)) where TResponse : class, IElasticsearchResponse, new();
+		///GET on /_xpack/rollup/job/{id} 
+		///The ID of the job(s) to fetch. Accepts glob patterns, or left blank for all jobs
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		TResponse XpackRollupGetJobs(string id, GetRollupJobRequestParameters requestParameters = null) where TResponse : class, IElasticsearchResponse, new();
+		///GET on /_xpack/rollup/job/{id} 
+		///The ID of the job(s) to fetch. Accepts glob patterns, or left blank for all jobs
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		Task XpackRollupGetJobsAsync(string id, GetRollupJobRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken)) where TResponse : class, IElasticsearchResponse, new();
+		///GET on /_xpack/rollup/job/ 
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		TResponse XpackRollupGetJobs(GetRollupJobRequestParameters requestParameters = null) where TResponse : class, IElasticsearchResponse, new();
+		///GET on /_xpack/rollup/job/ 
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		Task XpackRollupGetJobsAsync(GetRollupJobRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken)) where TResponse : class, IElasticsearchResponse, new();
+		///GET on /_xpack/rollup/data/{index} https://www.elastic.co/guide/en/elasticsearch/reference/current/rollup-get-rollup-caps.html
+		/// Index, indices or index-pattern to return rollup capabilities for. _all may be used to fetch rollup capabilities from all job
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		TResponse XpackRollupGetRollupCaps(string index, GetRollupCapabilitiesRequestParameters requestParameters = null) where TResponse : class, IElasticsearchResponse, new();
+		///GET on /_xpack/rollup/data/{index} https://www.elastic.co/guide/en/elasticsearch/reference/current/rollup-get-rollup-caps.html
+		/// Index, indices or index-pattern to return rollup capabilities for. _all may be used to fetch rollup capabilities from all job
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		Task XpackRollupGetRollupCapsAsync(string index, GetRollupCapabilitiesRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken)) where TResponse : class, IElasticsearchResponse, new();
+		///GET on /_xpack/rollup/data/ https://www.elastic.co/guide/en/elasticsearch/reference/current/rollup-get-rollup-caps.html
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		TResponse XpackRollupGetRollupCaps(GetRollupCapabilitiesRequestParameters requestParameters = null) where TResponse : class, IElasticsearchResponse, new();
+		///GET on /_xpack/rollup/data/ https://www.elastic.co/guide/en/elasticsearch/reference/current/rollup-get-rollup-caps.html
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		Task XpackRollupGetRollupCapsAsync(GetRollupCapabilitiesRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken)) where TResponse : class, IElasticsearchResponse, new();
+		///PUT on /_xpack/rollup/job/{id} 
+		///The ID of the job to create
+		///The job configuration
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		TResponse XpackRollupPutJob(string id, PostData body, CreateRollupJobRequestParameters requestParameters = null) where TResponse : class, IElasticsearchResponse, new();
+		///PUT on /_xpack/rollup/job/{id} 
+		///The ID of the job to create
+		///The job configuration
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		Task XpackRollupPutJobAsync(string id, PostData body, CreateRollupJobRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken)) where TResponse : class, IElasticsearchResponse, new();
+		///GET on /{index}/_rollup_search 
+		///The index or index-pattern (containing rollup or regular data) that should be searched
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		TResponse XpackRollupRollupSearchGet(string index, RollupSearchRequestParameters requestParameters = null) where TResponse : class, IElasticsearchResponse, new();
+		///GET on /{index}/_rollup_search 
+		///The index or index-pattern (containing rollup or regular data) that should be searched
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		Task XpackRollupRollupSearchGetAsync(string index, RollupSearchRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken)) where TResponse : class, IElasticsearchResponse, new();
+		///GET on /{index}/{type}/_rollup_search 
+		///The index or index-pattern (containing rollup or regular data) that should be searched
+		///The doc type inside the index
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		TResponse XpackRollupRollupSearchGet(string index, string type, RollupSearchRequestParameters requestParameters = null) where TResponse : class, IElasticsearchResponse, new();
+		///GET on /{index}/{type}/_rollup_search 
+		///The index or index-pattern (containing rollup or regular data) that should be searched
+		///The doc type inside the index
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		Task XpackRollupRollupSearchGetAsync(string index, string type, RollupSearchRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken)) where TResponse : class, IElasticsearchResponse, new();
+		///POST on /{index}/_rollup_search 
+		///The index or index-pattern (containing rollup or regular data) that should be searched
+		///The search request body
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		TResponse XpackRollupRollupSearch(string index, PostData body, RollupSearchRequestParameters requestParameters = null) where TResponse : class, IElasticsearchResponse, new();
+		///POST on /{index}/_rollup_search 
+		///The index or index-pattern (containing rollup or regular data) that should be searched
+		///The search request body
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		Task XpackRollupRollupSearchAsync(string index, PostData body, RollupSearchRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken)) where TResponse : class, IElasticsearchResponse, new();
+		///POST on /{index}/{type}/_rollup_search 
+		///The index or index-pattern (containing rollup or regular data) that should be searched
+		///The doc type inside the index
+		///The search request body
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		TResponse XpackRollupRollupSearch(string index, string type, PostData body, RollupSearchRequestParameters requestParameters = null) where TResponse : class, IElasticsearchResponse, new();
+		///POST on /{index}/{type}/_rollup_search 
+		///The index or index-pattern (containing rollup or regular data) that should be searched
+		///The doc type inside the index
+		///The search request body
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		Task XpackRollupRollupSearchAsync(string index, string type, PostData body, RollupSearchRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken)) where TResponse : class, IElasticsearchResponse, new();
+		///POST on /_xpack/rollup/job/{id}/_start 
+		///The ID of the job to start
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		TResponse XpackRollupStartJob(string id, StartRollupJobRequestParameters requestParameters = null) where TResponse : class, IElasticsearchResponse, new();
+		///POST on /_xpack/rollup/job/{id}/_start 
+		///The ID of the job to start
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		Task XpackRollupStartJobAsync(string id, StartRollupJobRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken)) where TResponse : class, IElasticsearchResponse, new();
+		///POST on /_xpack/rollup/job/{id}/_stop 
+		///The ID of the job to stop
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		TResponse XpackRollupStopJob(string id, StopRollupJobRequestParameters requestParameters = null) where TResponse : class, IElasticsearchResponse, new();
+		///POST on /_xpack/rollup/job/{id}/_stop 
+		///The ID of the job to stop
+		///A func that allows you to describe the querystring parameters & request specific connection settings.
+		Task XpackRollupStopJobAsync(string id, StopRollupJobRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken)) where TResponse : class, IElasticsearchResponse, new();
 		///GET on /_xpack/security/_authenticate https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-authenticate.html
 		///A func that allows you to describe the querystring parameters & request specific connection settings.
 		TResponse XpackSecurityAuthenticate(AuthenticateRequestParameters requestParameters = null) where TResponse : class, IElasticsearchResponse, new();
diff --git a/src/Nest/Document/Multiple/BulkAll/BulkAllObservable.cs b/src/Nest/Document/Multiple/BulkAll/BulkAllObservable.cs
index 736aac502cd..5014575e3f0 100644
--- a/src/Nest/Document/Multiple/BulkAll/BulkAllObservable.cs
+++ b/src/Nest/Document/Multiple/BulkAll/BulkAllObservable.cs
@@ -89,8 +89,10 @@ private void OnCompleted(Exception exception, IObserver observ
 
 		private void RefreshOnCompleted()
 		{
-			if (!this._partionedBulkRequest.RefreshOnCompleted) return;
-			var refresh = this._client.Refresh(this._partionedBulkRequest.Index);
+			if (!this._partionedBulkRequest.RefreshOnCompleted ) return;
+			var indices = this._partionedBulkRequest.RefreshIndices ?? this._partionedBulkRequest.Index;
+			if (indices == null) return;
+			var refresh = this._client.Refresh(indices);
 			if (!refresh.IsValid) throw Throw($"Refreshing after all documents have indexed failed", refresh.ApiCall);
 		}
 
diff --git a/src/Nest/Document/Multiple/BulkAll/BulkAllRequest.cs b/src/Nest/Document/Multiple/BulkAll/BulkAllRequest.cs
index 9876f5e0356..883c12d0fe2 100644
--- a/src/Nest/Document/Multiple/BulkAll/BulkAllRequest.cs
+++ b/src/Nest/Document/Multiple/BulkAll/BulkAllRequest.cs
@@ -41,11 +41,17 @@ public interface IBulkAllRequest where T : class
 		int? WaitForActiveShards { get; set; }
 
 		///Refresh the index after performing each operation (elasticsearch will refresh locally)
+		[Obsolete("This option is scheduled for deletion in 7.0, refreshing on each _bulk makes little sense for BulkAll")]
 		Refresh? Refresh { get; set; }
 
-		///Refresh the index after performing ALL the bulk operations (NOTE this is an additional request)
+		///
+		/// Refresh the index after performing ALL the bulk operations (NOTE this is an additional request)
+		/// 
 		bool RefreshOnCompleted { get; set; }
 
+		///The indices you wish to refresh after the bulk all completes, defaults to  
+		Indices RefreshIndices { get; set; }
+
 		///Specific per bulk operation routing value
 		Routing Routing { get; set; }
 
@@ -111,6 +117,8 @@ public class BulkAllRequest  : IBulkAllRequest
 		/// 
 		public Refresh? Refresh { get; set; }
 		/// 
+		public Indices RefreshIndices { get; set; }
+		/// 
 		public bool RefreshOnCompleted { get; set; }
 		/// 
 		public Routing Routing { get; set; }
@@ -151,6 +159,7 @@ public class BulkAllDescriptor : DescriptorBase, IBulkAl
 		TypeName IBulkAllRequest.Type { get; set; }
 		int? IBulkAllRequest.WaitForActiveShards { get; set; }
 		Refresh? IBulkAllRequest.Refresh { get; set; }
+		Indices IBulkAllRequest.RefreshIndices { get; set; }
 		bool IBulkAllRequest.RefreshOnCompleted { get; set; }
 		Routing IBulkAllRequest.Routing { get; set; }
 		Time IBulkAllRequest.Timeout { get; set; }
@@ -200,6 +209,9 @@ public BulkAllDescriptor BackOffRetries(int? backoffs) =>
 		/// 
 		public BulkAllDescriptor Refresh(Refresh? refresh) => Assign(p => p.Refresh = refresh);
 
+		/// 
+		public BulkAllDescriptor RefreshIndices(Indices indicesToRefresh) => Assign(a => a.RefreshIndices = indicesToRefresh);
+
 		/// 
 		public BulkAllDescriptor Routing(Routing routing) => Assign(p => p.Routing = routing);
 
diff --git a/src/Nest/Nest.csproj b/src/Nest/Nest.csproj
index d65c806c16d..528848e6b29 100644
--- a/src/Nest/Nest.csproj
+++ b/src/Nest/Nest.csproj
@@ -4,6 +4,7 @@
   
     netstandard2.0;net461
     netstandard2.0
+    latest
   
   
     
diff --git a/src/Nest/Search/Search/SearchRequest.cs b/src/Nest/Search/Search/SearchRequest.cs
index a573cfe4cbe..4e570ad5219 100644
--- a/src/Nest/Search/Search/SearchRequest.cs
+++ b/src/Nest/Search/Search/SearchRequest.cs
@@ -201,9 +201,7 @@ public SearchDescriptor Aggregations(AggregationDictionary aggregations) =>
 		public SearchDescriptor Source(Func, ISourceFilter> selector) =>
 			Assign(a => a.Source = new Union(selector?.Invoke(new SourceFilterDescriptor())));
 
-		/// 
-		/// The number of hits to return. Defaults to 10.
-		/// 
+		///  The number of hits to return. Defaults to 10. 
 		public SearchDescriptor Size(int? size) => Assign(a => a.Size = size);
 
 		/// 
diff --git a/src/Nest/XPack/RollUp/CreateRollupJob/CreateRollupJobRequest.cs b/src/Nest/XPack/RollUp/CreateRollupJob/CreateRollupJobRequest.cs
new file mode 100644
index 00000000000..8799355582d
--- /dev/null
+++ b/src/Nest/XPack/RollUp/CreateRollupJob/CreateRollupJobRequest.cs
@@ -0,0 +1,90 @@
+using System;
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	/// 
+	/// This API enables you to create a rollup job. The job will be created in a STOPPED state, and must be started with the Start Job API.
+	/// You must have manage or manage_rollup cluster privileges to use this API.
+	/// 
+	[MapsApi("xpack.rollup.put_job.json")]
+	[JsonConverter(typeof(ReadAsTypeJsonConverter))]
+	public partial interface ICreateRollupJobRequest
+	{
+		///  The index, or index pattern, that you wish to rollup. Supports wildcard-style patterns (logstash-*). 
+		[JsonProperty("index_pattern")]
+		string IndexPattern { get; set; }
+
+		///  The index that you wish to store rollup results into. Can be shared with other rollup jobs. 
+		[JsonProperty("rollup_index")]
+		IndexName RollupIndex { get; set; }
+
+		///  A cron string which defines when the rollup job should be executed. 
+		[JsonProperty("cron")]
+		string Cron { get; set; }
+
+		/// 
+		/// The number of bucket results that should be processed on each iteration of the rollup indexer.
+		/// A larger value will tend to execute faster, but will require more memory during processing.
+		/// 
+		[JsonProperty("page_size")]
+		long? PageSize { get; set; }
+
+		///  Defines the grouping fields that are defined for this rollup job 
+		[JsonProperty("groups")]
+		IRollupGroupings Groups { get; set; }
+
+		///  Defines the metrics that should be collected for each grouping tuple 
+		[JsonProperty("metrics")]
+		IEnumerable Metrics { get; set; }
+	}
+
+	/// 
+	public partial class CreateRollupJobRequest
+	{
+		/// 
+		public string IndexPattern { get; set; }
+		/// 
+		public IndexName RollupIndex { get; set; }
+		/// 
+		public string Cron { get; set; }
+		/// 
+		public long? PageSize { get; set; }
+		/// 
+		public IRollupGroupings Groups { get; set; }
+		/// 
+		public IEnumerable Metrics { get; set; }
+	}
+
+	/// >
+	public partial class CreateRollupJobDescriptor where T : class
+	{
+		string ICreateRollupJobRequest.IndexPattern { get; set; }
+		IndexName ICreateRollupJobRequest.RollupIndex { get; set; }
+		string ICreateRollupJobRequest.Cron { get; set; }
+		long? ICreateRollupJobRequest.PageSize { get; set; }
+		IRollupGroupings ICreateRollupJobRequest.Groups { get; set; }
+		IEnumerable ICreateRollupJobRequest.Metrics { get; set; }
+
+		/// 
+		public CreateRollupJobDescriptor IndexPattern(string indexPattern) => Assign(a => a.IndexPattern = indexPattern);
+
+		/// 
+		public CreateRollupJobDescriptor RollupIndex(IndexName index) => Assign(a => a.RollupIndex = index);
+
+		/// 
+		public CreateRollupJobDescriptor Cron(string cron) => Assign(a => a.Cron = cron);
+
+		/// 
+		public CreateRollupJobDescriptor PageSize(long? pageSize) => Assign(a => a.PageSize = pageSize);
+
+		/// 
+		public CreateRollupJobDescriptor Groups(Func, IRollupGroupings> selector) =>
+			Assign(a => a.Groups = selector?.Invoke(new RollupGroupingsDescriptor()));
+
+		/// 
+		public CreateRollupJobDescriptor Metrics(Func, IPromise >> selector) =>
+			Assign(a => a.Metrics = selector?.Invoke(new RollupFieldMetricsDescriptor())?.Value);
+	}
+}
diff --git a/src/Nest/XPack/RollUp/CreateRollupJob/CreateRollupJobResponse.cs b/src/Nest/XPack/RollUp/CreateRollupJob/CreateRollupJobResponse.cs
new file mode 100644
index 00000000000..925573f2dea
--- /dev/null
+++ b/src/Nest/XPack/RollUp/CreateRollupJob/CreateRollupJobResponse.cs
@@ -0,0 +1,9 @@
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	[JsonObject]
+	public interface ICreateRollupJobResponse : IAcknowledgedResponse {}
+
+	public class CreateRollupJobResponse : AcknowledgedResponseBase, ICreateRollupJobResponse {}
+}
diff --git a/src/Nest/XPack/RollUp/CreateRollupJob/ElasticClient-CreateRollupJob.cs b/src/Nest/XPack/RollUp/CreateRollupJob/ElasticClient-CreateRollupJob.cs
new file mode 100644
index 00000000000..4b1760bcc38
--- /dev/null
+++ b/src/Nest/XPack/RollUp/CreateRollupJob/ElasticClient-CreateRollupJob.cs
@@ -0,0 +1,55 @@
+using System;
+using System.Threading.Tasks;
+using Elasticsearch.Net;
+using System.Threading;
+
+namespace Nest
+{
+	public partial interface IElasticClient
+	{
+		/// 
+		/// Creates a rollup job. The job will be created in a STOPPED state, and must be started with StartRollupJob API
+		/// 
+		ICreateRollupJobResponse CreateRollupJob(Id id, Func, ICreateRollupJobRequest> selector)
+			where T : class;
+
+		/// 
+		ICreateRollupJobResponse CreateRollupJob(ICreateRollupJobRequest request);
+
+		/// 
+		Task CreateRollupJobAsync(Id id,
+			Func, ICreateRollupJobRequest> selector, CancellationToken cancellationToken = default)
+			where T : class;
+
+		/// 
+		Task CreateRollupJobAsync(ICreateRollupJobRequest request, CancellationToken cancellationToken = default);
+	}
+
+	public partial class ElasticClient
+	{
+		/// 
+		public ICreateRollupJobResponse CreateRollupJob(Id id, Func, ICreateRollupJobRequest> selector)
+			where T : class => this.CreateRollupJob(selector.InvokeOrDefault(new CreateRollupJobDescriptor(id)));
+
+		/// 
+		public ICreateRollupJobResponse CreateRollupJob(ICreateRollupJobRequest request) =>
+			this.Dispatcher.Dispatch(
+				request,
+				(p, d) =>this.LowLevelDispatch.XpackRollupPutJobDispatch(p, d)
+			);
+
+		/// 
+		public Task CreateRollupJobAsync(
+			Id id, Func, ICreateRollupJobRequest> selector, CancellationToken cancellationToken = default
+		) where T : class =>
+			this.CreateRollupJobAsync(selector.InvokeOrDefault(new CreateRollupJobDescriptor(id)), cancellationToken);
+
+		/// 
+		public Task CreateRollupJobAsync(ICreateRollupJobRequest request, CancellationToken cancellationToken = default) =>
+			this.Dispatcher.DispatchAsync(
+				request,
+				cancellationToken,
+				(p, d, c) => this.LowLevelDispatch.XpackRollupPutJobDispatchAsync(p, d, c)
+			);
+	}
+}
diff --git a/src/Nest/XPack/RollUp/DeleteRollupJob/DeleteRollupJobRequest.cs b/src/Nest/XPack/RollUp/DeleteRollupJob/DeleteRollupJobRequest.cs
new file mode 100644
index 00000000000..f2e183d7a9a
--- /dev/null
+++ b/src/Nest/XPack/RollUp/DeleteRollupJob/DeleteRollupJobRequest.cs
@@ -0,0 +1,15 @@
+namespace Nest
+{
+	[MapsApi("xpack.rollup.delete_job.json")]
+	public partial interface IDeleteRollupJobRequest
+	{
+	}
+
+	public partial class DeleteRollupJobRequest
+	{
+	}
+
+	public partial class DeleteRollupJobDescriptor
+	{
+	}
+}
diff --git a/src/Nest/XPack/RollUp/DeleteRollupJob/DeleteRollupJobResponse.cs b/src/Nest/XPack/RollUp/DeleteRollupJob/DeleteRollupJobResponse.cs
new file mode 100644
index 00000000000..c23ce78dc1b
--- /dev/null
+++ b/src/Nest/XPack/RollUp/DeleteRollupJob/DeleteRollupJobResponse.cs
@@ -0,0 +1,9 @@
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	[JsonObject]
+	public interface IDeleteRollupJobResponse : IAcknowledgedResponse {}
+
+	public class DeleteRollupJobResponse : AcknowledgedResponseBase, IDeleteRollupJobResponse {}
+}
diff --git a/src/Nest/XPack/RollUp/DeleteRollupJob/ElasticClient-DeleteRollupJob.cs b/src/Nest/XPack/RollUp/DeleteRollupJob/ElasticClient-DeleteRollupJob.cs
new file mode 100644
index 00000000000..9d5c92a089b
--- /dev/null
+++ b/src/Nest/XPack/RollUp/DeleteRollupJob/ElasticClient-DeleteRollupJob.cs
@@ -0,0 +1,54 @@
+using System;
+using System.Threading.Tasks;
+using Elasticsearch.Net;
+using System.Threading;
+
+namespace Nest
+{
+	public partial interface IElasticClient
+	{
+		/// 
+		/// Deletes an existing rollup job. The job can be started or stopped, in both cases it will be deleted.
+		/// Attempting to delete a non-existing job will throw an exception
+		/// 
+		IDeleteRollupJobResponse DeleteRollupJob(Id id, Func selector = null);
+
+		/// 
+		IDeleteRollupJobResponse DeleteRollupJob(IDeleteRollupJobRequest request);
+
+		/// 
+		Task DeleteRollupJobAsync(Id id,
+			Func selector = null, CancellationToken cancellationToken = default);
+
+		/// 
+		Task DeleteRollupJobAsync(IDeleteRollupJobRequest request, CancellationToken cancellationToken = default);
+	}
+
+	public partial class ElasticClient
+	{
+		/// 
+		public IDeleteRollupJobResponse DeleteRollupJob(Id id, Func selector = null) =>
+			this.DeleteRollupJob(selector.InvokeOrDefault(new DeleteRollupJobDescriptor(id)));
+
+		/// 
+		public IDeleteRollupJobResponse DeleteRollupJob(IDeleteRollupJobRequest request) =>
+			this.Dispatcher.Dispatch(
+				request,
+				(p, d) =>this.LowLevelDispatch.XpackRollupDeleteJobDispatch(p)
+			);
+
+		/// 
+		public Task DeleteRollupJobAsync(
+			Id id, Func selector = null, CancellationToken cancellationToken = default
+		)  =>
+			this.DeleteRollupJobAsync(selector.InvokeOrDefault(new DeleteRollupJobDescriptor(id)), cancellationToken);
+
+		/// 
+		public Task DeleteRollupJobAsync(IDeleteRollupJobRequest request, CancellationToken cancellationToken = default) =>
+			this.Dispatcher.DispatchAsync(
+				request,
+				cancellationToken,
+				(p, d, c) => this.LowLevelDispatch.XpackRollupDeleteJobDispatchAsync(p, c)
+			);
+	}
+}
diff --git a/src/Nest/XPack/RollUp/GetRollupCapabilities/ElasticClient-GetRollupCapabilities.cs b/src/Nest/XPack/RollUp/GetRollupCapabilities/ElasticClient-GetRollupCapabilities.cs
new file mode 100644
index 00000000000..09b3fcb019c
--- /dev/null
+++ b/src/Nest/XPack/RollUp/GetRollupCapabilities/ElasticClient-GetRollupCapabilities.cs
@@ -0,0 +1,62 @@
+using System;
+using System.Threading.Tasks;
+using Elasticsearch.Net;
+using System.Threading;
+
+namespace Nest
+{
+	public partial interface IElasticClient
+	{
+		/// 
+		/// Gets the rollup capabilities that have been configured for an index or index pattern.
+		/// This API is useful because a rollup job is often configured to rollup only a subset of fields from the
+		/// source index. Furthermore, only certain aggregations can be configured for various fields,
+		/// leading to a limited subset of functionality depending on that configuration.
+		/// 
+		/// 
+		/// This API will allow you to inspect an index and determine:
+		/// 
+		/// 1. Does this index have associated rollup data somewhere in the cluster?
+		/// 
+		/// 2. If yes to the first question, what fields were rolled up, what aggregations can be performed, and where does the data live?
+		/// 
+		IGetRollupCapabilitiesResponse GetRollupCapabilities(Func selector = null);
+
+		/// 
+		IGetRollupCapabilitiesResponse GetRollupCapabilities(IGetRollupCapabilitiesRequest request);
+
+		/// 
+		Task GetRollupCapabilitiesAsync(
+			Func selector = null, CancellationToken cancellationToken = default(CancellationToken));
+
+		/// 
+		Task GetRollupCapabilitiesAsync(IGetRollupCapabilitiesRequest request, CancellationToken cancellationToken = default(CancellationToken));
+	}
+
+	public partial class ElasticClient
+	{
+		/// 
+		public IGetRollupCapabilitiesResponse GetRollupCapabilities(Func selector = null) =>
+			this.GetRollupCapabilities(selector.InvokeOrDefault(new GetRollupCapabilitiesDescriptor()));
+
+		/// 
+		public IGetRollupCapabilitiesResponse GetRollupCapabilities(IGetRollupCapabilitiesRequest request) =>
+			this.Dispatcher.Dispatch(
+				request,
+				(p, d) => this.LowLevelDispatch.XpackRollupGetRollupCapsDispatch(p)
+			);
+
+		/// 
+		public Task GetRollupCapabilitiesAsync(
+			Func selector = null, CancellationToken cancellationToken = default(CancellationToken))  =>
+			this.GetRollupCapabilitiesAsync(selector.InvokeOrDefault(new GetRollupCapabilitiesDescriptor()), cancellationToken);
+
+		/// 
+		public Task GetRollupCapabilitiesAsync(IGetRollupCapabilitiesRequest request, CancellationToken cancellationToken = default(CancellationToken)) =>
+			this.Dispatcher.DispatchAsync(
+				request,
+				cancellationToken,
+				(p, d, c) => this.LowLevelDispatch.XpackRollupGetRollupCapsDispatchAsync(p, c)
+			);
+	}
+}
diff --git a/src/Nest/XPack/RollUp/GetRollupCapabilities/GetRollupCapabilitiesRequest.cs b/src/Nest/XPack/RollUp/GetRollupCapabilities/GetRollupCapabilitiesRequest.cs
new file mode 100644
index 00000000000..58c928a7c70
--- /dev/null
+++ b/src/Nest/XPack/RollUp/GetRollupCapabilities/GetRollupCapabilitiesRequest.cs
@@ -0,0 +1,15 @@
+namespace Nest
+{
+	[MapsApi("xpack.rollup.get_rollup_caps.json")]
+	public partial interface IGetRollupCapabilitiesRequest
+	{
+	}
+
+	public partial class GetRollupCapabilitiesRequest
+	{
+	}
+
+	public partial class GetRollupCapabilitiesDescriptor
+	{
+	}
+}
diff --git a/src/Nest/XPack/RollUp/GetRollupCapabilities/GetRollupCapabilitiesResponse.cs b/src/Nest/XPack/RollUp/GetRollupCapabilities/GetRollupCapabilitiesResponse.cs
new file mode 100644
index 00000000000..96088aa31b6
--- /dev/null
+++ b/src/Nest/XPack/RollUp/GetRollupCapabilities/GetRollupCapabilitiesResponse.cs
@@ -0,0 +1,62 @@
+using System;
+using System.Collections.Generic;
+using System.Linq.Expressions;
+using Elasticsearch.Net;
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	public interface IGetRollupCapabilitiesResponse : IResponse
+	{
+		IReadOnlyDictionary Indices { get; }
+	}
+
+	[JsonConverter(typeof(ResolvableDictionaryResponseJsonConverter))]
+	public class GetRollupCapabilitiesResponse : DictionaryResponseBase, IGetRollupCapabilitiesResponse
+	{
+		public IReadOnlyDictionary Indices => Self.BackingDictionary;
+	}
+
+	public class RollupCapabilities
+	{
+		[JsonProperty("rollup_jobs")]
+		public IReadOnlyCollection RollupJobs { get; internal set; }
+
+	}
+
+	public class RollupCapabilitiesJob
+	{
+		[JsonProperty("job_id")]
+		public string JobId { get; internal set; }
+
+		[JsonProperty("rollup_index")]
+		public string RollupIndex { get; internal set; }
+
+		[JsonProperty("index_pattern")]
+		public string IndexPattern { get; internal set; }
+
+		[JsonProperty("fields")]
+		public RollupFieldsCapabilitiesDictionary Fields { get; internal set; }
+	}
+
+	public class RollupFieldsCapabilities : IsADictionaryBase { }
+
+	[JsonConverter(typeof(RollupFieldsCapabilitiesDictionary.Converter))]
+	public class RollupFieldsCapabilitiesDictionary : ResolvableDictionaryProxy>
+	{
+		internal RollupFieldsCapabilitiesDictionary(IConnectionConfigurationValues c,
+			IReadOnlyDictionary> b) : base(c, b) { }
+
+		internal class Converter : ResolvableDictionaryJsonConverterBase
+			>
+		{
+			protected override RollupFieldsCapabilitiesDictionary Create(
+				IConnectionSettingsValues s, Dictionary> d
+			) => new RollupFieldsCapabilitiesDictionary(s, d);
+		}
+
+		public IReadOnlyCollection Field(Expression> selector) => this[selector];
+
+	}
+}
+
diff --git a/src/Nest/XPack/RollUp/GetRollupJob/ElasticClient-GetRollupJob.cs b/src/Nest/XPack/RollUp/GetRollupJob/ElasticClient-GetRollupJob.cs
new file mode 100644
index 00000000000..e8d5d17a06a
--- /dev/null
+++ b/src/Nest/XPack/RollUp/GetRollupJob/ElasticClient-GetRollupJob.cs
@@ -0,0 +1,56 @@
+using System;
+using System.Threading.Tasks;
+using Elasticsearch.Net;
+using System.Threading;
+
+namespace Nest
+{
+	public partial interface IElasticClient
+	{
+		/// 
+		/// Gets the configuration, stats and status of rollup jobs.
+		/// It can return the details for a single job, or for all jobs.
+		/// 
+ 		/// This API only returns active (both STARTED and STOPPED) jobs. If a job was created,
+ 		/// ran for a while then deleted, this API will not return any details about that job.
+		/// 
+		IGetRollupJobResponse GetRollupJob(Func selector = null);
+
+		/// 
+		IGetRollupJobResponse GetRollupJob(IGetRollupJobRequest request);
+
+		/// 
+		Task GetRollupJobAsync(
+			Func selector = null, CancellationToken cancellationToken = default(CancellationToken));
+
+		/// 
+		Task GetRollupJobAsync(IGetRollupJobRequest request, CancellationToken cancellationToken = default(CancellationToken));
+	}
+
+	public partial class ElasticClient
+	{
+		/// 
+		public IGetRollupJobResponse GetRollupJob(Func selector = null) =>
+			this.GetRollupJob(selector.InvokeOrDefault(new GetRollupJobDescriptor()));
+
+		/// 
+		public IGetRollupJobResponse GetRollupJob(IGetRollupJobRequest request) =>
+			this.Dispatcher.Dispatch(
+				request,
+				(p, d) =>this.LowLevelDispatch.XpackRollupGetJobsDispatch(p)
+			);
+
+		/// 
+		public Task GetRollupJobAsync(
+			Func selector = null, CancellationToken cancellationToken = default(CancellationToken))  =>
+			this.GetRollupJobAsync(selector.InvokeOrDefault(new GetRollupJobDescriptor()), cancellationToken);
+
+		/// 
+		public Task GetRollupJobAsync(IGetRollupJobRequest request, CancellationToken cancellationToken = default(CancellationToken)) =>
+			this.Dispatcher.DispatchAsync(
+				request,
+				cancellationToken,
+				(p, d, c) => this.LowLevelDispatch.XpackRollupGetJobsDispatchAsync(p, c)
+			);
+	}
+}
diff --git a/src/Nest/XPack/RollUp/GetRollupJob/GetRollupJobRequest.cs b/src/Nest/XPack/RollUp/GetRollupJob/GetRollupJobRequest.cs
new file mode 100644
index 00000000000..528d53bfa59
--- /dev/null
+++ b/src/Nest/XPack/RollUp/GetRollupJob/GetRollupJobRequest.cs
@@ -0,0 +1,15 @@
+namespace Nest
+{
+	[MapsApi("xpack.rollup.get_jobs.json")]
+	public partial interface IGetRollupJobRequest
+	{
+	}
+
+	public partial class GetRollupJobRequest
+	{
+	}
+
+	public partial class GetRollupJobDescriptor
+	{
+	}
+}
diff --git a/src/Nest/XPack/RollUp/GetRollupJob/GetRollupJobResponse.cs b/src/Nest/XPack/RollUp/GetRollupJob/GetRollupJobResponse.cs
new file mode 100644
index 00000000000..0a266cbb045
--- /dev/null
+++ b/src/Nest/XPack/RollUp/GetRollupJob/GetRollupJobResponse.cs
@@ -0,0 +1,17 @@
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	public interface IGetRollupJobResponse : IResponse
+	{
+		[JsonProperty("jobs")]
+		IReadOnlyCollection Jobs { get; }
+	}
+
+	public class GetRollupJobResponse : ResponseBase, IGetRollupJobResponse
+	{
+		public IReadOnlyCollection Jobs { get; internal set; } = EmptyReadOnly.Collection;
+	}
+}
+
diff --git a/src/Nest/XPack/RollUp/GetRollupJob/IndexingJobState.cs b/src/Nest/XPack/RollUp/GetRollupJob/IndexingJobState.cs
new file mode 100644
index 00000000000..d30a5692ed7
--- /dev/null
+++ b/src/Nest/XPack/RollUp/GetRollupJob/IndexingJobState.cs
@@ -0,0 +1,25 @@
+using System.Runtime.Serialization;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
+
+namespace Nest
+{
+	[JsonConverter(typeof(StringEnumConverter))]
+	public enum IndexingJobState
+	{
+		///  Indexer is running, but not actively indexing data (e.g. it's idle) 
+		[EnumMember(Value="started")] Started,
+
+		///  Indexer is actively indexing data 
+		[EnumMember(Value="indexing")] Indexing,
+
+		///  Transition state to where an indexer has acknowledged the stop but is still in process of halting 
+		[EnumMember(Value="stopping")] Stopping,
+
+		///  Indexer is "paused" and ignoring scheduled triggers 
+		[EnumMember(Value="stopped")] Stopped,
+
+		///  Something (internal or external) has requested the indexer abort and shutdown 
+		[EnumMember(Value="aborting")] Aborting
+	}
+}
\ No newline at end of file
diff --git a/src/Nest/XPack/RollUp/GetRollupJob/RollupJobConfiguration.cs b/src/Nest/XPack/RollUp/GetRollupJob/RollupJobConfiguration.cs
new file mode 100644
index 00000000000..f8a4379459b
--- /dev/null
+++ b/src/Nest/XPack/RollUp/GetRollupJob/RollupJobConfiguration.cs
@@ -0,0 +1,39 @@
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	public class RollupJobConfiguration
+	{
+		[JsonProperty("id")]
+		public string Id { get; internal set; }
+
+		[JsonProperty("timeout")]
+		public Time Timeout { get; internal set; }
+
+		/// 
+		[JsonProperty("index_pattern")]
+		public string IndexPattern { get; internal set; }
+
+		/// 
+		[JsonProperty("rollup_index")]
+		public IndexName RollupIndex { get; internal set; }
+
+		/// 
+		[JsonProperty("cron")]
+		public string Cron { get; internal set; }
+
+		/// 
+		[JsonProperty("page_size")]
+		public long? PageSize { get; internal set; }
+
+		/// 
+		[JsonProperty("groups")]
+		public IRollupGroupings Groups { get; internal set; }
+
+		/// 
+		[JsonProperty("metrics")]
+		public IEnumerable Metrics { get; internal set; }
+
+	}
+}
diff --git a/src/Nest/XPack/RollUp/GetRollupJob/RollupJobInformation.cs b/src/Nest/XPack/RollUp/GetRollupJob/RollupJobInformation.cs
new file mode 100644
index 00000000000..59581781ed5
--- /dev/null
+++ b/src/Nest/XPack/RollUp/GetRollupJob/RollupJobInformation.cs
@@ -0,0 +1,16 @@
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	public class RollupJobInformation
+	{
+		[JsonProperty("config")]
+		public RollupJobConfiguration Config { get; internal set; }
+
+		[JsonProperty("status")]
+		public RollupJobStatus Status { get; internal set; }
+
+		[JsonProperty("stats")]
+		public RollupJobStats Stats { get; internal set; }
+	}
+}
diff --git a/src/Nest/XPack/RollUp/GetRollupJob/RollupJobStats.cs b/src/Nest/XPack/RollUp/GetRollupJob/RollupJobStats.cs
new file mode 100644
index 00000000000..d68f89f5293
--- /dev/null
+++ b/src/Nest/XPack/RollUp/GetRollupJob/RollupJobStats.cs
@@ -0,0 +1,19 @@
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	public class RollupJobStats
+	{
+		[JsonProperty("pages_processed")]
+		public long PagesProcessed { get; internal set; }
+
+		[JsonProperty("documents_processed")]
+		public long DocumentsProcessed { get; internal set; }
+
+		[JsonProperty("rollups_indexed")]
+		public long RollupsIndexed { get;  internal set; }
+
+		[JsonProperty("trigger_count")]
+		public long TriggerCount { get; internal set; }
+	}
+}
diff --git a/src/Nest/XPack/RollUp/GetRollupJob/RollupJobStatus.cs b/src/Nest/XPack/RollUp/GetRollupJob/RollupJobStatus.cs
new file mode 100644
index 00000000000..d2ce154c755
--- /dev/null
+++ b/src/Nest/XPack/RollUp/GetRollupJob/RollupJobStatus.cs
@@ -0,0 +1,18 @@
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	public class RollupJobStatus
+	{
+		[JsonProperty("job_state")]
+		public IndexingJobState JobState { get; internal set; }
+
+		[JsonProperty("current_position")]
+		public IReadOnlyDictionary CurrentPosition { get; internal set; } =
+			EmptyReadOnly.Dictionary;
+
+		[JsonProperty("upgraded_doc_id")]
+		public bool UpgradedDocId { get; internal set; }
+	}
+}
diff --git a/src/Nest/XPack/RollUp/RollupConfiguration/DateHistogramRollupGrouping.cs b/src/Nest/XPack/RollUp/RollupConfiguration/DateHistogramRollupGrouping.cs
new file mode 100644
index 00000000000..9abfb6ee3a9
--- /dev/null
+++ b/src/Nest/XPack/RollUp/RollupConfiguration/DateHistogramRollupGrouping.cs
@@ -0,0 +1,82 @@
+using System;
+using System.Linq.Expressions;
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	/// 
+	/// A date_histogram group aggregates a date field into time-based buckets. The date_histogram group is mandatory ,
+	/// you currently cannot rollup documents without a timestamp and a date_histogram group
+	/// 
+	[JsonConverter(typeof(ReadAsTypeJsonConverter))]
+	public interface IDateHistogramRollupGrouping
+	{
+		/// 
+		/// The date field that is to be rolled up
+		/// 
+		[JsonProperty("field")]
+		Field Field { get; set; }
+
+		/// 
+		/// The interval of time buckets to be generated when rolling up. E.g. "60m" will produce 60 minute (hourly) rollups.
+		/// The interval defines the minimum interval that can be aggregated only.
+		/// 
+		[JsonProperty("interval")]
+		Time Interval { get; set; }
+
+		/// 
+		/// How long to wait before rolling up new documents. By default, the indexer attempts to roll up all data that is available.
+		/// 
+		[JsonProperty("delay")]
+		Time Delay { get; set; }
+
+		/// 
+		/// Defines what time_zone the rollup documents are stored as. Unlike raw data, which can shift timezones on the fly, rolled
+		/// documents have to be stored with a specific timezone. By default, rollup documents are stored in UT
+		/// 
+		[JsonProperty("time_zone")]
+		string TimeZone { get; set; }
+	}
+
+	/// 
+	public class DateHistogramRollupGrouping : IDateHistogramRollupGrouping
+	{
+		/// 
+		public Field Field { get; set; }
+
+		/// 
+		public Time Interval { get; set; }
+
+		/// 
+		public Time Delay { get; set; }
+
+		/// 
+		public string TimeZone { get; set; }
+	}
+
+	/// 
+	public class DateHistogramRollupGroupingDescriptor
+		: DescriptorBase, IDateHistogramRollupGrouping>, IDateHistogramRollupGrouping
+		where T : class
+	{
+		Field IDateHistogramRollupGrouping.Field { get; set; }
+		Time IDateHistogramRollupGrouping.Interval { get; set; }
+		Time IDateHistogramRollupGrouping.Delay { get; set; }
+		string IDateHistogramRollupGrouping.TimeZone { get; set; }
+
+		/// 
+		public DateHistogramRollupGroupingDescriptor Field(Field field) => Assign(a => a.Field = field);
+
+		/// 
+		public DateHistogramRollupGroupingDescriptor Field(Expression> field) => Assign(a => a.Field = field);
+
+		/// 
+		public DateHistogramRollupGroupingDescriptor Interval(Time interval) => Assign(a => a.Interval = interval);
+
+		/// 
+		public DateHistogramRollupGroupingDescriptor Delay(Time delay) => Assign(a => a.Delay = delay);
+
+		/// 
+		public DateHistogramRollupGroupingDescriptor TimeZone(string timeZone) => Assign(a => a.TimeZone = timeZone);
+	}
+}
diff --git a/src/Nest/XPack/RollUp/RollupConfiguration/HistogramRollupGrouping.cs b/src/Nest/XPack/RollUp/RollupConfiguration/HistogramRollupGrouping.cs
new file mode 100644
index 00000000000..95fa229fbb7
--- /dev/null
+++ b/src/Nest/XPack/RollUp/RollupConfiguration/HistogramRollupGrouping.cs
@@ -0,0 +1,52 @@
+using System;
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	///  The histogram group aggregates one or more numeric fields into numeric histogram intervals. 
+	[JsonConverter(typeof(ReadAsTypeJsonConverter))]
+	public interface IHistogramRollupGrouping
+	{
+		/// 
+		/// The set of fields that you wish to build histograms for. All fields specified must be some kind of numeric. Order does not matter
+		/// 
+		[JsonProperty("fields")]
+		Fields Fields { get; set; }
+
+		/// 
+		/// The interval of histogram buckets to be generated when rolling up. Note that only one interval can be specified in the
+		/// histogram group, meaning that all fields being grouped via the histogram must share the same interval.
+		/// 
+		[JsonProperty("interval")]
+		long? Interval { get; set; }
+	}
+
+	/// 
+	public class HistogramRollupGrouping : IHistogramRollupGrouping
+	{
+		/// 
+		public Fields Fields { get; set; }
+
+		/// 
+		public long? Interval { get; set; }
+	}
+
+	/// 
+	public class HistogramRollupGroupingDescriptor
+		: DescriptorBase, IHistogramRollupGrouping>, IHistogramRollupGrouping
+		where T : class
+	{
+		Fields IHistogramRollupGrouping.Fields { get; set; }
+		long? IHistogramRollupGrouping.Interval { get; set; }
+
+		/// 
+		public HistogramRollupGroupingDescriptor Fields(Func, IPromise> fields) =>
+			Assign(a => a.Fields = fields?.Invoke(new FieldsDescriptor())?.Value);
+
+		/// 
+		public HistogramRollupGroupingDescriptor Fields(Fields fields) => Assign(a => a.Fields = fields);
+
+		/// 
+		public HistogramRollupGroupingDescriptor Interval(long? interval) => Assign(a => a.Interval = interval);
+	}
+}
diff --git a/src/Nest/XPack/RollUp/RollupConfiguration/IRollupGroupings.cs b/src/Nest/XPack/RollUp/RollupConfiguration/IRollupGroupings.cs
new file mode 100644
index 00000000000..0b3f422b255
--- /dev/null
+++ b/src/Nest/XPack/RollUp/RollupConfiguration/IRollupGroupings.cs
@@ -0,0 +1,58 @@
+using System;
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	/// 
+	/// The groups section of the configuration is where you decide which fields should be grouped on, and with what aggregations.
+	/// These fields will then be available later for aggregating into buckets
+	/// 
+	[JsonConverter(typeof(ReadAsTypeJsonConverter))]
+	public interface IRollupGroupings
+	{
+		/// 
+		[JsonProperty("date_histogram")]
+		IDateHistogramRollupGrouping DateHistogram { get; set; }
+
+		/// 
+		[JsonProperty("histogram")]
+		IHistogramRollupGrouping Histogram { get; set; }
+
+		/// 
+		[JsonProperty("terms")]
+		ITermsRollupGrouping Terms { get; set; }
+	}
+
+	/// 
+	public class RollupGroupings : IRollupGroupings
+	{
+		/// 
+		public IDateHistogramRollupGrouping DateHistogram { get; set; }
+		/// 
+		public IHistogramRollupGrouping Histogram { get; set; }
+		/// 
+		public ITermsRollupGrouping Terms { get; set; }
+	}
+
+	/// 
+	public class RollupGroupingsDescriptor : DescriptorBase, IRollupGroupings>, IRollupGroupings
+		where T : class
+	{
+		IDateHistogramRollupGrouping IRollupGroupings.DateHistogram { get; set; }
+		IHistogramRollupGrouping IRollupGroupings.Histogram { get; set; }
+		ITermsRollupGrouping IRollupGroupings.Terms { get; set; }
+
+		/// 
+		public RollupGroupingsDescriptor DateHistogram(
+			Func, IDateHistogramRollupGrouping> selector) =>
+			Assign(a => a.DateHistogram = selector?.Invoke(new DateHistogramRollupGroupingDescriptor()));
+
+		/// 
+		public RollupGroupingsDescriptor Histogram(Func, IHistogramRollupGrouping> selector) =>
+			Assign(a => a.Histogram = selector?.Invoke(new HistogramRollupGroupingDescriptor()));
+
+		/// 
+		public RollupGroupingsDescriptor Terms(Func, ITermsRollupGrouping> selector) =>
+			Assign(a => a.Terms = selector?.Invoke(new TermsRollupGroupingDescriptor()));
+	}
+}
diff --git a/src/Nest/XPack/RollUp/RollupConfiguration/RollupFieldMetric.cs b/src/Nest/XPack/RollUp/RollupConfiguration/RollupFieldMetric.cs
new file mode 100644
index 00000000000..cfc9910f774
--- /dev/null
+++ b/src/Nest/XPack/RollUp/RollupConfiguration/RollupFieldMetric.cs
@@ -0,0 +1,40 @@
+using System;
+using System.Collections.Generic;
+using System.Linq.Expressions;
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	[JsonConverter(typeof(ReadAsTypeJsonConverter))]
+	public interface IRollupFieldMetric
+	{
+		[JsonProperty("field")]
+		Field Field { get; set; }
+		[JsonProperty("metrics")]
+		IEnumerable Metrics { get; set; }
+	}
+
+	public class RollupFieldMetric : IRollupFieldMetric
+	{
+		public Field Field { get; set; }
+		public IEnumerable Metrics { get; set; }
+	}
+
+	public class RollupFieldMetricsDescriptor : DescriptorPromiseBase, IList>
+		where T : class
+	{
+		public RollupFieldMetricsDescriptor() : base(new List()) { }
+
+		public RollupFieldMetricsDescriptor Field(Expression> field, params RollupMetric[] metrics) =>
+			Assign(a => a.Add(new RollupFieldMetric{ Field = field, Metrics = metrics}));
+
+		public RollupFieldMetricsDescriptor Field(Field field, params RollupMetric[] metrics) =>
+			Assign(a => a.Add(new RollupFieldMetric{ Field = field, Metrics = metrics}));
+
+		public RollupFieldMetricsDescriptor Field(Expression> field, IEnumerable metrics) =>
+			Assign(a => a.Add(new RollupFieldMetric{ Field = field, Metrics = metrics}));
+
+		public RollupFieldMetricsDescriptor Field(Field field, IEnumerable metrics) =>
+			Assign(a => a.Add(new RollupFieldMetric{ Field = field, Metrics = metrics}));
+	}
+}
diff --git a/src/Nest/XPack/RollUp/RollupConfiguration/RollupMetric.cs b/src/Nest/XPack/RollUp/RollupConfiguration/RollupMetric.cs
new file mode 100644
index 00000000000..8ae18c98f0f
--- /dev/null
+++ b/src/Nest/XPack/RollUp/RollupConfiguration/RollupMetric.cs
@@ -0,0 +1,16 @@
+using System.Runtime.Serialization;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
+
+namespace Nest
+{
+	[JsonConverter(typeof(StringEnumConverter))]
+	public enum RollupMetric
+	{
+		[EnumMember(Value = "min")] Min,
+		[EnumMember(Value = "max")] Max,
+		[EnumMember(Value = "sum")] Sum,
+		[EnumMember(Value = "avg")] Average,
+		[EnumMember(Value = "value_count")] ValueCount
+	}
+}
diff --git a/src/Nest/XPack/RollUp/RollupConfiguration/TermsRollupGrouping.cs b/src/Nest/XPack/RollUp/RollupConfiguration/TermsRollupGrouping.cs
new file mode 100644
index 00000000000..14317a542ab
--- /dev/null
+++ b/src/Nest/XPack/RollUp/RollupConfiguration/TermsRollupGrouping.cs
@@ -0,0 +1,43 @@
+using System;
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	/// 
+	/// The terms group can be used on keyword or numeric fields, to allow bucketing via the terms aggregation at a
+	/// later point. The terms group is optional. If defined, the indexer will enumerate and store all values of a field for
+	/// each time-period.
+	/// 
+	[JsonConverter(typeof(ReadAsTypeJsonConverter))]
+	public interface ITermsRollupGrouping
+	{
+		/// 
+		/// The set of fields that you wish to collect terms for. This array can contain fields that are both keyword and numerics.
+		/// Order does not matter
+		/// 
+		[JsonProperty("fields")]
+		Fields Fields { get; set; }
+	}
+
+	/// 
+	public class TermsRollupGrouping : ITermsRollupGrouping
+	{
+		/// 
+		public Fields Fields { get; set; }
+	}
+
+	/// 
+	public class TermsRollupGroupingDescriptor
+		: DescriptorBase, ITermsRollupGrouping>, ITermsRollupGrouping
+		where T : class
+	{
+		Fields ITermsRollupGrouping.Fields { get; set; }
+
+		/// 
+		public TermsRollupGroupingDescriptor Fields(Func, IPromise> fields) =>
+			Assign(a => a.Fields = fields?.Invoke(new FieldsDescriptor())?.Value);
+
+		/// 
+		public TermsRollupGroupingDescriptor Fields(Fields fields) => Assign(a => a.Fields = fields);
+	}
+}
diff --git a/src/Nest/XPack/RollUp/RollupSearch/ElasticClient-RollupSearch.cs b/src/Nest/XPack/RollUp/RollupSearch/ElasticClient-RollupSearch.cs
new file mode 100644
index 00000000000..272262ee1ba
--- /dev/null
+++ b/src/Nest/XPack/RollUp/RollupSearch/ElasticClient-RollupSearch.cs
@@ -0,0 +1,86 @@
+using System;
+using System.Threading.Tasks;
+using Elasticsearch.Net;
+using System.Threading;
+
+namespace Nest
+{
+	public partial interface IElasticClient
+	{
+		/// 
+		/// Searches rolled-up data using the standard query DSL. The Rollup Search endpoint is needed because,
+		/// internally, rolled-up documents utilize a different document structure than the original data.
+		/// The Rollup Search endpoint rewrites standard query DSL into a format that matches the rollup documents,
+		/// then takes the response and rewrites it back to what a client would expect given the original query.
+		/// 
+		IRollupSearchResponse RollupSearch(Indices indices, Func, IRollupSearchRequest> selector = null)
+			where THit : class;
+
+		/// 
+		IRollupSearchResponse RollupSearch(Indices indices, Func, IRollupSearchRequest> selector = null)
+			where THit : class
+			where T : class;
+
+		/// 
+		IRollupSearchResponse RollupSearch(IRollupSearchRequest request) where THit : class;
+
+		/// 
+		Task> RollupSearchAsync(Indices indices,
+			Func, IRollupSearchRequest> selector = null, CancellationToken cancellationToken = default)
+			where THit : class;
+
+		/// 
+		Task> RollupSearchAsync(Indices indices,
+			Func, IRollupSearchRequest> selector = null, CancellationToken cancellationToken = default)
+			where THit : class
+			where T : class;
+
+		/// 
+		Task> RollupSearchAsync(IRollupSearchRequest request, CancellationToken cancellationToken = default)
+			where THit : class;
+	}
+
+	public partial class ElasticClient
+	{
+		/// 
+		public IRollupSearchResponse RollupSearch(Indices indices, Func, IRollupSearchRequest> selector = null)
+			where THit : class =>
+			this.RollupSearch(selector.InvokeOrDefault(new RollupSearchDescriptor(indices)));
+
+		/// 
+		public IRollupSearchResponse RollupSearch(Indices indices, Func, IRollupSearchRequest> selector = null)
+			where T : class
+			where THit : class =>
+			this.RollupSearch(selector.InvokeOrDefault(new RollupSearchDescriptor(indices)));
+
+		/// 
+		public IRollupSearchResponse RollupSearch(IRollupSearchRequest request) where THit : class =>
+			this.Dispatcher.Dispatch>(
+				request,
+				(p, d) =>this.LowLevelDispatch.XpackRollupRollupSearchDispatch>(p, d)
+			);
+
+		/// 
+		public Task> RollupSearchAsync(
+			Indices indices, Func, IRollupSearchRequest> selector = null, CancellationToken cancellationToken = default
+		) where THit : class =>
+			this.RollupSearchAsync(selector.InvokeOrDefault(new RollupSearchDescriptor(indices)), cancellationToken);
+
+		/// 
+		public Task> RollupSearchAsync(
+			Indices indices, Func, IRollupSearchRequest> selector = null, CancellationToken cancellationToken = default
+		)
+			where T : class
+			where THit : class =>
+			this.RollupSearchAsync(selector.InvokeOrDefault(new RollupSearchDescriptor(indices)), cancellationToken);
+
+		/// 
+		public Task> RollupSearchAsync(IRollupSearchRequest request, CancellationToken cancellationToken = default)
+			where THit : class =>
+			this.Dispatcher.DispatchAsync, IRollupSearchResponse>(
+				request,
+				cancellationToken,
+				(p, d, c) => this.LowLevelDispatch.XpackRollupRollupSearchDispatchAsync>(p, d, c)
+			);
+	}
+}
diff --git a/src/Nest/XPack/RollUp/RollupSearch/RollupSearchRequest.cs b/src/Nest/XPack/RollUp/RollupSearch/RollupSearchRequest.cs
new file mode 100644
index 00000000000..cfe0cc132f8
--- /dev/null
+++ b/src/Nest/XPack/RollUp/RollupSearch/RollupSearchRequest.cs
@@ -0,0 +1,53 @@
+using System;
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	[MapsApi("xpack.rollup.rollup_search.json")]
+	public partial interface IRollupSearchRequest
+	{
+		/// When doing rollup searches against rolled up and live indices size needs to be set to 0 explicitly 
+		[JsonProperty("size")]
+		int? Size { get; set; }
+
+		///  Describe the query to perform using a query descriptor lambda
+		[JsonProperty("query")]
+		QueryContainer Query { get; set; }
+
+		///  Describe the aggregations to perform
+		[JsonProperty("aggs")]
+		AggregationDictionary Aggregations { get; set; }
+	}
+
+	public partial class RollupSearchRequest
+	{
+		/// 
+		public int? Size { get; set; }
+		/// 
+		public QueryContainer Query { get; set; }
+		/// 
+		public AggregationDictionary Aggregations { get; set; }
+	}
+
+	public partial class RollupSearchDescriptor where T : class
+	{
+		QueryContainer IRollupSearchRequest.Query { get; set; }
+		AggregationDictionary IRollupSearchRequest.Aggregations { get; set; }
+		int? IRollupSearchRequest.Size { get; set; }
+
+		/// 
+		public RollupSearchDescriptor Size(int? size) => Assign(a => a.Size = size);
+
+		/// 
+		public RollupSearchDescriptor Aggregations(Func, IAggregationContainer> aggregationsSelector) =>
+			Assign(a => a.Aggregations = aggregationsSelector(new AggregationContainerDescriptor())?.Aggregations);
+
+		/// 
+		public RollupSearchDescriptor Aggregations(AggregationDictionary aggregations) =>
+			Assign(a => a.Aggregations = aggregations);
+
+		/// 
+		public RollupSearchDescriptor Query(Func, QueryContainer> query) =>
+			Assign(a => a.Query = query?.Invoke(new QueryContainerDescriptor()));
+	}
+}
diff --git a/src/Nest/XPack/RollUp/RollupSearch/RollupSearchResponse.cs b/src/Nest/XPack/RollUp/RollupSearch/RollupSearchResponse.cs
new file mode 100644
index 00000000000..14558bfeda6
--- /dev/null
+++ b/src/Nest/XPack/RollUp/RollupSearch/RollupSearchResponse.cs
@@ -0,0 +1,13 @@
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	public interface IRollupSearchResponse : ISearchResponse where T : class
+	{
+	}
+
+	public class RollupSearchResponse : SearchResponse, IRollupSearchResponse
+		where T : class
+	{
+	}
+}
diff --git a/src/Nest/XPack/RollUp/StartRollupJob/ElasticClient-StartRollupJob.cs b/src/Nest/XPack/RollUp/StartRollupJob/ElasticClient-StartRollupJob.cs
new file mode 100644
index 00000000000..c9e41d11936
--- /dev/null
+++ b/src/Nest/XPack/RollUp/StartRollupJob/ElasticClient-StartRollupJob.cs
@@ -0,0 +1,53 @@
+using System;
+using System.Threading.Tasks;
+using Elasticsearch.Net;
+using System.Threading;
+
+namespace Nest
+{
+	public partial interface IElasticClient
+	{
+		/// Starts an existing, stopped rollup job. If the job does not exist an exception will be thrown.
+		/// Starting an already started job has no action.
+		/// 
+		IStartRollupJobResponse StartRollupJob(Id id, Func selector = null);
+
+		/// 
+		IStartRollupJobResponse StartRollupJob(IStartRollupJobRequest request);
+
+		/// 
+		Task StartRollupJobAsync(Id id,
+			Func selector = null, CancellationToken cancellationToken = default);
+
+		/// 
+		Task StartRollupJobAsync(IStartRollupJobRequest request, CancellationToken cancellationToken = default);
+	}
+
+	public partial class ElasticClient
+	{
+		/// 
+		public IStartRollupJobResponse StartRollupJob(Id id, Func selector = null) =>
+			this.StartRollupJob(selector.InvokeOrDefault(new StartRollupJobDescriptor(id)));
+
+		/// 
+		public IStartRollupJobResponse StartRollupJob(IStartRollupJobRequest request) =>
+			this.Dispatcher.Dispatch(
+				request,
+				(p, d) =>this.LowLevelDispatch.XpackRollupStartJobDispatch(p)
+			);
+
+		/// 
+		public Task StartRollupJobAsync(
+			Id id, Func selector = null, CancellationToken cancellationToken = default
+		)  =>
+			this.StartRollupJobAsync(selector.InvokeOrDefault(new StartRollupJobDescriptor(id)), cancellationToken);
+
+		/// 
+		public Task StartRollupJobAsync(IStartRollupJobRequest request, CancellationToken cancellationToken = default) =>
+			this.Dispatcher.DispatchAsync(
+				request,
+				cancellationToken,
+				(p, d, c) => this.LowLevelDispatch.XpackRollupStartJobDispatchAsync(p, c)
+			);
+	}
+}
diff --git a/src/Nest/XPack/RollUp/StartRollupJob/StartRollupJobRequest.cs b/src/Nest/XPack/RollUp/StartRollupJob/StartRollupJobRequest.cs
new file mode 100644
index 00000000000..609de02458f
--- /dev/null
+++ b/src/Nest/XPack/RollUp/StartRollupJob/StartRollupJobRequest.cs
@@ -0,0 +1,15 @@
+namespace Nest
+{
+	[MapsApi("xpack.rollup.start_job.json")]
+	public partial interface IStartRollupJobRequest
+	{
+	}
+
+	public partial class StartRollupJobRequest
+	{
+	}
+
+	public partial class StartRollupJobDescriptor
+	{
+	}
+}
diff --git a/src/Nest/XPack/RollUp/StartRollupJob/StartRollupJobResponse.cs b/src/Nest/XPack/RollUp/StartRollupJob/StartRollupJobResponse.cs
new file mode 100644
index 00000000000..f64e8b20b13
--- /dev/null
+++ b/src/Nest/XPack/RollUp/StartRollupJob/StartRollupJobResponse.cs
@@ -0,0 +1,16 @@
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	[JsonObject]
+	public interface IStartRollupJobResponse : IResponse
+	{
+		[JsonProperty("started")]
+		bool Started { get; set; }
+	}
+
+	public class StartRollupJobResponse : ResponseBase, IStartRollupJobResponse
+	{
+		public bool Started { get; set; }
+	}
+}
diff --git a/src/Nest/XPack/RollUp/StopRollupJob/ElasticClient-StopRollupJob.cs b/src/Nest/XPack/RollUp/StopRollupJob/ElasticClient-StopRollupJob.cs
new file mode 100644
index 00000000000..c83e78f60f3
--- /dev/null
+++ b/src/Nest/XPack/RollUp/StopRollupJob/ElasticClient-StopRollupJob.cs
@@ -0,0 +1,54 @@
+using System;
+using System.Threading.Tasks;
+using Elasticsearch.Net;
+using System.Threading;
+
+namespace Nest
+{
+	public partial interface IElasticClient
+	{
+		/// 
+		/// Stops an existing, started rollup job. If the job does not exist an exception will be thrown.
+		/// Stopping an already stopped job has no action.
+		/// 
+		IStopRollupJobResponse StopRollupJob(Id id, Func selector = null);
+
+		/// 
+		IStopRollupJobResponse StopRollupJob(IStopRollupJobRequest request);
+
+		/// 
+		Task StopRollupJobAsync(Id id,
+			Func selector = null, CancellationToken cancellationToken = default);
+
+		/// 
+		Task StopRollupJobAsync(IStopRollupJobRequest request, CancellationToken cancellationToken = default);
+	}
+
+	public partial class ElasticClient
+	{
+		/// 
+		public IStopRollupJobResponse StopRollupJob(Id id, Func selector = null) =>
+			this.StopRollupJob(selector.InvokeOrDefault(new StopRollupJobDescriptor(id)));
+
+		/// 
+		public IStopRollupJobResponse StopRollupJob(IStopRollupJobRequest request) =>
+			this.Dispatcher.Dispatch(
+				request,
+				(p, d) =>this.LowLevelDispatch.XpackRollupStopJobDispatch(p)
+			);
+
+		/// 
+		public Task StopRollupJobAsync(
+			Id id, Func selector = null, CancellationToken cancellationToken = default
+		)  =>
+			this.StopRollupJobAsync(selector.InvokeOrDefault(new StopRollupJobDescriptor(id)), cancellationToken);
+
+		/// 
+		public Task StopRollupJobAsync(IStopRollupJobRequest request, CancellationToken cancellationToken = default) =>
+			this.Dispatcher.DispatchAsync(
+				request,
+				cancellationToken,
+				(p, d, c) => this.LowLevelDispatch.XpackRollupStopJobDispatchAsync(p, c)
+			);
+	}
+}
diff --git a/src/Nest/XPack/RollUp/StopRollupJob/StopRollupJobRequest.cs b/src/Nest/XPack/RollUp/StopRollupJob/StopRollupJobRequest.cs
new file mode 100644
index 00000000000..ef92835be2f
--- /dev/null
+++ b/src/Nest/XPack/RollUp/StopRollupJob/StopRollupJobRequest.cs
@@ -0,0 +1,18 @@
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	[MapsApi("xpack.rollup.stop_job.json")]
+	public partial interface IStopRollupJobRequest
+	{
+	}
+
+	public partial class StopRollupJobRequest
+	{
+	}
+
+	public partial class StopRollupJobDescriptor
+	{
+	}
+}
diff --git a/src/Nest/XPack/RollUp/StopRollupJob/StopRollupJobResponse.cs b/src/Nest/XPack/RollUp/StopRollupJob/StopRollupJobResponse.cs
new file mode 100644
index 00000000000..7646c49b5d4
--- /dev/null
+++ b/src/Nest/XPack/RollUp/StopRollupJob/StopRollupJobResponse.cs
@@ -0,0 +1,16 @@
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	[JsonObject]
+	public interface IStopRollupJobResponse : IResponse
+	{
+		[JsonProperty("stopped")]
+		bool Stopped { get; set; }
+	}
+
+	public class StopRollupJobResponse : ResponseBase, IStopRollupJobResponse
+	{
+		public bool Stopped { get; set; }
+	}
+}
diff --git a/src/Nest/_Generated/_Descriptors.generated.cs b/src/Nest/_Generated/_Descriptors.generated.cs
index d11ecf2a867..bee24f39513 100644
--- a/src/Nest/_Generated/_Descriptors.generated.cs
+++ b/src/Nest/_Generated/_Descriptors.generated.cs
@@ -4383,6 +4383,116 @@ public MigrationUpgradeDescriptor(IndexName index) : base(r=>r.Required("index",
 		///Should the request block until the upgrade operation is completed
 		public MigrationUpgradeDescriptor WaitForCompletion(bool? waitForCompletion = true) => Qs("wait_for_completion", waitForCompletion);
 	}
+	///descriptor for XpackRollupDeleteJob 

+	public partial class DeleteRollupJobDescriptor  : RequestDescriptorBase, IDeleteRollupJobRequest
+	{ 
+		/// /_xpack/rollup/job/{id}
+		/// this parameter is required
+		public DeleteRollupJobDescriptor(Id id) : base(r=>r.Required("id", id)){}
+		// values part of the url path
+		Id IDeleteRollupJobRequest.Id => Self.RouteValues.Get("id");
+
+		// Request parameters
+
+	}
+	///descriptor for XpackRollupGetJobs 

+	public partial class GetRollupJobDescriptor  : RequestDescriptorBase, IGetRollupJobRequest
+	{ 
+		/// /_xpack/rollup/job/{id}
+		public GetRollupJobDescriptor() : base(){}
+		// values part of the url path
+		Id IGetRollupJobRequest.Id => Self.RouteValues.Get("id");
+
+		///The ID of the job(s) to fetch. Accepts glob patterns, or left blank for all jobs
+		public GetRollupJobDescriptor Id(Id id) => Assign(a=>a.RouteValues.Optional("id", id));
+
+		// Request parameters
+
+	}
+	///descriptor for XpackRollupGetRollupCaps 
https://www.elastic.co/guide/en/elasticsearch/reference/current/rollup-get-rollup-caps.html

+	public partial class GetRollupCapabilitiesDescriptor  : RequestDescriptorBase, IGetRollupCapabilitiesRequest
+	{ 
+		/// /_xpack/rollup/data/{index}
+		public GetRollupCapabilitiesDescriptor() : base(){}
+		// values part of the url path
+		Indices IGetRollupCapabilitiesRequest.Index => Self.RouteValues.Get("index");
+
+		/// Index, indices or index-pattern to return rollup capabilities for. _all may be used to fetch rollup capabilities from all job
+		public GetRollupCapabilitiesDescriptor Index(Indices index) => Assign(a=>a.RouteValues.Optional("index", index));
+
+		///a shortcut into calling Index(typeof(TOther))
+		public GetRollupCapabilitiesDescriptor Index() where TOther : class => Assign(a=>a.RouteValues.Optional("index", (Indices)typeof(TOther)));
+
+		///A shortcut into calling Index(Indices.All)
+		public GetRollupCapabilitiesDescriptor AllIndices() => this.Index(Indices.All);
+
+		// Request parameters
+
+	}
+	///descriptor for XpackRollupPutJob 

+	public partial class CreateRollupJobDescriptor  : RequestDescriptorBase,CreateRollupJobRequestParameters, ICreateRollupJobRequest>, ICreateRollupJobRequest
+	{ 
+		/// /_xpack/rollup/job/{id}
+		/// this parameter is required
+		public CreateRollupJobDescriptor(Id id) : base(r=>r.Required("id", id)){}
+		// values part of the url path
+		Id ICreateRollupJobRequest.Id => Self.RouteValues.Get("id");
+
+		// Request parameters
+
+	}
+	///descriptor for XpackRollupRollupSearch 

+	public partial class RollupSearchDescriptor  : RequestDescriptorBase,RollupSearchRequestParameters, IRollupSearchRequest>, IRollupSearchRequest
+	{ 
+		/// /{index}/_rollup_search
+		/// this parameter is required
+		public RollupSearchDescriptor(Indices index) : base(r=>r.Required("index", index)){}
+		// values part of the url path
+		Indices IRollupSearchRequest.Index => Self.RouteValues.Get("index");
+		TypeName IRollupSearchRequest.Type => Self.RouteValues.Get("type");
+
+		///The index or index-pattern (containing rollup or regular data) that should be searched
+		public RollupSearchDescriptor Index(Indices index) => Assign(a=>a.RouteValues.Required("index", index));
+
+		///a shortcut into calling Index(typeof(TOther))
+		public RollupSearchDescriptor Index() where TOther : class => Assign(a=>a.RouteValues.Required("index", (Indices)typeof(TOther)));
+
+		///A shortcut into calling Index(Indices.All)
+		public RollupSearchDescriptor AllIndices() => this.Index(Indices.All);
+
+		///The doc type inside the index
+		public RollupSearchDescriptor Type(TypeName type) => Assign(a=>a.RouteValues.Optional("type", type));
+
+		///a shortcut into calling Type(typeof(TOther))
+		public RollupSearchDescriptor Type() where TOther : class => Assign(a=>a.RouteValues.Optional("type", (TypeName)typeof(TOther)));
+
+		// Request parameters
+
+	}
+	///descriptor for XpackRollupStartJob 

+	public partial class StartRollupJobDescriptor  : RequestDescriptorBase, IStartRollupJobRequest
+	{ 
+		/// /_xpack/rollup/job/{id}/_start
+		/// this parameter is required
+		public StartRollupJobDescriptor(Id id) : base(r=>r.Required("id", id)){}
+		// values part of the url path
+		Id IStartRollupJobRequest.Id => Self.RouteValues.Get("id");
+
+		// Request parameters
+
+	}
+	///descriptor for XpackRollupStopJob 

+	public partial class StopRollupJobDescriptor  : RequestDescriptorBase, IStopRollupJobRequest
+	{ 
+		/// /_xpack/rollup/job/{id}/_stop
+		/// this parameter is required
+		public StopRollupJobDescriptor(Id id) : base(r=>r.Required("id", id)){}
+		// values part of the url path
+		Id IStopRollupJobRequest.Id => Self.RouteValues.Get("id");
+
+		// Request parameters
+
+	}
 	///descriptor for XpackSecurityAuthenticate 
https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-authenticate.html

 	public partial class AuthenticateDescriptor  : RequestDescriptorBase, IAuthenticateRequest
 	{ 
diff --git a/src/Nest/_Generated/_LowLevelDispatch.generated.cs b/src/Nest/_Generated/_LowLevelDispatch.generated.cs
index 36e891b31dd..0f41a95e230 100644
--- a/src/Nest/_Generated/_LowLevelDispatch.generated.cs
+++ b/src/Nest/_Generated/_LowLevelDispatch.generated.cs
@@ -3838,6 +3838,170 @@ internal partial class LowLevelDispatch
 			throw InvalidDispatch("XpackMigrationUpgrade", p, new [] { POST }, "/_xpack/migration/upgrade/{index}");
 		}
 		
+		internal TResponse XpackRollupDeleteJobDispatch(IRequest p) where TResponse : class, IElasticsearchResponse, new()
+		{
+			switch(p.HttpMethod)
+			{
+				case DELETE:
+					if (AllSetNoFallback(p.RouteValues.Id)) return _lowLevel.XpackRollupDeleteJob(p.RouteValues.Id,p.RequestParameters);
+					break;
+			}
+			throw InvalidDispatch("XpackRollupDeleteJob", p, new [] { DELETE }, "/_xpack/rollup/job/{id}");
+		}
+		
+		internal Task XpackRollupDeleteJobDispatchAsync(IRequest p, CancellationToken ct) where TResponse : class, IElasticsearchResponse, new()
+		{
+			switch(p.HttpMethod)
+			{
+				case DELETE:
+					if (AllSetNoFallback(p.RouteValues.Id)) return _lowLevel.XpackRollupDeleteJobAsync(p.RouteValues.Id,p.RequestParameters,ct);
+					break;
+			}
+			throw InvalidDispatch("XpackRollupDeleteJob", p, new [] { DELETE }, "/_xpack/rollup/job/{id}");
+		}
+		
+		internal TResponse XpackRollupGetJobsDispatch(IRequest p) where TResponse : class, IElasticsearchResponse, new()
+		{
+			switch(p.HttpMethod)
+			{
+				case GET:
+					if (AllSet(p.RouteValues.Id)) return _lowLevel.XpackRollupGetJobs(p.RouteValues.Id,p.RequestParameters);
+						return _lowLevel.XpackRollupGetJobs(p.RequestParameters);
+			}
+			throw InvalidDispatch("XpackRollupGetJobs", p, new [] { GET }, "/_xpack/rollup/job/{id}", "/_xpack/rollup/job/");
+		}
+		
+		internal Task XpackRollupGetJobsDispatchAsync(IRequest p, CancellationToken ct) where TResponse : class, IElasticsearchResponse, new()
+		{
+			switch(p.HttpMethod)
+			{
+				case GET:
+					if (AllSet(p.RouteValues.Id)) return _lowLevel.XpackRollupGetJobsAsync(p.RouteValues.Id,p.RequestParameters,ct);
+						return _lowLevel.XpackRollupGetJobsAsync(p.RequestParameters,ct);
+			}
+			throw InvalidDispatch("XpackRollupGetJobs", p, new [] { GET }, "/_xpack/rollup/job/{id}", "/_xpack/rollup/job/");
+		}
+		
+		internal TResponse XpackRollupGetRollupCapsDispatch(IRequest p) where TResponse : class, IElasticsearchResponse, new()
+		{
+			switch(p.HttpMethod)
+			{
+				case GET:
+					if (AllSet(p.RouteValues.Index)) return _lowLevel.XpackRollupGetRollupCaps(p.RouteValues.Index,p.RequestParameters);
+						return _lowLevel.XpackRollupGetRollupCaps(p.RequestParameters);
+			}
+			throw InvalidDispatch("XpackRollupGetRollupCaps", p, new [] { GET }, "/_xpack/rollup/data/{index}", "/_xpack/rollup/data/");
+		}
+		
+		internal Task XpackRollupGetRollupCapsDispatchAsync(IRequest p, CancellationToken ct) where TResponse : class, IElasticsearchResponse, new()
+		{
+			switch(p.HttpMethod)
+			{
+				case GET:
+					if (AllSet(p.RouteValues.Index)) return _lowLevel.XpackRollupGetRollupCapsAsync(p.RouteValues.Index,p.RequestParameters,ct);
+						return _lowLevel.XpackRollupGetRollupCapsAsync(p.RequestParameters,ct);
+			}
+			throw InvalidDispatch("XpackRollupGetRollupCaps", p, new [] { GET }, "/_xpack/rollup/data/{index}", "/_xpack/rollup/data/");
+		}
+		
+		internal TResponse XpackRollupPutJobDispatch(IRequest p,SerializableData body) where TResponse : class, IElasticsearchResponse, new()
+		{
+			switch(p.HttpMethod)
+			{
+				case PUT:
+					if (AllSetNoFallback(p.RouteValues.Id)) return _lowLevel.XpackRollupPutJob(p.RouteValues.Id,body,p.RequestParameters);
+					break;
+			}
+			throw InvalidDispatch("XpackRollupPutJob", p, new [] { PUT }, "/_xpack/rollup/job/{id}");
+		}
+		
+		internal Task XpackRollupPutJobDispatchAsync(IRequest p,SerializableData body, CancellationToken ct) where TResponse : class, IElasticsearchResponse, new()
+		{
+			switch(p.HttpMethod)
+			{
+				case PUT:
+					if (AllSetNoFallback(p.RouteValues.Id)) return _lowLevel.XpackRollupPutJobAsync(p.RouteValues.Id,body,p.RequestParameters,ct);
+					break;
+			}
+			throw InvalidDispatch("XpackRollupPutJob", p, new [] { PUT }, "/_xpack/rollup/job/{id}");
+		}
+		
+		internal TResponse XpackRollupRollupSearchDispatch(IRequest p,SerializableData body) where TResponse : class, IElasticsearchResponse, new()
+		{
+			switch(p.HttpMethod)
+			{
+				case GET:
+					if (AllSet(p.RouteValues.Index, p.RouteValues.Type)) return _lowLevel.XpackRollupRollupSearchGet(p.RouteValues.Index,p.RouteValues.Type,p.RequestParameters);
+					if (AllSetNoFallback(p.RouteValues.Index)) return _lowLevel.XpackRollupRollupSearchGet(p.RouteValues.Index,p.RequestParameters);
+					break;
+				case POST:
+					if (AllSet(p.RouteValues.Index, p.RouteValues.Type)) return _lowLevel.XpackRollupRollupSearch(p.RouteValues.Index,p.RouteValues.Type,body,p.RequestParameters);
+					if (AllSetNoFallback(p.RouteValues.Index)) return _lowLevel.XpackRollupRollupSearch(p.RouteValues.Index,body,p.RequestParameters);
+					break;
+			}
+			throw InvalidDispatch("XpackRollupRollupSearch", p, new [] { GET, POST }, "/{index}/_rollup_search", "/{index}/{type}/_rollup_search");
+		}
+		
+		internal Task XpackRollupRollupSearchDispatchAsync(IRequest p,SerializableData body, CancellationToken ct) where TResponse : class, IElasticsearchResponse, new()
+		{
+			switch(p.HttpMethod)
+			{
+				case GET:
+					if (AllSet(p.RouteValues.Index, p.RouteValues.Type)) return _lowLevel.XpackRollupRollupSearchGetAsync(p.RouteValues.Index,p.RouteValues.Type,p.RequestParameters,ct);
+					if (AllSetNoFallback(p.RouteValues.Index)) return _lowLevel.XpackRollupRollupSearchGetAsync(p.RouteValues.Index,p.RequestParameters,ct);
+					break;
+				case POST:
+					if (AllSet(p.RouteValues.Index, p.RouteValues.Type)) return _lowLevel.XpackRollupRollupSearchAsync(p.RouteValues.Index,p.RouteValues.Type,body,p.RequestParameters,ct);
+					if (AllSetNoFallback(p.RouteValues.Index)) return _lowLevel.XpackRollupRollupSearchAsync(p.RouteValues.Index,body,p.RequestParameters,ct);
+					break;
+			}
+			throw InvalidDispatch("XpackRollupRollupSearch", p, new [] { GET, POST }, "/{index}/_rollup_search", "/{index}/{type}/_rollup_search");
+		}
+		
+		internal TResponse XpackRollupStartJobDispatch(IRequest p) where TResponse : class, IElasticsearchResponse, new()
+		{
+			switch(p.HttpMethod)
+			{
+				case POST:
+					if (AllSetNoFallback(p.RouteValues.Id)) return _lowLevel.XpackRollupStartJob(p.RouteValues.Id,p.RequestParameters);
+					break;
+			}
+			throw InvalidDispatch("XpackRollupStartJob", p, new [] { POST }, "/_xpack/rollup/job/{id}/_start");
+		}
+		
+		internal Task XpackRollupStartJobDispatchAsync(IRequest p, CancellationToken ct) where TResponse : class, IElasticsearchResponse, new()
+		{
+			switch(p.HttpMethod)
+			{
+				case POST:
+					if (AllSetNoFallback(p.RouteValues.Id)) return _lowLevel.XpackRollupStartJobAsync(p.RouteValues.Id,p.RequestParameters,ct);
+					break;
+			}
+			throw InvalidDispatch("XpackRollupStartJob", p, new [] { POST }, "/_xpack/rollup/job/{id}/_start");
+		}
+		
+		internal TResponse XpackRollupStopJobDispatch(IRequest p) where TResponse : class, IElasticsearchResponse, new()
+		{
+			switch(p.HttpMethod)
+			{
+				case POST:
+					if (AllSetNoFallback(p.RouteValues.Id)) return _lowLevel.XpackRollupStopJob(p.RouteValues.Id,p.RequestParameters);
+					break;
+			}
+			throw InvalidDispatch("XpackRollupStopJob", p, new [] { POST }, "/_xpack/rollup/job/{id}/_stop");
+		}
+		
+		internal Task XpackRollupStopJobDispatchAsync(IRequest p, CancellationToken ct) where TResponse : class, IElasticsearchResponse, new()
+		{
+			switch(p.HttpMethod)
+			{
+				case POST:
+					if (AllSetNoFallback(p.RouteValues.Id)) return _lowLevel.XpackRollupStopJobAsync(p.RouteValues.Id,p.RequestParameters,ct);
+					break;
+			}
+			throw InvalidDispatch("XpackRollupStopJob", p, new [] { POST }, "/_xpack/rollup/job/{id}/_stop");
+		}
+		
 		internal TResponse XpackSecurityAuthenticateDispatch(IRequest p) where TResponse : class, IElasticsearchResponse, new()
 		{
 			switch(p.HttpMethod)
diff --git a/src/Nest/_Generated/_Requests.generated.cs b/src/Nest/_Generated/_Requests.generated.cs
index 5706dea4f33..15b9d689106 100644
--- a/src/Nest/_Generated/_Requests.generated.cs
+++ b/src/Nest/_Generated/_Requests.generated.cs
@@ -1460,6 +1460,23 @@ public CreateRequest(DocumentPath document, IndexName index = null, T
 		public string Pipeline { get => Q("pipeline"); set => Q("pipeline", value); }
 	}
 	[JsonObject(MemberSerialization = MemberSerialization.OptIn)]
+	public partial interface ICreateRollupJobRequest : IRequest
+	{
+		Id Id { get; }
+	}
+	///Request parameters for XpackRollupPutJob 

+	public partial class CreateRollupJobRequest : PlainRequestBase, ICreateRollupJobRequest
+	{
+		protected ICreateRollupJobRequest Self => this;
+		////_xpack/rollup/job/{id}
+		///this parameter is required
+		public CreateRollupJobRequest(Id id) : base(r=>r.Required("id", id)){}
+		// values part of the url path
+		Id ICreateRollupJobRequest.Id => Self.RouteValues.Get("id");
+
+		// Request parameters
+	}
+	[JsonObject(MemberSerialization = MemberSerialization.OptIn)]
 	public partial interface IDeactivateWatchRequest : IRequest
 	{
 		Id WatchId { get; }
@@ -2017,6 +2034,23 @@ public DeleteRoleRequest(Name name) : base(r=>r.Required("name", name)){}
 		public Refresh? Refresh { get => Q("refresh"); set => Q("refresh", value); }
 	}
 	[JsonObject(MemberSerialization = MemberSerialization.OptIn)]
+	public partial interface IDeleteRollupJobRequest : IRequest
+	{
+		Id Id { get; }
+	}
+	///Request parameters for XpackRollupDeleteJob 

+	public partial class DeleteRollupJobRequest : PlainRequestBase, IDeleteRollupJobRequest
+	{
+		protected IDeleteRollupJobRequest Self => this;
+		////_xpack/rollup/job/{id}
+		///this parameter is required
+		public DeleteRollupJobRequest(Id id) : base(r=>r.Required("id", id)){}
+		// values part of the url path
+		Id IDeleteRollupJobRequest.Id => Self.RouteValues.Get("id");
+
+		// Request parameters
+	}
+	[JsonObject(MemberSerialization = MemberSerialization.OptIn)]
 	public partial interface IDeleteScriptRequest : IRequest
 	{
 		Id Id { get; }
@@ -3128,6 +3162,44 @@ public GetRoleRequest() : base(){}
 		// Request parameters
 	}
 	[JsonObject(MemberSerialization = MemberSerialization.OptIn)]
+	public partial interface IGetRollupCapabilitiesRequest : IRequest
+	{
+		Indices Index { get; }
+	}
+	///Request parameters for XpackRollupGetRollupCaps 
https://www.elastic.co/guide/en/elasticsearch/reference/current/rollup-get-rollup-caps.html

+	public partial class GetRollupCapabilitiesRequest : PlainRequestBase, IGetRollupCapabilitiesRequest
+	{
+		protected IGetRollupCapabilitiesRequest Self => this;
+		////_xpack/rollup/data/{index}
+		///Optional, accepts null
+		public GetRollupCapabilitiesRequest(Indices index) : base(r=>r.Optional("index", index)){}
+		////_xpack/rollup/data/
+		public GetRollupCapabilitiesRequest() : base(){}
+		// values part of the url path
+		Indices IGetRollupCapabilitiesRequest.Index => Self.RouteValues.Get("index");
+
+		// Request parameters
+	}
+	[JsonObject(MemberSerialization = MemberSerialization.OptIn)]
+	public partial interface IGetRollupJobRequest : IRequest
+	{
+		Id Id { get; }
+	}
+	///Request parameters for XpackRollupGetJobs 

+	public partial class GetRollupJobRequest : PlainRequestBase, IGetRollupJobRequest
+	{
+		protected IGetRollupJobRequest Self => this;
+		////_xpack/rollup/job/{id}
+		///Optional, accepts null
+		public GetRollupJobRequest(Id id) : base(r=>r.Optional("id", id)){}
+		////_xpack/rollup/job/
+		public GetRollupJobRequest() : base(){}
+		// values part of the url path
+		Id IGetRollupJobRequest.Id => Self.RouteValues.Get("id");
+
+		// Request parameters
+	}
+	[JsonObject(MemberSerialization = MemberSerialization.OptIn)]
 	public partial interface IGetScriptRequest : IRequest
 	{
 		Id Id { get; }
@@ -4572,6 +4644,29 @@ public RolloverIndexRequest(Name alias, IndexName new_index) : base(r=>r.Require
 		public string WaitForActiveShards { get => Q("wait_for_active_shards"); set => Q("wait_for_active_shards", value); }
 	}
 	[JsonObject(MemberSerialization = MemberSerialization.OptIn)]
+	public partial interface IRollupSearchRequest : IRequest
+	{
+		Indices Index { get; }
+		TypeName Type { get; }
+	}
+	///Request parameters for XpackRollupRollupSearch 

+	public partial class RollupSearchRequest : PlainRequestBase, IRollupSearchRequest
+	{
+		protected IRollupSearchRequest Self => this;
+		////{index}/_rollup_search
+		///this parameter is required
+		public RollupSearchRequest(Indices index) : base(r=>r.Required("index", index)){}
+		////{index}/{type}/_rollup_search
+		///this parameter is required
+		///Optional, accepts null
+		public RollupSearchRequest(Indices index, TypeName type) : base(r=>r.Required("index", index).Optional("type", type)){}
+		// values part of the url path
+		Indices IRollupSearchRequest.Index => Self.RouteValues.Get("index");
+		TypeName IRollupSearchRequest.Type => Self.RouteValues.Get("type");
+
+		// Request parameters
+	}
+	[JsonObject(MemberSerialization = MemberSerialization.OptIn)]
 	public partial interface IRootNodeInfoRequest : IRequest
 	{
 	}
@@ -5289,6 +5384,23 @@ public StartDatafeedRequest(Id datafeed_id) : base(r=>r.Required("datafeed_id",
 		// Request parameters
 	}
 	[JsonObject(MemberSerialization = MemberSerialization.OptIn)]
+	public partial interface IStartRollupJobRequest : IRequest
+	{
+		Id Id { get; }
+	}
+	///Request parameters for XpackRollupStartJob 

+	public partial class StartRollupJobRequest : PlainRequestBase, IStartRollupJobRequest
+	{
+		protected IStartRollupJobRequest Self => this;
+		////_xpack/rollup/job/{id}/_start
+		///this parameter is required
+		public StartRollupJobRequest(Id id) : base(r=>r.Required("id", id)){}
+		// values part of the url path
+		Id IStartRollupJobRequest.Id => Self.RouteValues.Get("id");
+
+		// Request parameters
+	}
+	[JsonObject(MemberSerialization = MemberSerialization.OptIn)]
 	public partial interface IStartTrialLicenseRequest : IRequest
 	{
 	}
@@ -5336,6 +5448,23 @@ public StopDatafeedRequest(Id datafeed_id) : base(r=>r.Required("datafeed_id", d
 		public bool? AllowNoDatafeeds { get => Q("allow_no_datafeeds"); set => Q("allow_no_datafeeds", value); }
 	}
 	[JsonObject(MemberSerialization = MemberSerialization.OptIn)]
+	public partial interface IStopRollupJobRequest : IRequest
+	{
+		Id Id { get; }
+	}
+	///Request parameters for XpackRollupStopJob 

+	public partial class StopRollupJobRequest : PlainRequestBase, IStopRollupJobRequest
+	{
+		protected IStopRollupJobRequest Self => this;
+		////_xpack/rollup/job/{id}/_stop
+		///this parameter is required
+		public StopRollupJobRequest(Id id) : base(r=>r.Required("id", id)){}
+		// values part of the url path
+		Id IStopRollupJobRequest.Id => Self.RouteValues.Get("id");
+
+		// Request parameters
+	}
+	[JsonObject(MemberSerialization = MemberSerialization.OptIn)]
 	public partial interface IStopWatcherRequest : IRequest
 	{
 	}
diff --git a/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/BenchmarkCluster.cs b/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/BenchmarkCluster.cs
deleted file mode 100644
index 4e57c44bf23..00000000000
--- a/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/BenchmarkCluster.cs
+++ /dev/null
@@ -1,5 +0,0 @@
-namespace Tests.Core.ManagedElasticsearch.Clusters
-{
-	public class BenchmarkCluster : ClientTestClusterBase { }
-
-}
diff --git a/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/TimeSeriesCluster.cs b/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/TimeSeriesCluster.cs
new file mode 100644
index 00000000000..8db45a34cb6
--- /dev/null
+++ b/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/TimeSeriesCluster.cs
@@ -0,0 +1,58 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using Nest;
+using Tests.Domain;
+
+namespace Tests.Core.ManagedElasticsearch.Clusters
+{
+	public class TimeSeriesCluster : XPackCluster
+	{
+		protected override void SeedCluster() =>
+			new TimeSeriesSeeder(this.Client).SeedNode();
+	}
+
+	public class TimeSeriesSeeder
+	{
+		private readonly IElasticClient _client;
+
+		public TimeSeriesSeeder(IElasticClient client) => this._client = client;
+
+		public static readonly string IndicesWildCard = "logs-*";
+
+		public void SeedNode()
+		{
+			_client.PutIndexTemplate("logs-template", p => p
+				.Create()
+				.Mappings(map => map
+					.Map(m => m.AutoMap())
+				)
+				.IndexPatterns(IndicesWildCard)
+				.Settings(s => s
+					.NumberOfShards(1)
+					.NumberOfReplicas(0)
+				)
+			);
+
+			var logs = Log.Generator.GenerateLazy(200_000);
+			var sw = Stopwatch.StartNew();
+			var dropped = new List();
+			var bulkAll = _client.BulkAll(new BulkAllRequest(logs)
+			{
+				Size = 10_000,
+				MaxDegreeOfParallelism = 8,
+				RefreshOnCompleted = true,
+				RefreshIndices = IndicesWildCard,
+				DroppedDocumentCallback = (d, l) => dropped.Add(l),
+				BufferToBulk = (b, buffer) => b.IndexMany(buffer, (i, l) => i.Index($"logs-{l.Timestamp:yyyy-MM-dd}"))
+			});
+			bulkAll.Wait(TimeSpan.FromMinutes(1), delegate { });
+			Console.WriteLine($"Completed in {sw.Elapsed} with {dropped.Count} dropped logs");
+
+			var countResult = _client.Count(s => s.Index(IndicesWildCard));
+			Console.WriteLine($"Stored {countResult.Count} in {IndicesWildCard} indices");
+
+
+		}
+	}
+}
diff --git a/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/WatcherStateCluster.cs b/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/WatcherStateCluster.cs
index 443b7e8698a..d1ce816456e 100644
--- a/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/WatcherStateCluster.cs
+++ b/src/Tests/Tests.Core/ManagedElasticsearch/Clusters/WatcherStateCluster.cs
@@ -1,17 +1,11 @@
 using System;
-using System.Threading;
 using Elastic.Managed.Configuration;
 
 namespace Tests.Core.ManagedElasticsearch.Clusters
 {
-	/// 
-	/// Cluster that modifies the state of the Watcher Service
-	/// 
+	///  Cluster that modifies the state of the Watcher Service
 	public class WatcherStateCluster : XPackCluster
 	{
-		protected override void ModifyNodeConfiguration(NodeConfiguration n, int port)
-		{
-			n.WaitForShutdown = TimeSpan.FromSeconds(30);
-		}
+		protected override void ModifyNodeConfiguration(NodeConfiguration n, int port) => n.WaitForShutdown = TimeSpan.FromSeconds(30);
 	}
 }
diff --git a/src/Tests/Tests.Core/Xunit/Generators.cs b/src/Tests/Tests.Core/Xunit/Generators.cs
index 27f38abaade..0cbaa6b6d4e 100644
--- a/src/Tests/Tests.Core/Xunit/Generators.cs
+++ b/src/Tests/Tests.Core/Xunit/Generators.cs
@@ -10,7 +10,7 @@ public static class Generators
 		static Generators()
 		{
 			var tags = Tag.Generator;
-			var messages = Message.Generator;
+			var messages = Log.Generator;
 			var people = Person.People;
 			var developers = Developer.Developers;
 			var projects = Project.Projects;
diff --git a/src/Tests/Tests.Domain/Log.cs b/src/Tests/Tests.Domain/Log.cs
new file mode 100644
index 00000000000..42e5bb3f5d8
--- /dev/null
+++ b/src/Tests/Tests.Domain/Log.cs
@@ -0,0 +1,80 @@
+using System;
+using System.Linq;
+using System.Text;
+using Bogus;
+using Nest;
+using Tests.Configuration;
+
+namespace Tests.Domain
+{
+	public class Log
+	{
+		public DateTime Timestamp { get; set; }
+
+		public string Body { get; set; }
+		[Keyword] public string Tag { get; set; }
+		[Keyword] public string UserAgent { get; set; }
+		[Keyword] public string User { get; set; }
+		[Keyword] public string Url { get; set; }
+		[Keyword] public string Section { get; set; }
+		public long Temperature { get; set; }
+		public double Voltage { get; set; }
+		public double Load { get; set; }
+		public double NetIn { get; set; }
+		public double NetOut { get; set; }
+
+		public static Faker Generator { get; } =
+			new Faker()
+				.UseSeed(TestConfiguration.Instance.Seed)
+				.RuleFor(m => m.Timestamp, m => m.Date.Between(DateTime.UtcNow.AddDays(-7), DateTime.UtcNow))
+				.RuleFor(m => m.Body, m => GetMessageText())
+				.RuleFor(m => m.Tag, m => m.Hacker.Abbreviation())
+				.RuleFor(m => m.UserAgent, m => m.Internet.UserAgent())
+				.RuleFor(m => m.User, m => m.Internet.UserName())
+				.RuleFor(m => m.Url, m => m.Internet.UrlWithPath())
+				.RuleFor(m => m.Temperature, m => m.Random.Number(-10, 45))
+				.RuleFor(m => m.Voltage, m => m.Random.Double(0, 10.0))
+				.RuleFor(m => m.Section, m => m.PickRandom(Sections))
+				.RuleFor(m => m.Load, m => m.Random.Double(100, 500))
+				.RuleFor(m => m.NetIn, m => m.Random.Double(1000, 10000))
+				.RuleFor(m => m.NetOut, m => m.Random.Double(1000, 10000))
+			;
+
+		private static readonly Random Random = new Random(15842);
+		public static readonly string[] Words =
+		{
+			"molestie", "vel", "metus", "neque", "dui", "volutpat", "sollicitudin", "sociis", "ac", "imperdiet", "tristique", "et",
+			"nascetur", "ad", "rhoncus", "viverra", "ornare", "consectetur", "ultrices", "orci", "parturient", "lorem", "massa", "quis",
+			"platea", "aenean", "fermentum", "augue", "placerat", "auctor", "natoque", "habitasse", "pharetra", "ridiculus", "leo", "sit",
+			"cras", "est", "venenatis", "aptent", "nibh", "magnis", "sodales", "malesuada", "praesent", "potenti", "lobortis", "justo",
+			"quam", "cubilia", "pellentesque", "porttitor", "pretium", "adipiscing", "phasellus", "lectus", "vivamus", "id", "mi",
+			"bibendum", "feugiat", "odio", "rutrum", "vestibulum", "posuere", "elementum", "suscipit", "purus", "accumsan", "egestas",
+			"mus", "varius", "a", "arcu", "commodo", "dis", "lacinia", "tellus", "cursus", "aliquet", "interdum", "turpis", "maecenas",
+			"dapibus", "cum", "fames", "montes", "iaculis", "erat", "euismod", "hac", "faucibus", "mauris", "tempus", "primis", "velit",
+			"sem", "duis", "luctus", "penatibus", "sapien", "blandit", "eros", "suspendisse", "urna", "ipsum", "congue", "nulla", "taciti",
+			"mollis", "facilisis", "at", "amet", "laoreet", "dignissim", "fringilla", "in", "nostra", "quisque", "donec", "enim",
+			"eleifend", "nisl", "morbi", "felis", "torquent", "eget", "convallis", "etiam", "tincidunt", "facilisi", "pulvinar",
+			"vulputate", "integre", "himenaeos", "netus", "senectus", "non", "litora", "per", "curae", "ultricies", "nec", "nam", "eu",
+			"ante", "mattis", "vehicula", "sociosqu", "nunc", "semper", "lacus", "proin", "risus", "condimentum", "scelerisque", "conubia",
+			"consequat", "dolor", "libero", "diam", "ut", "inceptos", "porta", "nullam", "dictumst", "magna", "tempor", "fusce", "vitae",
+			"aliquam", "curabitur", "ligula", "habitant", "class", "hendrerit", "sagittis", "gravida", "nisi", "tortor", "ullamcorper",
+			"dictum", "elit", "sed"
+		};
+
+		public static readonly string[] Sections = Words.Where(w=>w.Length > 3).Take(10).ToArray();
+
+		private static string GetMessageText()
+		{
+			var numWords = Random.Next(1, 20);
+
+			var sb = new StringBuilder();
+
+			for (var i = 0; i < numWords; i++)
+			{
+				sb.Append(Words[Random.Next(0, Words.Length)]).Append(" ");
+			}
+
+			return sb.ToString().TrimEnd();
+		}
+	}
+}
diff --git a/src/Tests/Tests.Domain/Message.cs b/src/Tests/Tests.Domain/Message.cs
deleted file mode 100644
index d8172786c61..00000000000
--- a/src/Tests/Tests.Domain/Message.cs
+++ /dev/null
@@ -1,41 +0,0 @@
-using System;
-using System.Text;
-using Bogus;
-using Tests.Configuration;
-
-namespace Tests.Domain
-{
-	public class Message
-	{
-		public string Body { get; set; }
-
-		public Guid Id { get; set; }
-
-		public DateTime Timestamp { get; set; }
-
-		private static readonly Random Random = new Random(15842);
-		private static readonly string[] Words = { "molestie", "vel", "metus", "neque", "dui", "volutpat", "sollicitudin", "sociis", "ac", "imperdiet", "tristique", "et", "nascetur", "ad", "rhoncus", "viverra", "ornare", "consectetur", "ultrices", "orci", "parturient", "lorem", "massa", "quis", "platea", "aenean", "fermentum", "augue", "placerat", "auctor", "natoque", "habitasse", "pharetra", "ridiculus", "leo", "sit", "cras", "est", "venenatis", "aptent", "nibh", "magnis", "sodales", "malesuada", "praesent", "potenti", "lobortis", "justo", "quam", "cubilia", "pellentesque", "porttitor", "pretium", "adipiscing", "phasellus", "lectus", "vivamus", "id", "mi", "bibendum", "feugiat", "odio", "rutrum", "vestibulum", "posuere", "elementum", "suscipit", "purus", "accumsan", "egestas", "mus", "varius", "a", "arcu", "commodo", "dis", "lacinia", "tellus", "cursus", "aliquet", "interdum", "turpis", "maecenas", "dapibus", "cum", "fames", "montes", "iaculis", "erat", "euismod", "hac", "faucibus", "mauris", "tempus", "primis", "velit", "sem", "duis", "luctus", "penatibus", "sapien", "blandit", "eros", "suspendisse", "urna", "ipsum", "congue", "nulla", "taciti", "mollis", "facilisis", "at", "amet", "laoreet", "dignissim", "fringilla", "in", "nostra", "quisque", "donec", "enim", "eleifend", "nisl", "morbi", "felis", "torquent", "eget", "convallis", "etiam", "tincidunt", "facilisi", "pulvinar", "vulputate", "integre", "himenaeos", "netus", "senectus", "non", "litora", "per", "curae", "ultricies", "nec", "nam", "eu", "ante", "mattis", "vehicula", "sociosqu", "nunc", "semper", "lacus", "proin", "risus", "condimentum", "scelerisque", "conubia", "consequat", "dolor", "libero", "diam", "ut", "inceptos", "porta", "nullam", "dictumst", "magna", "tempor", "fusce", "vitae", "aliquam", "curabitur", "ligula", "habitant", "class", "hendrerit", "sagittis", "gravida", "nisi", "tortor", "ullamcorper", "dictum", "elit", "sed" };
-
-		public static Faker Generator { get; } =
-			new Faker()
-				.UseSeed(TestConfiguration.Instance.Seed)
-				.RuleFor(m => m.Id, m => Guid.NewGuid())
-				.RuleFor(m => m.Body, m => GetMessageText())
-				.RuleFor(m => m.Timestamp, m => DateTime.UtcNow)
-			;
-
-		private static string GetMessageText()
-		{
-			var numWords = Random.Next(1, 20);
-
-			var sb = new StringBuilder();
-
-			for (var i = 0; i < numWords; i++)
-			{
-				sb.Append(Words[Random.Next(0, Words.Length)]).Append(" ");
-			}
-
-			return sb.ToString().TrimEnd();
-		}
-	}
-}
diff --git a/src/Tests/Tests/Framework/EndpointTests/CrudTestBase.cs b/src/Tests/Tests/Framework/EndpointTests/CrudTestBase.cs
index f6b63a02aee..e446af571fa 100644
--- a/src/Tests/Tests/Framework/EndpointTests/CrudTestBase.cs
+++ b/src/Tests/Tests/Framework/EndpointTests/CrudTestBase.cs
@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
+using System.Linq;
 using System.Threading.Tasks;
 using Elastic.Managed.Ephemeral;
 using Elastic.Xunit.XunitPlumbing;
@@ -45,6 +46,7 @@ public abstract class CrudTestBase false;
+    
 		// https://youtrack.jetbrains.com/issue/RIDER-19912
 		[I] protected override Task CreateCallIsValid() => base.CreateCallIsValid();
 		[I] protected override Task GetAfterCreateIsValid() => base.GetAfterCreateIsValid();
@@ -56,6 +58,7 @@ protected CrudTestBase(WritableCluster cluster, EndpointUsage usage) : base(clus
 		[I] protected override Task ExistsAfterDeleteIsValid() => base.ExistsAfterDeleteIsValid();
 		[I] protected override Task DeleteNotFoundIsNotValid() => base.DeleteNotFoundIsNotValid();
 	}
+  
 	public abstract class CrudTestBase
 		: CrudTestBase
 			where TCluster : IEphemeralCluster, INestTestCluster, new()
@@ -66,6 +69,7 @@ public abstract class CrudTestBase false;
+
 		// https://youtrack.jetbrains.com/issue/RIDER-19912
 		[I] protected override Task CreateCallIsValid() => base.CreateCallIsValid();
 		[I] protected override Task GetAfterCreateIsValid() => base.GetAfterCreateIsValid();
@@ -90,6 +94,7 @@ public abstract class CrudTestBase _afterCreateResponses = new Dictionary();
 		private readonly LazyResponses _createExistsResponse;
 		private readonly LazyResponses _updateResponse;
 		private readonly LazyResponses _updateGetResponse;
@@ -98,26 +103,33 @@ public abstract class CrudTestBase LazyResponses.Empty;
 		protected virtual LazyResponses Delete() => LazyResponses.Empty;
+		protected virtual IDictionary> AfterCreateCalls() => new Dictionary>();
 
 		private static string RandomFluent { get; } = $"fluent-{RandomString()}";
 		private static string RandomFluentAsync { get; } = $"fluentasync-{RandomString()}";
@@ -126,6 +138,9 @@ protected CrudTestBase(TCluster cluster, EndpointUsage usage)
 
 		protected virtual bool SupportsDeletes => true;
 		protected virtual bool SupportsExists => true;
+		protected virtual bool SupportsUpdates => true;
+		/// Helpful if you want to capture a reproduce trace with e.g fiddler
+		protected virtual bool TestOnlyOneMethod => false;
 
 		protected virtual void IntegrationSetup(IElasticClient client) { }
 
@@ -150,20 +165,43 @@ Func> requestAsync
 				if (TestClient.Configuration.RunIntegrationTests)
 				{
 					this.IntegrationSetup(client);
-					this._usage.CalledSetup = true;
+					_usage.CalledSetup = true;
 				}
 
-				var sf = Sanitize(RandomFluent);
-				dict.Add(Integration.ClientMethod.Fluent, fluent(sf, client, f => fluentBody(sf, f)));
+				var sf = this.Sanitize(RandomFluent);
+				dict.Add(ClientMethod.Fluent, fluent(sf, client, f => fluentBody(sf, f)));
+				if (this.TestOnlyOneMethod) return dict;
 
-				var sfa = Sanitize(RandomFluentAsync);
-				dict.Add(Integration.ClientMethod.FluentAsync, await fluentAsync(sfa, client, f => fluentBody(sfa, f)));
+				var sfa = this.Sanitize(RandomFluentAsync);
+				dict.Add(ClientMethod.FluentAsync, await fluentAsync(sfa, client, f => fluentBody(sfa, f)));
 
-				var si = Sanitize(RandomInitializer);
-				dict.Add(Integration.ClientMethod.Initializer, request(si, client, initializerBody(si)));
+				var si = this.Sanitize(RandomInitializer);
+				dict.Add(ClientMethod.Initializer, request(si, client, initializerBody(si)));
 
-				var sia = Sanitize(RandomInitializerAsync);
-				dict.Add(Integration.ClientMethod.InitializerAsync, await requestAsync(sia, client, initializerBody(sia)));
+				var sia = this.Sanitize(RandomInitializerAsync);
+				dict.Add(ClientMethod.InitializerAsync, await requestAsync(sia, client, initializerBody(sia)));
+				return dict;
+			});
+		}
+		protected LazyResponses Call(
+			Func, IElasticClient, Task> requestAsync
+		)
+			where TResponse : class, IResponse
+		{
+			var client = this.Client;
+			return new LazyResponses(async () =>
+			{
+				var dict = new Dictionary();
+
+				var values = new List() {
+					this.Sanitize(RandomFluent)};
+				if (!this.TestOnlyOneMethod)
+				{
+					values.Add(this.Sanitize(RandomFluentAsync));
+					values.Add(this.Sanitize(RandomInitializer));
+					values.Add(this.Sanitize(RandomInitializerAsync));
+				}
+				dict.Add(ClientMethod.InitializerAsync, await requestAsync(values, client));
 				return dict;
 			});
 		}
@@ -174,25 +212,19 @@ Func> requestAsync
 		protected TCluster Cluster { get; }
 		protected virtual IElasticClient Client => this.Cluster.Client;
 
+		protected async Task AssertOnAfterCreateResponse(string name, Action assert)
+			where TResponse : class, IResponse
+		{
+			await this.ExecuteOnceInOrder();
+			if (_afterCreateResponses.ContainsKey(name))
+				await this.AssertOnAllResponses(_afterCreateResponses[name], assert);
+			else throw new Exception($"{name} is not a keyed after create response");
+		}
+
 		protected async Task AssertOnAllResponses(LazyResponses responses, Action assert)
 			where TResponse : class, IResponse
 		{
-			//hack to make sure these are resolved in the right order, calling twice yields cached results so
-			//should be fast
-			await this._createResponse;
-			await this._createGetResponse;
-			if (this.SupportsExists)
-				await this._createExistsResponse;
-			await this._updateResponse;
-			await this._updateGetResponse;
-			if (this.SupportsDeletes)
-			{
-				await this._deleteResponse;
-				await this._deleteGetResponse;
-				if (this.SupportsExists)
-					await this._deleteExistsResponse;
-				await this._deleteNotFoundResponse;
-			}
+			await this.ExecuteOnceInOrder();
 
 			foreach (var kv in await responses)
 			{
@@ -211,37 +243,76 @@ protected async Task AssertOnAllResponses(LazyResponses responses, Ac
 			}
 		}
 
-		protected async Task AssertOnCreate(Action assert) => await this.AssertOnAllResponses(this._createResponse, assert);
-		protected async Task AssertOnUpdate(Action assert) => await this.AssertOnAllResponses(this._updateResponse, assert);
+		private async Task ExecuteOnceInOrder()
+		{
+			//hack to make sure these are resolved in the right order, calling twice yields cached results so  should be fast
+			await _createResponse;
+			await _createGetResponse;
+			if (this.SupportsExists)
+				await _createExistsResponse;
+
+			foreach (var kv in _afterCreateResponses) await kv.Value;
+			if (this.SupportsUpdates)
+			{
+				await _updateResponse;
+				await _updateGetResponse;
+			}
+
+			if (this.SupportsDeletes)
+			{
+				await _deleteResponse;
+				await _deleteGetResponse;
+				if (this.SupportsExists)
+					await _deleteExistsResponse;
+				await _deleteNotFoundResponse;
+			}
+		}
+
+		protected async Task AssertOnCreate(Action assert) => await this.AssertOnAllResponses(_createResponse, assert);
+		protected async Task AssertOnUpdate(Action assert)
+		{
+			if (!this.SupportsUpdates) return;
+			await this.AssertOnAllResponses(_updateResponse, assert);
+		}
+
 		protected async Task AssertOnDelete(Action assert)
 		{
 			if (!this.SupportsDeletes) return;
-			await this.AssertOnAllResponses(this._deleteResponse, assert);
+			await this.AssertOnAllResponses(_deleteResponse, assert);
+		}
+
+		protected async Task AssertOnGetAfterCreate(Action assert)
+		{
+			await this.AssertOnAllResponses(_createGetResponse, assert);
+		}
+
+		protected async Task AssertOnGetAfterUpdate(Action assert)
+		{
+			if (!this.SupportsUpdates) return;
+			await this.AssertOnAllResponses(_updateGetResponse, assert);
 		}
 
-		protected async Task AssertOnGetAfterCreate(Action assert) => await this.AssertOnAllResponses(this._createGetResponse, assert);
-		protected async Task AssertOnGetAfterUpdate(Action assert) => await this.AssertOnAllResponses(this._updateGetResponse, assert);
 		protected async Task AssertOnGetAfterDelete(Action assert)
 		{
 			if (!this.SupportsDeletes) return;
-			await this.AssertOnAllResponses(this._deleteGetResponse, assert);
+			await this.AssertOnAllResponses(_deleteGetResponse, assert);
 		}
 
 		protected async Task AssertOnExistsAfterCreate(Action assert)
 		{
 			if (!this.SupportsExists) return;
-			await this.AssertOnAllResponses(this._createExistsResponse, assert);
+			await this.AssertOnAllResponses(_createExistsResponse, assert);
 		}
 		protected async Task AssertOnExistsAfterDelete(Action assert)
 		{
 			if (!this.SupportsExists) return;
-			await this.AssertOnAllResponses(this._deleteExistsResponse, assert);
+			await this.AssertOnAllResponses(_deleteExistsResponse, assert);
 		}
 
 		protected async Task AssertOnDeleteNotFoundAfterDelete(Action assert)
 		{
 			if (!this.SupportsDeletes) return;
-			await this.AssertOnAllResponses(this._deleteNotFoundResponse, assert);
+			await this.AssertOnAllResponses(_deleteNotFoundResponse, assert);
 		}
 
 		protected virtual void ExpectAfterCreate(TReadResponse response) { }
@@ -253,19 +324,19 @@ protected virtual void ExpectExistsAfterDelete(TExistsResponse response) { }
 		[I] protected virtual async Task CreateCallIsValid() => await this.AssertOnCreate(r => r.ShouldBeValid());
 		[I] protected virtual async Task GetAfterCreateIsValid() => await this.AssertOnGetAfterCreate(r => {
 			r.ShouldBeValid();
-			ExpectAfterCreate(r);
+			this.ExpectAfterCreate(r);
 		});
 		[I] protected virtual async Task ExistsAfterCreateIsValid() => await this.AssertOnExistsAfterCreate(r => {
 			r.ShouldBeValid();
 			r.Exists.Should().BeTrue();
-			ExpectExistsAfterCreate(r);
+			this.ExpectExistsAfterCreate(r);
 		});
 
 		[I] protected virtual async Task UpdateCallIsValid() => await this.AssertOnUpdate(r => r.ShouldBeValid());
 
 		[I] protected virtual async Task GetAfterUpdateIsValid() => await this.AssertOnGetAfterUpdate(r => {
 			r.ShouldBeValid();
-			ExpectAfterUpdate(r);
+			this.ExpectAfterUpdate(r);
 		});
 
 		[I] protected virtual async Task DeleteCallIsValid() => await this.AssertOnDelete(r => r.ShouldBeValid());
@@ -273,12 +344,12 @@ [I] protected virtual async Task GetAfterUpdateIsValid() => await this.AssertOnG
 		[I] protected virtual async Task ExistsAfterDeleteIsValid() => await this.AssertOnExistsAfterDelete(r => {
 			r.ShouldBeValid();
 			r.Exists.Should().BeFalse();
-			ExpectExistsAfterDelete(r);
+			this.ExpectExistsAfterDelete(r);
 		});
 		[I] protected virtual async Task DeleteNotFoundIsNotValid() => await this.AssertOnDeleteNotFoundAfterDelete(r =>
 		{
 			r.ShouldNotBeValid();
-			ExpectDeleteNotFoundResponse(r);
+			this.ExpectDeleteNotFoundResponse(r);
 		});
 	}
 }
diff --git a/src/Tests/Tests/XPack/Rollup/CreateRollupJob/CreateRollupUrlTests.cs b/src/Tests/Tests/XPack/Rollup/CreateRollupJob/CreateRollupUrlTests.cs
new file mode 100644
index 00000000000..595f1fb2a48
--- /dev/null
+++ b/src/Tests/Tests/XPack/Rollup/CreateRollupJob/CreateRollupUrlTests.cs
@@ -0,0 +1,22 @@
+using System.Threading.Tasks;
+using Elastic.Xunit.XunitPlumbing;
+using Nest;
+using Tests.Domain;
+using Tests.Framework;
+using static Tests.Framework.UrlTester;
+
+namespace Tests.XPack.Rollup.CreateRollupJob
+{
+	public class CreateRollupUrlTests : UrlTestsBase
+	{
+		[U] public override async Task Urls()
+		{
+			const string id = "rollup-id";
+			await PUT($"/_xpack/rollup/job/{id}")
+				.Fluent(c => c.CreateRollupJob(id, s => s))
+				.Request(c => c.CreateRollupJob(new CreateRollupJobRequest(id)))
+				.FluentAsync(c => c.CreateRollupJobAsync(id, s => s))
+				.RequestAsync(c => c.CreateRollupJobAsync(new CreateRollupJobRequest(id)));
+		}
+	}
+}
diff --git a/src/Tests/Tests/XPack/Rollup/DeleteRollupJob/DeleteRollupUrlTests.cs b/src/Tests/Tests/XPack/Rollup/DeleteRollupJob/DeleteRollupUrlTests.cs
new file mode 100644
index 00000000000..350be775758
--- /dev/null
+++ b/src/Tests/Tests/XPack/Rollup/DeleteRollupJob/DeleteRollupUrlTests.cs
@@ -0,0 +1,21 @@
+using System.Threading.Tasks;
+using Elastic.Xunit.XunitPlumbing;
+using Nest;
+using Tests.Framework;
+using static Tests.Framework.UrlTester;
+
+namespace Tests.XPack.Rollup.DeleteRollupJob
+{
+	public class DeleteRollupUrlTests : UrlTestsBase
+	{
+		[U] public override async Task Urls()
+		{
+			const string id = "rollup-id";
+			await DELETE($"/_xpack/rollup/job/{id}")
+				.Fluent(c => c.DeleteRollupJob(id))
+				.Request(c => c.DeleteRollupJob(new DeleteRollupJobRequest(id)))
+				.FluentAsync(c => c.DeleteRollupJobAsync(id))
+				.RequestAsync(c => c.DeleteRollupJobAsync(new DeleteRollupJobRequest(id)));
+		}
+	}
+}
diff --git a/src/Tests/Tests/XPack/Rollup/GetRollupCapabilities/GetRollupCapabilitiesTests.cs b/src/Tests/Tests/XPack/Rollup/GetRollupCapabilities/GetRollupCapabilitiesTests.cs
new file mode 100644
index 00000000000..e85e17b8d62
--- /dev/null
+++ b/src/Tests/Tests/XPack/Rollup/GetRollupCapabilities/GetRollupCapabilitiesTests.cs
@@ -0,0 +1,27 @@
+using System.Threading.Tasks;
+using Elastic.Xunit.XunitPlumbing;
+using Nest;
+using Tests.Framework;
+using static Tests.Framework.UrlTester;
+
+namespace Tests.XPack.Rollup.GetRollupCapabilities
+{
+	public class GetRollupCapabilitiesTests : UrlTestsBase
+	{
+		[U] public override async Task Urls()
+		{
+			const string index = "rollup-index";
+			await GET($"_xpack/rollup/data/{index}")
+				.Fluent(c => c.GetRollupCapabilities(j => j.Index(index)))
+				.Request(c => c.GetRollupCapabilities(new GetRollupCapabilitiesRequest(index)))
+				.FluentAsync(c => c.GetRollupCapabilitiesAsync(j => j.Index(index)))
+				.RequestAsync(c => c.GetRollupCapabilitiesAsync(new GetRollupCapabilitiesRequest(index)));
+
+			await GET($"_xpack/rollup/data/")
+				.Fluent(c => c.GetRollupCapabilities())
+				.Request(c => c.GetRollupCapabilities(new GetRollupCapabilitiesRequest()))
+				.FluentAsync(c => c.GetRollupCapabilitiesAsync())
+				.RequestAsync(c => c.GetRollupCapabilitiesAsync(new GetRollupCapabilitiesRequest()));
+		}
+	}
+}
diff --git a/src/Tests/Tests/XPack/Rollup/GetRollupJob/GetRollupUrlTests.cs b/src/Tests/Tests/XPack/Rollup/GetRollupJob/GetRollupUrlTests.cs
new file mode 100644
index 00000000000..d386953fdcf
--- /dev/null
+++ b/src/Tests/Tests/XPack/Rollup/GetRollupJob/GetRollupUrlTests.cs
@@ -0,0 +1,27 @@
+using System.Threading.Tasks;
+using Elastic.Xunit.XunitPlumbing;
+using Nest;
+using Tests.Framework;
+using static Tests.Framework.UrlTester;
+
+namespace Tests.XPack.Rollup.GetRollupJob
+{
+	public class GetRollupUrlTests : UrlTestsBase
+	{
+		[U] public override async Task Urls()
+		{
+			const string id = "rollup-id";
+			await GET($"/_xpack/rollup/job/{id}")
+				.Fluent(c => c.GetRollupJob(j => j.Id(id)))
+				.Request(c => c.GetRollupJob(new GetRollupJobRequest(id)))
+				.FluentAsync(c => c.GetRollupJobAsync(j => j.Id(id)))
+				.RequestAsync(c => c.GetRollupJobAsync(new GetRollupJobRequest(id)));
+
+			await GET($"/_xpack/rollup/job/")
+				.Fluent(c => c.GetRollupJob())
+				.Request(c => c.GetRollupJob(new GetRollupJobRequest()))
+				.FluentAsync(c => c.GetRollupJobAsync())
+				.RequestAsync(c => c.GetRollupJobAsync(new GetRollupJobRequest()));
+		}
+	}
+}
diff --git a/src/Tests/Tests/XPack/Rollup/RollupJobCrudTests.cs b/src/Tests/Tests/XPack/Rollup/RollupJobCrudTests.cs
new file mode 100644
index 00000000000..41ef30c97fe
--- /dev/null
+++ b/src/Tests/Tests/XPack/Rollup/RollupJobCrudTests.cs
@@ -0,0 +1,289 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reactive.Threading.Tasks;
+using System.Threading.Tasks;
+using Bogus.DataSets;
+using Elastic.Xunit.XunitPlumbing;
+using FluentAssertions;
+using Nest;
+using Tests.Core.Extensions;
+using Tests.Core.ManagedElasticsearch.Clusters;
+using Tests.Domain;
+using Tests.Framework;
+using Tests.Framework.Integration;
+
+namespace Tests.XPack.Rollup
+{
+	public class RollupJobCrudTests
+		: CrudTestBase
+	{
+		public RollupJobCrudTests(TimeSeriesCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
+
+		private static string CreateRollupName(string s) => $"rollup-logs-{s}";
+		private static readonly string CronPeriod = "*/2 * * * * ?";
+
+		protected override bool SupportsUpdates => false;
+		protected override bool TestOnlyOneMethod => true;
+
+		protected override LazyResponses Create() => this.Calls, CreateRollupJobRequest, ICreateRollupJobRequest, ICreateRollupJobResponse>(
+			this.CreateInitializer,
+			this.CreateFluent,
+			fluent: (s, c, f) => c.CreateRollupJob(CreateRollupName(s), f),
+			fluentAsync: (s, c, f) => c.CreateRollupJobAsync(CreateRollupName(s), f),
+			request: (s, c, r) => c.CreateRollupJob(r),
+			requestAsync: (s, c, r) => c.CreateRollupJobAsync(r)
+		);
+
+		protected CreateRollupJobRequest CreateInitializer(string role) => new CreateRollupJobRequest(CreateRollupName(role))
+		{
+			PageSize = 1000,
+			IndexPattern = TimeSeriesSeeder.IndicesWildCard,
+			RollupIndex = CreateRollupName(role),
+			Cron = CronPeriod,
+			Groups = new RollupGroupings
+			{
+				DateHistogram = new DateHistogramRollupGrouping
+				{
+					Field = Infer.Field(p=>p.Timestamp),
+					Interval = TimeSpan.FromHours(1)
+				},
+				Terms = new TermsRollupGrouping
+				{
+					Fields = Infer.Field(p=>p.Section).And(p=>p.Tag)
+				},
+				Histogram = new HistogramRollupGrouping
+				{
+					Fields = Infer.Field(p => p.Load).And(p => p.NetIn).And(p => p.NetOut),
+					Interval = 5
+				}
+			},
+			Metrics = new List
+			{
+				new RollupFieldMetric
+				{
+					Field = Infer.Field(p=>p.Temperature),
+					Metrics = new [] {RollupMetric.Average,RollupMetric.Min,RollupMetric.Max}
+				},
+			}
+		};
+		protected ICreateRollupJobRequest CreateFluent(string role, CreateRollupJobDescriptor d) => d
+			.PageSize(1000)
+			.IndexPattern(TimeSeriesSeeder.IndicesWildCard)
+			.RollupIndex(CreateRollupName(role))
+			.Cron(CronPeriod)
+			.Groups(g => g
+				.DateHistogram(dh => dh.Field(p => p.Timestamp).Interval(TimeSpan.FromHours(1)))
+				.Terms(t => t.Fields(f => f.Field(p => p.Section).Field(p => p.Tag)))
+				.Histogram(h => h.Fields(f => f.Field(p => p.Load).Field(p => p.NetIn).Field(p => p.NetOut)).Interval(5))
+			)
+			.Metrics(m=>m
+				.Field(p => p.Temperature, RollupMetric.Average, RollupMetric.Min, RollupMetric.Max)
+			);
+
+		protected override LazyResponses Read() => this.Calls(
+			this.ReadInitializer,
+			this.ReadFluent,
+			fluent: (s, c, f) => c.GetRollupJob(f),
+			fluentAsync: (s, c, f) => c.GetRollupJobAsync(f),
+			request: (s, c, r) => c.GetRollupJob(r),
+			requestAsync: (s, c, r) => c.GetRollupJobAsync(r)
+		);
+
+		protected GetRollupJobRequest ReadInitializer(string role) => new GetRollupJobRequest(CreateRollupName(role));
+		protected IGetRollupJobRequest ReadFluent(string role, GetRollupJobDescriptor d) => d.Id(CreateRollupName(role));
+
+		protected override IDictionary> AfterCreateCalls() => new Dictionary>
+		{
+			{ "start", () => this.Calls(
+				this.StartInitializer,
+				this.StartFluent,
+				fluent: (s, c, f) => c.StartRollupJob(CreateRollupName(s), f),
+				fluentAsync: (s, c, f) => c.StartRollupJobAsync(CreateRollupName(s), f),
+				request: (s, c, r) => c.StartRollupJob(r),
+				requestAsync: (s, c, r) => c.StartRollupJobAsync(r)
+			)},
+			{ "wait_for_finish", () => this.Call(this.WaitForFinish) },
+			{ "rollup_search", () => this.Calls, RollupSearchRequest, IRollupSearchRequest, IRollupSearchResponse>(
+				this.RollupSearchInitializer,
+				this.RollupSearchFluent,
+				fluent: (s, c, f) => c.RollupSearch(CreateRollupSearchIndices(s), f),
+				fluentAsync: (s, c, f) => c.RollupSearchAsync(CreateRollupSearchIndices(s), f),
+				request: (s, c, r) => c.RollupSearch(r),
+				requestAsync: (s, c, r) => c.RollupSearchAsync(r)
+			)},
+			{ "rollup_caps", () => this.Calls(
+				this.CapsInitializer,
+				this.CapsFluent,
+				fluent: (s, c, f) => c.GetRollupCapabilities(f),
+				fluentAsync: (s, c, f) => c.GetRollupCapabilitiesAsync(f),
+				request: (s, c, r) => c.GetRollupCapabilities(r),
+				requestAsync: (s, c, r) => c.GetRollupCapabilitiesAsync(r)
+			)},
+			{ "stop", () => this.Calls(
+				this.StopInitializer,
+				this.StopFluent,
+				fluent: (s, c, f) => c.StopRollupJob(CreateRollupName(s), f),
+				fluentAsync: (s, c, f) => c.StopRollupJobAsync(CreateRollupName(s), f),
+				request: (s, c, r) => c.StopRollupJob(r),
+				requestAsync: (s, c, r) => c.StopRollupJobAsync(r)
+			)},
+
+		};
+		protected StartRollupJobRequest StartInitializer(string role) => new StartRollupJobRequest(CreateRollupName(role));
+		protected IStartRollupJobRequest StartFluent(string role, StartRollupJobDescriptor d) => d;
+
+		protected StopRollupJobRequest StopInitializer(string role) => new StopRollupJobRequest(CreateRollupName(role));
+		protected IStopRollupJobRequest StopFluent(string role, StopRollupJobDescriptor d) => d;
+
+		protected GetRollupCapabilitiesRequest CapsInitializer(string role) => new GetRollupCapabilitiesRequest(TimeSeriesSeeder.IndicesWildCard);
+		protected IGetRollupCapabilitiesRequest CapsFluent(string role, GetRollupCapabilitiesDescriptor d) => d.Index(TimeSeriesSeeder.IndicesWildCard);
+
+		[I] public async Task StartsJob() =>
+			await this.AssertOnAfterCreateResponse("start", r => r.Started.Should().BeTrue());
+
+		[I] public async Task StopsJob() =>
+			await this.AssertOnAfterCreateResponse("stop", r => r.Stopped.Should().BeTrue());
+
+		private async Task WaitForFinish(List allJobs, IElasticClient client)
+		{
+			var tasks = new List>(4);
+			foreach(var job in allJobs)
+				tasks.Add(WaitForFinish(CreateRollupName(job), client));
+			await Task.WhenAll(tasks);
+			return tasks[0].Result;
+		}
+		private static async Task WaitForFinish(string job, IElasticClient client)
+		{
+			IGetRollupJobResponse response;
+			var stillRunning = true;
+			long processed = 0;
+			do
+			{
+				//we can do this because we know new data is no longer indexed into these indexes
+				response = await client.GetRollupJobAsync(j => j.Id(job));
+				var validResponseWithJobs = response.IsValid && response.Jobs.Count > 0;
+				if (!validResponseWithJobs) break;
+
+				var processedNow = response.Jobs.First().Stats.DocumentsProcessed;
+				if (processed > 0 && processedNow == processed) break;
+
+				stillRunning = response.Jobs.All(j => j.Status.JobState != IndexingJobState.Stopped);
+				processed = processedNow;
+				await Task.Delay(TimeSpan.FromSeconds(2));
+			} while (stillRunning);
+
+			return response;
+		}
+
+		//make sure we query a rolled up index and a live index
+		protected static Nest.Indices CreateRollupSearchIndices(string rollupIndex) => $"{CreateRollupName(rollupIndex)},logs-{DateTime.UtcNow:yyyy-MM-dd}";
+
+		protected RollupSearchRequest RollupSearchInitializer(string index) => new RollupSearchRequest(CreateRollupSearchIndices(index))
+		{
+			Size = 0,
+			Query = new MatchAllQuery() {},
+			Aggregations = new MaxAggregation("max_temp", Infer.Field(p=>p.Temperature))
+				&& new AverageAggregation("avg_temp", Infer.Field(p=>p.Temperature))
+		};
+
+		protected IRollupSearchRequest RollupSearchFluent(string index, RollupSearchDescriptor d) => d
+			.Size(0)
+			.Query(q=>q.MatchAll())
+			.Aggregations(aggs=>
+				aggs.Max("max_temp", m=>m.Field(p=>p.Temperature))
+				&& aggs.Min("avg_temp", m=>m.Field(p=>p.Temperature))
+			);
+
+		[I] public async Task RollupSearchReturnsAggregations() =>
+			await this.AssertOnAfterCreateResponse>("rollup_search", r =>
+			{
+				r.ShouldBeValid();
+				var avg = r.Aggregations.Average("avg_temp");
+				avg.Should().NotBeNull();
+				avg.Value.Should().HaveValue().And.BeInRange(-10, 45);
+				var max = r.Aggregations.Max("max_temp");
+				max.Should().NotBeNull();
+				max.Value.Should().HaveValue().And.BeInRange(-10, 45);
+			});
+
+		[I] public async Task GetRollupCapabilities() =>
+			await this.AssertOnAfterCreateResponse("rollup_caps", r =>
+			{
+				r.IsValid.Should().BeTrue();
+				r.Indices.Should().NotBeEmpty().And.ContainKey(TimeSeriesSeeder.IndicesWildCard);
+
+				var indexCaps = r.Indices[TimeSeriesSeeder.IndicesWildCard];
+				indexCaps.RollupJobs.Should().NotBeEmpty();
+				var job = indexCaps.RollupJobs.First();
+				job.JobId.Should().NotBeNullOrWhiteSpace();
+				job.RollupIndex.Should().NotBeNullOrWhiteSpace();
+				job.IndexPattern.Should().Be(TimeSeriesSeeder.IndicesWildCard);
+				job.Fields.Should().NotBeEmpty();
+				var capabilities = job.Fields.Field(p => p.Temperature);
+				capabilities.Should().NotBeEmpty();
+				foreach (var c in capabilities)
+					c.Should().ContainKey("agg");
+			});
+
+
+		// ignored because we mark SupportsUpdates => false
+		protected override LazyResponses Update() => null;
+
+		protected override LazyResponses Delete() => this.Calls(
+			this.DeleteInitializer,
+			this.DeleteFluent,
+			fluent: (s, c, f) => c.DeleteRollupJob(CreateRollupName(s), f),
+			fluentAsync: (s, c, f) => c.DeleteRollupJobAsync(CreateRollupName(s), f),
+			request: (s, c, r) => c.DeleteRollupJob(r),
+			requestAsync: (s, c, r) => c.DeleteRollupJobAsync(r)
+		);
+
+		protected DeleteRollupJobRequest DeleteInitializer(string role) => new DeleteRollupJobRequest(CreateRollupName(role));
+		protected IDeleteRollupJobRequest DeleteFluent(string role, DeleteRollupJobDescriptor d) => d;
+
+		protected override void ExpectAfterCreate(IGetRollupJobResponse response)
+		{
+			response.Jobs.Should().NotBeNull().And.NotBeEmpty();
+			foreach (var j in response.Jobs)
+			{
+				j.Config.Should().NotBeNull("job should have config");
+				j.Config.Cron.Should().NotBeNullOrWhiteSpace();
+				j.Config.Id.Should().NotBeNullOrWhiteSpace();
+				j.Config.Timeout.Should().NotBeNull().And.Be("20s");
+				j.Config.IndexPattern.Should().NotBeNull().And.Be(TimeSeriesSeeder.IndicesWildCard);
+				j.Config.RollupIndex.Should().NotBeNull();
+				j.Config.PageSize.Should().Be(1000);
+				j.Config.Groups.Should().NotBeNull();
+				j.Config.Groups.Terms.Should().NotBeNull();
+				j.Config.Groups.Terms.Fields.Should().NotBeNull();
+				j.Config.Groups.DateHistogram.Should().NotBeNull();
+				j.Config.Groups.DateHistogram.Field.Should().NotBeNull().And.Be("timestamp");
+				j.Config.Groups.DateHistogram.Interval.Should().NotBeNull().And.Be("1h");
+				j.Config.Groups.DateHistogram.Interval.Should().NotBeNull().And.Be(TimeSpan.FromHours(1));
+				j.Config.Groups.Histogram.Should().NotBeNull();
+				j.Config.Groups.Histogram.Fields.Should().NotBeNull();
+				j.Config.Groups.Histogram.Interval.Should().NotBeNull().And.Be(5);
+
+				j.Config.Metrics.Should().NotBeEmpty("config should have metrics");
+				foreach (var m in j.Config.Metrics)
+				{
+					m.Field.Should().NotBeNull("metric field");
+					m.Metrics.Should().NotBeEmpty("metric metrics");
+				}
+
+				j.Stats.Should().NotBeNull("job should have stats");
+				j.Stats.DocumentsProcessed.Should().Be(0);
+				j.Stats.PagesProcessed.Should().Be(0);
+				j.Stats.RollupsIndexed.Should().Be(0);
+				j.Stats.TriggerCount.Should().Be(0);
+				j.Status.Should().NotBeNull("job should have status");
+				j.Status.JobState.Should().Be(IndexingJobState.Stopped);
+				j.Status.UpgradedDocId.Should().BeTrue("indexing status upgrade doc id");
+			}
+		}
+
+		[I, SkipVersion("<6.4.1", "https://github.com/elastic/elasticsearch/issues/34292")]
+		protected override async Task GetAfterDeleteIsValid() => await this.AssertOnGetAfterDelete(r => r.ShouldBeValid());
+	}
+}
diff --git a/src/Tests/Tests/XPack/Rollup/RollupSearch/RollupSearchUrlTests.cs b/src/Tests/Tests/XPack/Rollup/RollupSearch/RollupSearchUrlTests.cs
new file mode 100644
index 00000000000..a6fde9b0f0e
--- /dev/null
+++ b/src/Tests/Tests/XPack/Rollup/RollupSearch/RollupSearchUrlTests.cs
@@ -0,0 +1,28 @@
+using System.Threading.Tasks;
+using Elastic.Xunit.XunitPlumbing;
+using Nest;
+using Tests.Domain;
+using Tests.Framework;
+using static Tests.Framework.UrlTester;
+
+namespace Tests.XPack.Rollup.RollupSearch
+{
+	public class RollupSearchUrlTests : UrlTestsBase
+	{
+		[U] public override async Task Urls()
+		{
+			const string index = "default-index";
+			await POST($"/{index}/_rollup_search")
+				.Fluent(c => c.RollupSearch(index, s => s))
+				.Request(c => c.RollupSearch(new RollupSearchRequest(index)))
+				.FluentAsync(c => c.RollupSearchAsync(index, s => s))
+				.RequestAsync(c => c.RollupSearchAsync(new RollupSearchRequest(index)));
+
+			await POST($"/_all/_rollup_search")
+				.Fluent(c => c.RollupSearch(Nest.Indices.All, s => s))
+				.Request(c => c.RollupSearch(new RollupSearchRequest(Nest.Indices.All)))
+				.FluentAsync(c => c.RollupSearchAsync(Nest.Indices.All, s => s))
+				.RequestAsync(c => c.RollupSearchAsync(new RollupSearchRequest(Nest.Indices.All)));
+		}
+	}
+}
diff --git a/src/Tests/Tests/XPack/Rollup/StartRollupJob/StartRollupUrlTests.cs b/src/Tests/Tests/XPack/Rollup/StartRollupJob/StartRollupUrlTests.cs
new file mode 100644
index 00000000000..388c68a0915
--- /dev/null
+++ b/src/Tests/Tests/XPack/Rollup/StartRollupJob/StartRollupUrlTests.cs
@@ -0,0 +1,21 @@
+using System.Threading.Tasks;
+using Elastic.Xunit.XunitPlumbing;
+using Nest;
+using Tests.Framework;
+using static Tests.Framework.UrlTester;
+
+namespace Tests.XPack.Rollup.StartRollupJob
+{
+	public class StartRollupUrlTests : UrlTestsBase
+	{
+		[U] public override async Task Urls()
+		{
+			const string id = "rollup-id";
+			await POST($"/_xpack/rollup/job/{id}/_start")
+				.Fluent(c => c.StartRollupJob(id))
+				.Request(c => c.StartRollupJob(new StartRollupJobRequest(id)))
+				.FluentAsync(c => c.StartRollupJobAsync(id))
+				.RequestAsync(c => c.StartRollupJobAsync(new StartRollupJobRequest(id)));
+		}
+	}
+}
diff --git a/src/Tests/Tests/XPack/Rollup/StopRollupJob/StopRollupUrlTests.cs b/src/Tests/Tests/XPack/Rollup/StopRollupJob/StopRollupUrlTests.cs
new file mode 100644
index 00000000000..0b937653275
--- /dev/null
+++ b/src/Tests/Tests/XPack/Rollup/StopRollupJob/StopRollupUrlTests.cs
@@ -0,0 +1,21 @@
+using System.Threading.Tasks;
+using Elastic.Xunit.XunitPlumbing;
+using Nest;
+using Tests.Framework;
+using static Tests.Framework.UrlTester;
+
+namespace Tests.XPack.Rollup.StopRollupJob
+{
+	public class StopRollupUrlTests : UrlTestsBase
+	{
+		[U] public override async Task Urls()
+		{
+			const string id = "rollup-id";
+			await POST($"/_xpack/rollup/job/{id}/_stop")
+				.Fluent(c => c.StopRollupJob(id))
+				.Request(c => c.StopRollupJob(new StopRollupJobRequest(id)))
+				.FluentAsync(c => c.StopRollupJobAsync(id))
+				.RequestAsync(c => c.StopRollupJobAsync(new StopRollupJobRequest(id)));
+		}
+	}
+}

From 84d77b6dd784e1cbf00a5b245bdd586ed8473270 Mon Sep 17 00:00:00 2001
From: Russ Cam 
Date: Thu, 18 Oct 2018 12:08:40 +1000
Subject: [PATCH 49/63] Generate APIs from 6.4 branch

This commit re-runs the API generation against the 6.4 branch.
Ignore the new nodes.reload_secure_settings.json endpoint for now
---
 .../ApiGenerator/ApiGenerator.cs              |    7 +-
 .../Core/nodes.reload_secure_settings.json    |   23 +
 .../RestSpecification/Core/root.html          | 1055 +++++++++++------
 .../ApiGenerator/last_downloaded_version.txt  |    2 +-
 4 files changed, 725 insertions(+), 362 deletions(-)
 create mode 100644 src/CodeGeneration/ApiGenerator/RestSpecification/Core/nodes.reload_secure_settings.json

diff --git a/src/CodeGeneration/ApiGenerator/ApiGenerator.cs b/src/CodeGeneration/ApiGenerator/ApiGenerator.cs
index c71340f3b11..6f1f9ea141c 100644
--- a/src/CodeGeneration/ApiGenerator/ApiGenerator.cs
+++ b/src/CodeGeneration/ApiGenerator/ApiGenerator.cs
@@ -59,6 +59,7 @@ public static void Generate(string downloadBranch, params string[] folders)
 			"xpack.ml.get_filters.json",
 			"xpack.ml.put_filter.json",
 			"rank_eval.json",
+
 			// these API's are new and need to be mapped
 			"xpack.license.get_basic_status.json",
 			"xpack.license.post_start_basic.json",
@@ -72,20 +73,16 @@ public static void Generate(string downloadBranch, params string[] folders)
 			"xpack.ml.put_calendar.json",
 			"xpack.ml.put_calendar_job.json",
 			"xpack.ml.get_calendar_job.json",
-
-			"xpack.sql.clear_cursor.json",
-			"xpack.sql.query.json",
-			"xpack.sql.translate.json",
 			"xpack.ssl.certificates.json",
 
 			// 6.4 new  API's
-
 			"xpack.ml.update_filter.json",
 			"xpack.security.delete_privileges.json",
 			"xpack.security.get_privileges.json",
 			"xpack.security.has_privileges.json",
 			"xpack.security.put_privilege.json",
 			"xpack.security.put_privileges.json",
+			"nodes.reload_secure_settings.json"
 		};
 
 		private static RestApiSpec CreateRestApiSpecModel(string downloadBranch, string[] folders)
diff --git a/src/CodeGeneration/ApiGenerator/RestSpecification/Core/nodes.reload_secure_settings.json b/src/CodeGeneration/ApiGenerator/RestSpecification/Core/nodes.reload_secure_settings.json
new file mode 100644
index 00000000000..487beaba865
--- /dev/null
+++ b/src/CodeGeneration/ApiGenerator/RestSpecification/Core/nodes.reload_secure_settings.json
@@ -0,0 +1,23 @@
+{
+  "nodes.reload_secure_settings": {
+    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-nodes-reload-secure-settings.html",
+    "methods": ["POST"],
+    "url": {
+      "path": "/_nodes/reload_secure_settings",
+      "paths": ["/_nodes/reload_secure_settings", "/_nodes/{node_id}/reload_secure_settings"],
+      "parts": {
+        "node_id": {
+          "type": "list",
+          "description": "A comma-separated list of node IDs to span the reload/reinit call. Should stay empty because reloading usually involves all cluster nodes."
+        }
+      },
+      "params": {
+        "timeout": {
+          "type" : "time",
+          "description" : "Explicit operation timeout"
+        }
+      }
+    },
+    "body": null
+  }
+}
diff --git a/src/CodeGeneration/ApiGenerator/RestSpecification/Core/root.html b/src/CodeGeneration/ApiGenerator/RestSpecification/Core/root.html
index f4b99b9bc20..5731743b1a3 100644
--- a/src/CodeGeneration/ApiGenerator/RestSpecification/Core/root.html
+++ b/src/CodeGeneration/ApiGenerator/RestSpecification/Core/root.html
@@ -18,47 +18,49 @@
 
 
 
-  
-  
+  
+  
   
   
-  
+  
   
 
   
   
-  elasticsearch/rest-api-spec/src/main/resources/rest-api-spec/api at 6.2 · elastic/elasticsearch · GitHub
-    
+  elasticsearch/rest-api-spec/src/main/resources/rest-api-spec/api at 6.4 · elastic/elasticsearch · GitHub
+    
     
   
   
 
     
-    
+    
 
   
   
   
   
-  
+  
 
 
   
 
   
 
-    
-  
-  
-    
+      
+    
+    
 
-
+  
 
 
 
 
+    
+
+
+
 
-  
 
 
   
@@ -67,24 +69,23 @@
     
 
       
-    
+    
 
-    
+    
 
-  
+  
 
-  
+  
   
 
-      
+      
 
-  
   
 
   
 
 
-    
+    
 
 
   
@@ -98,7 +99,7 @@
 
 
 
-
+  
 
   
 
@@ -115,7 +116,8 @@
 
 
 
-