diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolver.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolver.java index 0270ee22ca8c5..89a40711c9a19 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolver.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolver.java @@ -11,7 +11,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.commons.compress.utils.Lists; import org.gradle.jvm.toolchain.JavaLanguageVersion; import org.gradle.jvm.toolchain.JavaToolchainDownload; import org.gradle.jvm.toolchain.JavaToolchainRequest; @@ -21,17 +20,17 @@ import java.io.IOException; import java.net.URI; import java.net.URL; -import java.util.Comparator; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.StreamSupport; import static org.gradle.jvm.toolchain.JavaToolchainDownload.fromUri; public abstract class AdoptiumJdkToolchainResolver extends AbstractCustomJavaToolchainResolver { // package protected for better testing - final Map> CACHED_SEMVERS = new ConcurrentHashMap<>(); + final Map> CACHED_RELEASES = new ConcurrentHashMap<>(); @Override public Optional resolve(JavaToolchainRequest request) { @@ -39,7 +38,7 @@ public Optional resolve(JavaToolchainRequest request) { return Optional.empty(); } AdoptiumVersionRequest versionRequestKey = toVersionRequest(request); - Optional versionInfo = CACHED_SEMVERS.computeIfAbsent( + Optional versionInfo = CACHED_RELEASES.computeIfAbsent( versionRequestKey, (r) -> resolveAvailableVersion(versionRequestKey) ); @@ -54,12 +53,12 @@ private AdoptiumVersionRequest toVersionRequest(JavaToolchainRequest request) { return new AdoptiumVersionRequest(platform, arch, javaLanguageVersion); } - private Optional resolveAvailableVersion(AdoptiumVersionRequest requestKey) { + private Optional resolveAvailableVersion(AdoptiumVersionRequest requestKey) { ObjectMapper mapper = new ObjectMapper(); try { int languageVersion = requestKey.languageVersion.asInt(); URL source = new URL( - "https://api.adoptium.net/v3/info/release_versions?architecture=" + "https://api.adoptium.net/v3/info/release_names?architecture=" + requestKey.arch + "&image_type=jdk&os=" + requestKey.platform @@ -71,14 +70,8 @@ private Optional resolveAvailableVersion(AdoptiumVersionReq + ")" ); JsonNode jsonNode = mapper.readTree(source); - JsonNode versionsNode = jsonNode.get("versions"); - return Optional.of( - Lists.newArrayList(versionsNode.iterator()) - .stream() - .map(this::toVersionInfo) - .max(Comparator.comparing(AdoptiumVersionInfo::semver)) - .get() - ); + JsonNode versionsNode = jsonNode.get("releases"); + return StreamSupport.stream(versionsNode.spliterator(), false).map(JsonNode::textValue).findFirst(); } catch (FileNotFoundException e) { // request combo not supported (e.g. aarch64 + windows return Optional.empty(); @@ -87,21 +80,10 @@ private Optional resolveAvailableVersion(AdoptiumVersionReq } } - private AdoptiumVersionInfo toVersionInfo(JsonNode node) { - return new AdoptiumVersionInfo( - node.get("build").asInt(), - node.get("major").asInt(), - node.get("minor").asInt(), - node.get("openjdk_version").asText(), - node.get("security").asInt(), - node.get("semver").asText() - ); - } - - private URI resolveDownloadURI(AdoptiumVersionRequest request, AdoptiumVersionInfo versionInfo) { + private URI resolveDownloadURI(AdoptiumVersionRequest request, String version) { return URI.create( - "https://api.adoptium.net/v3/binary/version/jdk-" - + versionInfo.semver + "https://api.adoptium.net/v3/binary/version/" + + version + "/" + request.platform + "/" @@ -118,7 +100,5 @@ private boolean requestIsSupported(JavaToolchainRequest request) { return anyVendorOr(request.getJavaToolchainSpec().getVendor().get(), JvmVendorSpec.ADOPTIUM); } - record AdoptiumVersionInfo(int build, int major, int minor, String openjdkVersion, int security, String semver) {} - record AdoptiumVersionRequest(String platform, String arch, JavaLanguageVersion languageVersion) {} } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java index 818cb040c172e..162895fd486cf 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java @@ -39,11 +39,7 @@ record JdkBuild(JavaLanguageVersion languageVersion, String version, String buil ); // package private so it can be replaced by tests - List builds = List.of( - getBundledJdkBuild(), - // 22 release candidate - new JdkBuild(JavaLanguageVersion.of(22), "22", "36", "830ec9fcccef480bb3e73fb7ecafe059") - ); + List builds = List.of(getBundledJdkBuild()); private JdkBuild getBundledJdkBuild() { String bundledJdkVersion = VersionProperties.getBundledJdkVersion(); diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolverSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolverSpec.groovy index 6383d577f027f..fe4a644ddfc1d 100644 --- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolverSpec.groovy +++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolverSpec.groovy @@ -11,7 +11,6 @@ package org.elasticsearch.gradle.internal.toolchain import org.gradle.api.services.BuildServiceParameters import org.gradle.jvm.toolchain.JavaLanguageVersion import org.gradle.jvm.toolchain.JavaToolchainResolver -import org.gradle.platform.OperatingSystem import static org.elasticsearch.gradle.internal.toolchain.AbstractCustomJavaToolchainResolver.toArchString import static org.elasticsearch.gradle.internal.toolchain.AbstractCustomJavaToolchainResolver.toOsString @@ -38,12 +37,7 @@ class AdoptiumJdkToolchainResolverSpec extends AbstractToolchainResolverSpec { toOsString(it[2], it[1]), toArchString(it[3]), languageVersion); - resolver.CACHED_SEMVERS.put(request, Optional.of(new AdoptiumJdkToolchainResolver.AdoptiumVersionInfo(languageVersion.asInt(), - 1, - 1, - "" + languageVersion.asInt() + ".1.1.1+37", - 0, "" + languageVersion.asInt() + ".1.1.1+37.1" - ))) + resolver.CACHED_RELEASES.put(request, Optional.of('jdk-' + languageVersion.asInt() + '.1.1.1+37.1')) } return resolver diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index 044f6c07c756e..a2744e89174b2 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -2,7 +2,7 @@ elasticsearch = 8.15.0 lucene = 9.10.0 bundled_jdk_vendor = openjdk -bundled_jdk = 21.0.2+13@f2283984656d49d69e91c558476027ac +bundled_jdk = 22.0.1+8@c7ec1332f7bb44aeba2eb341ae18aca4 # optional dependencies spatial4j = 0.7 jts = 1.15.0 diff --git a/docs/changelog/108409.yaml b/docs/changelog/108409.yaml new file mode 100644 index 0000000000000..6cff86cf93930 --- /dev/null +++ b/docs/changelog/108409.yaml @@ -0,0 +1,6 @@ +pr: 108409 +summary: Support multiple associated groups for TopN +area: Application +type: enhancement +issues: + - 108018 diff --git a/docs/changelog/108574.yaml b/docs/changelog/108574.yaml new file mode 100644 index 0000000000000..b3c957721e01e --- /dev/null +++ b/docs/changelog/108574.yaml @@ -0,0 +1,5 @@ +pr: 108574 +summary: "[ESQL] CBRT function" +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/108600.yaml b/docs/changelog/108600.yaml new file mode 100644 index 0000000000000..59177bf34114c --- /dev/null +++ b/docs/changelog/108600.yaml @@ -0,0 +1,15 @@ +pr: 108600 +summary: "Prevent DLS/FLS if `replication` is assigned" +area: Security +type: breaking +issues: [ ] +breaking: + title: "Prevent DLS/FLS if `replication` is assigned" + area: REST API + details: For cross-cluster API keys, {es} no longer allows specifying document-level security (DLS) + or field-level security (FLS) in the `search` field, if `replication` is also specified. + {es} likewise blocks the use of any existing cross-cluster API keys that meet this condition. + impact: Remove any document-level security (DLS) or field-level security (FLS) definitions from the `search` field + for cross-cluster API keys that also have a `replication` field, or create two separate cross-cluster API keys, + one for search and one for replication. + notable: false diff --git a/docs/changelog/108602.yaml b/docs/changelog/108602.yaml new file mode 100644 index 0000000000000..d544c89980123 --- /dev/null +++ b/docs/changelog/108602.yaml @@ -0,0 +1,5 @@ +pr: 108602 +summary: "[Inference API] Extract optional long instead of integer in `RateLimitSettings#of`" +area: Machine Learning +type: bug +issues: [] diff --git a/docs/changelog/108643.yaml b/docs/changelog/108643.yaml new file mode 100644 index 0000000000000..f71a943673326 --- /dev/null +++ b/docs/changelog/108643.yaml @@ -0,0 +1,6 @@ +pr: 108643 +summary: Use `scheduleUnlessShuttingDown` in `LeaderChecker` +area: Cluster Coordination +type: bug +issues: + - 108642 diff --git a/docs/changelog/108651.yaml b/docs/changelog/108651.yaml new file mode 100644 index 0000000000000..e629c114dac51 --- /dev/null +++ b/docs/changelog/108651.yaml @@ -0,0 +1,29 @@ +pr: 108651 +summary: Add support for the 'ISP' database to the geoip processor +area: Ingest Node +type: enhancement +issues: [] +highlight: + title: Add support for the 'ISP' database to the geoip processor + body: |- + Follow on to https://github.com/elastic/elasticsearch/pull/107287, + https://github.com/elastic/elasticsearch/pull/107377, and + https://github.com/elastic/elasticsearch/pull/108639 + + Adds support for the ['GeoIP2 + ISP'](https://dev.maxmind.com/geoip/docs/databases/isp) database from + MaxMind to the geoip processor. + + The geoip processor will automatically download the [various 'GeoLite2' + databases](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data), + but the 'GeoIP2 ISP' database is not a 'GeoLite2' database -- it's a + commercial database available to those with a suitable license from + MaxMind. + + The support that is being added for it in this PR is in line with the + support that we already have for MaxMind's 'GeoIP2 City' and 'GeoIP2 + Country' databases -- that is, one would need to arrange their own + download management via some custom endpoint or otherwise arrange for + the relevant file(s) to be in the $ES_CONFIG/ingest-geoip directory on + the nodes of the cluster. + notable: true diff --git a/docs/changelog/108654.yaml b/docs/changelog/108654.yaml new file mode 100644 index 0000000000000..9afae6a19ca80 --- /dev/null +++ b/docs/changelog/108654.yaml @@ -0,0 +1,5 @@ +pr: 108654 +summary: Update bundled JDK to Java 22 (again) +area: Packaging +type: upgrade +issues: [] diff --git a/docs/changelog/108672.yaml b/docs/changelog/108672.yaml new file mode 100644 index 0000000000000..e1261fcf6f232 --- /dev/null +++ b/docs/changelog/108672.yaml @@ -0,0 +1,5 @@ +pr: 108672 +summary: Add bounds checking to parsing ISO8601 timezone offset values +area: Infra/Core +type: bug +issues: [] diff --git a/docs/reference/connector/apis/update-connector-filtering-api.asciidoc b/docs/reference/connector/apis/update-connector-filtering-api.asciidoc index f864b68f65395..7ae80276d3151 100644 --- a/docs/reference/connector/apis/update-connector-filtering-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-filtering-api.asciidoc @@ -6,7 +6,7 @@ beta::[] -Updates the draft `filtering` configuration of a connector and marks the draft validation state as `edited`. The filtering configuration can be activated once validated by the Elastic connector service. +Updates the draft `filtering` configuration of a connector and marks the draft validation state as `edited`. The filtering draft is activated once validated by the running Elastic connector service. The filtering property is used to configure sync rules (both basic and advanced) for a connector. Learn more in the {enterprise-search-ref}/sync-rules.html[sync rules documentation]. @@ -15,14 +15,13 @@ The filtering property is used to configure sync rules (both basic and advanced) `PUT _connector//_filtering` -`PUT _connector//_filtering/_activate` - [[update-connector-filtering-api-prereq]] ==== {api-prereq-title} * To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. * The `connector_id` parameter should reference an existing connector. -* To activate filtering rules, the `draft.validation.state` must be `valid`. +* Filtering draft is activated once validated by the running Elastic connector service, the `draft.validation.state` must be `valid`. +* If, after a validation attempt, the `draft.validation.state` equals to `invalid`, inspect `draft.validation.errors` and fix any issues. [[update-connector-filtering-api-path-params]] ==== {api-path-parms-title} @@ -185,20 +184,4 @@ PUT _connector/my-sql-connector/_filtering/_validation Note, you can also update draft `rules` and `advanced_snippet` in a single request. -Once the draft is updated, its validation state is set to `edited`. The connector service will then validate the rules and report the validation state as either `invalid` or `valid`. If the state is `valid`, the draft filtering can be activated with: - - -[source,console] ----- -PUT _connector/my-sql-connector/_filtering/_activate ----- -// TEST[continued] - -[source,console-result] ----- -{ - "result": "updated" -} ----- - -Once filtering rules are activated, they will be applied to all subsequent full or incremental syncs. +Once the draft is updated, its validation state is set to `edited`. The connector service will then validate the rules and report the validation state as either `invalid` or `valid`. If the state is `valid`, the draft filtering will be activated by the running Elastic connector service. diff --git a/docs/reference/esql/esql-language.asciidoc b/docs/reference/esql/esql-language.asciidoc index 77f5e79753fdd..a6b4da47f249f 100644 --- a/docs/reference/esql/esql-language.asciidoc +++ b/docs/reference/esql/esql-language.asciidoc @@ -14,6 +14,7 @@ Detailed reference documentation for the {esql} language: * <> * <> * <> +* <> include::esql-syntax.asciidoc[] include::esql-commands.asciidoc[] @@ -23,3 +24,4 @@ include::esql-index-options.asciidoc[] include::multivalued-fields.asciidoc[] include::esql-process-data-with-dissect-grok.asciidoc[] include::esql-enrich-data.asciidoc[] +include::implicit-casting.asciidoc[] diff --git a/docs/reference/esql/functions/description/cbrt.asciidoc b/docs/reference/esql/functions/description/cbrt.asciidoc new file mode 100644 index 0000000000000..836dec8a87d69 --- /dev/null +++ b/docs/reference/esql/functions/description/cbrt.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Returns the cube root of a number. The input can be any numeric value, the return value is always a double. Cube roots of infinities are null. diff --git a/docs/reference/esql/functions/description/sqrt.asciidoc b/docs/reference/esql/functions/description/sqrt.asciidoc index 61e4f9b64fcd1..b9f354a33541f 100644 --- a/docs/reference/esql/functions/description/sqrt.asciidoc +++ b/docs/reference/esql/functions/description/sqrt.asciidoc @@ -2,4 +2,4 @@ *Description* -Returns the square root of a number. The input can be any numeric value, the return value is always a double. Square roots of negative numbers and infinites are null. +Returns the square root of a number. The input can be any numeric value, the return value is always a double. Square roots of negative numbers and infinities are null. diff --git a/docs/reference/esql/functions/examples/cbrt.asciidoc b/docs/reference/esql/functions/examples/cbrt.asciidoc new file mode 100644 index 0000000000000..56f1ef0a819e0 --- /dev/null +++ b/docs/reference/esql/functions/examples/cbrt.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/math.csv-spec[tag=cbrt] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/math.csv-spec[tag=cbrt-result] +|=== + diff --git a/docs/reference/esql/functions/kibana/definition/cbrt.json b/docs/reference/esql/functions/kibana/definition/cbrt.json new file mode 100644 index 0000000000000..600174e17ca0c --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/cbrt.json @@ -0,0 +1,59 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "cbrt", + "description" : "Returns the cube root of a number. The input can be any numeric value, the return value is always a double.\nCube roots of infinities are null.", + "signatures" : [ + { + "params" : [ + { + "name" : "number", + "type" : "double", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "integer", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "unsigned_long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + } + ], + "examples" : [ + "ROW d = 1000.0\n| EVAL c = cbrt(d)" + ] +} diff --git a/docs/reference/esql/functions/kibana/definition/sqrt.json b/docs/reference/esql/functions/kibana/definition/sqrt.json index e990049a9ce67..7d9111036402d 100644 --- a/docs/reference/esql/functions/kibana/definition/sqrt.json +++ b/docs/reference/esql/functions/kibana/definition/sqrt.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "eval", "name" : "sqrt", - "description" : "Returns the square root of a number. The input can be any numeric value, the return value is always a double.\nSquare roots of negative numbers and infinites are null.", + "description" : "Returns the square root of a number. The input can be any numeric value, the return value is always a double.\nSquare roots of negative numbers and infinities are null.", "signatures" : [ { "params" : [ diff --git a/docs/reference/esql/functions/kibana/docs/cbrt.md b/docs/reference/esql/functions/kibana/docs/cbrt.md new file mode 100644 index 0000000000000..50cdad02818e8 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/cbrt.md @@ -0,0 +1,12 @@ + + +### CBRT +Returns the cube root of a number. The input can be any numeric value, the return value is always a double. +Cube roots of infinities are null. + +``` +ROW d = 1000.0 +| EVAL c = cbrt(d) +``` diff --git a/docs/reference/esql/functions/kibana/docs/sqrt.md b/docs/reference/esql/functions/kibana/docs/sqrt.md index 264abe53921c4..fccec95a4884d 100644 --- a/docs/reference/esql/functions/kibana/docs/sqrt.md +++ b/docs/reference/esql/functions/kibana/docs/sqrt.md @@ -4,7 +4,7 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ ### SQRT Returns the square root of a number. The input can be any numeric value, the return value is always a double. -Square roots of negative numbers and infinites are null. +Square roots of negative numbers and infinities are null. ``` ROW d = 100.0 diff --git a/docs/reference/esql/functions/layout/cbrt.asciidoc b/docs/reference/esql/functions/layout/cbrt.asciidoc new file mode 100644 index 0000000000000..18106f0e6ca35 --- /dev/null +++ b/docs/reference/esql/functions/layout/cbrt.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-cbrt]] +=== `CBRT` + +*Syntax* + +[.text-center] +image::esql/functions/signature/cbrt.svg[Embedded,opts=inline] + +include::../parameters/cbrt.asciidoc[] +include::../description/cbrt.asciidoc[] +include::../types/cbrt.asciidoc[] +include::../examples/cbrt.asciidoc[] diff --git a/docs/reference/esql/functions/math-functions.asciidoc b/docs/reference/esql/functions/math-functions.asciidoc index 9aa5cd2db1927..db907c8d54061 100644 --- a/docs/reference/esql/functions/math-functions.asciidoc +++ b/docs/reference/esql/functions/math-functions.asciidoc @@ -13,6 +13,7 @@ * <> * <> * <> +* <> * <> * <> * <> @@ -37,6 +38,7 @@ include::layout/acos.asciidoc[] include::layout/asin.asciidoc[] include::layout/atan.asciidoc[] include::layout/atan2.asciidoc[] +include::layout/cbrt.asciidoc[] include::layout/ceil.asciidoc[] include::layout/cos.asciidoc[] include::layout/cosh.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/cbrt.asciidoc b/docs/reference/esql/functions/parameters/cbrt.asciidoc new file mode 100644 index 0000000000000..65013f4c21265 --- /dev/null +++ b/docs/reference/esql/functions/parameters/cbrt.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`number`:: +Numeric expression. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/signature/cbrt.svg b/docs/reference/esql/functions/signature/cbrt.svg new file mode 100644 index 0000000000000..ba96c276caaa0 --- /dev/null +++ b/docs/reference/esql/functions/signature/cbrt.svg @@ -0,0 +1 @@ +CBRT(number) \ No newline at end of file diff --git a/docs/reference/esql/functions/type-conversion-functions.asciidoc b/docs/reference/esql/functions/type-conversion-functions.asciidoc index 2fec7f40bde8b..96c29a776bc2b 100644 --- a/docs/reference/esql/functions/type-conversion-functions.asciidoc +++ b/docs/reference/esql/functions/type-conversion-functions.asciidoc @@ -5,6 +5,11 @@ Type conversion functions ++++ +[TIP] +==== +{esql} supports implicit casting from string literals to certain data types. Refer to <> for details. +==== + {esql} supports these type conversion functions: // tag::type_list[] diff --git a/docs/reference/esql/functions/types/cbrt.asciidoc b/docs/reference/esql/functions/types/cbrt.asciidoc new file mode 100644 index 0000000000000..7cda278abdb56 --- /dev/null +++ b/docs/reference/esql/functions/types/cbrt.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +number | result +double | double +integer | double +long | double +unsigned_long | double +|=== diff --git a/docs/reference/esql/implicit-casting.asciidoc b/docs/reference/esql/implicit-casting.asciidoc new file mode 100644 index 0000000000000..f0c0aa3d82063 --- /dev/null +++ b/docs/reference/esql/implicit-casting.asciidoc @@ -0,0 +1,53 @@ +[[esql-implicit-casting]] +=== {esql} implicit casting + +++++ +Implicit casting +++++ + +Often users will input `datetime`, `ip`, `version`, or geospatial objects as simple strings in their queries for use in predicates, functions, or expressions. {esql} provides <> to explicitly convert these strings into the desired data types. + +Without implicit casting users must explicitly code these `to_X` functions in their queries, when string literals don't match the target data types they are assigned or compared to. Here is an example of using `to_datetime` to explicitly perform a data type conversion. + +[source.merge.styled,esql] +---- +FROM employees +| EVAL dd_ns1=date_diff("day", to_datetime("2023-12-02T11:00:00.00Z"), birth_date) +| SORT emp_no +| KEEP dd_ns1 +| LIMIT 1 +---- + +Implicit casting improves usability, by automatically converting string literals to the target data type. This is most useful when the target data type is `datetime`, `ip`, `version` or a geo spatial. It is natural to specify these as a string in queries. + +The first query can be coded without calling the `to_datetime` function, as follows: + +[source.merge.styled,esql] +---- +FROM employees +| EVAL dd_ns1=date_diff("day", "2023-12-02T11:00:00.00Z", birth_date) +| SORT emp_no +| KEEP dd_ns1 +| LIMIT 1 +---- + +[float] +=== Implicit casting support + +The following table details which {esql} operations support implicit casting for different data types. + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +||ScalarFunction|BinaryComparison|ArithmeticOperation|InListPredicate|AggregateFunction +|DATETIME|Y|Y|Y|Y|N +|DOUBLE|Y|N|N|N|N +|LONG|Y|N|N|N|N +|INTEGER|Y|N|N|N|N +|IP|Y|Y|Y|Y|N +|VERSION|Y|Y|Y|Y|N +|GEO_POINT|Y|N|N|N|N +|GEO_SHAPE|Y|N|N|N|N +|CARTESIAN_POINT|Y|N|N|N|N +|CARTESIAN_SHAPE|Y|N|N|N|N +|BOOLEAN|Y|Y|Y|Y|N +|=== diff --git a/docs/reference/ingest/processors/geoip.asciidoc b/docs/reference/ingest/processors/geoip.asciidoc index 3348ae9cbfee9..4fbf9678f2fc7 100644 --- a/docs/reference/ingest/processors/geoip.asciidoc +++ b/docs/reference/ingest/processors/geoip.asciidoc @@ -60,10 +60,15 @@ in `properties`. `hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, and `residential_proxy`. The fields actually added depend on what has been found and which properties were configured in `properties`. * If the GeoIP2 Domain database is used, then the following fields may be added under the `target_field`: `ip`, and `domain`. +The fields actually added depend on what has been found and which properties were configured in `properties`. +* If the GeoIP2 ISP database is used, then the following fields may be added under the `target_field`: `ip`, `asn`, +`organization_name`, `network`, `isp`, `isp_organization`, `mobile_country_code`, and `mobile_network_code`. The fields actually added +depend on what has been found and which properties were configured in `properties`. * If the GeoIP2 Enterprise database is used, then the following fields may be added under the `target_field`: `ip`, `country_iso_code`, `country_name`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `timezone`, `location`, `asn`, -`organization_name`, `network`, `hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, and `residential_proxy`. -The fields actually added depend on what has been found and which properties were configured in `properties`. +`organization_name`, `network`, `hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, `residential_proxy`, +`isp`, `isp_organization`, `mobile_country_code`, and `mobile_network_code`. The fields actually added depend on what has been found +and which properties were configured in `properties`. Here is an example that uses the default city database and adds the geographical information to the `geoip` field based on the `ip` field: diff --git a/docs/reference/search/retriever.asciidoc b/docs/reference/search/retriever.asciidoc index c47ccd60afc05..590df272cc89e 100644 --- a/docs/reference/search/retriever.asciidoc +++ b/docs/reference/search/retriever.asciidoc @@ -12,6 +12,11 @@ allows for complex behavior to be depicted in a tree-like structure, called the retriever tree, to better clarify the order of operations that occur during a search. +[TIP] +==== +Refer to <> for a high level overview of the retrievers abstraction. +==== + The following retrievers are available: `standard`:: diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 00c6ec4bd4a2e..53db6f13a31b3 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -1720,6 +1720,27 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/modules/data-streams/build.gradle b/modules/data-streams/build.gradle index 8acdb0f156af1..b9d38551a2674 100644 --- a/modules/data-streams/build.gradle +++ b/modules/data-streams/build.gradle @@ -20,6 +20,7 @@ restResources { dependencies { testImplementation project(path: ':test:test-clusters') + internalClusterTestImplementation project(":modules:mapper-extras") } tasks.named('yamlRestTest') { diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index 2b1a8e1c0e318..f79eea8676b3e 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -1281,7 +1281,7 @@ public void testSearchAllResolvesDataStreams() throws Exception { public void testGetDataStream() throws Exception { Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, maximumNumberOfReplicas() + 2).build(); DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder().dataRetention(randomMillisUpToYear9999()).build(); - putComposableIndexTemplate("template_for_foo", null, List.of("metrics-foo*"), settings, null, null, lifecycle); + putComposableIndexTemplate("template_for_foo", null, List.of("metrics-foo*"), settings, null, null, lifecycle, false); int numDocsFoo = randomIntBetween(2, 16); indexDocs("metrics-foo", numDocsFoo); @@ -1642,7 +1642,8 @@ public void testCreateDataStreamWithSameNameAsDataStreamAlias() throws Exception null, null, Map.of("my-alias", AliasMetadata.builder("my-alias").build()), - null + null, + false ); var request = new CreateDataStreamAction.Request("my-ds"); assertAcked(client().execute(CreateDataStreamAction.INSTANCE, request).actionGet()); @@ -1675,7 +1676,8 @@ public void testCreateDataStreamAliasWithSameNameAsIndexAlias() throws Exception null, null, Map.of("logs", AliasMetadata.builder("logs").build()), - null + null, + false ); var request = new CreateDataStreamAction.Request("logs-es"); @@ -1712,7 +1714,8 @@ public void testCreateDataStreamAliasWithSameNameAsIndex() throws Exception { null, null, Map.of("logs", AliasMetadata.builder("logs").build()), - null + null, + false ) ); assertThat( @@ -1902,7 +1905,11 @@ static void verifyDocs(String dataStream, long expectedNumHits, long minGenerati } public static void putComposableIndexTemplate(String id, List patterns) throws IOException { - putComposableIndexTemplate(id, null, patterns, null, null); + putComposableIndexTemplate(id, patterns, false); + } + + public static void putComposableIndexTemplate(String id, List patterns, boolean withFailureStore) throws IOException { + putComposableIndexTemplate(id, null, patterns, null, null, null, null, withFailureStore); } public void testPartitionedTemplate() throws IOException { @@ -2277,7 +2284,7 @@ static void putComposableIndexTemplate( @Nullable Settings settings, @Nullable Map metadata ) throws IOException { - putComposableIndexTemplate(id, mappings, patterns, settings, metadata, null, null); + putComposableIndexTemplate(id, mappings, patterns, settings, metadata, null, null, false); } static void putComposableIndexTemplate( @@ -2287,7 +2294,8 @@ static void putComposableIndexTemplate( @Nullable Settings settings, @Nullable Map metadata, @Nullable Map aliases, - @Nullable DataStreamLifecycle lifecycle + @Nullable DataStreamLifecycle lifecycle, + boolean withFailureStore ) throws IOException { TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request(id); request.indexTemplate( @@ -2295,7 +2303,7 @@ static void putComposableIndexTemplate( .indexPatterns(patterns) .template(new Template(settings, mappings == null ? null : CompressedXContent.fromJSON(mappings), aliases, lifecycle)) .metadata(metadata) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, withFailureStore)) .build() ); client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java index da782cfd86ce2..1bd4d54b9c804 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java @@ -36,6 +36,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.index.Index; +import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; @@ -81,13 +82,17 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase { private String dsBackingIndexName; private String otherDsBackingIndexName; + private String fsBackingIndexName; + private String fsFailureIndexName; private String ds2BackingIndexName; private String otherDs2BackingIndexName; + private String fs2BackingIndexName; + private String fs2FailureIndexName; private String id; @Override protected Collection> nodePlugins() { - return List.of(MockRepository.Plugin.class, DataStreamsPlugin.class); + return List.of(MockRepository.Plugin.class, DataStreamsPlugin.class, MapperExtrasPlugin.class); } @Before @@ -97,6 +102,18 @@ public void setup() throws Exception { createRepository(REPO, "fs", location); DataStreamIT.putComposableIndexTemplate("t1", List.of("ds", "other-ds")); + DataStreamIT.putComposableIndexTemplate("t2", """ + { + "properties": { + "@timestamp": { + "type": "date", + "format": "date_optional_time||epoch_millis" + }, + "flag": { + "type": "boolean" + } + } + }""", List.of("with-fs"), null, null, null, null, true); CreateDataStreamAction.Request request = new CreateDataStreamAction.Request("ds"); AcknowledgedResponse response = client.execute(CreateDataStreamAction.INSTANCE, request).get(); @@ -106,15 +123,30 @@ public void setup() throws Exception { response = client.execute(CreateDataStreamAction.INSTANCE, request).get(); assertTrue(response.isAcknowledged()); + request = new CreateDataStreamAction.Request("with-fs"); + response = client.execute(CreateDataStreamAction.INSTANCE, request).get(); + assertTrue(response.isAcknowledged()); + // Resolve backing index names after data streams have been created: // (these names have a date component, and running around midnight could lead to test failures otherwise) GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" }); GetDataStreamAction.Response getDataStreamResponse = client.execute(GetDataStreamAction.INSTANCE, getDataStreamRequest).actionGet(); dsBackingIndexName = getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(); otherDsBackingIndexName = getDataStreamResponse.getDataStreams().get(1).getDataStream().getIndices().get(0).getName(); + fsBackingIndexName = getDataStreamResponse.getDataStreams().get(2).getDataStream().getIndices().get(0).getName(); + fsFailureIndexName = getDataStreamResponse.getDataStreams() + .get(2) + .getDataStream() + .getFailureIndices() + .getIndices() + .get(0) + .getName(); + // Will be used in some tests, to test renaming while restoring a snapshot: ds2BackingIndexName = dsBackingIndexName.replace("-ds-", "-ds2-"); otherDs2BackingIndexName = otherDsBackingIndexName.replace("-other-ds-", "-other-ds2-"); + fs2BackingIndexName = fsBackingIndexName.replace("-with-fs-", "-with-fs2-"); + fs2FailureIndexName = fsFailureIndexName.replace("-with-fs-", "-with-fs2-"); DocWriteResponse indexResponse = client.prepareIndex("ds") .setOpType(DocWriteRequest.OpType.CREATE) @@ -232,12 +264,16 @@ public void testSnapshotAndRestoreAllDataStreamsInPlace() throws Exception { GetDataStreamAction.Response ds = client.execute(GetDataStreamAction.INSTANCE, getDataSteamRequest).get(); assertThat( ds.getDataStreams().stream().map(e -> e.getDataStream().getName()).collect(Collectors.toList()), - contains(equalTo("ds"), equalTo("other-ds")) + contains(equalTo("ds"), equalTo("other-ds"), equalTo("with-fs")) ); List backingIndices = ds.getDataStreams().get(0).getDataStream().getIndices(); assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(dsBackingIndexName)); backingIndices = ds.getDataStreams().get(1).getDataStream().getIndices(); assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(otherDsBackingIndexName)); + backingIndices = ds.getDataStreams().get(2).getDataStream().getIndices(); + assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(fsBackingIndexName)); + List failureIndices = ds.getDataStreams().get(2).getDataStream().getFailureIndices().getIndices(); + assertThat(failureIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(fsFailureIndexName)); } public void testSnapshotAndRestoreInPlace() { @@ -295,13 +331,72 @@ public void testSnapshotAndRestoreInPlace() { // The backing index created as part of rollover should still exist (but just not part of the data stream) assertThat(indexExists(backingIndexAfterSnapshot), is(true)); - // An additional rollover should create a new backing index (3th generation) and leave .ds-ds-...-2 index as is: + // An additional rollover should create a new backing index (3rd generation) and leave .ds-ds-...-2 index as is: rolloverRequest = new RolloverRequest("ds", null); rolloverResponse = client.admin().indices().rolloverIndex(rolloverRequest).actionGet(); assertThat(rolloverResponse.isRolledOver(), is(true)); assertThat(rolloverResponse.getNewIndex(), equalTo(DataStream.getDefaultBackingIndexName("ds", 3))); } + public void testFailureStoreSnapshotAndRestore() throws Exception { + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(REPO, SNAPSHOT) + .setWaitForCompletion(true) + .setIndices("with-fs") + .setIncludeGlobalState(false) + .get(); + + RestStatus status = createSnapshotResponse.getSnapshotInfo().status(); + assertEquals(RestStatus.OK, status); + + assertThat(getSnapshot(REPO, SNAPSHOT).indices(), containsInAnyOrder(fsBackingIndexName, fsFailureIndexName)); + + assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request("with-fs"))); + + { + RestoreSnapshotResponse restoreSnapshotResponse = client.admin() + .cluster() + .prepareRestoreSnapshot(REPO, SNAPSHOT) + .setWaitForCompletion(true) + .setIndices("with-fs") + .get(); + + assertEquals(2, restoreSnapshotResponse.getRestoreInfo().successfulShards()); + + GetDataStreamAction.Response ds = client.execute( + GetDataStreamAction.INSTANCE, + new GetDataStreamAction.Request(new String[] { "with-fs" }) + ).get(); + assertEquals(1, ds.getDataStreams().size()); + assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size()); + assertEquals(fsBackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName()); + assertEquals(fsFailureIndexName, ds.getDataStreams().get(0).getDataStream().getFailureIndices().getIndices().get(0).getName()); + } + { + // With rename pattern + RestoreSnapshotResponse restoreSnapshotResponse = client.admin() + .cluster() + .prepareRestoreSnapshot(REPO, SNAPSHOT) + .setWaitForCompletion(true) + .setIndices("with-fs") + .setRenamePattern("-fs") + .setRenameReplacement("-fs2") + .get(); + + assertEquals(2, restoreSnapshotResponse.getRestoreInfo().successfulShards()); + + GetDataStreamAction.Response ds = client.execute( + GetDataStreamAction.INSTANCE, + new GetDataStreamAction.Request(new String[] { "with-fs2" }) + ).get(); + assertEquals(1, ds.getDataStreams().size()); + assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size()); + assertEquals(fs2BackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName()); + assertEquals(fs2FailureIndexName, ds.getDataStreams().get(0).getDataStream().getFailureIndices().getIndices().get(0).getName()); + } + } + public void testSnapshotAndRestoreAllIncludeSpecificDataStream() throws Exception { DocWriteResponse indexResponse = client.prepareIndex("other-ds") .setOpType(DocWriteRequest.OpType.CREATE) @@ -338,10 +433,13 @@ public void testSnapshotAndRestoreAllIncludeSpecificDataStream() throws Exceptio if (filterDuringSnapshotting) { assertThat(getSnapshot(REPO, SNAPSHOT).indices(), containsInAnyOrder(backingIndexName)); } else { - assertThat(getSnapshot(REPO, SNAPSHOT).indices(), containsInAnyOrder(dsBackingIndexName, otherDsBackingIndexName)); + assertThat( + getSnapshot(REPO, SNAPSHOT).indices(), + containsInAnyOrder(dsBackingIndexName, otherDsBackingIndexName, fsBackingIndexName, fsFailureIndexName) + ); } - assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "*" })).get()); + assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request("*")).get()); assertAcked(client.admin().indices().prepareDelete("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN)); RestoreSnapshotRequest restoreSnapshotRequest = new RestoreSnapshotRequest(REPO, SNAPSHOT); @@ -395,7 +493,10 @@ public void testSnapshotAndRestoreReplaceAll() throws Exception { RestStatus status = createSnapshotResponse.getSnapshotInfo().status(); assertEquals(RestStatus.OK, status); - assertThat(getSnapshot(REPO, SNAPSHOT).indices(), containsInAnyOrder(dsBackingIndexName, otherDsBackingIndexName)); + assertThat( + getSnapshot(REPO, SNAPSHOT).indices(), + containsInAnyOrder(dsBackingIndexName, otherDsBackingIndexName, fsBackingIndexName, fsFailureIndexName) + ); assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "*" })).get()); assertAcked(client.admin().indices().prepareDelete("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN)); @@ -403,7 +504,7 @@ public void testSnapshotAndRestoreReplaceAll() throws Exception { var restoreSnapshotRequest = new RestoreSnapshotRequest(REPO, SNAPSHOT).waitForCompletion(true).includeGlobalState(false); RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().restoreSnapshot(restoreSnapshotRequest).actionGet(); - assertEquals(2, restoreSnapshotResponse.getRestoreInfo().successfulShards()); + assertEquals(4, restoreSnapshotResponse.getRestoreInfo().successfulShards()); assertEquals(DOCUMENT_SOURCE, client.prepareGet(dsBackingIndexName, id).get().getSourceAsMap()); assertResponse(client.prepareSearch("ds"), response -> { @@ -416,10 +517,10 @@ public void testSnapshotAndRestoreReplaceAll() throws Exception { GetDataStreamAction.INSTANCE, new GetDataStreamAction.Request(new String[] { "*" }) ).get(); - assertEquals(2, ds.getDataStreams().size()); + assertEquals(3, ds.getDataStreams().size()); assertThat( ds.getDataStreams().stream().map(i -> i.getDataStream().getName()).collect(Collectors.toList()), - containsInAnyOrder("ds", "other-ds") + containsInAnyOrder("ds", "other-ds", "with-fs") ); GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("my-alias")).actionGet(); @@ -451,14 +552,17 @@ public void testSnapshotAndRestoreAll() throws Exception { RestStatus status = createSnapshotResponse.getSnapshotInfo().status(); assertEquals(RestStatus.OK, status); - assertThat(getSnapshot(REPO, SNAPSHOT).indices(), containsInAnyOrder(dsBackingIndexName, otherDsBackingIndexName)); + assertThat( + getSnapshot(REPO, SNAPSHOT).indices(), + containsInAnyOrder(dsBackingIndexName, otherDsBackingIndexName, fsBackingIndexName, fsFailureIndexName) + ); - assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "*" })).get()); + assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request("*")).get()); assertAcked(client.admin().indices().prepareDelete("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN)); var restoreSnapshotRequest = new RestoreSnapshotRequest(REPO, SNAPSHOT).waitForCompletion(true).includeGlobalState(false); RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().restoreSnapshot(restoreSnapshotRequest).actionGet(); - assertEquals(2, restoreSnapshotResponse.getRestoreInfo().successfulShards()); + assertEquals(4, restoreSnapshotResponse.getRestoreInfo().successfulShards()); assertEquals(DOCUMENT_SOURCE, client.prepareGet(dsBackingIndexName, id).get().getSourceAsMap()); assertResponse(client.prepareSearch("ds"), response -> { @@ -471,11 +575,15 @@ public void testSnapshotAndRestoreAll() throws Exception { GetDataStreamAction.INSTANCE, new GetDataStreamAction.Request(new String[] { "*" }) ).get(); - assertEquals(2, ds.getDataStreams().size()); + assertEquals(3, ds.getDataStreams().size()); assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size()); assertEquals(dsBackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName()); assertEquals(1, ds.getDataStreams().get(1).getDataStream().getIndices().size()); assertEquals(otherDsBackingIndexName, ds.getDataStreams().get(1).getDataStream().getIndices().get(0).getName()); + assertEquals(1, ds.getDataStreams().get(2).getDataStream().getIndices().size()); + assertEquals(fsBackingIndexName, ds.getDataStreams().get(2).getDataStream().getIndices().get(0).getName()); + assertEquals(1, ds.getDataStreams().get(2).getDataStream().getFailureIndices().getIndices().size()); + assertEquals(fsFailureIndexName, ds.getDataStreams().get(2).getDataStream().getFailureIndices().getIndices().get(0).getName()); GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("my-alias")).actionGet(); assertThat(getAliasesResponse.getDataStreamAliases().keySet(), containsInAnyOrder("ds", "other-ds")); @@ -507,16 +615,19 @@ public void testSnapshotAndRestoreIncludeAliasesFalse() throws Exception { RestStatus status = createSnapshotResponse.getSnapshotInfo().status(); assertEquals(RestStatus.OK, status); - assertThat(getSnapshot(REPO, SNAPSHOT).indices(), containsInAnyOrder(dsBackingIndexName, otherDsBackingIndexName)); + assertThat( + getSnapshot(REPO, SNAPSHOT).indices(), + containsInAnyOrder(dsBackingIndexName, otherDsBackingIndexName, fsBackingIndexName, fsFailureIndexName) + ); - assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "*" })).get()); + assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request("*")).get()); assertAcked(client.admin().indices().prepareDelete("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN)); var restoreSnapshotRequest = new RestoreSnapshotRequest(REPO, SNAPSHOT).waitForCompletion(true) .includeGlobalState(false) .includeAliases(false); RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().restoreSnapshot(restoreSnapshotRequest).actionGet(); - assertEquals(2, restoreSnapshotResponse.getRestoreInfo().successfulShards()); + assertEquals(4, restoreSnapshotResponse.getRestoreInfo().successfulShards()); assertEquals(DOCUMENT_SOURCE, client.prepareGet(dsBackingIndexName, id).get().getSourceAsMap()); assertResponse(client.prepareSearch("ds"), response -> { @@ -529,11 +640,15 @@ public void testSnapshotAndRestoreIncludeAliasesFalse() throws Exception { GetDataStreamAction.INSTANCE, new GetDataStreamAction.Request(new String[] { "*" }) ).get(); - assertEquals(2, ds.getDataStreams().size()); + assertEquals(3, ds.getDataStreams().size()); assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size()); assertEquals(dsBackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName()); assertEquals(1, ds.getDataStreams().get(1).getDataStream().getIndices().size()); assertEquals(otherDsBackingIndexName, ds.getDataStreams().get(1).getDataStream().getIndices().get(0).getName()); + assertEquals(1, ds.getDataStreams().get(2).getDataStream().getIndices().size()); + assertEquals(fsBackingIndexName, ds.getDataStreams().get(2).getDataStream().getIndices().get(0).getName()); + assertEquals(1, ds.getDataStreams().get(2).getDataStream().getIndices().size()); + assertEquals(fsFailureIndexName, ds.getDataStreams().get(2).getDataStream().getFailureIndices().getIndices().get(0).getName()); GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("*")).actionGet(); assertThat(getAliasesResponse.getDataStreamAliases(), anEmptyMap()); @@ -930,7 +1045,32 @@ public void testPartialRestoreSnapshotThatIncludesDataStream() { .prepareRestoreSnapshot(REPO, snapshot) .setIndices(indexWithoutDataStream) .setWaitForCompletion(true) - .setRestoreGlobalState(randomBoolean()) + .setRestoreGlobalState(false) + .get() + .getRestoreInfo(); + assertThat(restoreInfo.failedShards(), is(0)); + assertThat(restoreInfo.successfulShards(), is(1)); + } + + /** + * This test is a copy of the {@link #testPartialRestoreSnapshotThatIncludesDataStream()} the only difference + * is that one include the global state and one doesn't. In general this shouldn't matter that's why it used to be + * a random parameter of the test, but because of #107515 it fails when we include the global state. Keep them + * separate until this is fixed. + */ + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107515") + public void testPartialRestoreSnapshotThatIncludesDataStreamWithGlobalState() { + final String snapshot = "test-snapshot"; + final String indexWithoutDataStream = "test-idx-no-ds"; + createIndexWithContent(indexWithoutDataStream); + createFullSnapshot(REPO, snapshot); + assertAcked(client.admin().indices().prepareDelete(indexWithoutDataStream)); + RestoreInfo restoreInfo = client.admin() + .cluster() + .prepareRestoreSnapshot(REPO, snapshot) + .setIndices(indexWithoutDataStream) + .setWaitForCompletion(true) + .setRestoreGlobalState(true) .get() .getRestoreInfo(); assertThat(restoreInfo.failedShards(), is(0)); @@ -1027,7 +1167,32 @@ public void testExcludeDSFromSnapshotWhenExcludingItsIndices() { .cluster() .prepareRestoreSnapshot(REPO, snapshot) .setWaitForCompletion(true) - .setRestoreGlobalState(randomBoolean()) + .setRestoreGlobalState(false) + .get() + .getRestoreInfo(); + assertThat(restoreInfo.failedShards(), is(0)); + assertThat(restoreInfo.successfulShards(), is(1)); + } + + /** + * This test is a copy of the {@link #testExcludeDSFromSnapshotWhenExcludingItsIndices()} the only difference + * is that one include the global state and one doesn't. In general this shouldn't matter that's why it used to be + * a random parameter of the test, but because of #107515 it fails when we include the global state. Keep them + * separate until this is fixed. + */ + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107515") + public void testExcludeDSFromSnapshotWhenExcludingItsIndicesWithGlobalState() { + final String snapshot = "test-snapshot"; + final String indexWithoutDataStream = "test-idx-no-ds"; + createIndexWithContent(indexWithoutDataStream); + final SnapshotInfo snapshotInfo = createSnapshot(REPO, snapshot, List.of("*", "-.*")); + assertThat(snapshotInfo.dataStreams(), empty()); + assertAcked(client.admin().indices().prepareDelete(indexWithoutDataStream)); + RestoreInfo restoreInfo = client.admin() + .cluster() + .prepareRestoreSnapshot(REPO, snapshot) + .setWaitForCompletion(true) + .setRestoreGlobalState(true) .get() .getRestoreInfo(); assertThat(restoreInfo.failedShards(), is(0)); @@ -1051,7 +1216,7 @@ public void testRestoreSnapshotFully() throws Exception { assertEquals(RestStatus.OK, restoreSnapshotResponse.status()); GetDataStreamAction.Request getRequest = new GetDataStreamAction.Request(new String[] { "*" }); - assertThat(client.execute(GetDataStreamAction.INSTANCE, getRequest).get().getDataStreams(), hasSize(2)); + assertThat(client.execute(GetDataStreamAction.INSTANCE, getRequest).get().getDataStreams(), hasSize(3)); assertNotNull(client.admin().indices().prepareGetIndex().setIndices(indexName).get()); } diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java index d43dad87a6067..7712be94b4326 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.datastreams.CreateDataStreamAction; import org.elasticsearch.action.datastreams.lifecycle.GetDataStreamLifecycleAction; import org.elasticsearch.action.datastreams.lifecycle.PutDataStreamLifecycleAction; +import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.core.TimeValue; import org.elasticsearch.datastreams.DataStreamsPlugin; @@ -229,6 +230,8 @@ public void testDeleteLifecycle() throws Exception { // Remove lifecycle from concrete data stream { DeleteDataStreamLifecycleAction.Request deleteDataLifecycleRequest = new DeleteDataStreamLifecycleAction.Request( + TimeValue.THIRTY_SECONDS, + AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, new String[] { "with-lifecycle-1" } ); assertThat( @@ -254,6 +257,8 @@ public void testDeleteLifecycle() throws Exception { // Remove lifecycle from all data streams { DeleteDataStreamLifecycleAction.Request deleteDataLifecycleRequest = new DeleteDataStreamLifecycleAction.Request( + TimeValue.THIRTY_SECONDS, + AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, new String[] { "*" } ); assertThat( diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java index 7252d31d838c5..97c6c1ddff977 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java @@ -203,6 +203,7 @@ public void testSystemDataStreamRetention() throws Exception { client().execute( PutDataStreamGlobalRetentionAction.INSTANCE, new PutDataStreamGlobalRetentionAction.Request( + TimeValue.THIRTY_SECONDS, TimeValue.timeValueSeconds(globalRetentionSeconds), TimeValue.timeValueSeconds(globalRetentionSeconds) ) @@ -290,7 +291,10 @@ public void testSystemDataStreamRetention() throws Exception { client().execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(SYSTEM_DATA_STREAM_NAME)).actionGet(); } finally { - client().execute(DeleteDataStreamGlobalRetentionAction.INSTANCE, new DeleteDataStreamGlobalRetentionAction.Request()); + client().execute( + DeleteDataStreamGlobalRetentionAction.INSTANCE, + new DeleteDataStreamGlobalRetentionAction.Request(TimeValue.THIRTY_SECONDS) + ); } } finally { dataStreamLifecycleServices.forEach(dataStreamLifecycleService -> dataStreamLifecycleService.setNowSupplier(clock::millis)); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java index 2723637b2959b..35ee41fca18e8 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java @@ -213,6 +213,7 @@ public void testSystemExplainLifecycle() throws Exception { client().execute( PutDataStreamGlobalRetentionAction.INSTANCE, new PutDataStreamGlobalRetentionAction.Request( + TimeValue.THIRTY_SECONDS, TimeValue.timeValueSeconds(globalRetentionSeconds), TimeValue.timeValueSeconds(globalRetentionSeconds) ) @@ -260,7 +261,10 @@ public void testSystemExplainLifecycle() throws Exception { ); } } finally { - client().execute(DeleteDataStreamGlobalRetentionAction.INSTANCE, new DeleteDataStreamGlobalRetentionAction.Request()); + client().execute( + DeleteDataStreamGlobalRetentionAction.INSTANCE, + new DeleteDataStreamGlobalRetentionAction.Request(TimeValue.THIRTY_SECONDS) + ); } } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamGlobalRetentionAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamGlobalRetentionAction.java index e3cdd6a8c14d9..92cb855b7cb4e 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamGlobalRetentionAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamGlobalRetentionAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.datastreams.lifecycle.UpdateDataStreamGlobalRetentionService; import org.elasticsearch.features.FeatureService; import org.elasticsearch.tasks.Task; @@ -64,8 +65,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(dryRun); } - public Request() { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT); + public Request(TimeValue masterNodeTimeout) { + super(masterNodeTimeout); } public boolean dryRun() { diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java index 3bd100a106dd6..70f822ddee72a 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; import java.io.IOException; import java.util.Arrays; @@ -47,8 +48,8 @@ public void writeTo(StreamOutput out) throws IOException { indicesOptions.writeIndicesOptions(out); } - public Request(String[] names) { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT); + public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String[] names) { + super(masterNodeTimeout, ackTimeout); this.names = names; } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamGlobalRetentionAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamGlobalRetentionAction.java index 5816823ed710a..1d1064dd42b1a 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamGlobalRetentionAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamGlobalRetentionAction.java @@ -47,10 +47,6 @@ private GetDataStreamGlobalRetentionAction() {/* no instances */} public static final class Request extends MasterNodeReadRequest { - public Request() { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT); - } - public Request(StreamInput in) throws IOException { super(in); } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleStatsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleStatsAction.java index cc61c7fe664be..6e930defd4e0b 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleStatsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleStatsAction.java @@ -43,8 +43,8 @@ public Request(StreamInput in) throws IOException { super(in); } - public Request() { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT); + public Request(TimeValue masterNodeTimeout) { + super(masterNodeTimeout); } @Override diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/PutDataStreamGlobalRetentionAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/PutDataStreamGlobalRetentionAction.java index 65ca34a99da23..cd9156ad8b2c8 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/PutDataStreamGlobalRetentionAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/PutDataStreamGlobalRetentionAction.java @@ -32,9 +32,6 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xcontent.ConstructingObjectParser; -import org.elasticsearch.xcontent.ObjectParser; -import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; import java.util.List; @@ -53,34 +50,9 @@ private PutDataStreamGlobalRetentionAction() {/* no instances */} public static final class Request extends MasterNodeRequest { - public static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>( - "put_data_stream_global_retention_request", - args -> new PutDataStreamGlobalRetentionAction.Request((TimeValue) args[0], (TimeValue) args[1]) - ); - - static { - PARSER.declareField( - ConstructingObjectParser.optionalConstructorArg(), - (p, c) -> TimeValue.parseTimeValue(p.textOrNull(), DataStreamGlobalRetention.DEFAULT_RETENTION_FIELD.getPreferredName()), - DataStreamGlobalRetention.DEFAULT_RETENTION_FIELD, - ObjectParser.ValueType.STRING_OR_NULL - ); - PARSER.declareField( - ConstructingObjectParser.optionalConstructorArg(), - (p, c) -> TimeValue.parseTimeValue(p.textOrNull(), DataStreamGlobalRetention.MAX_RETENTION_FIELD.getPreferredName()), - DataStreamGlobalRetention.MAX_RETENTION_FIELD, - ObjectParser.ValueType.STRING_OR_NULL - ); - } - private final DataStreamGlobalRetention globalRetention; private boolean dryRun = false; - public static PutDataStreamGlobalRetentionAction.Request parseRequest(XContentParser parser) { - return PARSER.apply(parser, null); - } - public Request(StreamInput in) throws IOException { super(in); globalRetention = DataStreamGlobalRetention.read(in); @@ -107,8 +79,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(dryRun); } - public Request(@Nullable TimeValue defaultRetention, @Nullable TimeValue maxRetention) { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT); + public Request(TimeValue masterNodeTimeout, @Nullable TimeValue defaultRetention, @Nullable TimeValue maxRetention) { + super(masterNodeTimeout); this.globalRetention = new DataStreamGlobalRetention(defaultRetention, maxRetention); } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDataStreamLifecycleStatsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDataStreamLifecycleStatsAction.java index a10a955b33975..a3959ae818218 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDataStreamLifecycleStatsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDataStreamLifecycleStatsAction.java @@ -36,8 +36,7 @@ public List routes() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { - GetDataStreamLifecycleStatsAction.Request request = new GetDataStreamLifecycleStatsAction.Request(); - request.masterNodeTimeout(getMasterNodeTimeout(restRequest)); + final var request = new GetDataStreamLifecycleStatsAction.Request(getMasterNodeTimeout(restRequest)); return channel -> client.execute( GetDataStreamLifecycleStatsAction.INSTANCE, request, diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDeleteDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDeleteDataStreamLifecycleAction.java index b624892ac6bba..a8a64eaf5cfa3 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDeleteDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDeleteDataStreamLifecycleAction.java @@ -8,6 +8,7 @@ package org.elasticsearch.datastreams.lifecycle.rest; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.datastreams.lifecycle.action.DeleteDataStreamLifecycleAction; @@ -20,6 +21,7 @@ import java.util.List; import static org.elasticsearch.rest.RestRequest.Method.DELETE; +import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout; @ServerlessScope(Scope.INTERNAL) public class RestDeleteDataStreamLifecycleAction extends BaseRestHandler { @@ -36,7 +38,9 @@ public List routes() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { - DeleteDataStreamLifecycleAction.Request deleteDataLifecycleRequest = new DeleteDataStreamLifecycleAction.Request( + final var deleteDataLifecycleRequest = new DeleteDataStreamLifecycleAction.Request( + getMasterNodeTimeout(request), + request.paramAsTime("timeout", AcknowledgedRequest.DEFAULT_ACK_TIMEOUT), Strings.splitStringByCommaToArray(request.param("name")) ); deleteDataLifecycleRequest.indicesOptions(IndicesOptions.fromRequest(request, deleteDataLifecycleRequest.indicesOptions())); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java index 5a9b00dde58cc..641d8bad2b135 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java @@ -96,7 +96,11 @@ enum Database { Property.ANONYMOUS, Property.PUBLIC_PROXY, Property.RESIDENTIAL_PROXY, - Property.DOMAIN + Property.DOMAIN, + Property.ISP, + Property.ISP_ORGANIZATION_NAME, + Property.MOBILE_COUNTRY_CODE, + Property.MOBILE_NETWORK_CODE ), Set.of( Property.COUNTRY_ISO_CODE, @@ -107,6 +111,28 @@ enum Database { Property.CITY_NAME, Property.LOCATION ) + ), + Isp( + Set.of( + Property.IP, + Property.ASN, + Property.ORGANIZATION_NAME, + Property.NETWORK, + Property.ISP, + Property.ISP_ORGANIZATION_NAME, + Property.MOBILE_COUNTRY_CODE, + Property.MOBILE_NETWORK_CODE + ), + Set.of( + Property.IP, + Property.ASN, + Property.ORGANIZATION_NAME, + Property.NETWORK, + Property.ISP, + Property.ISP_ORGANIZATION_NAME, + Property.MOBILE_COUNTRY_CODE, + Property.MOBILE_NETWORK_CODE + ) ); private static final String CITY_DB_SUFFIX = "-City"; @@ -115,6 +141,7 @@ enum Database { private static final String ANONYMOUS_IP_DB_SUFFIX = "-Anonymous-IP"; private static final String DOMAIN_DB_SUFFIX = "-Domain"; private static final String ENTERPRISE_DB_SUFFIX = "-Enterprise"; + private static final String ISP_DB_SUFFIX = "-ISP"; /** * Parses the passed-in databaseType (presumably from the passed-in databaseFile) and return the Database instance that is @@ -140,6 +167,8 @@ public static Database getDatabase(final String databaseType, final String datab database = Database.Domain; } else if (databaseType.endsWith(Database.ENTERPRISE_DB_SUFFIX)) { database = Database.Enterprise; + } else if (databaseType.endsWith(Database.ISP_DB_SUFFIX)) { + database = Database.Isp; } } @@ -215,7 +244,11 @@ enum Property { ANONYMOUS, PUBLIC_PROXY, RESIDENTIAL_PROXY, - DOMAIN; + DOMAIN, + ISP, + ISP_ORGANIZATION_NAME, + MOBILE_COUNTRY_CODE, + MOBILE_NETWORK_CODE; /** * Parses a string representation of a property into an actual Property instance. Not all properties that exist are diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java index 97b90f612ea92..a6f2be9c1fb79 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java @@ -18,6 +18,7 @@ import com.maxmind.geoip2.model.CountryResponse; import com.maxmind.geoip2.model.DomainResponse; import com.maxmind.geoip2.model.EnterpriseResponse; +import com.maxmind.geoip2.model.IspResponse; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -190,6 +191,12 @@ public EnterpriseResponse getEnterprise(InetAddress ipAddress) { return getResponse(ipAddress, DatabaseReader::tryEnterprise); } + @Nullable + @Override + public IspResponse getIsp(InetAddress ipAddress) { + return getResponse(ipAddress, DatabaseReader::tryIsp); + } + boolean preLookup() { return currentUsages.updateAndGet(current -> current < 0 ? current : current + 1) > 0; } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDatabase.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDatabase.java index 7cbd423a5f2e9..681751f0f525a 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDatabase.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDatabase.java @@ -14,6 +14,7 @@ import com.maxmind.geoip2.model.CountryResponse; import com.maxmind.geoip2.model.DomainResponse; import com.maxmind.geoip2.model.EnterpriseResponse; +import com.maxmind.geoip2.model.IspResponse; import org.elasticsearch.core.Nullable; @@ -65,6 +66,9 @@ public interface GeoIpDatabase { @Nullable EnterpriseResponse getEnterprise(InetAddress ipAddress); + @Nullable + IspResponse getIsp(InetAddress ipAddress); + /** * Releases the current database object. Called after processing a single document. Databases should be closed or returned to a * resource pool. No further interactions should be expected. diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java index 16485987176b7..6d420b0547293 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java @@ -15,6 +15,7 @@ import com.maxmind.geoip2.model.CountryResponse; import com.maxmind.geoip2.model.DomainResponse; import com.maxmind.geoip2.model.EnterpriseResponse; +import com.maxmind.geoip2.model.IspResponse; import com.maxmind.geoip2.record.City; import com.maxmind.geoip2.record.Continent; import com.maxmind.geoip2.record.Country; @@ -178,6 +179,7 @@ private Map getGeoData(GeoIpDatabase geoIpDatabase, String ip) t case AnonymousIp -> retrieveAnonymousIpGeoData(geoIpDatabase, ipAddress); case Domain -> retrieveDomainGeoData(geoIpDatabase, ipAddress); case Enterprise -> retrieveEnterpriseGeoData(geoIpDatabase, ipAddress); + case Isp -> retrieveIspGeoData(geoIpDatabase, ipAddress); }; } @@ -424,6 +426,11 @@ private Map retrieveEnterpriseGeoData(GeoIpDatabase geoIpDatabas String organization_name = response.getTraits().getAutonomousSystemOrganization(); Network network = response.getTraits().getNetwork(); + String isp = response.getTraits().getIsp(); + String ispOrganization = response.getTraits().getOrganization(); + String mobileCountryCode = response.getTraits().getMobileCountryCode(); + String mobileNetworkCode = response.getTraits().getMobileNetworkCode(); + boolean isHostingProvider = response.getTraits().isHostingProvider(); boolean isTorExitNode = response.getTraits().isTorExitNode(); boolean isAnonymousVpn = response.getTraits().isAnonymousVpn(); @@ -531,6 +538,84 @@ private Map retrieveEnterpriseGeoData(GeoIpDatabase geoIpDatabas geoData.put("domain", domain); } } + case ISP -> { + if (isp != null) { + geoData.put("isp", isp); + } + } + case ISP_ORGANIZATION_NAME -> { + if (ispOrganization != null) { + geoData.put("isp_organization", ispOrganization); + } + } + case MOBILE_COUNTRY_CODE -> { + if (mobileCountryCode != null) { + geoData.put("mobile_country_code", mobileCountryCode); + } + } + case MOBILE_NETWORK_CODE -> { + if (mobileNetworkCode != null) { + geoData.put("mobile_network_code", mobileNetworkCode); + } + } + } + } + return geoData; + } + + private Map retrieveIspGeoData(GeoIpDatabase geoIpDatabase, InetAddress ipAddress) { + IspResponse response = geoIpDatabase.getIsp(ipAddress); + if (response == null) { + return Map.of(); + } + + String isp = response.getIsp(); + String ispOrganization = response.getOrganization(); + String mobileNetworkCode = response.getMobileNetworkCode(); + String mobileCountryCode = response.getMobileCountryCode(); + Long asn = response.getAutonomousSystemNumber(); + String organization_name = response.getAutonomousSystemOrganization(); + Network network = response.getNetwork(); + + Map geoData = new HashMap<>(); + for (Property property : this.properties) { + switch (property) { + case IP -> geoData.put("ip", NetworkAddress.format(ipAddress)); + case ASN -> { + if (asn != null) { + geoData.put("asn", asn); + } + } + case ORGANIZATION_NAME -> { + if (organization_name != null) { + geoData.put("organization_name", organization_name); + } + } + case NETWORK -> { + if (network != null) { + geoData.put("network", network.toString()); + } + } + case ISP -> { + if (isp != null) { + geoData.put("isp", isp); + } + } + case ISP_ORGANIZATION_NAME -> { + if (ispOrganization != null) { + geoData.put("isp_organization", ispOrganization); + } + } + case MOBILE_COUNTRY_CODE -> { + if (mobileCountryCode != null) { + geoData.put("mobile_country_code", mobileCountryCode); + } + } + case MOBILE_NETWORK_CODE -> { + if (mobileNetworkCode != null) { + geoData.put("mobile_network_code", mobileNetworkCode); + } + } } } return geoData; diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java index cd6737cced308..3f1216c515f5d 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java @@ -387,7 +387,7 @@ public void testEnterprise() throws Exception { assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); @SuppressWarnings("unchecked") Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData.size(), equalTo(19)); + assertThat(geoData.size(), equalTo(21)); assertThat(geoData.get("ip"), equalTo(ip)); assertThat(geoData.get("country_iso_code"), equalTo("US")); assertThat(geoData.get("country_name"), equalTo("United States")); @@ -410,6 +410,42 @@ public void testEnterprise() throws Exception { assertThat(geoData.get("public_proxy"), equalTo(false)); assertThat(geoData.get("residential_proxy"), equalTo(false)); assertThat(geoData.get("domain"), equalTo("frpt.net")); + assertThat(geoData.get("isp"), equalTo("Fairpoint Communications")); + assertThat(geoData.get("isp_organization"), equalTo("Fairpoint Communications")); + } + + public void testIsp() throws Exception { + String ip = "149.101.100.1"; + GeoIpProcessor processor = new GeoIpProcessor( + randomAlphaOfLength(10), + null, + "source_field", + loader("/GeoIP2-ISP-Test.mmdb"), + () -> true, + "target_field", + ALL_PROPERTIES, + false, + false, + "filename" + ); + + Map document = new HashMap<>(); + document.put("source_field", ip); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + processor.execute(ingestDocument); + + assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); + @SuppressWarnings("unchecked") + Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); + assertThat(geoData.size(), equalTo(8)); + assertThat(geoData.get("ip"), equalTo(ip)); + assertThat(geoData.get("asn"), equalTo(6167L)); + assertThat(geoData.get("organization_name"), equalTo("CELLCO-PART")); + assertThat(geoData.get("network"), equalTo("149.101.100.0/28")); + assertThat(geoData.get("isp"), equalTo("Verizon Wireless")); + assertThat(geoData.get("isp_organization"), equalTo("Verizon Wireless")); + assertThat(geoData.get("mobile_network_code"), equalTo("004")); + assertThat(geoData.get("mobile_country_code"), equalTo("310")); } public void testAddressIsNotInTheDatabase() throws Exception { diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java index 07ea7f59eb521..a396995663da7 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java @@ -220,7 +220,11 @@ public class MaxMindSupportTests extends ESTestCase { "traits.autonomousSystemOrganization", "traits.domain", "traits.hostingProvider", + "traits.isp", + "traits.mobileCountryCode", + "traits.mobileNetworkCode", "traits.network", + "traits.organization", "traits.publicProxy", "traits.residentialProxy", "traits.torExitNode" @@ -273,17 +277,25 @@ public class MaxMindSupportTests extends ESTestCase { "traits.anycast", "traits.connectionType", "traits.ipAddress", - "traits.isp", "traits.legitimateProxy", - "traits.mobileCountryCode", - "traits.mobileNetworkCode", - "traits.organization", "traits.satelliteProvider", "traits.staticIpScore", "traits.userCount", "traits.userType" ); + private static final Set ISP_SUPPORTED_FIELDS = Set.of( + "autonomousSystemNumber", + "autonomousSystemOrganization", + "network", + "isp", + "mobileCountryCode", + "mobileNetworkCode", + "organization" + ); + + private static final Set ISP_UNSUPPORTED_FIELDS = Set.of("ipAddress"); + private static final Map> TYPE_TO_SUPPORTED_FIELDS_MAP = Map.of( Database.AnonymousIp, ANONYMOUS_IP_SUPPORTED_FIELDS, @@ -296,7 +308,9 @@ public class MaxMindSupportTests extends ESTestCase { Database.Domain, DOMAIN_SUPPORTED_FIELDS, Database.Enterprise, - ENTERPRISE_SUPPORTED_FIELDS + ENTERPRISE_SUPPORTED_FIELDS, + Database.Isp, + ISP_SUPPORTED_FIELDS ); private static final Map> TYPE_TO_UNSUPPORTED_FIELDS_MAP = Map.of( Database.AnonymousIp, @@ -310,7 +324,9 @@ public class MaxMindSupportTests extends ESTestCase { Database.Domain, DOMAIN_UNSUPPORTED_FIELDS, Database.Enterprise, - ENTERPRISE_UNSUPPORTED_FIELDS + ENTERPRISE_UNSUPPORTED_FIELDS, + Database.Isp, + ISP_UNSUPPORTED_FIELDS ); private static final Map> TYPE_TO_MAX_MIND_CLASS = Map.of( Database.AnonymousIp, @@ -324,12 +340,13 @@ public class MaxMindSupportTests extends ESTestCase { Database.Domain, DomainResponse.class, Database.Enterprise, - EnterpriseResponse.class + EnterpriseResponse.class, + Database.Isp, + IspResponse.class ); private static final Set> KNOWN_UNSUPPORTED_RESPONSE_CLASSES = Set.of( ConnectionTypeResponse.class, - IspResponse.class, IpRiskResponse.class ); diff --git a/modules/ingest-geoip/src/test/resources/GeoIP2-ISP-Test.mmdb b/modules/ingest-geoip/src/test/resources/GeoIP2-ISP-Test.mmdb new file mode 100644 index 0000000000000..d16b0eee4c5e5 Binary files /dev/null and b/modules/ingest-geoip/src/test/resources/GeoIP2-ISP-Test.mmdb differ diff --git a/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java b/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java index 275666eec5c42..98eb69aa9e21e 100644 --- a/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java +++ b/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java @@ -8,6 +8,8 @@ package org.elasticsearch.kibana; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchRequest; @@ -15,12 +17,15 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor; import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.threadpool.ThreadPool; +import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Locale; @@ -42,7 +47,9 @@ * threads that wait on a phaser. This lets us verify that operations on system indices * are being directed to other thread pools.

*/ +@TestLogging(reason = "investigate", value = "org.elasticsearch.kibana.KibanaThreadPoolIT:DEBUG") public class KibanaThreadPoolIT extends ESIntegTestCase { + private static final Logger logger = LogManager.getLogger(KibanaThreadPoolIT.class); @Override protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { @@ -195,10 +202,21 @@ private static void fillThreadPoolQueues(String threadPoolName, ThreadPool threa try { threadPool.executor(threadPoolName).execute(() -> {}); } catch (EsRejectedExecutionException e) { + logger.debug("Exception when filling the queue " + threadPoolName, e); + logThreadPoolQueue(threadPoolName, threadPool); // we can't be sure that some other task won't get queued in a test cluster // but the threadpool's thread is already blocked } } + + logThreadPoolQueue(threadPoolName, threadPool); + } + + private static void logThreadPoolQueue(String threadPoolName, ThreadPool threadPool) { + if (threadPool.executor(threadPoolName) instanceof EsThreadPoolExecutor tpe) { + logger.debug("Thread pool details " + threadPoolName + " " + tpe); + logger.debug(Arrays.toString(tpe.getTasks().toArray())); + } } } diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java index 81ac8ab1200f6..40752e5b296bc 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java @@ -1211,6 +1211,7 @@ private List listPlugins() { /** * Check that readiness listener works */ + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/108523") public void test500Readiness() throws Exception { assertFalse(readinessProbe(9399)); // Disabling security so we wait for green diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index f1232d2442c8b..b1c43a478f94f 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -165,6 +165,9 @@ static TransportVersion def(int id) { public static final TransportVersion JOIN_STATUS_AGE_SERIALIZATION = def(8_656_00_0); public static final TransportVersion ML_RERANK_DOC_OPTIONAL = def(8_657_00_0); public static final TransportVersion FAILURE_STORE_FIELD_PARITY = def(8_658_00_0); + public static final TransportVersion ML_INFERENCE_AZURE_AI_STUDIO = def(8_659_00_0); + public static final TransportVersion ML_INFERENCE_COHERE_COMPLETION_ADDED = def(8_660_00_0); + /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java index 5d20443fa3989..c2fd49eb91a42 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; @@ -65,7 +66,9 @@ public class CreateSnapshotRequest extends MasterNodeRequest