Merge commit 'db70623' into fix_swallow_send_request

Tim-Brooks · May 15, 2024 · 1fa129e · 1fa129e
2 parents f33f236 + db70623
commit 1fa129e
Show file tree

Hide file tree

Showing 133 changed files with 2,581 additions and 631 deletions.
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.LowercaseNormalizer;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.mapper.MapperMetrics;
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
@@ -71,7 +72,8 @@ public static MapperService create(String mappings) {
                 public <T> T compile(Script script, ScriptContext<T> scriptContext) {
                     throw new UnsupportedOperationException();
                 }
-            }
+            },
+            MapperMetrics.NOOP
         );
 
         try {

diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.mapper.MapperMetrics;
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParsedDocument;
@@ -154,7 +155,8 @@ protected SearchExecutionContext buildSearchExecutionContext() {
             null,
             () -> true,
             null,
-            Collections.emptyMap()
+            Collections.emptyMap(),
+            MapperMetrics.NOOP
         );
     }
 
@@ -186,7 +188,8 @@ protected final MapperService createMapperService(String mappings) {
                 public <T> T compile(Script script, ScriptContext<T> scriptContext) {
                     throw new UnsupportedOperationException();
                 }
-            }
+            },
+            MapperMetrics.NOOP
         );
 
         try {

diff --git a/...ternal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java b/...ternal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java
@@ -60,6 +60,7 @@
 import static org.gradle.api.JavaVersion.VERSION_20;
 import static org.gradle.api.JavaVersion.VERSION_21;
 import static org.gradle.api.JavaVersion.VERSION_22;
+import static org.gradle.api.JavaVersion.VERSION_23;
 
 @CacheableTask
 public abstract class ThirdPartyAuditTask extends DefaultTask {
@@ -336,8 +337,8 @@ private String runForbiddenAPIsCli() throws IOException {
                 spec.setExecutable(javaHome.get() + "/bin/java");
             }
             spec.classpath(getForbiddenAPIsClasspath(), classpath);
-            // Enable explicitly for each release as appropriate. Just JDK 20/21/22 for now, and just the vector module.
-            if (isJavaVersion(VERSION_20) || isJavaVersion(VERSION_21) || isJavaVersion(VERSION_22)) {
+            // Enable explicitly for each release as appropriate. Just JDK 20/21/22/23 for now, and just the vector module.
+            if (isJavaVersion(VERSION_20) || isJavaVersion(VERSION_21) || isJavaVersion(VERSION_22) || isJavaVersion(VERSION_23)) {
                 spec.jvmArgs("--add-modules", "jdk.incubator.vector");
             }
             spec.jvmArgs("-Xmx1g");

diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
@@ -1107,11 +1107,11 @@ private void logFileContents(String description, Path from, boolean tailLogs) {
             return;
         }
 
-        boolean foundNettyLeaks = false;
+        boolean foundLeaks = false;
         for (String logLine : errorsAndWarnings.keySet()) {
-            if (logLine.contains("ResourceLeakDetector]")) {
+            if (logLine.contains("ResourceLeakDetector") || logLine.contains("LeakTracker")) {
                 tailLogs = true;
-                foundNettyLeaks = true;
+                foundLeaks = true;
                 break;
             }
         }
@@ -1140,8 +1140,8 @@ private void logFileContents(String description, Path from, boolean tailLogs) {
                 });
             }
         }
-        if (foundNettyLeaks) {
-            throw new TestClustersException("Found Netty ByteBuf leaks in node logs.");
+        if (foundLeaks) {
+            throw new TestClustersException("Found resource leaks in node logs.");
         }
     }
 

diff --git a/client/test/build.gradle b/client/test/build.gradle
@@ -27,9 +27,9 @@ dependencies {
   api "org.hamcrest:hamcrest:${versions.hamcrest}"
 
   // mockito
-  api 'org.mockito:mockito-core:5.9.0'
-  api 'org.mockito:mockito-subclass:5.9.0'
-  api 'net.bytebuddy:byte-buddy:1.14.11'
+  api 'org.mockito:mockito-core:5.11.0'
+  api 'org.mockito:mockito-subclass:5.11.0'
+  api 'net.bytebuddy:byte-buddy:1.14.12'
   api 'org.objenesis:objenesis:3.3'
 }
 

diff --git a/...ibution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java b/...ibution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java
@@ -27,61 +27,64 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
         boolean isHotspot = sysprops.getOrDefault("sun.management.compiler", "").contains("HotSpot");
         String libraryPath = findLibraryPath(sysprops);
 
-        return Stream.of(
-            /*
-             * Cache ttl in seconds for positive DNS lookups noting that this overrides the JDK security property networkaddress.cache.ttl;
-             * can be set to -1 to cache forever.
-             */
-            "-Des.networkaddress.cache.ttl=60",
-            /*
-             * Cache ttl in seconds for negative DNS lookups noting that this overrides the JDK security property
-             * networkaddress.cache.negative ttl; set to -1 to cache forever.
-             */
-            "-Des.networkaddress.cache.negative.ttl=10",
-            // Allow to set the security manager.
-            "-Djava.security.manager=allow",
-            // pre-touch JVM emory pages during initialization
-            "-XX:+AlwaysPreTouch",
-            // explicitly set the stack size
-            "-Xss1m",
-            // set to headless, just in case,
-            "-Djava.awt.headless=true",
-            // ensure UTF-8 encoding by default (e.g., filenames)
-            "-Dfile.encoding=UTF-8",
-            // use our provided JNA always versus the system one
-            "-Djna.nosys=true",
-            /*
-             * Turn off a JDK optimization that throws away stack traces for common exceptions because stack traces are important for
-             * debugging.
-             */
-            "-XX:-OmitStackTraceInFastThrow",
-            // flags to configure Netty
-            "-Dio.netty.noUnsafe=true",
-            "-Dio.netty.noKeySetOptimization=true",
-            "-Dio.netty.recycler.maxCapacityPerThread=0",
-            // log4j 2
-            "-Dlog4j.shutdownHookEnabled=false",
-            "-Dlog4j2.disable.jmx=true",
-            "-Dlog4j2.formatMsgNoLookups=true",
-            /*
-             * Due to internationalization enhancements in JDK 9 Elasticsearch need to set the provider to COMPAT otherwise time/date
-             * parsing will break in an incompatible way for some date patterns and locales.
-             */
-            "-Djava.locale.providers=SPI,COMPAT",
-            /*
-             * Temporarily suppress illegal reflective access in searchable snapshots shared cache preallocation; this is temporary while we
-             * explore alternatives. See org.elasticsearch.xpack.searchablesnapshots.preallocate.Preallocate.
-             */
-            "--add-opens=java.base/java.io=org.elasticsearch.preallocate",
-            "--add-opens=org.apache.lucene.core/org.apache.lucene.store=org.elasticsearch.vec",
-            maybeEnableNativeAccess(),
-            maybeOverrideDockerCgroup(distroType),
-            maybeSetActiveProcessorCount(nodeSettings),
-            setReplayFile(distroType, isHotspot),
-            "-Djava.library.path=" + libraryPath,
-            "-Djna.library.path=" + libraryPath,
-            // Pass through distribution type
-            "-Des.distribution.type=" + distroType
+        return Stream.concat(
+            Stream.of(
+                /*
+                 * Cache ttl in seconds for positive DNS lookups noting that this overrides the JDK security property
+                 * networkaddress.cache.ttl can be set to -1 to cache forever.
+                 */
+                "-Des.networkaddress.cache.ttl=60",
+                /*
+                 * Cache ttl in seconds for negative DNS lookups noting that this overrides the JDK security property
+                 * networkaddress.cache.negative ttl; set to -1 to cache forever.
+                 */
+                "-Des.networkaddress.cache.negative.ttl=10",
+                // Allow to set the security manager.
+                "-Djava.security.manager=allow",
+                // pre-touch JVM emory pages during initialization
+                "-XX:+AlwaysPreTouch",
+                // explicitly set the stack size
+                "-Xss1m",
+                // set to headless, just in case,
+                "-Djava.awt.headless=true",
+                // ensure UTF-8 encoding by default (e.g., filenames)
+                "-Dfile.encoding=UTF-8",
+                // use our provided JNA always versus the system one
+                "-Djna.nosys=true",
+                /*
+                 * Turn off a JDK optimization that throws away stack traces for common exceptions because stack traces are important for
+                 * debugging.
+                 */
+                "-XX:-OmitStackTraceInFastThrow",
+                // flags to configure Netty
+                "-Dio.netty.noUnsafe=true",
+                "-Dio.netty.noKeySetOptimization=true",
+                "-Dio.netty.recycler.maxCapacityPerThread=0",
+                // log4j 2
+                "-Dlog4j.shutdownHookEnabled=false",
+                "-Dlog4j2.disable.jmx=true",
+                "-Dlog4j2.formatMsgNoLookups=true",
+                /*
+                 * Due to internationalization enhancements in JDK 9 Elasticsearch need to set the provider to COMPAT otherwise time/date
+                 * parsing will break in an incompatible way for some date patterns and locales.
+                 */
+                "-Djava.locale.providers=SPI,COMPAT",
+                /*
+                 * Temporarily suppress illegal reflective access in searchable snapshots shared cache preallocation; this is temporary
+                 * while we explore alternatives. See org.elasticsearch.xpack.searchablesnapshots.preallocate.Preallocate.
+                 */
+                "--add-opens=java.base/java.io=org.elasticsearch.preallocate",
+                "--add-opens=org.apache.lucene.core/org.apache.lucene.store=org.elasticsearch.vec",
+                maybeEnableNativeAccess(),
+                maybeOverrideDockerCgroup(distroType),
+                maybeSetActiveProcessorCount(nodeSettings),
+                setReplayFile(distroType, isHotspot),
+                "-Djava.library.path=" + libraryPath,
+                "-Djna.library.path=" + libraryPath,
+                // Pass through distribution type
+                "-Des.distribution.type=" + distroType
+            ),
+            maybeWorkaroundG1Bug()
         ).filter(e -> e.isEmpty() == false).collect(Collectors.toList());
     }
 
@@ -137,6 +140,17 @@ private static String maybeEnableNativeAccess() {
         return "";
     }
 
+    /*
+     * Only affects 22 and 22.0.1, see https://bugs.openjdk.org/browse/JDK-8329528
+     */
+    private static Stream<String> maybeWorkaroundG1Bug() {
+        Runtime.Version v = Runtime.version();
+        if (v.feature() == 22 && v.update() <= 1) {
+            return Stream.of("-XX:+UnlockDiagnosticVMOptions", "-XX:G1NumCollectionsKeepPinned=10000000");
+        }
+        return Stream.of();
+    }
+
     private static String findLibraryPath(Map<String, String> sysprops) {
         // working dir is ES installation, so we use relative path here
         Path platformDir = Paths.get("lib", "platform");

diff --git a/docs/changelog/106486.yaml b/docs/changelog/106486.yaml
@@ -0,0 +1,17 @@
+pr: 106486
+summary: Create custom parser for ISO-8601 datetimes
+area: Infra/Core
+type: enhancement
+issues:
+  - 102063
+highlight:
+  title: New custom parser for ISO-8601 datetimes
+  body: |-
+    This introduces a new custom parser for ISO-8601 datetimes, for the `iso8601`, `strict_date_optional_time`, and
+    `strict_date_optional_time_nanos` built-in date formats. This provides a performance improvement over the
+    default Java date-time parsing. Whilst it maintains much of the same behaviour,
+    the new parser does not accept nonsensical date-time strings that have multiple fractional seconds fields
+    or multiple timezone specifiers. If the new parser fails to parse a string, it will then use the previous parser
+    to parse it. If a large proportion of the input data consists of these invalid strings, this may cause
+    a small performance degradation. If you wish to force the use of the old parsers regardless,
+    set the JVM property `es.datetime.java_time_parsers=true` on all ES nodes.
diff --git a/docs/changelog/108571.yaml b/docs/changelog/108571.yaml
@@ -0,0 +1,5 @@
+pr: 108571
+summary: Workaround G1 bug for JDK 22 and 22.0.1
+area: Infra/CLI
+type: bug
+issues: []
diff --git a/docs/changelog/108639.yaml b/docs/changelog/108639.yaml
@@ -0,0 +1,28 @@
+pr: 108639
+summary: Add support for the 'Domain' database to the geoip processor
+area: Ingest Node
+type: enhancement
+issues: []
+highlight:
+  title: Add support for the 'Domain' database to the geoip processor
+  body: |-
+    Follow on to #107287 and #107377
+
+    Adds support for the ['GeoIP2
+    Domain'](https://dev.maxmind.com/geoip/docs/databases/domain) database
+    from MaxMind to the `geoip` processor.
+
+    The `geoip` processor will automatically download the [various
+    'GeoLite2'
+    databases](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data),
+    but the 'GeoIP2 Domain' database is not a 'GeoLite2' database -- it's a
+    commercial database available to those with a suitable license from
+    MaxMind.
+
+    The support that is being added for it in this PR is in line with the
+    support that we already have for MaxMind's 'GeoIP2 City' and 'GeoIP2
+    Country' databases -- that is, one would need to arrange their own
+    download management via some custom endpoint or otherwise arrange for
+    the relevant file(s) to be in the `$ES_CONFIG/ingest-geoip` directory on
+    the nodes of the cluster.
+  notable: true
diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc
@@ -268,7 +268,7 @@ used for abuse detection.
 =====
 `return_documents`::
 (Optional, boolean)
-For `cohere` service only. Specify whether to return doc text within the 
+For `cohere` service only. Specify whether to return doc text within the
 results.
 
 `top_n`::
@@ -307,16 +307,6 @@ For `openai` and `azureopenai` service only. Specifies the user issuing the
 request, which can be used for abuse detection.
 
 =====
-+
-.`task_settings` for the `completion` task type
-[%collapsible%closed]
-=====
-`user`:::
-(optional, string)
-For `openai` service only. Specifies the user issuing the request, which can be used for abuse detection.
-=====
-
-
 [discrete]
 [[put-inference-api-example]]
 ==== {api-examples-title}
@@ -351,11 +341,11 @@ The following example shows how to create an {infer} endpoint called
 
 [source,console]
 ------------------------------------------------------------
-PUT _inference/rerank/cohere-rerank 
+PUT _inference/rerank/cohere-rerank
 {
     "service": "cohere",
     "service_settings": {
-        "api_key": "<API-KEY>", 
+        "api_key": "<API-KEY>",
         "model_id": "rerank-english-v3.0"
     },
     "task_settings": {
@@ -366,7 +356,7 @@ PUT _inference/rerank/cohere-rerank
 ------------------------------------------------------------
 // TEST[skip:TBD]
 
-For more examples, also review the 
+For more examples, also review the
 https://docs.cohere.com/docs/elasticsearch-and-cohere#rerank-search-results-with-cohere-and-elasticsearch[Cohere documentation].
 
 

diff --git a/docs/reference/ingest/processors/geoip.asciidoc b/docs/reference/ingest/processors/geoip.asciidoc
@@ -59,6 +59,7 @@ in `properties`.
 * If the GeoIP2 Anonymous IP database is used, then the following fields may be added under the `target_field`: `ip`,
 `hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, and `residential_proxy`. The fields actually added
 depend on what has been found and which properties were configured in `properties`.
+* If the GeoIP2 Domain database is used, then the following fields may be added under the `target_field`: `ip`, and `domain`.
 * If the GeoIP2 Enterprise database is used, then the following fields may be added under the `target_field`: `ip`,
 `country_iso_code`, `country_name`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `timezone`, `location`, `asn`,
 `organization_name`, `network`, `hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, and `residential_proxy`.

diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
@@ -1589,9 +1589,9 @@
             <sha256 value="1a1ac9ce65eddcea54ead958387bb0b3863d02a2ffe856ab6a57ac79737c19cf" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="net.bytebuddy" name="byte-buddy" version="1.14.11">
-         <artifact name="byte-buddy-1.14.11.jar">
-            <sha256 value="62ae28187ed2b062813da6a9d567bfee733c341582699b62dd980230729a0313" origin="Generated by Gradle"/>
+      <component group="net.bytebuddy" name="byte-buddy" version="1.14.12">
+         <artifact name="byte-buddy-1.14.12.jar">
+            <sha256 value="970636134d61c183b19f8f58fa631e30d2f2abca344b37848a393cac7863dd70" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="net.java.dev.jets3t" name="jets3t" version="0.9.0">
@@ -3701,13 +3701,13 @@
             <sha256 value="f97483ba0944b9fa133aa29638764ddbeadb51ec3dbc02074c58fa2caecd07fa" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.mockito" name="mockito-core" version="5.9.0">
-         <artifact name="mockito-core-5.9.0.jar">
-            <sha256 value="bbad9185ed734965fac7e367f0e51596f69531e51d8b2cbcec1048dd6fb41f2c" origin="Generated by Gradle"/>
+      <component group="org.mockito" name="mockito-core" version="5.11.0">
+         <artifact name="mockito-core-5.11.0.jar">
+            <sha256 value="f076c96b1f49b8d9bc42e46b0969aaf5684c40c8b5b679d400e5d880073a0e00" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.mockito" name="mockito-subclass" version="5.9.0">
-         <artifact name="mockito-subclass-5.9.0.jar">
+      <component group="org.mockito" name="mockito-subclass" version="5.11.0">
+         <artifact name="mockito-subclass-5.11.0.jar">
             <sha256 value="61e65116bf6178cd3a5f1cecc0d73d68395e7c175d07a7f9478650f55b36fb1d" origin="Generated by Gradle"/>
          </artifact>
       </component>