Merge branch 'main' into fix_flatobj

Signed-off-by: Shinsuke Sugaya <shinsuke@apache.org>

.ci/bwcVersions

@@ -19,3 +19,4 @@ BWC_VERSION:
   - "2.7.0"
   - "2.7.1"
   - "2.8.0"
+  - "2.9.0"

.github/workflows/publish-maven-snapshots.yml

@@ -5,8 +5,8 @@ on:
   push:
       branches:
         - main
-        - '1.3'
-        - 2.x
+        - '[0-9]+.[0-9]+'
+        - '[0-9]+.x'
 
 jobs:
   build-and-publish-snapshots:

CHANGELOG.md

@@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add getter for path field in NestedQueryBuilder ([#4636](https://github.com/opensearch-project/OpenSearch/pull/4636))
 - Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151))
 - Add events correlation engine plugin ([#6854](https://github.com/opensearch-project/OpenSearch/issues/6854))
+- Add support for ignoring missing Javadoc on generated code using annotation ([#7604](https://github.com/opensearch-project/OpenSearch/pull/7604))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
@@ -40,6 +41,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.netflix.nebula:nebula-publishing-plugin` from 19.2.0 to 20.3.0
 - Bump `com.diffplug.spotless` from 6.17.0 to 6.18.0
 - Bump `io.opencensus:opencensus-api` from 0.18.0 to 0.31.1 ([#7291](https://github.com/opensearch-project/OpenSearch/pull/7291))
+- Add  `com.github.luben:zstd-jni` version 1.5.5-3 ([#2996](https://github.com/opensearch-project/OpenSearch/pull/2996))
+- OpenJDK Update (April 2023 Patch releases) ([#7344](https://github.com/opensearch-project/OpenSearch/pull/7344)
 
 ### Changed
 - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
@@ -48,6 +51,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Change http code on create index API with bad input raising NotXContentException from 500 to 400 ([#4773](https://github.com/opensearch-project/OpenSearch/pull/4773))
 - Change http code for DecommissioningFailedException from 500 to 400 ([#5283](https://github.com/opensearch-project/OpenSearch/pull/5283))
 - Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792))
+- Reduce memory copy in zstd compression ([#7681](https://github.com/opensearch-project/OpenSearch/pull/7681))
+- Add ZSTD compression for snapshotting ([#2996](https://github.com/opensearch-project/OpenSearch/pull/2996))
 
 ### Deprecated
 
@@ -78,74 +83,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 2.x]
 ### Added
-- [Extensions] Moving Extensions APIs to support cross versions via protobuf. ([#7402](https://github.com/opensearch-project/OpenSearch/issues/7402))
-- [Extensions] Add IdentityPlugin into core to support Extension identities ([#7246](https://github.com/opensearch-project/OpenSearch/pull/7246))
-- Add connectToNodeAsExtension in TransportService ([#6866](https://github.com/opensearch-project/OpenSearch/pull/6866))
-- [Search Pipelines] Accept pipelines defined in search source ([#7253](https://github.com/opensearch-project/OpenSearch/pull/7253))
-- [Search Pipelines] Add `default_search_pipeline` index setting ([#7470](https://github.com/opensearch-project/OpenSearch/pull/7470))
-- [Search Pipelines] Add RenameFieldResponseProcessor for Search Pipelines ([#7377](https://github.com/opensearch-project/OpenSearch/pull/7377))
-- [Search Pipelines] Split search pipeline processor factories by type ([#7597](https://github.com/opensearch-project/OpenSearch/pull/7597))
-- [Search Pipelines] Add script processor ([#7607](https://github.com/opensearch-project/OpenSearch/pull/7607))
-- Add descending order search optimization through reverse segment read. ([#7244](https://github.com/opensearch-project/OpenSearch/pull/7244))
-- Add 'unsigned_long' numeric field type ([#6237](https://github.com/opensearch-project/OpenSearch/pull/6237))
-- Add back primary shard preference for queries ([#7375](https://github.com/opensearch-project/OpenSearch/pull/7375))
-- Add task cancellation timestamp in task API ([#7455](https://github.com/opensearch-project/OpenSearch/pull/7455))
-- Adds ExtensionsManager.lookupExtensionSettingsById ([#7466](https://github.com/opensearch-project/OpenSearch/pull/7466))
-- SegRep with Remote: Add hook for publishing checkpoint notifications after segment upload to remote store ([#7394](https://github.com/opensearch-project/OpenSearch/pull/7394))
-- Add search_after query optimizations with shard/segment short cutting ([#7453](https://github.com/opensearch-project/OpenSearch/pull/7453))
-- Provide mechanism to configure XContent parsing constraints (after update to Jackson 2.15.0 and above) ([#7550](https://github.com/opensearch-project/OpenSearch/pull/7550))
-- Support to clear filecache using clear indices cache API ([#7498](https://github.com/opensearch-project/OpenSearch/pull/7498))
-- Create NamedRoute to map extension routes to a shortened name ([#6870](https://github.com/opensearch-project/OpenSearch/pull/6870))
-- Added @dbwiddis as on OpenSearch maintainer ([#7665](https://github.com/opensearch-project/OpenSearch/pull/7665))
 
 ### Dependencies
-- Bump `com.netflix.nebula:gradle-info-plugin` from 12.0.0 to 12.1.3 (#7564)
-- Bump `com.netflix.nebula:nebula-publishing-plugin` from 19.2.0 to 20.2.0
-- Bump `com.google.protobuf:protobuf-java` from 3.22.2 to 3.22.3
-- Bump `jackson` from 2.14.2 to 2.15.0 ([#7286](https://github.com/opensearch-project/OpenSearch/pull/7286))
-- Bump `com.netflix.nebula:nebula-publishing-plugin` from 20.2.0 to 20.3.0
-- Bump `com.netflix.nebula.ospackage-base` from 11.0.0 to 11.3.0
-- Bump `gradle.plugin.com.github.johnrengelman:shadow` from 7.1.2 to 8.0.0
-- Bump `jna` from 5.11.0 to 5.13.0
-- Bump `commons-io:commons-io` from 2.7 to 2.12.0 (#7661, #7658, #7656)
-- Bump `org.apache.shiro:shiro-core` from 1.9.1 to 1.11.0 ([#7397](https://github.com/opensearch-project/OpenSearch/pull/7397))
-- Bump `jetty-server` in hdfs-fixture from 9.4.49.v20220914 to 9.4.51.v20230217 ([#7405](https://github.com/opensearch-project/OpenSearch/pull/7405))
-- OpenJDK Update (April 2023 Patch releases) ([#7448](https://github.com/opensearch-project/OpenSearch/pull/7448)
-- Bump `org.apache.commons:commons-compress` from 1.22 to 1.23.0 (#7462)
-- Bump `com.azure:azure-core` from 1.34.0 to 1.39.0
-- Bump `com.networknt:json-schema-validator` from 1.0.78 to 1.0.81 (#7460)
-- Bump Apache Lucene to 9.6.0 ([#7505](https://github.com/opensearch-project/OpenSearch/pull/7505))
-- Bump `com.google.cloud:google-cloud-core-http` from 1.93.3 to 2.17.0 (#7488)
-- Bump `com.google.guava:guava` from 30.1.1-jre to 31.1-jre (#7565)
-- Bump `com.azure:azure-storage-common` from 12.20.0 to 12.21.0 (#7566)
-- Bump `org.apache.commons:commons-compress` from 1.22 to 1.23.0 (#7563)
-- Bump `jackson` from 2.15.0 to 2.15.1 ([#7603](https://github.com/opensearch-project/OpenSearch/pull/7603))
-- Bump `net.minidev:json-smart` from 2.4.10 to 2.4.11 (#7660)
-- Bump `io.projectreactor.netty:reactor-netty-core` from 1.1.5 to 1.1.7 (#7657)
-- Bump `org.apache.maven:maven-model` from 3.9.1 to 3.9.2 (#7655)
-- Bump `com.google.api:gax` from 2.17.0 to 2.27.0 (#7697)
-- Bump `io.projectreactor.netty:reactor-netty` from 1.1.4 to 1.1.7 ([#7725](https://github.com/opensearch-project/OpenSearch/pull/7725))
-- Bump `io.projectreactor.netty:reactor-netty-http` from 1.1.4 to 1.1.7 ([#7725](https://github.com/opensearch-project/OpenSearch/pull/7725))
 
 ### Changed
-- Enable `./gradlew build` on MacOS by disabling bcw tests ([#7303](https://github.com/opensearch-project/OpenSearch/pull/7303))
-- Moved concurrent-search from sandbox plugin to server module behind feature flag ([#7203](https://github.com/opensearch-project/OpenSearch/pull/7203))
-- Allow access to indices cache clear APIs for read only indexes ([#7303](https://github.com/opensearch-project/OpenSearch/pull/7303))
-- Changed concurrent-search threadpool type to be resizable and support task resource tracking ([#7502](https://github.com/opensearch-project/OpenSearch/pull/7502))
-- Default search preference to _primary for searchable snapshot indices ([#7628](https://github.com/opensearch-project/OpenSearch/pull/7628))
-- [Segment Replication] Remove codec name string match check for checkpoints ([#7741](https://github.com/opensearch-project/OpenSearch/pull/7741))
+- Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836))
+- Add min, max, average and thread info to resource stats in tasks API ([#7673](https://github.com/opensearch-project/OpenSearch/pull/7673))
 
 ### Deprecated
 
 ### Removed
 
 ### Fixed
-- Add more index blocks check for resize APIs ([#6774](https://github.com/opensearch-project/OpenSearch/pull/6774))
-- Replaces ZipInputStream with ZipFile to fix Zip Slip vulnerability ([#7230](https://github.com/opensearch-project/OpenSearch/pull/7230))
-- Add missing validation/parsing of SearchBackpressureMode of SearchBackpressureSettings ([#7541](https://github.com/opensearch-project/OpenSearch/pull/7541))
 - Improve indexing performance for flat_object type ([#7855](https://github.com/opensearch-project/OpenSearch/pull/7855))
 
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.7...2.x
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.8...2.x

build.gradle

@@ -54,7 +54,7 @@ plugins {
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
   id "com.diffplug.spotless" version "6.18.0" apply false
-  id "org.gradle.test-retry" version "1.5.2" apply false
+  id "org.gradle.test-retry" version "1.5.3" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'
 }

buildSrc/src/main/java/org/opensearch/gradle/JdkDownloadPlugin.java

@@ -129,13 +129,9 @@ private void setupRepository(Project project, Jdk jdk) {
                 // To distinguish between those, the GA releases have only major version component (fe 17+32),
                 // the updates always have minor/patch components (fe 17.0.1+12), checking for the presence of
                 // version separator '.' should be enough.
-                artifactPattern = "jdk-"
-                    + jdk.getBaseVersion()
-                    + "+"
-                    + jdk.getBuild()
-                    + "/OpenJDK"
-                    + jdk.getMajor()
-                    + (jdk.getBaseVersion().contains(".") ? "U" : "")
+                artifactPattern = "jdk-" + jdk.getBaseVersion() + "+" + jdk.getBuild() + "/OpenJDK" + jdk.getMajor()
+                // JDK-20 does use 'U' suffix all the time, no matter it is update or GA release
+                    + (jdk.getBaseVersion().contains(".") || jdk.getBaseVersion().matches("^2\\d+$") ? "U" : "")
                     + "-jdk_[classifier]_[module]_hotspot_"
                     + jdk.getBaseVersion()
                     + "_"

buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java

@@ -31,6 +31,8 @@
 
 package org.opensearch.gradle;
 
+import org.apache.commons.lang3.StringUtils;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.HashMap;
@@ -54,22 +56,26 @@ public static String getLucene() {
         return lucene;
     }
 
-    public static String getBundledJdk(final String platform) {
+    public static String getBundledJdk(final String platform, final String arch) {
         switch (platform) {
             case "darwin": // fall trough
             case "mac":
                 return bundledJdkDarwin;
             case "freebsd":
                 return bundledJdkFreeBSD;
             case "linux":
-                return bundledJdkLinux;
+                return getBundledJdkLinux(arch);
             case "windows":
                 return bundledJdkWindows;
             default:
                 throw new IllegalArgumentException("unknown platform [" + platform + "]");
         }
     }
 
+    public static String getBundledJdk(final String platform) {
+        return getBundledJdk(platform, null);
+    }
+
     public static String getBundledJdkVendor() {
         return bundledJdkVendor;
     }
@@ -84,6 +90,10 @@ public static Map<String, String> getVersions() {
     private static final String bundledJdkFreeBSD;
     private static final String bundledJdkLinux;
     private static final String bundledJdkWindows;
+    private static final String bundledJdkLinux_arm64;
+    private static final String bundledJdkLinux_x64;
+    private static final String bundledJdkLinux_s390x;
+    private static final String bundledJdkLinux_ppc64le;
     private static final String bundledJdkVendor;
     private static final Map<String, String> versions = new HashMap<String, String>();
 
@@ -98,6 +108,12 @@ public static Map<String, String> getVersions() {
         bundledJdkLinux = props.getProperty("bundled_jdk_linux", bundledJdk);
         bundledJdkWindows = props.getProperty("bundled_jdk_windows", bundledJdk);
 
+        // Bundled JDKs per architecture (linux platform)
+        bundledJdkLinux_arm64 = props.getProperty("bundled_jdk_linux_arm64", bundledJdkLinux);
+        bundledJdkLinux_x64 = props.getProperty("bundled_jdk_linux_x64", bundledJdkLinux);
+        bundledJdkLinux_s390x = props.getProperty("bundled_jdk_linux_s390x", bundledJdkLinux);
+        bundledJdkLinux_ppc64le = props.getProperty("bundled_jdk_linux_ppc64le", bundledJdkLinux);
+
         for (String property : props.stringPropertyNames()) {
             versions.put(property, props.getProperty(property));
         }
@@ -119,4 +135,24 @@ private static Properties getVersionProperties() {
     public static boolean isOpenSearchSnapshot() {
         return opensearch.endsWith("-SNAPSHOT");
     }
+
+    private static String getBundledJdkLinux(String arch) {
+        if (StringUtils.isBlank(arch)) {
+            return bundledJdkLinux;
+        }
+
+        switch (arch) {
+            case "aarch64":
+            case "arm64":
+                return bundledJdkLinux_arm64;
+            case "x64":
+                return bundledJdkLinux_x64;
+            case "s390x":
+                return bundledJdkLinux_s390x;
+            case "ppc64le":
+                return bundledJdkLinux_ppc64le;
+            default:
+                throw new IllegalArgumentException("unknown platform [" + arch + "] for 'linux' platform");
+        }
+    }
 }

buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java

@@ -75,9 +75,9 @@
 import java.util.stream.Stream;
 
 public class DistroTestPlugin implements Plugin<Project> {
-    private static final String SYSTEM_JDK_VERSION = "11.0.18+10";
+    private static final String SYSTEM_JDK_VERSION = "11.0.19+7";
     private static final String SYSTEM_JDK_VENDOR = "adoptium";
-    private static final String GRADLE_JDK_VERSION = "17.0.6+10";
+    private static final String GRADLE_JDK_VERSION = "17.0.7+7";
     private static final String GRADLE_JDK_VENDOR = "adoptium";
 
     // all distributions used by distro tests. this is temporary until tests are per distribution

buildSrc/version.properties

@@ -2,8 +2,9 @@ opensearch        = 3.0.0
 lucene            = 9.7.0-snapshot-4d1ed9e
 
 bundled_jdk_vendor = adoptium
-bundled_jdk = 19.0.2+7
-
+bundled_jdk = 20.0.1+9
+# See please https://github.com/adoptium/temurin-build/issues/3371
+bundled_jdk_linux_ppc64le = 20+36
 
 # optional dependencies
 spatial4j         = 0.7
@@ -23,6 +24,7 @@ kotlin            = 1.7.10
 antlr4            = 4.11.1
 guava             = 31.1-jre
 protobuf          = 3.22.3
+jakarta_annotation = 1.3.5
 
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.5.0
@@ -56,3 +58,6 @@ bytebuddy         = 1.14.3
 
 # benchmark dependencies
 jmh               = 1.35
+
+# compression
+zstd              = 1.5.5-3

client/rest-high-level/src/test/java/org/opensearch/client/core/tasks/GetTaskResponseTests.java

@@ -43,6 +43,7 @@
 import org.opensearch.tasks.Task;
 import org.opensearch.tasks.TaskId;
 import org.opensearch.tasks.TaskInfo;
+import org.opensearch.tasks.TaskThreadUsage;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.io.IOException;
@@ -138,12 +139,12 @@ private static RawTaskStatus randomRawTaskStatus() {
     }
 
     private static TaskResourceStats randomResourceStats() {
-        return randomBoolean() ? null : new TaskResourceStats(new HashMap<String, TaskResourceUsage>() {
+        return randomBoolean() ? null : new TaskResourceStats(new HashMap<>() {
             {
                 for (int i = 0; i < randomInt(5); i++) {
                     put(randomAlphaOfLength(5), new TaskResourceUsage(randomNonNegativeLong(), randomNonNegativeLong()));
                 }
             }
-        });
+        }, new TaskThreadUsage(randomInt(10), randomInt(10)));
     }
 }

distribution/build.gradle

@@ -283,7 +283,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
       (platform == 'linux' || platform == 'darwin' ? ['x64', 'aarch64', 's390x', 'ppc64le'] : ['x64']).each { architecture ->
         "bundled_${platform}_${architecture}" {
           it.platform = platform
-          it.version = VersionProperties.getBundledJdk(platform)
+          it.version = VersionProperties.getBundledJdk(platform, architecture)
           it.vendor = VersionProperties.bundledJdkVendor
           it.architecture = architecture
         }

distribution/src/config/opensearch.yml

@@ -102,10 +102,9 @@ ${path.logs}
 #
 # ---------------------------------- Experimental Features -----------------------------------
 #
-# Gates the visibility of the index setting that allows changing of replication type.
-# Once the feature is ready for production release, this feature flag can be removed.
+# Gates the visibility of the experimental segment replication features until they are production ready.
 #
-#opensearch.experimental.feature.replication_type.enabled: false
+#opensearch.experimental.feature.segment_replication_experimental.enabled: false
 #
 #
 # Gates the visibility of the index setting that allows persisting data to remote store along with local disk.

distribution/tools/keystore-cli/build.gradle

@@ -35,5 +35,5 @@ dependencies {
   compileOnly project(":libs:opensearch-cli")
   testImplementation project(":test:framework")
   testImplementation 'com.google.jimfs:jimfs:1.2'
-  testRuntimeOnly 'com.google.guava:guava:31.1-jre'
+  testRuntimeOnly 'com.google.guava:guava:32.0.0-jre'
 }

distribution/tools/plugin-cli/build.gradle

@@ -39,7 +39,7 @@ dependencies {
   api "org.bouncycastle:bc-fips:1.0.2.3"
   testImplementation project(":test:framework")
   testImplementation 'com.google.jimfs:jimfs:1.2'
-  testRuntimeOnly 'com.google.guava:guava:31.1-jre'
+  testRuntimeOnly 'com.google.guava:guava:32.0.0-jre'
 
   implementation 'org.apache.commons:commons-compress:1.23.0'
 }
@@ -82,9 +82,6 @@ thirdPartyAudit.ignoreViolations(
 )
 
 thirdPartyAudit.ignoreMissingClasses(
-  'com.github.luben.zstd.BufferPool',
-  'com.github.luben.zstd.ZstdInputStream',
-  'com.github.luben.zstd.ZstdOutputStream',
   'org.brotli.dec.BrotliInputStream',
   'org.objectweb.asm.AnnotationVisitor',
   'org.objectweb.asm.Attribute',

distribution/tools/upgrade-cli/build.gradle

@@ -19,7 +19,7 @@ dependencies {
   implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
   testImplementation project(":test:framework")
   testImplementation 'com.google.jimfs:jimfs:1.2'
-  testRuntimeOnly 'com.google.guava:guava:31.1-jre'
+  testRuntimeOnly 'com.google.guava:guava:32.0.0-jre'
 }
 
 tasks.named("dependencyLicenses").configure {

doc-tools/missing-doclet/build.gradle

@@ -5,10 +5,6 @@ plugins {
 group 'org.opensearch'
 version '1.0.0-SNAPSHOT'
 
-repositories {
-    mavenCentral()
-}
-
 tasks.withType(JavaCompile) {
   options.compilerArgs += ["--release", targetCompatibility.toString()]
   options.encoding = "UTF-8"