Merge branch 'main' into updateHDFS

Signed-off-by: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com>

CHANGELOG.md

@@ -41,6 +41,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.bouncycastle:bcmail-jdk15on` to `org.bouncycastle:bcmail-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247))
 - Bump `org.bouncycastle:bcpkix-jdk15on` to `org.bouncycastle:bcpkix-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247))
 - Bump `net.minidev:json-smart` from 2.4.11 to 2.5.0 ([#8576](https://github.com/opensearch-project/OpenSearch/pull/8576))
+- Bump `com.google.cloud:google-cloud-core-http` from 2.17.0 to 2.21.0 ([#8586](https://github.com/opensearch-project/OpenSearch/pull/8586))
+
 
 
 ### Changed
@@ -105,7 +107,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ### Dependencies
 - Bump `com.azure:azure-storage-common` from 12.21.0 to 12.21.1 (#7566, #7814)
-- Bump `com.google.guava:guava` from 30.1.1-jre to 32.1.1-jre (#7565, #7811, #7807, #7808, #8402, #8400, #8401)
+- Bump `com.google.guava:guava` from 30.1.1-jre to 32.1.1-jre (#7565, #7811, #7807, #7808, #8402, #8400, #8401, #8581)
 - Bump `net.minidev:json-smart` from 2.4.10 to 2.4.11 (#7660, #7812)
 - Bump `org.gradle.test-retry` from 1.5.2 to 1.5.3 (#7810)
 - Bump `com.diffplug.spotless` from 6.17.0 to 6.18.0 (#7896)
@@ -125,7 +127,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `io.projectreactor:reactor-core` from 3.4.18 to 3.5.6 in /plugins/repository-azure ([#8016](https://github.com/opensearch-project/OpenSearch/pull/8016))
 - Bump `spock-core` from 2.1-groovy-3.0 to 2.3-groovy-3.0 ([#8122](https://github.com/opensearch-project/OpenSearch/pull/8122))
 - Bump `com.networknt:json-schema-validator` from 1.0.83 to 1.0.84 (#8141)
-- Bump `com.netflix.nebula:gradle-info-plugin` from 12.1.3 to 12.1.4 (#8139)
+- Bump `com.netflix.nebula:gradle-info-plugin` from 12.1.3 to 12.1.5 (#8139, #8568)
 - Bump `commons-io:commons-io` from 2.12.0 to 2.13.0 in /plugins/discovery-azure-classic ([#8140](https://github.com/opensearch-project/OpenSearch/pull/8140))
 - Bump `mockito` from 5.2.0 to 5.4.0 ([#8181](https://github.com/opensearch-project/OpenSearch/pull/8181))
 - Bump `netty` from 4.1.93.Final to 4.1.94.Final ([#8191](https://github.com/opensearch-project/OpenSearch/pull/8191))
@@ -137,6 +139,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Update Apache HttpCore/ HttpClient and Apache HttpCore5 / HttpClient5 dependencies ([#8434](https://github.com/opensearch-project/OpenSearch/pull/8434))
 - Bump `org.apache.maven:maven-model` from 3.9.2 to 3.9.3 (#8403)
 - Bump `io.projectreactor.netty:reactor-netty` and `io.projectreactor.netty:reactor-netty-core` from 1.1.7 to 1.1.8 (#8405)
+- Bump `com.azure:azure-storage-blob` from 12.22.2 to 12.22.3 (#8572)
 
 ### Changed
 - Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836))
@@ -158,6 +161,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Search Pipelines] Pass pipeline creation context to processor factories ([#8164](https://github.com/opensearch-project/OpenSearch/pull/8164))
 - Enabling compression levels for zstd and zstd_no_dict ([#8312](https://github.com/opensearch-project/OpenSearch/pull/8312))
 - Optimize Metadata build() to skip redundant computations as part of ClusterState build ([#7853](https://github.com/opensearch-project/OpenSearch/pull/7853))
+- Add safeguard limits for file cache during node level allocation ([#8208](https://github.com/opensearch-project/OpenSearch/pull/8208))
 
 ### Deprecated
 
@@ -172,8 +176,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix mapping char_filter when mapping a hashtag ([#7591](https://github.com/opensearch-project/OpenSearch/pull/7591))
 - Fix NPE in multiterms aggregations involving empty buckets ([#7318](https://github.com/opensearch-project/OpenSearch/pull/7318))
 - Precise system clock time in MasterService debug logs ([#7902](https://github.com/opensearch-project/OpenSearch/pull/7902))
+- Improve indexing performance for flat_object type ([#7855](https://github.com/opensearch-project/OpenSearch/pull/7855))
 
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.8...2.x
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.8...2.x

buildSrc/build.gradle

@@ -114,7 +114,7 @@ dependencies {
   api 'org.apache.ant:ant:1.10.13'
   api 'com.netflix.nebula:gradle-extra-configurations-plugin:10.0.0'
   api 'com.netflix.nebula:nebula-publishing-plugin:20.3.0'
-  api 'com.netflix.nebula:gradle-info-plugin:12.1.4'
+  api 'com.netflix.nebula:gradle-info-plugin:12.1.5'
   api 'org.apache.rat:apache-rat:0.15'
   api 'commons-io:commons-io:2.13.0'
   api "net.java.dev.jna:jna:5.13.0"

buildSrc/version.properties

@@ -22,7 +22,7 @@ jettison          = 1.5.4
 woodstox          = 6.4.0
 kotlin            = 1.7.10
 antlr4            = 4.11.1
-guava             = 32.0.1-jre
+guava             = 32.1.1-jre
 protobuf          = 3.22.3
 jakarta_annotation = 1.3.5
 

distribution/tools/keystore-cli/build.gradle

@@ -35,7 +35,7 @@ dependencies {
   compileOnly project(":libs:opensearch-cli")
   testImplementation project(":test:framework")
   testImplementation 'com.google.jimfs:jimfs:1.2'
-  testRuntimeOnly('com.google.guava:guava:32.1.1-jre') {
+  testRuntimeOnly("com.google.guava:guava:${versions.guava}") {
     transitive = false
   }
 }

distribution/tools/plugin-cli/build.gradle

@@ -41,7 +41,7 @@ dependencies {
   api "org.bouncycastle:bc-fips:1.0.2.3"
   testImplementation project(":test:framework")
   testImplementation 'com.google.jimfs:jimfs:1.2'
-  testRuntimeOnly('com.google.guava:guava:32.1.1-jre') {
+  testRuntimeOnly("com.google.guava:guava:${versions.guava}") {
     transitive = false
   }
 

distribution/tools/upgrade-cli/build.gradle

@@ -21,7 +21,7 @@ dependencies {
   implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
   testImplementation project(":test:framework")
   testImplementation 'com.google.jimfs:jimfs:1.2'
-  testRuntimeOnly('com.google.guava:guava:32.1.1-jre') {
+  testRuntimeOnly("com.google.guava:guava:${versions.guava}") {
     transitive = false
   }
 }

modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/30_search.yml

@@ -482,100 +482,3 @@
                         }]
     - match: { error.root_cause.0.type: "illegal_argument_exception" }
     - match: { error.root_cause.0.reason: "script score function must not produce negative scores, but got: [-9.0]"}
-
----
-
-"Flat-object fields from within the scripting":
-  - skip:
-      version: " - 2.6.99"
-      reason: "flat_object is introduced in 2.7.0"
-
-  - do:
-      indices.create:
-        index: test
-        body:
-          mappings:
-            properties:
-              flat:
-                type : "flat_object"
-
-  # This document has 6 distinct parts in its flat_object field paths:
-  # - flat.field_1
-  # - flat.field_2
-  # - flat.field_3
-  # - flat.inner
-  # - flat.field_A
-  # - flat.field_B
-  - do:
-      index:
-        index: test
-        id:    1
-        body:  {
-          "flat": {
-            "field_1": "John Doe",
-            "field_2": 33,
-            "field_3": false,
-            "inner": {
-              "field_A": ["foo", "bar"],
-              "field_B": false
-            }
-          }
-        }
-
-  - do:
-      index:
-        index: test
-        id:    2
-        body:  {
-          "flat": {
-            "field_1": "Joe Public",
-            "field_2": 45
-          }
-        }
-
-  - do:
-      indices.refresh:
-        index: test
-
-  # It is possible to filter based on the number of distinct parts of flat_object field paths
-  - do:
-      search:
-        body: {
-          _source: true,
-          query: {
-            bool: {
-              filter: {
-                script: {
-                  script: {
-                    source: "doc['flat'].size() == 6",
-                    lang: "painless"
-                  }
-                }
-              }
-            }
-          }
-        }
-
-  - length:   { hits.hits: 1  }
-  - match: { hits.hits.0._source.flat.field_1: "John Doe" }
-
-  - do:
-      search:
-        body: {
-          _source: true,
-          query: {
-            bool: {
-              filter: {
-                script: {
-                  script: {
-                    source: "doc['flat'].size() < 6",
-                    lang: "painless"
-                  }
-                }
-              }
-            }
-          }
-        }
-
-  - length:   { hits.hits: 1  }
-  - match: { hits.hits.0._source.flat.field_1: "Joe Public" }

plugins/ingest-attachment/licenses/guava-32.1.1-jre.jar.sha1

@@ -0,0 +1 @@
+ad575652d84153075dd41ec6177ccb15251262b2

plugins/repository-azure/build.gradle

@@ -55,7 +55,7 @@ dependencies {
   api "io.netty:netty-resolver-dns:${versions.netty}"
   api "io.netty:netty-transport-native-unix-common:${versions.netty}"
   implementation project(':modules:transport-netty4')
-  api 'com.azure:azure-storage-blob:12.22.2'
+  api 'com.azure:azure-storage-blob:12.22.3'
   api 'org.reactivestreams:reactive-streams:1.0.4'
   api 'io.projectreactor:reactor-core:3.5.6'
   api 'io.projectreactor.netty:reactor-netty:1.1.8'

plugins/repository-azure/licenses/azure-storage-blob-12.22.3.jar.sha1

@@ -0,0 +1 @@
+0df12462c2eac3beaf25d283f707a0560853228b

plugins/repository-azure/licenses/guava-32.1.1-jre.jar.sha1

@@ -0,0 +1 @@
+ad575652d84153075dd41ec6177ccb15251262b2

plugins/repository-gcs/build.gradle

@@ -67,7 +67,7 @@ dependencies {
   api "com.google.auth:google-auth-library-oauth2-http:${versions.google_auth}"
 
   api 'com.google.cloud:google-cloud-core:2.5.10'
-  api 'com.google.cloud:google-cloud-core-http:2.17.0'
+  api 'com.google.cloud:google-cloud-core-http:2.21.0'
   api 'com.google.cloud:google-cloud-storage:1.113.1'
 
   api 'com.google.code.gson:gson:2.9.0'

plugins/repository-gcs/licenses/google-cloud-core-http-2.21.0.jar.sha1

@@ -0,0 +1 @@
+07da4710ccdbcfee253672c0b9e00e7370626c26

plugins/repository-gcs/licenses/guava-32.1.1-jre.jar.sha1

@@ -0,0 +1 @@
+ad575652d84153075dd41ec6177ccb15251262b2

plugins/repository-hdfs/licenses/guava-32.1.1-jre.jar.sha1

@@ -0,0 +1 @@
+ad575652d84153075dd41ec6177ccb15251262b2

server/src/internalClusterTest/java/org/opensearch/cluster/ClusterInfoServiceIT.java

@@ -50,6 +50,7 @@
 import org.opensearch.core.common.Strings;
 import org.opensearch.index.IndexService;
 import org.opensearch.index.shard.IndexShard;
+import org.opensearch.index.store.remote.filecache.FileCacheStats;
 import org.opensearch.index.store.Store;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.SystemIndexDescriptor;
@@ -192,6 +193,11 @@ public void testClusterInfoServiceCollectsInformation() {
             logger.info("--> shard size: {}", size);
             assertThat("shard size is greater than 0", size, greaterThanOrEqualTo(0L));
         }
+
+        final Map<String, FileCacheStats> nodeFileCacheStats = info.nodeFileCacheStats;
+        assertNotNull(nodeFileCacheStats);
+        assertThat("file cache is empty on non search nodes", nodeFileCacheStats.size(), Matchers.equalTo(0));
+
         ClusterService clusterService = internalTestCluster.getInstance(ClusterService.class, internalTestCluster.getClusterManagerName());
         ClusterState state = clusterService.state();
         for (ShardRouting shard : state.routingTable().allShards()) {
@@ -209,6 +215,28 @@ public void testClusterInfoServiceCollectsInformation() {
         }
     }
 
+    public void testClusterInfoServiceCollectsFileCacheInformation() {
+        internalCluster().startNodes(1);
+        internalCluster().ensureAtLeastNumSearchAndDataNodes(2);
+
+        InternalTestCluster internalTestCluster = internalCluster();
+        // Get the cluster info service on the cluster-manager node
+        final InternalClusterInfoService infoService = (InternalClusterInfoService) internalTestCluster.getInstance(
+            ClusterInfoService.class,
+            internalTestCluster.getClusterManagerName()
+        );
+        infoService.setUpdateFrequency(TimeValue.timeValueMillis(200));
+        ClusterInfo info = infoService.refresh();
+        assertNotNull("info should not be null", info);
+        final Map<String, FileCacheStats> nodeFileCacheStats = info.nodeFileCacheStats;
+        assertNotNull(nodeFileCacheStats);
+        assertThat("file cache is enabled on both search nodes", nodeFileCacheStats.size(), Matchers.equalTo(2));
+
+        for (FileCacheStats fileCacheStats : nodeFileCacheStats.values()) {
+            assertThat("file cache is non empty", fileCacheStats.getTotal().getBytes(), greaterThan(0L));
+        }
+    }
+
     public void testClusterInfoServiceInformationClearOnError() {
         internalCluster().startNodes(
             2,