Merge branch 'main' into fix_fs_stats

Signed-off-by: Jay Deng <jayd0104@gmail.com>

CHANGELOG.md

@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add fingerprint ingest processor ([#13724](https://github.com/opensearch-project/OpenSearch/pull/13724))
 - [Remote Store] Rate limiter for remote store low priority uploads ([#14374](https://github.com/opensearch-project/OpenSearch/pull/14374/))
 - Apply the date histogram rewrite optimization to range aggregation ([#13865](https://github.com/opensearch-project/OpenSearch/pull/13865))
+- [Writable Warm] Add composite directory implementation and integrate it with FileCache ([12782](https://github.com/opensearch-project/OpenSearch/pull/12782))
 
 ### Dependencies
 - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442))
@@ -35,6 +36,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix handling of Short and Byte data types in ScriptProcessor ingest pipeline ([#14379](https://github.com/opensearch-project/OpenSearch/issues/14379))
 - Switch to iterative version of WKT format parser ([#14086](https://github.com/opensearch-project/OpenSearch/pull/14086))
 - Fix the computed max shards of cluster to avoid int overflow ([#14155](https://github.com/opensearch-project/OpenSearch/pull/14155))
+- Fixed rest-high-level client searchTemplate & mtermVectors endpoints to have a leading slash ([#14465](https://github.com/opensearch-project/OpenSearch/pull/14465))
 - Fix fs info reporting negative available size ([#11573](https://github.com/opensearch-project/OpenSearch/pull/11573))
 
 ### Security

client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java

@@ -540,7 +540,7 @@ static Request searchTemplate(SearchTemplateRequest searchTemplateRequest) throw
         Request request;
 
         if (searchTemplateRequest.isSimulate()) {
-            request = new Request(HttpGet.METHOD_NAME, "_render/template");
+            request = new Request(HttpGet.METHOD_NAME, "/_render/template");
         } else {
             SearchRequest searchRequest = searchTemplateRequest.getRequest();
             String endpoint = endpoint(searchRequest.indices(), "_search/template");
@@ -803,8 +803,7 @@ static Request termVectors(TermVectorsRequest tvrequest) throws IOException {
     }
 
     static Request mtermVectors(MultiTermVectorsRequest mtvrequest) throws IOException {
-        String endpoint = "_mtermvectors";
-        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+        Request request = new Request(HttpGet.METHOD_NAME, "/_mtermvectors");
         request.setEntity(createEntity(mtvrequest, REQUEST_BODY_CONTENT_TYPE));
         return request;
     }

client/rest-high-level/src/test/java/org/opensearch/client/IndicesClientIT.java

@@ -701,7 +701,7 @@ public void testOpenExistingIndex() throws IOException {
         closeIndex(index);
         ResponseException exception = expectThrows(
             ResponseException.class,
-            () -> client().performRequest(new Request(HttpGet.METHOD_NAME, index + "/_search"))
+            () -> client().performRequest(new Request(HttpGet.METHOD_NAME, "/" + index + "/_search"))
         );
         assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.BAD_REQUEST.getStatus()));
         assertThat(exception.getMessage().contains(index), equalTo(true));
@@ -714,7 +714,7 @@ public void testOpenExistingIndex() throws IOException {
         );
         assertTrue(openIndexResponse.isAcknowledged());
 
-        Response response = client().performRequest(new Request(HttpGet.METHOD_NAME, index + "/_search"));
+        Response response = client().performRequest(new Request(HttpGet.METHOD_NAME, "/" + index + "/_search"));
         assertThat(response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
     }
 
@@ -771,7 +771,7 @@ public void testCloseExistingIndex() throws IOException {
 
             ResponseException exception = expectThrows(
                 ResponseException.class,
-                () -> client().performRequest(new Request(HttpGet.METHOD_NAME, indexResult.getIndex() + "/_search"))
+                () -> client().performRequest(new Request(HttpGet.METHOD_NAME, "/" + indexResult.getIndex() + "/_search"))
             );
             assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.BAD_REQUEST.getStatus()));
             assertThat(exception.getMessage().contains(indexResult.getIndex()), equalTo(true));
@@ -1270,7 +1270,7 @@ public void testGetAliasesNonExistentIndexOrAlias() throws IOException {
             assertThat(getAliasesResponse.getException(), nullValue());
         }
         createIndex(index, Settings.EMPTY);
-        client().performRequest(new Request(HttpPut.METHOD_NAME, index + "/_alias/" + alias));
+        client().performRequest(new Request(HttpPut.METHOD_NAME, "/" + index + "/_alias/" + alias));
         {
             GetAliasesRequest getAliasesRequest = new GetAliasesRequest().indices(index, "non_existent_index");
             GetAliasesResponse getAliasesResponse = execute(

client/rest-high-level/src/test/java/org/opensearch/client/RankEvalIT.java

@@ -121,7 +121,7 @@ public void testRankEvalRequest() throws IOException {
         }
 
         // now try this when test2 is closed
-        client().performRequest(new Request("POST", "index2/_close"));
+        client().performRequest(new Request("POST", "/index2/_close"));
         rankEvalRequest.indicesOptions(IndicesOptions.fromParameters(null, "true", null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
         response = execute(rankEvalRequest, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync);
     }

client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java

@@ -1399,7 +1399,7 @@ public void testRenderSearchTemplate() throws Exception {
 
         // Verify that the resulting REST request looks as expected.
         Request request = RequestConverters.searchTemplate(searchTemplateRequest);
-        String endpoint = "_render/template";
+        String endpoint = "/_render/template";
 
         assertEquals(HttpGet.METHOD_NAME, request.getMethod());
         assertEquals(endpoint, request.getEndpoint());
@@ -1565,7 +1565,7 @@ public void testMultiTermVectors() throws IOException {
 
         Request request = RequestConverters.mtermVectors(mtvRequest);
         assertEquals(HttpGet.METHOD_NAME, request.getMethod());
-        assertEquals("_mtermvectors", request.getEndpoint());
+        assertEquals("/_mtermvectors", request.getEndpoint());
         assertToXContentBody(mtvRequest, request.getEntity());
     }
 
@@ -1585,7 +1585,7 @@ public void testMultiTermVectorsWithType() throws IOException {
 
         Request request = RequestConverters.mtermVectors(mtvRequest);
         assertEquals(HttpGet.METHOD_NAME, request.getMethod());
-        assertEquals("_mtermvectors", request.getEndpoint());
+        assertEquals("/_mtermvectors", request.getEndpoint());
         assertToXContentBody(mtvRequest, request.getEntity());
     }
 

client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java

@@ -727,7 +727,7 @@ public void testSearchWithSuggest() throws IOException {
     }
 
     public void testSearchWithWeirdScriptFields() throws Exception {
-        Request doc = new Request("PUT", "test/_doc/1");
+        Request doc = new Request("PUT", "/test/_doc/1");
         doc.setJsonEntity("{\"field\":\"value\"}");
         client().performRequest(doc);
         client().performRequest(new Request("POST", "/test/_refresh"));
@@ -774,7 +774,7 @@ public void testSearchWithWeirdScriptFields() throws Exception {
     public void testSearchWithDerivedFields() throws Exception {
         // Just testing DerivedField definition from SearchSourceBuilder derivedField()
         // We are not testing the full functionality here
-        Request doc = new Request("PUT", "test/_doc/1");
+        Request doc = new Request("PUT", "/test/_doc/1");
         doc.setJsonEntity("{\"field\":\"value\"}");
         client().performRequest(doc);
         client().performRequest(new Request("POST", "/test/_refresh"));

client/rest-high-level/src/test/java/org/opensearch/client/documentation/SearchDocumentationIT.java

@@ -998,7 +998,7 @@ public void onFailure(Exception e) {
 
     protected void registerQueryScript(RestClient restClient) throws IOException {
         // tag::register-script
-        Request scriptRequest = new Request("POST", "_scripts/title_search");
+        Request scriptRequest = new Request("POST", "/_scripts/title_search");
         scriptRequest.setJsonEntity(
             "{" +
             "  \"script\": {" +

client/rest-high-level/src/test/java/org/opensearch/client/documentation/SnapshotClientDocumentationIT.java

@@ -827,7 +827,7 @@ private void createTestIndex() throws IOException {
     }
 
     private void createTestSnapshots() throws IOException {
-        Request createSnapshot = new Request("put", String.format(Locale.ROOT, "_snapshot/%s/%s", repositoryName, snapshotName));
+        Request createSnapshot = new Request("put", String.format(Locale.ROOT, "/_snapshot/%s/%s", repositoryName, snapshotName));
         createSnapshot.addParameter("wait_for_completion", "true");
         createSnapshot.setJsonEntity("{\"indices\":\"" + indexName + "\"}");
         Response response = highLevelClient().getLowLevelClient().performRequest(createSnapshot);

server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheCleanupIT.java

@@ -66,6 +66,8 @@
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0, supportsDedicatedMasters = false)
 public class IndicesRequestCacheCleanupIT extends OpenSearchIntegTestCase {
 
+    private static final long MAX_ITERATIONS = 5;
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Arrays.asList(InternalSettingsPlugin.class);
@@ -74,23 +76,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     public void testCacheWithInvalidation() throws Exception {
         Client client = client();
         String index = "index";
-        assertAcked(
-            client.admin()
-                .indices()
-                .prepareCreate(index)
-                .setMapping("k", "type=keyword")
-                .setSettings(
-                    Settings.builder()
-                        .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
-                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-                        .put("index.refresh_interval", -1)
-                        // Disable index refreshing to avoid cache being invalidated mid-test
-                        .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(-1))
-                )
-                .get()
-        );
-        indexRandom(false, client.prepareIndex(index).setSource("k", "hello"));
+        setupIndex(client, index);
         ensureSearchable(index);
         // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
         forceMerge(client, index);
@@ -125,8 +111,8 @@ public void testCacheClearAPIRemovesStaleKeysWhenStalenessThresholdIsLow() throw
                 .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.10)
                 .put(
                     IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
-                    // setting intentionally high to avoid cache cleaner interfering
-                    TimeValue.timeValueMillis(300)
+                    // Set interval much larger than test timeout to effectively disable it
+                    TimeValue.timeValueDays(1)
                 )
         );
         Client client = client(node);
@@ -210,7 +196,7 @@ public void testStaleKeysCleanupWithLowThreshold() throws Exception {
             assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
             // cache cleaner should NOT have cleaned from index 1
             assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+        }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
         // sleep until cache cleaner would have cleaned up the stale key from index 2
     }
 
@@ -260,7 +246,7 @@ public void testCacheCleanupOnEqualStalenessAndThreshold() throws Exception {
             assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
             // cache cleaner should NOT have cleaned from index 1
             assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+        }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
     }
 
     // when staleness threshold is higher than staleness, it should NOT clean the cache
@@ -308,7 +294,7 @@ public void testCacheCleanupSkipsWithHighStalenessThreshold() throws Exception {
             assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
             // cache cleaner should NOT have cleaned from index 1
             assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+        }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
     }
 
     // when staleness threshold is explicitly set to 0, cache cleaner regularly cleans up stale keys.
@@ -356,7 +342,7 @@ public void testCacheCleanupOnZeroStalenessThreshold() throws Exception {
             assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
             // cache cleaner should NOT have cleaned from index 1
             assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+        }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
     }
 
     // when staleness threshold is not explicitly set, cache cleaner regularly cleans up stale keys
@@ -403,7 +389,7 @@ public void testStaleKeysRemovalWithoutExplicitThreshold() throws Exception {
             assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
             // cache cleaner should NOT have cleaned from index 1
             assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+        }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
     }
 
     // when cache cleaner interval setting is not set, cache cleaner is configured appropriately with the fall-back setting
@@ -447,7 +433,7 @@ public void testCacheCleanupWithDefaultSettings() throws Exception {
             assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
             // cache cleaner should NOT have cleaned from index 1
             assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+        }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
     }
 
     // staleness threshold updates flows through to the cache cleaner
@@ -490,7 +476,7 @@ public void testDynamicStalenessThresholdUpdate() throws Exception {
         assertBusy(() -> {
             // cache cleaner should NOT have cleaned up the stale key from index 2
             assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+        }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
 
         // Update indices.requests.cache.cleanup.staleness_threshold to "10%"
         ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
@@ -505,7 +491,7 @@ public void testDynamicStalenessThresholdUpdate() throws Exception {
             assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
             // cache cleaner should NOT have cleaned from index 1
             assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+        }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
     }
 
     // staleness threshold dynamic updates should throw exceptions on invalid input
@@ -557,7 +543,7 @@ public void testCacheClearanceAfterIndexClosure() throws Exception {
         assertBusy(() -> {
             // cache cleaner should have cleaned up the stale keys from index
             assertEquals(0, getNodeCacheStats(client).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+        }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
     }
 
     // deleting the Index after caching will clean up from Indices Request Cache
@@ -598,7 +584,7 @@ public void testCacheCleanupAfterIndexDeletion() throws Exception {
         assertBusy(() -> {
             // cache cleaner should have cleaned up the stale keys from index
             assertEquals(0, getNodeCacheStats(client).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+        }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
     }
 
     // when staleness threshold is lower than staleness, it should clean the cache from all indices having stale keys
@@ -645,7 +631,7 @@ public void testStaleKeysCleanupWithMultipleIndices() throws Exception {
         // Assert cache is cleared up
         assertBusy(
             () -> { assertEquals(0, getRequestCacheStats(client, index1).getMemorySizeInBytes()); },
-            cacheCleanIntervalInMillis * 2,
+            cacheCleanIntervalInMillis * MAX_ITERATIONS,
             TimeUnit.MILLISECONDS
         );
 
@@ -667,7 +653,7 @@ public void testStaleKeysCleanupWithMultipleIndices() throws Exception {
             long currentMemorySizeInBytesForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
             // assert the memory size of index1 to only contain 1 entry added after flushAndRefresh
             assertEquals(memorySizeForIndex1With1Entries, currentMemorySizeInBytesForIndex1);
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+        }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
     }
 
     private void setupIndex(Client client, String index) throws Exception {