From f1ec1fc00bdb233523b5253db89bf3c9f09fecd7 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Tue, 13 Jul 2021 17:10:27 -0400 Subject: [PATCH 1/2] Stabilize IndicesRequestCacheIT test This test fails from time to time on 6.8 This patch tries to address failures in 2 ways: 1. Disables automatic refreshes. Request cache gets invalidated when refresh happens (as IndexReader's version may change), which may lead to test failurs. 2. Makes predictable indexing. Before we were using random indexing requests that could lead to multiple segments with deleted documents. This patch ensures that we are indexing only necessary docs in a single bulk request which should create only a single segment for each index. Relates to #61565 --- .../indices/IndicesRequestCacheIT.java | 71 +++++++++---------- 1 file changed, 34 insertions(+), 37 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java b/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java index bef0768185d63..cbb74f88a904f 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java @@ -21,6 +21,8 @@ import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.client.Client; @@ -41,6 +43,7 @@ import java.util.Arrays; import java.util.List; +import static org.elasticsearch.index.IndexSettings.INDEX_REFRESH_INTERVAL_SETTING; import static org.elasticsearch.search.aggregations.AggregationBuilders.dateHistogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.dateRange; import static org.elasticsearch.search.aggregations.AggregationBuilders.filter; @@ -185,29 +188,27 @@ public void testQueryRewriteMissingValues() throws Exception { assertCacheState(client, "index", 2, 1); } - public void testQueryRewriteDates() throws Exception { + public void testQueryRewriteDates() { Client client = client(); assertAcked(client.admin().indices().prepareCreate("index").addMapping("type", "d", "type=date") .setSettings(Settings.builder().put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true) - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)).get()); - indexRandom(true, client.prepareIndex("index", "type", "1").setSource("d", "2014-01-01T00:00:00"), - client.prepareIndex("index", "type", "2").setSource("d", "2014-02-01T00:00:00"), - client.prepareIndex("index", "type", "3").setSource("d", "2014-03-01T00:00:00"), - client.prepareIndex("index", "type", "4").setSource("d", "2014-04-01T00:00:00"), - client.prepareIndex("index", "type", "5").setSource("d", "2014-05-01T00:00:00"), - client.prepareIndex("index", "type", "6").setSource("d", "2014-06-01T00:00:00"), - client.prepareIndex("index", "type", "7").setSource("d", "2014-07-01T00:00:00"), - client.prepareIndex("index", "type", "8").setSource("d", "2014-08-01T00:00:00"), - client.prepareIndex("index", "type", "9").setSource("d", "2014-09-01T00:00:00")); - ensureSearchable("index"); - assertCacheState(client, "index", 0, 0); + .put(INDEX_REFRESH_INTERVAL_SETTING.getKey(), "-1") // disable automatic refreshes + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)).get()); - // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache - ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index").setFlush(true).get(); - ElasticsearchAssertions.assertAllSuccessful(forceMergeResponse); - refresh(); + BulkRequestBuilder bulkBuilder = client().prepareBulk(); + bulkBuilder.add(client.prepareIndex("index", "type", "1").setSource("d", "2014-01-01T00:00:00")); + bulkBuilder.add(client.prepareIndex("index", "type", "2").setSource("d", "2014-02-01T00:00:00")); + bulkBuilder.add(client.prepareIndex("index", "type", "3").setSource("d", "2014-03-01T00:00:00")); + bulkBuilder.add(client.prepareIndex("index", "type", "4").setSource("d", "2014-04-01T00:00:00")); + bulkBuilder.add(client.prepareIndex("index", "type", "5").setSource("d", "2014-05-01T00:00:00")); + bulkBuilder.add(client.prepareIndex("index", "type", "6").setSource("d", "2014-06-01T00:00:00")); + bulkBuilder.add(client.prepareIndex("index", "type", "7").setSource("d", "2014-07-01T00:00:00")); + bulkBuilder.add(client.prepareIndex("index", "type", "8").setSource("d", "2014-08-01T00:00:00")); + bulkBuilder.add(client.prepareIndex("index", "type", "9").setSource("d", "2014-09-01T00:00:00")); + BulkResponse actionGet = bulkBuilder.execute().actionGet(); + assertThat(actionGet.hasFailures() ? actionGet.buildFailureMessage() : "", actionGet.hasFailures(), equalTo(false)); ensureSearchable("index"); - + refresh("index"); assertCacheState(client, "index", 0, 0); final SearchResponse r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) @@ -232,9 +233,10 @@ public void testQueryRewriteDates() throws Exception { assertCacheState(client, "index", 2, 1); } - public void testQueryRewriteDatesWithNow() throws Exception { + public void testQueryRewriteDatesWithNow() { Client client = client(); Settings settings = Settings.builder().put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true) + .put(INDEX_REFRESH_INTERVAL_SETTING.getKey(), "-1") // disable automatic refreshes .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).build(); assertAcked(client.admin().indices().prepareCreate("index-1").addMapping("type", "d", "type=date") .setSettings(settings).get()); @@ -243,26 +245,21 @@ public void testQueryRewriteDatesWithNow() throws Exception { assertAcked(client.admin().indices().prepareCreate("index-3").addMapping("type", "d", "type=date") .setSettings(settings).get()); ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); - indexRandom(true, client.prepareIndex("index-1", "type", "1").setSource("d", now), - client.prepareIndex("index-1", "type", "2").setSource("d", now.minusDays(1)), - client.prepareIndex("index-1", "type", "3").setSource("d", now.minusDays(2)), - client.prepareIndex("index-2", "type", "4").setSource("d", now.minusDays(3)), - client.prepareIndex("index-2", "type", "5").setSource("d", now.minusDays(4)), - client.prepareIndex("index-2", "type", "6").setSource("d", now.minusDays(5)), - client.prepareIndex("index-3", "type", "7").setSource("d", now.minusDays(6)), - client.prepareIndex("index-3", "type", "8").setSource("d", now.minusDays(7)), - client.prepareIndex("index-3", "type", "9").setSource("d", now.minusDays(8))); - ensureSearchable("index-1", "index-2", "index-3"); - assertCacheState(client, "index-1", 0, 0); - assertCacheState(client, "index-2", 0, 0); - assertCacheState(client, "index-3", 0, 0); - // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache - ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index-1", "index-2", "index-3").setFlush(true) - .get(); - ElasticsearchAssertions.assertAllSuccessful(forceMergeResponse); - refresh(); + BulkRequestBuilder bulkBuilder = client().prepareBulk(); + bulkBuilder.add(client.prepareIndex("index-1", "type", "1").setSource("d", now)); + bulkBuilder.add(client.prepareIndex("index-1", "type", "2").setSource("d", now.minusDays(1))); + bulkBuilder.add(client.prepareIndex("index-1", "type", "3").setSource("d", now.minusDays(2))); + bulkBuilder.add(client.prepareIndex("index-2", "type", "4").setSource("d", now.minusDays(3))); + bulkBuilder.add(client.prepareIndex("index-2", "type", "5").setSource("d", now.minusDays(4))); + bulkBuilder.add(client.prepareIndex("index-2", "type", "6").setSource("d", now.minusDays(5))); + bulkBuilder.add(client.prepareIndex("index-3", "type", "7").setSource("d", now.minusDays(6))); + bulkBuilder.add(client.prepareIndex("index-3", "type", "8").setSource("d", now.minusDays(7))); + bulkBuilder.add(client.prepareIndex("index-3", "type", "9").setSource("d", now.minusDays(8))); + BulkResponse actionGet = bulkBuilder.execute().actionGet(); + assertThat(actionGet.hasFailures() ? actionGet.buildFailureMessage() : "", actionGet.hasFailures(), equalTo(false)); ensureSearchable("index-1", "index-2", "index-3"); + refresh("index-1", "index-2", "index-3"); assertCacheState(client, "index-1", 0, 0); assertCacheState(client, "index-2", 0, 0); From 7f125e75efd8f697a6fae8820e9712ed8b36a25d Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Thu, 15 Jul 2021 09:50:16 -0400 Subject: [PATCH 2/2] Rename variable --- .../org/elasticsearch/indices/IndicesRequestCacheIT.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java b/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java index cbb74f88a904f..8301ae9ca665a 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java @@ -205,8 +205,8 @@ public void testQueryRewriteDates() { bulkBuilder.add(client.prepareIndex("index", "type", "7").setSource("d", "2014-07-01T00:00:00")); bulkBuilder.add(client.prepareIndex("index", "type", "8").setSource("d", "2014-08-01T00:00:00")); bulkBuilder.add(client.prepareIndex("index", "type", "9").setSource("d", "2014-09-01T00:00:00")); - BulkResponse actionGet = bulkBuilder.execute().actionGet(); - assertThat(actionGet.hasFailures() ? actionGet.buildFailureMessage() : "", actionGet.hasFailures(), equalTo(false)); + BulkResponse bulkResponse = bulkBuilder.execute().actionGet(); + assertThat(bulkResponse.hasFailures() ? bulkResponse.buildFailureMessage() : "", bulkResponse.hasFailures(), equalTo(false)); ensureSearchable("index"); refresh("index"); assertCacheState(client, "index", 0, 0); @@ -256,8 +256,8 @@ public void testQueryRewriteDatesWithNow() { bulkBuilder.add(client.prepareIndex("index-3", "type", "7").setSource("d", now.minusDays(6))); bulkBuilder.add(client.prepareIndex("index-3", "type", "8").setSource("d", now.minusDays(7))); bulkBuilder.add(client.prepareIndex("index-3", "type", "9").setSource("d", now.minusDays(8))); - BulkResponse actionGet = bulkBuilder.execute().actionGet(); - assertThat(actionGet.hasFailures() ? actionGet.buildFailureMessage() : "", actionGet.hasFailures(), equalTo(false)); + BulkResponse bulkResponse = bulkBuilder.execute().actionGet(); + assertThat(bulkResponse.hasFailures() ? bulkResponse.buildFailureMessage() : "", bulkResponse.hasFailures(), equalTo(false)); ensureSearchable("index-1", "index-2", "index-3"); refresh("index-1", "index-2", "index-3");