From 7aeec2106b724205367d6bee8bac403542d4531d Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 16 Jan 2019 11:35:14 -0800 Subject: [PATCH 1/4] add test for running certain ILM actions during snapshotting --- x-pack/plugin/ilm/qa/multi-node/build.gradle | 6 +- .../TimeSeriesLifecycleActionsIT.java | 133 ++++++++++++++++++ 2 files changed, 138 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/build.gradle b/x-pack/plugin/ilm/qa/multi-node/build.gradle index edd7f3aad472e..5f033626932df 100644 --- a/x-pack/plugin/ilm/qa/multi-node/build.gradle +++ b/x-pack/plugin/ilm/qa/multi-node/build.gradle @@ -5,6 +5,11 @@ dependencies { testCompile project(path: xpackProject('plugin').path, configuration: 'testArtifacts') } +integTestRunner { + /* To support taking index snapshots, we have to set path.repo setting */ + systemProperty 'tests.path.repo', new File(buildDir, "cluster/shared/repo") +} + integTestCluster { numNodes = 4 clusterName = 'ilm' @@ -16,5 +21,4 @@ integTestCluster { setting 'xpack.ml.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'indices.lifecycle.poll_interval', '1000ms' - } diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java index 779a737c88279..954debcb6a914 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.FrozenEngine; import org.elasticsearch.test.rest.ESRestTestCase; @@ -58,6 +59,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.startsWith; public class TimeSeriesLifecycleActionsIT extends ESRestTestCase { private String index; @@ -357,6 +359,46 @@ public void testDeleteOnlyShouldNotMakeIndexReadonly() throws Exception { indexDocument(); } + public void testDeleteDuringSnapshot() throws Exception { + // Create the repository before taking the snapshot. + Request request = new Request("PUT", "/_snapshot/repo"); + request.setJsonEntity(Strings + .toString(JsonXContent.contentBuilder() + .startObject() + .field("type", "fs") + .startObject("settings") + .field("compress", randomBoolean()) + .field("location", System.getProperty("tests.path.repo")) + .field("max_snapshot_bytes_per_sec", "256b") + .endObject() + .endObject())); + assertOK(client().performRequest(request)); + // create delete policy + createNewSingletonPolicy("delete", new DeleteAction(), TimeValue.timeValueMillis(0)); + // create index without policy + createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)); + // index document so snapshot actually does something + indexDocument(); + // start snapshot + request = new Request("PUT", "/_snapshot/repo/snapshot"); + request.addParameter("wait_for_completion", "false"); + request.setJsonEntity("{\"indices\": \"" + index + "\"}"); + assertOK(client().performRequest(request)); + // add policy and expect it to trigger delete immediately (while snapshot in progress) + updatePolicy(index, policy); + // assert that exception was thrown + assertBusy(() -> { + assertThat(getStepKeyForIndex(index).getName(), equalTo("ERROR")); + assertThat(getReasonForIndex(index), startsWith("Cannot delete indices that are being snapshotted")); + }); + assertThat(getSnapshotState("snapshot"), equalTo("IN_PROGRESS")); + assertOK(client().performRequest(new Request("DELETE", "/_snapshot/repo/snapshot"))); + ResponseException e = expectThrows(ResponseException.class, + () -> client().performRequest(new Request("GET", "/_snapshot/repo/snapshot"))); + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); + } + public void testReadOnly() throws Exception { createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)); @@ -426,6 +468,46 @@ public void testShrinkAction() throws Exception { expectThrows(ResponseException.class, this::indexDocument); } + public void testShrinkDuringSnapshot() throws Exception { + // Create the repository before taking the snapshot. + Request request = new Request("PUT", "/_snapshot/repo"); + request.setJsonEntity(Strings + .toString(JsonXContent.contentBuilder() + .startObject() + .field("type", "fs") + .startObject("settings") + .field("compress", randomBoolean()) + .field("location", System.getProperty("tests.path.repo")) + .field("max_snapshot_bytes_per_sec", "256b") + .endObject() + .endObject())); + assertOK(client().performRequest(request)); + // create delete policy + createNewSingletonPolicy("warm", new ShrinkAction(1), TimeValue.timeValueMillis(0)); + // create index without policy + createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)); + // index document so snapshot actually does something + indexDocument(); + // start snapshot + request = new Request("PUT", "/_snapshot/repo/snapshot"); + request.addParameter("wait_for_completion", "false"); + request.setJsonEntity("{\"indices\": \"" + index + "\"}"); + assertOK(client().performRequest(request)); + // add policy and expect it to trigger delete immediately (while snapshot in progress) + updatePolicy(index, policy); + // assert that exception was thrown + assertBusy(() -> { + assertThat(getStepKeyForIndex(index).getName(), equalTo("ERROR")); + assertThat(getReasonForIndex(index), startsWith("Cannot delete indices that are being snapshotted")); + }); + assertThat(getSnapshotState("snapshot"), equalTo("IN_PROGRESS")); + assertOK(client().performRequest(new Request("DELETE", "/_snapshot/repo/snapshot"))); + ResponseException e = expectThrows(ResponseException.class, + () -> client().performRequest(new Request("GET", "/_snapshot/repo/snapshot"))); + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); + } + public void testFreezeAction() throws Exception { createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)); @@ -440,6 +522,46 @@ public void testFreezeAction() throws Exception { }); } + public void testFreezeDuringSnapshot() throws Exception { + // Create the repository before taking the snapshot. + Request request = new Request("PUT", "/_snapshot/repo"); + request.setJsonEntity(Strings + .toString(JsonXContent.contentBuilder() + .startObject() + .field("type", "fs") + .startObject("settings") + .field("compress", randomBoolean()) + .field("location", System.getProperty("tests.path.repo")) + .field("max_snapshot_bytes_per_sec", "256b") + .endObject() + .endObject())); + assertOK(client().performRequest(request)); + // create delete policy + createNewSingletonPolicy("cold", new FreezeAction(), TimeValue.timeValueMillis(0)); + // create index without policy + createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)); + // index document so snapshot actually does something + indexDocument(); + // start snapshot + request = new Request("PUT", "/_snapshot/repo/snapshot"); + request.addParameter("wait_for_completion", "false"); + request.setJsonEntity("{\"indices\": \"" + index + "\"}"); + assertOK(client().performRequest(request)); + // add policy and expect it to trigger delete immediately (while snapshot in progress) + updatePolicy(index, policy); + // assert that exception was thrown + assertBusy(() -> { + assertThat(getStepKeyForIndex(index).getName(), equalTo("ERROR")); + assertThat(getReasonForIndex(index), startsWith("Cannot close indices that are being snapshotted")); + }); + assertThat(getSnapshotState("snapshot"), equalTo("IN_PROGRESS")); + assertOK(client().performRequest(new Request("DELETE", "/_snapshot/repo/snapshot"))); + ResponseException e = expectThrows(ResponseException.class, + () -> client().performRequest(new Request("GET", "/_snapshot/repo/snapshot"))); + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); + } + @SuppressWarnings("unchecked") public void testNonexistentPolicy() throws Exception { String indexPrefix = randomAlphaOfLengthBetween(5,15).toLowerCase(Locale.ROOT); @@ -732,4 +854,15 @@ private void indexDocument() throws IOException { Response response = client().performRequest(indexRequest); logger.info(response.getStatusLine()); } + + private String getSnapshotState(String snapshot) throws IOException { + Response response = client().performRequest(new Request("GET", "/_snapshot/repo/snapshot")); + Map responseMap; + try (InputStream is = response.getEntity().getContent()) { + responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true); + } + Map snapResponse = ((List>) responseMap.get("snapshots")).get(0); + assertThat(snapResponse.get("snapshot"), equalTo("snapshot")); + return (String) snapResponse.get("state"); + } } From a97d0eb7e110f60ca821016c9d2c2ebb544d03c8 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 16 Jan 2019 22:27:16 -0800 Subject: [PATCH 2/4] swap the tests to awaitsfix and test that things succeed --- .../TimeSeriesLifecycleActionsIT.java | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java index 954debcb6a914..cfedc7b8b2109 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java @@ -359,6 +359,7 @@ public void testDeleteOnlyShouldNotMakeIndexReadonly() throws Exception { indexDocument(); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37541") public void testDeleteDuringSnapshot() throws Exception { // Create the repository before taking the snapshot. Request request = new Request("PUT", "/_snapshot/repo"); @@ -387,11 +388,9 @@ public void testDeleteDuringSnapshot() throws Exception { assertOK(client().performRequest(request)); // add policy and expect it to trigger delete immediately (while snapshot in progress) updatePolicy(index, policy); - // assert that exception was thrown - assertBusy(() -> { - assertThat(getStepKeyForIndex(index).getName(), equalTo("ERROR")); - assertThat(getReasonForIndex(index), startsWith("Cannot delete indices that are being snapshotted")); - }); + // assert that index was deleted + assertBusy(() -> assertFalse(indexExists(index))); + // assert that snapshot is still in progress and clean up assertThat(getSnapshotState("snapshot"), equalTo("IN_PROGRESS")); assertOK(client().performRequest(new Request("DELETE", "/_snapshot/repo/snapshot"))); ResponseException e = expectThrows(ResponseException.class, @@ -468,7 +467,9 @@ public void testShrinkAction() throws Exception { expectThrows(ResponseException.class, this::indexDocument); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37541") public void testShrinkDuringSnapshot() throws Exception { + String shrunkenIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + index; // Create the repository before taking the snapshot. Request request = new Request("PUT", "/_snapshot/repo"); request.setJsonEntity(Strings @@ -496,11 +497,17 @@ public void testShrinkDuringSnapshot() throws Exception { assertOK(client().performRequest(request)); // add policy and expect it to trigger delete immediately (while snapshot in progress) updatePolicy(index, policy); - // assert that exception was thrown + // assert that index was shrunk and original index was deleted assertBusy(() -> { - assertThat(getStepKeyForIndex(index).getName(), equalTo("ERROR")); - assertThat(getReasonForIndex(index), startsWith("Cannot delete indices that are being snapshotted")); + assertTrue(indexExists(shrunkenIndex)); + assertTrue(aliasExists(shrunkenIndex, index)); + Map settings = getOnlyIndexSettings(shrunkenIndex); + assertThat(getStepKeyForIndex(shrunkenIndex), equalTo(TerminalPolicyStep.KEY)); + assertThat(settings.get(IndexMetaData.SETTING_NUMBER_OF_SHARDS), equalTo(String.valueOf(1))); + assertThat(settings.get(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.getKey()), equalTo("true")); }); + expectThrows(ResponseException.class, this::indexDocument); + // assert that snapshot is still in progress and clean up assertThat(getSnapshotState("snapshot"), equalTo("IN_PROGRESS")); assertOK(client().performRequest(new Request("DELETE", "/_snapshot/repo/snapshot"))); ResponseException e = expectThrows(ResponseException.class, @@ -522,6 +529,7 @@ public void testFreezeAction() throws Exception { }); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37541") public void testFreezeDuringSnapshot() throws Exception { // Create the repository before taking the snapshot. Request request = new Request("PUT", "/_snapshot/repo"); @@ -550,11 +558,15 @@ public void testFreezeDuringSnapshot() throws Exception { assertOK(client().performRequest(request)); // add policy and expect it to trigger delete immediately (while snapshot in progress) updatePolicy(index, policy); - // assert that exception was thrown + // assert that the index froze assertBusy(() -> { - assertThat(getStepKeyForIndex(index).getName(), equalTo("ERROR")); - assertThat(getReasonForIndex(index), startsWith("Cannot close indices that are being snapshotted")); + Map settings = getOnlyIndexSettings(index); + assertThat(getStepKeyForIndex(index), equalTo(TerminalPolicyStep.KEY)); + assertThat(settings.get(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.getKey()), equalTo("true")); + assertThat(settings.get(IndexSettings.INDEX_SEARCH_THROTTLED.getKey()), equalTo("true")); + assertThat(settings.get(FrozenEngine.INDEX_FROZEN.getKey()), equalTo("true")); }); + // assert that snapshot is still in progress and clean up assertThat(getSnapshotState("snapshot"), equalTo("IN_PROGRESS")); assertOK(client().performRequest(new Request("DELETE", "/_snapshot/repo/snapshot"))); ResponseException e = expectThrows(ResponseException.class, From e8c43dee5b7800495d268905241bdbca59dd600d Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 16 Jan 2019 22:35:21 -0800 Subject: [PATCH 3/4] fix getSnapshotState --- .../xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java index cfedc7b8b2109..a525008aedda1 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java @@ -868,13 +868,13 @@ private void indexDocument() throws IOException { } private String getSnapshotState(String snapshot) throws IOException { - Response response = client().performRequest(new Request("GET", "/_snapshot/repo/snapshot")); + Response response = client().performRequest(new Request("GET", "/_snapshot/repo/" + snapshot)); Map responseMap; try (InputStream is = response.getEntity().getContent()) { responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true); } Map snapResponse = ((List>) responseMap.get("snapshots")).get(0); - assertThat(snapResponse.get("snapshot"), equalTo("snapshot")); + assertThat(snapResponse.get("snapshot"), equalTo(snapshot)); return (String) snapResponse.get("state"); } } From 273b932c2aee4e528225ef81cb0145cc92cc407d Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 16 Jan 2019 22:55:06 -0800 Subject: [PATCH 4/4] fix checkstyle --- .../xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java index a525008aedda1..43eb7cd0b2ab6 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java @@ -59,7 +59,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.startsWith; public class TimeSeriesLifecycleActionsIT extends ESRestTestCase { private String index;