Dry up Searchable Snapshots ITs (#63190) (#63321)

Just a few spots where we can dry up these tests using the snapshot test infrastructure in core that I found while studying the existing searchable snapshot tests.
elastic · Oct 6, 2020 · 2aa80f9 · 2aa80f9
1 parent 82096d3
commit 2aa80f9
Show file tree

Hide file tree

Showing 13 changed files with 85 additions and 318 deletions.
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java
@@ -1196,8 +1196,7 @@ public void testConcurrentSnapshotWorksWithOldVersionRepo() throws Exception {
         final String indexFast = "index-fast";
         createIndexWithContent(indexFast, dataNode2, dataNode);
 
-        final ActionFuture<CreateSnapshotResponse> createFastSnapshot =
-                client().admin().cluster().prepareCreateSnapshot(repoName, "fast-snapshot").setWaitForCompletion(true).execute();
+        final ActionFuture<CreateSnapshotResponse> createFastSnapshot = startFullSnapshot(repoName, "fast-snapshot");
 
         assertThat(createSlowFuture.isDone(), is(false));
         unblockNode(repoName, dataNode);

diff --git a/...ternalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/...ternalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
@@ -580,10 +580,7 @@ public void testRestoreIndexWithShardsMissingInLocalGateway() throws Exception {
 
         indexRandomDocs("test-idx", 100);
 
-        logger.info("--> start snapshot");
-        assertThat(clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap-1").setIndices("test-idx")
-                .setWaitForCompletion(true).get().getSnapshotInfo().state(),
-            equalTo(SnapshotState.SUCCESS));
+        createSnapshot("test-repo", "test-snap-1", Collections.singletonList("test-idx"));
 
         logger.info("--> close the index");
         assertAcked(client().admin().indices().prepareClose("test-idx"));
@@ -796,9 +793,7 @@ public void testRestoreShrinkIndex() throws Exception {
         assertAcked(client().admin().indices().prepareResizeIndex(sourceIdx, shrunkIdx).get());
 
         logger.info("--> snapshot the shrunk index");
-        assertSuccessful(clusterAdmin()
-            .prepareCreateSnapshot(repo, snapshot)
-            .setWaitForCompletion(true).setIndices(shrunkIdx).execute());
+        createSnapshot(repo, snapshot, Collections.singletonList(shrunkIdx));
 
         logger.info("--> delete index and stop the data node");
         assertAcked(client().admin().indices().prepareDelete(sourceIdx).get());
@@ -1095,11 +1090,7 @@ public void testRetentionLeasesClearedOnRestore() throws Exception {
         assertTrue(shardStats + ": " + retentionLeases, retentionLeases.contains(leaseId));
 
         final String snapshotName = "snapshot-retention-leases";
-        logger.debug("-->  create snapshot {}:{}", repoName, snapshotName);
-        CreateSnapshotResponse createResponse = clusterAdmin().prepareCreateSnapshot(repoName, snapshotName)
-            .setWaitForCompletion(true).setIndices(indexName).get();
-        assertThat(createResponse.getSnapshotInfo().successfulShards(), equalTo(shardCount));
-        assertThat(createResponse.getSnapshotInfo().failedShards(), equalTo(0));
+        createSnapshot(repoName, snapshotName, Collections.singletonList(indexName));
 
         if (randomBoolean()) {
             final int extraDocCount = iterations(10, 1000);

diff --git a/.../internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/.../internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -76,6 +76,7 @@
 import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -259,16 +260,7 @@ public void testSingleGetAfterRestore() throws Exception {
         index(indexName, typeName, docId, "value", expectedValue);
 
         createRepository(repoName, "fs", absolutePath);
-
-        logger.info("--> snapshot");
-        CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(repoName, snapshotName)
-                .setWaitForCompletion(true)
-                .setIndices(indexName)
-                .get();
-        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
-        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(),
-            equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
-        assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS));
+        createSnapshot(repoName, snapshotName, Collections.singletonList(indexName));
 
         RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(repoName, snapshotName)
                 .setWaitForCompletion(true)
@@ -289,11 +281,7 @@ public void testFreshIndexUUID() {
         assertTrue(originalIndexUUID, originalIndexUUID != null);
         assertFalse(originalIndexUUID, originalIndexUUID.equals(IndexMetadata.INDEX_UUID_NA_VALUE));
         ensureGreen();
-        CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap")
-            .setWaitForCompletion(true).setIndices("test").get();
-        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
-        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(),
-            equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
+        createSnapshot("test-repo", "test-snap", Collections.singletonList("test"));
         NumShards numShards = getNumShards("test");
 
         cluster().wipeIndices("test");
@@ -456,12 +444,7 @@ public void testDataFileFailureDuringRestore() throws Exception {
 
         indexRandomDocs("test-idx", 100);
 
-        logger.info("--> snapshot");
-        CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap")
-            .setWaitForCompletion(true).setIndices("test-idx").get();
-        assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS));
-        assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(),
-            equalTo(createSnapshotResponse.getSnapshotInfo().successfulShards()));
+        createSnapshot("test-repo", "test-snap", Collections.singletonList("test-idx"));
 
         createRepository("test-repo", "mock",
             Settings.builder().put("location", repositoryLocation).put("random", randomAlphaOfLength(10))
@@ -505,12 +488,7 @@ public void testDataFileCorruptionDuringRestore() throws Exception {
         createRepository("test-repo", "fs", repositoryLocation);
         createIndexWithRandomDocs("test-idx", 100);
 
-        logger.info("--> snapshot");
-        CreateSnapshotResponse createSnapshotResponse = client.admin().cluster()
-            .prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get();
-        assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS));
-        assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(),
-            equalTo(createSnapshotResponse.getSnapshotInfo().successfulShards()));
+        createSnapshot("test-repo", "test-snap", Collections.singletonList("test-idx"));
 
         createRepository("test-repo", "mock",
             Settings.builder().put("location", repositoryLocation).put("random", randomAlphaOfLength(10))
@@ -602,14 +580,8 @@ private void unrestorableUseCase(final String indexName,
 
         // create a snapshot
         final NumShards numShards = getNumShards(indexName);
-        CreateSnapshotResponse snapshotResponse = clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap")
-                                                                            .setWaitForCompletion(true)
-                                                                            .setIndices(indexName)
-                                                                            .get();
-
-        assertThat(snapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS));
-        assertThat(snapshotResponse.getSnapshotInfo().successfulShards(), equalTo(numShards.numPrimaries));
-        assertThat(snapshotResponse.getSnapshotInfo().failedShards(), equalTo(0));
+        final SnapshotInfo snapshotInfo = createSnapshot("test-repo", "test-snap", Collections.singletonList(indexName));
+        assertThat(snapshotInfo.successfulShards(), equalTo(numShards.numPrimaries));
 
         // delete the test index
         assertAcked(client().admin().indices().prepareDelete(indexName));
@@ -681,12 +653,7 @@ public void testDeletionOfFailingToRecoverIndexShouldStopRestore() throws Except
 
         createIndexWithRandomDocs("test-idx", 100);
 
-        logger.info("--> snapshot");
-        CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap")
-            .setWaitForCompletion(true).setIndices("test-idx").get();
-        assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS));
-        assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(),
-            equalTo(createSnapshotResponse.getSnapshotInfo().successfulShards()));
+        createSnapshot("test-repo", "test-snap", Collections.singletonList("test-idx"));
 
         logger.info("-->  update repository with mock version");
         createRepository("test-repo", "mock", Settings.builder()
@@ -760,12 +727,7 @@ public void testDeleteSnapshot() throws Exception {
                 index("test-idx", "_doc", Integer.toString(i * 10 + j), "foo", "bar" + i * 10 + j);
             }
             refresh();
-            logger.info("--> snapshot {}", i);
-            CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-" + i)
-                .setWaitForCompletion(true).setIndices("test-idx").get();
-            assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
-            assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(),
-                equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
+            createSnapshot("test-repo", "test-snap-" + i, Collections.singletonList("test-idx"));
             // Store number of files after each snapshot
             numberOfFiles[i] = numberOfFiles(repo);
         }
@@ -948,12 +910,7 @@ public void testReadonlyRepository() throws Exception {
 
         createIndexWithRandomDocs("test-idx", 100);
 
-        logger.info("--> snapshot");
-        CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap")
-            .setWaitForCompletion(true).setIndices("test-idx").get();
-        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
-        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(),
-            equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
+        createSnapshot("test-repo", "test-snap", Collections.singletonList("test-idx"));
 
         assertThat(getSnapshot("test-repo", "test-snap").state(), equalTo(SnapshotState.SUCCESS));
 
@@ -1002,13 +959,7 @@ public void testThrottling() throws Exception {
             .put("max_snapshot_bytes_per_sec", throttleSnapshot ? "10k" : "0"));
 
         createIndexWithRandomDocs("test-idx", 100);
-
-        logger.info("--> snapshot");
-        CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap")
-            .setWaitForCompletion(true).setIndices("test-idx").get();
-        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
-        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(),
-            equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
+        createSnapshot("test-repo", "test-snap", Collections.singletonList("test-idx"));
 
         logger.info("--> delete index");
         cluster().wipeIndices("test-idx");
@@ -1195,11 +1146,7 @@ public void testSnapshotMoreThanOnce() throws InterruptedException {
         indexRandomDocs("test", randomIntBetween(10, 100));
         assertNoFailures(client().admin().indices().prepareForceMerge("test").setFlush(true).setMaxNumSegments(1).get());
 
-        CreateSnapshotResponse createSnapshotResponseFirst = client.admin().cluster().prepareCreateSnapshot("test-repo", "test")
-            .setWaitForCompletion(true).setIndices("test").get();
-        assertThat(createSnapshotResponseFirst.getSnapshotInfo().successfulShards(), greaterThan(0));
-        assertThat(createSnapshotResponseFirst.getSnapshotInfo().successfulShards(),
-            equalTo(createSnapshotResponseFirst.getSnapshotInfo().totalShards()));
+        createSnapshot("test-repo", "test", Collections.singletonList("test"));
         assertThat(getSnapshot("test-repo", "test").state(), equalTo(SnapshotState.SUCCESS));
         {
             SnapshotStatus snapshotStatus = client.admin().cluster().prepareSnapshotStatus("test-repo")
@@ -1210,11 +1157,7 @@ public void testSnapshotMoreThanOnce() throws InterruptedException {
             }
         }
 
-        CreateSnapshotResponse createSnapshotResponseSecond = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-1")
-            .setWaitForCompletion(true).setIndices("test").get();
-        assertThat(createSnapshotResponseSecond.getSnapshotInfo().successfulShards(), greaterThan(0));
-        assertThat(createSnapshotResponseSecond.getSnapshotInfo().successfulShards(),
-            equalTo(createSnapshotResponseSecond.getSnapshotInfo().totalShards()));
+        createSnapshot("test-repo", "test-1", Collections.singletonList("test"));
         assertThat(getSnapshot("test-repo", "test-1").state(), equalTo(SnapshotState.SUCCESS));
         {
             SnapshotStatus snapshotStatus = client.admin().cluster().prepareSnapshotStatus("test-repo")
@@ -1226,11 +1169,7 @@ public void testSnapshotMoreThanOnce() throws InterruptedException {
         }
 
         client().prepareDelete("test", "_doc", "1").get();
-        CreateSnapshotResponse createSnapshotResponseThird = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-2")
-            .setWaitForCompletion(true).setIndices("test").get();
-        assertThat(createSnapshotResponseThird.getSnapshotInfo().successfulShards(), greaterThan(0));
-        assertThat(createSnapshotResponseThird.getSnapshotInfo().successfulShards(),
-            equalTo(createSnapshotResponseThird.getSnapshotInfo().totalShards()));
+        createSnapshot("test-repo", "test-2", Collections.singletonList("test"));
         assertThat(getSnapshot("test-repo", "test-2").state(), equalTo(SnapshotState.SUCCESS));
         {
             SnapshotStatus snapshotStatus = client.admin().cluster().prepareSnapshotStatus("test-repo")
@@ -1791,11 +1730,7 @@ public void testGetSnapshotsFromIndexBlobOnly() throws Exception {
             }
             final String snapshotName = "test-snap-" + i;
             indicesPerSnapshot.put(snapshotName, indices);
-            client.admin().cluster()
-                .prepareCreateSnapshot("test-repo", snapshotName)
-                .setWaitForCompletion(true)
-                .setIndices(indices.toArray(new String[indices.size()]))
-                .get();
+            createSnapshot("test-repo", snapshotName, indices);
         }
 
         logger.info("--> verify _all returns snapshot info");
@@ -1855,12 +1790,7 @@ public void testSnapshottingWithMissingSequenceNumbers() {
 
         refresh();
 
-        logger.info("--> snapshot");
-        CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshotName)
-            .setWaitForCompletion(true).setIndices(indexName).get();
-        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
-        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(),
-            equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
+        createSnapshot(repositoryName, snapshotName, Collections.singletonList(indexName));
 
         logger.info("--> delete indices");
         assertAcked(client.admin().indices().prepareDelete(indexName));

diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
@@ -83,6 +83,8 @@
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
 
@@ -386,6 +388,22 @@ protected SnapshotInfo createFullSnapshot(String repoName, String snapshotName)
         return snapshotInfo;
     }
 
+    protected SnapshotInfo createSnapshot(String repositoryName, String snapshot, List<String> indices) {
+        logger.info("--> creating snapshot [{}] of {} in [{}]", snapshot, indices, repositoryName);
+        final CreateSnapshotResponse response = client().admin()
+                .cluster()
+                .prepareCreateSnapshot(repositoryName, snapshot)
+                .setIndices(indices.toArray(Strings.EMPTY_ARRAY))
+                .setWaitForCompletion(true)
+                .get();
+
+        final SnapshotInfo snapshotInfo = response.getSnapshotInfo();
+        assertThat(snapshotInfo.state(), is(SnapshotState.SUCCESS));
+        assertThat(snapshotInfo.successfulShards(), greaterThan(0));
+        assertThat(snapshotInfo.failedShards(), equalTo(0));
+        return snapshotInfo;
+    }
+
     protected void createIndexWithRandomDocs(String indexName, int docCount) throws InterruptedException {
         createIndex(indexName);
         ensureGreen();

diff --git a/...t/java/org/elasticsearch/blobstore/cache/SearchableSnapshotsBlobStoreCacheIntegTests.java b/...t/java/org/elasticsearch/blobstore/cache/SearchableSnapshotsBlobStoreCacheIntegTests.java
@@ -50,6 +50,7 @@
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
@@ -118,9 +119,9 @@ public void testBlobStoreCache() throws Exception {
 
         final String repositoryName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
         final Path repositoryLocation = randomRepoPath();
-        createFsRepository(repositoryName, repositoryLocation);
+        createRepository(repositoryName, "fs", repositoryLocation);
 
-        final SnapshotId snapshot = createSnapshot(repositoryName, org.elasticsearch.common.collect.List.of(indexName));
+        final SnapshotId snapshot = createSnapshot(repositoryName, "test-snapshot", Collections.singletonList(indexName)).snapshotId();
         assertAcked(client().admin().indices().prepareDelete(indexName));
 
         // extract the list of blobs per shard from the snapshot directory on disk