Skip to content

Commit

Permalink
Include hidden indices in snapshots by default (#57325)
Browse files Browse the repository at this point in the history 
Previously, hidden indices were not included in snapshots by default, unless
specified using one of the usual methods for doing so: naming indices directly,
using index patterns starting with a ., or specifying expand_wildcards to
a value that includes hidden (e.g. all or hidden,open).

This commit changes the default expand_wildcards value to include hidden
indices.

Also includes test changes from #57661, as they are necessary to prevent test
failures with these changes, due to code sharing with the HLRC and the new 
defaults using values that are only accepted by >=7.7.0
  • Loading branch information
@gwbrown
gwbrown committed Jun 9, 2020
1 parent 7ecd4af commit 7301509
Show file tree
Hide file tree
Showing 5 changed files with 188 additions and 3 deletions.
43 changes: 43 additions & 0 deletions client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest;
import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.repositories.fs.FsRepository;
Expand Down Expand Up @@ -286,6 +287,48 @@ public void testRestoreSnapshot() throws IOException {
assertThat(restoreInfo.failedShards(), equalTo(0));
}

public void testSnapshotHidden() throws IOException {
String testRepository = "test";
String testSnapshot = "snapshot_1";
String testIndex = "test_index";

AcknowledgedResponse putRepositoryResponse = createTestRepository(testRepository, FsRepository.TYPE, "{\"location\": \".\"}");
assertTrue(putRepositoryResponse.isAcknowledged());

createIndex(testIndex, Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1,3))
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexMetadata.SETTING_INDEX_HIDDEN, true)
.build());
assertTrue("index [" + testIndex + "] should have been created", indexExists(testIndex));

CreateSnapshotRequest createSnapshotRequest = new CreateSnapshotRequest(testRepository, testSnapshot);
createSnapshotRequest.indices("*");
createSnapshotRequest.waitForCompletion(true);
if (randomBoolean()) {
createSnapshotRequest.userMetadata(randomUserMetadata());
}
CreateSnapshotResponse createSnapshotResponse = createTestSnapshot(createSnapshotRequest);
assertEquals(RestStatus.OK, createSnapshotResponse.status());

deleteIndex(testIndex);
assertFalse("index [" + testIndex + "] should have been deleted", indexExists(testIndex));

RestoreSnapshotRequest request = new RestoreSnapshotRequest(testRepository, testSnapshot);
request.waitForCompletion(true);
request.indices(randomFrom(testIndex, "test_*"));
request.renamePattern(testIndex);

RestoreSnapshotResponse response = execute(request, highLevelClient().snapshot()::restore,
highLevelClient().snapshot()::restoreAsync);

RestoreInfo restoreInfo = response.getRestoreInfo();
assertThat(restoreInfo.name(), equalTo(testSnapshot));
assertThat(restoreInfo.indices(), equalTo(Collections.singletonList(testIndex)));
assertThat(restoreInfo.successfulShards(), greaterThan(0));
assertThat(restoreInfo.failedShards(), equalTo(0));
}

public void testDeleteSnapshot() throws IOException {
String repository = "test_repository";
String snapshot = "test_snapshot";
Expand Down
7 changes: 5 additions & 2 deletions ...ulti-version/src/test/java/org/elasticsearch/upgrades/MultiVersionRepositoryAccessIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@

import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus;
import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest;
Expand Down Expand Up @@ -291,7 +290,11 @@ private static void createRepository(RestHighLevelClient client, String repoName
}

private static void createSnapshot(RestHighLevelClient client, String repoName, String name, String index) throws IOException {
client.snapshot().create(new CreateSnapshotRequest(repoName, name).waitForCompletion(true).indices(index), RequestOptions.DEFAULT);
final Request createSnapshotRequest = new Request("PUT", "/_snapshot/" + repoName + "/" + name);
createSnapshotRequest.addParameter("wait_for_completion", "true");
createSnapshotRequest.setJsonEntity("{ \"indices\" : \"" + index + "\"}");
final Response response = client.getLowLevelClient().performRequest(createSnapshotRequest);
assertThat(response.getStatusLine().getStatusCode(), is(HttpURLConnection.HTTP_OK));
}

private void createIndex(RestHighLevelClient client, String name, int shards) throws IOException {
Expand Down
129 changes: 129 additions & 0 deletions .../internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertRequestBuilderThrows;
import static org.hamcrest.Matchers.allOf;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.endsWith;
Expand Down Expand Up @@ -3733,6 +3734,134 @@ public void testBulkDeleteWithOverlappingPatterns() {
assertThat(getSnapshotsResponse.getSnapshots(), empty());
}

public void testHiddenIndicesIncludedInSnapshot() {
Client client = client();
final String normalIndex = "normal-index";
final String hiddenIndex = "hidden-index";
final String dottedHiddenIndex = ".index-hidden";
final String repoName = "test-repo";

logger.info("--> creating repository");
assertAcked(client.admin().cluster().preparePutRepository(repoName).setType("fs").setSettings(randomRepoSettings()));

logger.info("--> creating indices");
createIndex(normalIndex, Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1,3))
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.build());
createIndex(hiddenIndex, Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1,3))
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexMetadata.SETTING_INDEX_HIDDEN, true)
.build());
createIndex(dottedHiddenIndex, Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1,3))
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexMetadata.SETTING_INDEX_HIDDEN, true)
.build());
ensureGreen();

logger.info("--> indexing some data");
for (int i = 0; i < 100; i++) {
index(normalIndex, "_doc", Integer.toString(i), "foo", "bar" + i);
index(hiddenIndex, "_doc", Integer.toString(i), "foo", "baz" + i);
index(dottedHiddenIndex, "_doc", Integer.toString(i), "foo", "baz" + i);
}
refresh();
assertHitCount(client.prepareSearch(normalIndex).setSize(0).get(), 100L);
assertHitCount(client.prepareSearch(hiddenIndex).setSize(0).get(), 100L);
assertHitCount(client.prepareSearch(dottedHiddenIndex).setSize(0).get(), 100L);

logger.info("--> taking a snapshot");
final String snapName = "test-snap";
CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repoName, snapName)
.setWaitForCompletion(true).setIndices(randomFrom("*", "_all")).get();
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(),
equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));

List<SnapshotInfo> snapshotInfos = client.admin().cluster().prepareGetSnapshots(repoName)
.setSnapshots(randomFrom(snapName, "_all", "*", "*-snap", "test*")).get().getSnapshots();
assertThat(snapshotInfos.size(), equalTo(1));
SnapshotInfo snapshotInfo = snapshotInfos.get(0);
assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS));
assertThat(snapshotInfo.version(), equalTo(Version.CURRENT));

logger.info("--> deleting indices");
cluster().wipeIndices(normalIndex, hiddenIndex, dottedHiddenIndex);

// Verify that hidden indices get restored with a wildcard restore
{
RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster()
.prepareRestoreSnapshot(repoName, snapName)
.setWaitForCompletion(true)
.setIndices("*")
.execute().actionGet();
assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
assertThat(restoreSnapshotResponse.getRestoreInfo().successfulShards(),
equalTo(restoreSnapshotResponse.getRestoreInfo().totalShards()));
assertThat(restoreSnapshotResponse.getRestoreInfo().indices(), containsInAnyOrder(normalIndex, hiddenIndex, dottedHiddenIndex));
ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
assertThat(clusterState.getMetadata().hasIndex(normalIndex), equalTo(true));
assertThat(clusterState.getMetadata().hasIndex(hiddenIndex), equalTo(true));
assertThat(clusterState.getMetadata().hasIndex(dottedHiddenIndex), equalTo(true));
cluster().wipeIndices(normalIndex, hiddenIndex, dottedHiddenIndex);
}

// Verify that exclusions work on hidden indices
{
RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster()
.prepareRestoreSnapshot(repoName, snapName)
.setWaitForCompletion(true)
.setIndices("*", "-.*")
.execute().actionGet();
assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
assertThat(restoreSnapshotResponse.getRestoreInfo().successfulShards(),
equalTo(restoreSnapshotResponse.getRestoreInfo().totalShards()));
assertThat(restoreSnapshotResponse.getRestoreInfo().indices(), containsInAnyOrder(normalIndex, hiddenIndex));
ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
assertThat(clusterState.getMetadata().hasIndex(normalIndex), equalTo(true));
assertThat(clusterState.getMetadata().hasIndex(hiddenIndex), equalTo(true));
assertThat(clusterState.getMetadata().hasIndex(dottedHiddenIndex), equalTo(false));
cluster().wipeIndices(normalIndex, hiddenIndex);
}

// Verify that hidden indices can be restored with a non-star pattern
{
RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster()
.prepareRestoreSnapshot(repoName, snapName)
.setWaitForCompletion(true)
.setIndices("hid*")
.execute().actionGet();
assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
assertThat(restoreSnapshotResponse.getRestoreInfo().successfulShards(),
equalTo(restoreSnapshotResponse.getRestoreInfo().totalShards()));
assertThat(restoreSnapshotResponse.getRestoreInfo().indices(), containsInAnyOrder(hiddenIndex));
ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
assertThat(clusterState.getMetadata().hasIndex(normalIndex), equalTo(false));
assertThat(clusterState.getMetadata().hasIndex(hiddenIndex), equalTo(true));
assertThat(clusterState.getMetadata().hasIndex(dottedHiddenIndex), equalTo(false));
cluster().wipeIndices(hiddenIndex);
}

// Verify that hidden indices can be restored by fully specified name
{
RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster()
.prepareRestoreSnapshot(repoName, snapName)
.setWaitForCompletion(true)
.setIndices(dottedHiddenIndex)
.get();
assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
assertThat(restoreSnapshotResponse.getRestoreInfo().successfulShards(),
equalTo(restoreSnapshotResponse.getRestoreInfo().totalShards()));
assertThat(restoreSnapshotResponse.getRestoreInfo().indices(), containsInAnyOrder(dottedHiddenIndex));
ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
assertThat(clusterState.getMetadata().hasIndex(normalIndex), equalTo(false));
assertThat(clusterState.getMetadata().hasIndex(hiddenIndex), equalTo(false));
assertThat(clusterState.getMetadata().hasIndex(dottedHiddenIndex), equalTo(true));
}
}

private void verifySnapshotInfo(final GetSnapshotsResponse response, final Map<String, List<String>> indicesPerSnapshot) {
for (SnapshotInfo snapshotInfo : response.getSnapshots()) {
final List<String> expected = snapshotInfo.indices();
Expand Down
2 changes: 1 addition & 1 deletion ...n/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ public class CreateSnapshotRequest extends MasterNodeRequest<CreateSnapshotReque

private String[] indices = EMPTY_ARRAY;

private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen();
private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenHidden();

private boolean partial = false;

Expand Down
10 changes: 10 additions & 0 deletions server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,8 @@ public enum Option {

public static final IndicesOptions STRICT_EXPAND_OPEN =
new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES), EnumSet.of(WildcardStates.OPEN));
public static final IndicesOptions STRICT_EXPAND_OPEN_HIDDEN =
new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES), EnumSet.of(WildcardStates.OPEN, WildcardStates.HIDDEN));
public static final IndicesOptions LENIENT_EXPAND_OPEN =
new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES, Option.IGNORE_UNAVAILABLE),
EnumSet.of(WildcardStates.OPEN));
Expand Down Expand Up @@ -386,6 +388,14 @@ public static IndicesOptions strictExpandOpen() {
return STRICT_EXPAND_OPEN;
}

/**
* @return indices options that requires every specified index to exist, expands wildcards only to open indices, includes hidden
* indices, and allows that no indices are resolved from wildcard expressions (not returning an error).
*/
public static IndicesOptions strictExpandOpenHidden() {
return STRICT_EXPAND_OPEN_HIDDEN;
}

/**
* @return indices options that requires every specified index to exist, expands wildcards only to open indices,
* allows that no indices are resolved from wildcard expressions (not returning an error) and forbids the
Expand Down

0 comments on commit 7301509

Please sign in to comment.