Skip to content

Commit

Permalink
Validate that snapshot repository exists for ILM policies during Gene…
Browse files Browse the repository at this point in the history 
…rateSnapshotNameStep (#77657)
  • Loading branch information
@joegallo
joegallo committed Sep 30, 2021
1 parent 6788f79 commit 35cb0ea
Show file tree
Hide file tree
Showing 2 changed files with 119 additions and 28 deletions.
51 changes: 31 additions & 20 deletions .../plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/GenerateSnapshotNameStep.java
View file
Expand Up @@ -13,9 +13,10 @@
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.core.Nullable;
import org.elasticsearch.cluster.metadata.RepositoriesMetadata;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.core.Nullable;
import org.elasticsearch.index.Index;

import java.util.Collections;
Expand Down Expand Up @@ -61,29 +62,39 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
return clusterState;
}

ClusterState.Builder newClusterStateBuilder = ClusterState.builder(clusterState);

LifecycleExecutionState lifecycleState = fromIndexMetadata(indexMetaData);
assert lifecycleState.getSnapshotName() == null : "index " + index.getName() + " should not have a snapshot generated by " +
"the ilm policy but has " + lifecycleState.getSnapshotName();
LifecycleExecutionState.Builder newCustomData = LifecycleExecutionState.builder(lifecycleState);
String policy = indexMetaData.getSettings().get(LifecycleSettings.LIFECYCLE_NAME);
String snapshotNamePrefix = ("<{now/d}-" + index.getName() + "-" + policy + ">").toLowerCase(Locale.ROOT);
String snapshotName = generateSnapshotName(snapshotNamePrefix);
ActionRequestValidationException validationException = validateGeneratedSnapshotName(snapshotNamePrefix, snapshotName);
if (validationException != null) {
logger.warn("unable to generate a snapshot name as part of policy [{}] for index [{}] due to [{}]",
policy, index.getName(), validationException.getMessage());
throw validationException;
LifecycleExecutionState lifecycleState = fromIndexMetadata(indexMetaData);

// validate that the snapshot repository exists -- because policies are refreshed on later retries, and because
// this fails prior to the snapshot repository being recorded in the ilm metadata, the policy can just be corrected
// and everything will pass on the subsequent retry
if (clusterState.metadata().custom(RepositoriesMetadata.TYPE, RepositoriesMetadata.EMPTY).repository(snapshotRepository) == null) {
throw new IllegalStateException("repository [" + snapshotRepository + "] is missing. [" + policy + "] policy for " +
"index [" + index.getName() + "] cannot continue until the repository is created or the policy is changed");
}
newCustomData.setSnapshotName(snapshotName);
newCustomData.setSnapshotRepository(snapshotRepository);

LifecycleExecutionState.Builder newCustomData = LifecycleExecutionState.builder(lifecycleState);
newCustomData.setSnapshotIndexName(index.getName());
newCustomData.setSnapshotRepository(snapshotRepository);
if (lifecycleState.getSnapshotName() == null) {
// generate and validate the snapshotName
String snapshotNamePrefix = ("<{now/d}-" + index.getName() + "-" + policy + ">").toLowerCase(Locale.ROOT);
String snapshotName = generateSnapshotName(snapshotNamePrefix);
ActionRequestValidationException validationException = validateGeneratedSnapshotName(snapshotNamePrefix, snapshotName);
if (validationException != null) {
logger.warn("unable to generate a snapshot name as part of policy [{}] for index [{}] due to [{}]",
policy, index.getName(), validationException.getMessage());
throw validationException;
}

newCustomData.setSnapshotName(snapshotName);
}

IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(indexMetaData);
indexMetadataBuilder.putCustom(ILM_CUSTOM_METADATA_KEY, newCustomData.build().asMap());
newClusterStateBuilder.metadata(Metadata.builder(clusterState.getMetadata()).put(indexMetadataBuilder));
return newClusterStateBuilder.build();
return ClusterState.builder(clusterState)
.metadata(Metadata.builder(clusterState.getMetadata())
.put(IndexMetadata.builder(indexMetaData)
.putCustom(ILM_CUSTOM_METADATA_KEY, newCustomData.build().asMap())))
.build();
}

@Override
Expand Down
96 changes: 88 additions & 8 deletions ...in/core/src/test/java/org/elasticsearch/xpack/core/ilm/GenerateSnapshotNameStepTests.java
View file
Expand Up @@ -12,12 +12,17 @@
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.cluster.metadata.RepositoriesMetadata;
import org.elasticsearch.cluster.metadata.RepositoryMetadata;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;

import java.util.Collections;
import java.util.Locale;

import static org.elasticsearch.xpack.core.ilm.GenerateSnapshotNameStep.generateSnapshotName;
import static org.elasticsearch.xpack.core.ilm.GenerateSnapshotNameStep.validateGeneratedSnapshotName;
import static org.elasticsearch.xpack.core.ilm.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.greaterThan;
Expand Down Expand Up @@ -63,23 +68,98 @@ protected GenerateSnapshotNameStep copyInstance(GenerateSnapshotNameStep instanc
public void testPerformAction() {
String indexName = randomAlphaOfLength(10);
String policyName = "test-ilm-policy";
IndexMetadata.Builder indexMetadataBuilder =
IndexMetadata.builder(indexName).settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
.numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5));

IndexMetadata indexMetadata = indexMetadataBuilder.build();
ClusterState clusterState =
ClusterState.builder(emptyClusterState()).metadata(Metadata.builder().put(indexMetadata, true).build()).build();
final IndexMetadata indexMetadata = IndexMetadata.builder(indexName)
.settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
.numberOfShards(randomIntBetween(1, 5))
.numberOfReplicas(randomIntBetween(0, 5))
.build();

GenerateSnapshotNameStep generateSnapshotNameStep = createRandomInstance();
ClusterState newClusterState = generateSnapshotNameStep.performAction(indexMetadata.getIndex(), clusterState);

// generate a snapshot repository with the expected name
RepositoryMetadata repo = new RepositoryMetadata(generateSnapshotNameStep.getSnapshotRepository(), "fs", Settings.EMPTY);

ClusterState clusterState = ClusterState.builder(emptyClusterState())
.metadata(Metadata.builder()
.put(indexMetadata, true)
.putCustom(RepositoriesMetadata.TYPE, new RepositoriesMetadata(Collections.singletonList(repo)))
.build()).build();

ClusterState newClusterState;

// the snapshot index name, snapshot repository, and snapshot name are generated as expected
newClusterState = generateSnapshotNameStep.performAction(indexMetadata.getIndex(), clusterState);
LifecycleExecutionState executionState = LifecycleExecutionState.fromIndexMetadata(newClusterState.metadata().index(indexName));
assertThat(executionState.getSnapshotIndexName(), is(indexName));
assertThat("the " + GenerateSnapshotNameStep.NAME + " step must generate a snapshot name", executionState.getSnapshotName(),
notNullValue());
assertThat(executionState.getSnapshotRepository(), is(generateSnapshotNameStep.getSnapshotRepository()));
assertThat(executionState.getSnapshotName(), containsString(indexName.toLowerCase(Locale.ROOT)));
assertThat(executionState.getSnapshotName(), containsString(policyName.toLowerCase(Locale.ROOT)));

// re-running this step results in no change to the important outputs
newClusterState = generateSnapshotNameStep.performAction(indexMetadata.getIndex(), newClusterState);
LifecycleExecutionState repeatedState = LifecycleExecutionState.fromIndexMetadata(newClusterState.metadata().index(indexName));
assertThat(repeatedState.getSnapshotIndexName(), is(executionState.getSnapshotIndexName()));
assertThat(repeatedState.getSnapshotRepository(), is(executionState.getSnapshotRepository()));
assertThat(repeatedState.getSnapshotName(), is(executionState.getSnapshotName()));
}

public void testPerformActionRejectsNonexistentRepository() {
String indexName = randomAlphaOfLength(10);
String policyName = "test-ilm-policy";
final IndexMetadata indexMetadata = IndexMetadata.builder(indexName)
.settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
.numberOfShards(randomIntBetween(1, 5))
.numberOfReplicas(randomIntBetween(0, 5))
.build();

GenerateSnapshotNameStep generateSnapshotNameStep = createRandomInstance();

ClusterState clusterState = ClusterState.builder(emptyClusterState())
.metadata(Metadata.builder()
.put(indexMetadata, false)
.putCustom(RepositoriesMetadata.TYPE, RepositoriesMetadata.EMPTY)
.build()).build();

IllegalStateException illegalStateException = expectThrows(IllegalStateException.class,
() -> generateSnapshotNameStep.performAction(indexMetadata.getIndex(), clusterState));
assertThat(illegalStateException.getMessage(), is("repository [" + generateSnapshotNameStep.getSnapshotRepository() + "] " +
"is missing. [test-ilm-policy] policy for index [" + indexName + "] cannot continue until the repository " +
"is created or the policy is changed"));
}

public void testPerformActionWillOverwriteCachedRepository() {
String indexName = randomAlphaOfLength(10);
String policyName = "test-ilm-policy";

LifecycleExecutionState.Builder newCustomData = LifecycleExecutionState.builder();
newCustomData.setSnapshotName("snapshot-name-is-not-touched");
newCustomData.setSnapshotRepository("snapshot-repository-will-be-reset");

final IndexMetadata indexMetadata = IndexMetadata.builder(indexName)
.settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
.numberOfShards(randomIntBetween(1, 5))
.numberOfReplicas(randomIntBetween(0, 5))
.putCustom(ILM_CUSTOM_METADATA_KEY, newCustomData.build().asMap())
.build();

GenerateSnapshotNameStep generateSnapshotNameStep = createRandomInstance();

// generate a snapshot repository with the expected name
RepositoryMetadata repo = new RepositoryMetadata(generateSnapshotNameStep.getSnapshotRepository(), "fs", Settings.EMPTY);

ClusterState clusterState = ClusterState.builder(emptyClusterState())
.metadata(Metadata.builder()
.put(indexMetadata, false)
.putCustom(RepositoriesMetadata.TYPE, new RepositoriesMetadata(Collections.singletonList(repo)))
.build()).build();

ClusterState newClusterState = generateSnapshotNameStep.performAction(indexMetadata.getIndex(), clusterState);

LifecycleExecutionState executionState = LifecycleExecutionState.fromIndexMetadata(newClusterState.metadata().index(indexName));
assertThat(executionState.getSnapshotName(), is("snapshot-name-is-not-touched"));
assertThat(executionState.getSnapshotRepository(), is(generateSnapshotNameStep.getSnapshotRepository()));
}

public void testNameGeneration() {
Expand Down

0 comments on commit 35cb0ea

Please sign in to comment.