diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java index 8f83da053b2154..8fda5686819f9d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java @@ -34,7 +34,6 @@ import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -391,8 +390,8 @@ public CreateSnapshotRequest source(Map source) { if (name.equals("indices")) { if (entry.getValue() instanceof String) { indices(Strings.splitStringByCommaToArray((String) entry.getValue())); - } else if (entry.getValue() instanceof ArrayList) { - indices((ArrayList) entry.getValue()); + } else if (entry.getValue() instanceof List) { + indices((List) entry.getValue()); } else { throw new IllegalArgumentException("malformed indices section, should be an array of strings"); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 050d97ba54cf04..abea780be58e59 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -541,7 +541,7 @@ boolean isPatternMatchingAllIndices(MetaData metaData, String[] indicesOrAliases return false; } - static final class Context { + public static class Context { private final ClusterState state; private final IndicesOptions options; @@ -561,7 +561,7 @@ static final class Context { this(state, options, startTime, false, false); } - Context(ClusterState state, IndicesOptions options, long startTime, boolean preserveAliases, boolean resolveToWriteIndex) { + protected Context(ClusterState state, IndicesOptions options, long startTime, boolean preserveAliases, boolean resolveToWriteIndex) { this.state = state; this.options = options; this.startTime = startTime; @@ -817,7 +817,7 @@ private static List resolveEmptyOrTrivialWildcard(IndicesOptions options } } - static final class DateMathExpressionResolver implements ExpressionResolver { + public static final class DateMathExpressionResolver implements ExpressionResolver { private static final DateFormatter DEFAULT_DATE_FORMATTER = DateFormatter.forPattern("uuuu.MM.dd"); private static final String EXPRESSION_LEFT_BOUND = "<"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index a745215fa55331..531e553bdfd885 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -57,9 +57,9 @@ import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata; import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction; import org.elasticsearch.xpack.core.indexlifecycle.LifecycleType; -import org.elasticsearch.xpack.core.indexlifecycle.SetPriorityAction; import org.elasticsearch.xpack.core.indexlifecycle.ReadOnlyAction; import org.elasticsearch.xpack.core.indexlifecycle.RolloverAction; +import org.elasticsearch.xpack.core.indexlifecycle.SetPriorityAction; import org.elasticsearch.xpack.core.indexlifecycle.ShrinkAction; import org.elasticsearch.xpack.core.indexlifecycle.TimeseriesLifecycleType; import org.elasticsearch.xpack.core.indexlifecycle.UnfollowAction; @@ -186,6 +186,10 @@ import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchAction; import org.elasticsearch.xpack.core.watcher.transport.actions.service.WatcherServiceAction; import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsAction; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecycleMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.DeleteSnapshotLifecycleAction; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.GetSnapshotLifecycleAction; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.PutSnapshotLifecycleAction; import java.util.ArrayList; import java.util.Arrays; @@ -363,6 +367,10 @@ public List> getClientActions() { RemoveIndexLifecyclePolicyAction.INSTANCE, MoveToStepAction.INSTANCE, RetryAction.INSTANCE, + PutSnapshotLifecycleAction.INSTANCE, + GetSnapshotLifecycleAction.INSTANCE, + DeleteSnapshotLifecycleAction.INSTANCE, + // Freeze TransportFreezeIndexAction.FreezeIndexAction.INSTANCE ); } @@ -431,6 +439,9 @@ public List getNamedWriteables() { new NamedWriteableRegistry.Entry(MetaData.Custom.class, IndexLifecycleMetadata.TYPE, IndexLifecycleMetadata::new), new NamedWriteableRegistry.Entry(NamedDiff.class, IndexLifecycleMetadata.TYPE, IndexLifecycleMetadata.IndexLifecycleMetadataDiff::new), + new NamedWriteableRegistry.Entry(MetaData.Custom.class, SnapshotLifecycleMetadata.TYPE, SnapshotLifecycleMetadata::new), + new NamedWriteableRegistry.Entry(NamedDiff.class, SnapshotLifecycleMetadata.TYPE, + SnapshotLifecycleMetadata.SnapshotLifecycleMetadataDiff::new), // ILM - LifecycleTypes new NamedWriteableRegistry.Entry(LifecycleType.class, TimeseriesLifecycleType.TYPE, (in) -> TimeseriesLifecycleType.INSTANCE), diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecycleMetadata.java similarity index 71% rename from x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleMetadata.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecycleMetadata.java index 3de5a3002bdef8..08df7d2baf9009 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecycleMetadata.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.snapshotlifecycle; +package org.elasticsearch.xpack.core.snapshotlifecycle; import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractDiffable; @@ -12,9 +12,11 @@ import org.elasticsearch.cluster.DiffableUtils; import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.XPackPlugin.XPackMetaDataCustom; @@ -22,8 +24,11 @@ import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.TreeMap; +import java.util.function.Function; +import java.util.stream.Collectors; /** * Custom cluster state metadata that stores all the snapshot lifecycle @@ -32,6 +37,20 @@ public class SnapshotLifecycleMetadata implements XPackMetaDataCustom { public static final String TYPE = "snapshot_lifecycle"; + public static final ParseField POLICIES_FIELD = new ParseField("policies"); + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(TYPE, + a -> new SnapshotLifecycleMetadata( + ((List) a[0]).stream() + .collect(Collectors.toMap(m -> m.getPolicy().getId(), Function.identity())))); + + static { + PARSER.declareNamedObjects(ConstructingObjectParser.constructorArg(), (p, c, n) -> SnapshotLifecyclePolicyMetadata.parse(p, n), + v -> { + throw new IllegalArgumentException("ordered " + POLICIES_FIELD.getPreferredName() + " are not supported"); + }, POLICIES_FIELD); + } private final Map snapshotConfigurations; @@ -75,7 +94,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field("policies", this.snapshotConfigurations); + builder.field(POLICIES_FIELD.getPreferredName(), this.snapshotConfigurations); return builder; } @@ -93,6 +112,12 @@ public static class SnapshotLifecycleMetadataDiff implements NamedDiff newLifecycles = new TreeMap<>( @@ -110,8 +135,8 @@ public void writeTo(StreamOutput out) throws IOException { lifecycles.writeTo(out); } - static Diff readLifecyclePolicyDiffFrom(StreamInput in) throws IOException { - return AbstractDiffable.readDiffFrom(SnapshotLifecyclePolicy::new, in); + static Diff readLifecyclePolicyDiffFrom(StreamInput in) throws IOException { + return AbstractDiffable.readDiffFrom(SnapshotLifecyclePolicyMetadata::new, in); } } } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicy.java similarity index 68% rename from x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicy.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicy.java index 2e5d6495f5b39a..e1837978541a23 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicy.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicy.java @@ -4,13 +4,18 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.snapshotlifecycle; +package org.elasticsearch.xpack.core.snapshotlifecycle; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.AbstractDiffable; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.Diffable; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.Context; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -21,6 +26,9 @@ import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -42,6 +50,8 @@ public class SnapshotLifecyclePolicy extends AbstractDiffable PARSER = @@ -103,8 +113,29 @@ public ValidationException validate() { return null; } + /** + * Since snapshots need to be uniquely named, this method will resolve any date math used in + * the provided name, as well as appending a unique identifier so expressions that may overlap + * still result in unique snapshot names. + */ + public String generateSnapshotName(Context context) { + List candidates = DATE_MATH_RESOLVER.resolve(context, Collections.singletonList(this.name)); + if (candidates.size() != 1) { + throw new IllegalStateException("resolving snapshot name " + this.name + " generated more than one candidate: " + candidates); + } + // TODO: we are breaking the rules of UUIDs by lowercasing this here, find an alternative (snapshot names must be lowercase) + return candidates.get(0) + "-" + UUIDs.randomBase64UUID().toLowerCase(Locale.ROOT); + } + + /** + * Generate a new create snapshot request from this policy. The name of the snapshot is + * generated at this time based on any date math expressions in the "name" field. + */ public CreateSnapshotRequest toRequest() { - throw new UnsupportedOperationException("implement me"); + CreateSnapshotRequest req = new CreateSnapshotRequest(repository, generateSnapshotName(new ResolverContext())); + req.source(configuration); + req.waitForCompletion(false); + return req; } public static SnapshotLifecyclePolicy parse(XContentParser parser, String id) { @@ -157,4 +188,29 @@ public boolean equals(Object obj) { public String toString() { return Strings.toString(this); } + + /** + * This is a context for the DateMathExpressionResolver, which does not require + * {@code IndicesOptions} or {@code ClusterState} since it only uses the start + * time to resolve expressions + */ + public static final class ResolverContext extends Context { + public ResolverContext() { + this(System.currentTimeMillis()); + } + + public ResolverContext(long startTime) { + super(null, null, startTime, false, false); + } + + @Override + public ClusterState getState() { + throw new UnsupportedOperationException("should never be called"); + } + + @Override + public IndicesOptions getOptions() { + throw new UnsupportedOperationException("should never be called"); + } + } } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicyMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicyMetadata.java similarity index 96% rename from x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicyMetadata.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicyMetadata.java index ab52ab84d9944e..1c2e1956707f2a 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicyMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicyMetadata.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.snapshotlifecycle; +package org.elasticsearch.xpack.core.snapshotlifecycle; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.Diffable; @@ -59,6 +59,10 @@ public class SnapshotLifecyclePolicyMetadata extends AbstractDiffable headers, long version, long modifiedDate) { this.policy = policy; this.headers = headers; diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/DeleteSnapshotLifecycleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/action/DeleteSnapshotLifecycleAction.java similarity index 89% rename from x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/DeleteSnapshotLifecycleAction.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/action/DeleteSnapshotLifecycleAction.java index eca85152a11121..8a5295b0d1edab 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/DeleteSnapshotLifecycleAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/action/DeleteSnapshotLifecycleAction.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.snapshotlifecycle.action; +package org.elasticsearch.xpack.core.snapshotlifecycle.action; import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionRequestValidationException; @@ -26,19 +26,18 @@ protected DeleteSnapshotLifecycleAction() { } @Override - public DeleteSnapshotLifecycleAction.Response newResponse() { + public Response newResponse() { return new Response(); } - public static class Request extends AcknowledgedRequest { + public static class Request extends AcknowledgedRequest { private String lifecycleId; - Request(String lifecycleId) { - this.lifecycleId = Objects.requireNonNull(lifecycleId, "id may not be null"); - } + public Request() { } - Request() { + public Request(String lifecycleId) { + this.lifecycleId = Objects.requireNonNull(lifecycleId, "id may not be null"); } public String getLifecycleId() { diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/GetSnapshotLifecycleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/action/GetSnapshotLifecycleAction.java similarity index 95% rename from x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/GetSnapshotLifecycleAction.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/action/GetSnapshotLifecycleAction.java index 7994c4e194b188..5606bf837e2c4f 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/GetSnapshotLifecycleAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/action/GetSnapshotLifecycleAction.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.snapshotlifecycle.action; +package org.elasticsearch.xpack.core.snapshotlifecycle.action; import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionRequestValidationException; @@ -17,8 +17,8 @@ import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.xpack.snapshotlifecycle.SnapshotLifecyclePolicy; -import org.elasticsearch.xpack.snapshotlifecycle.SnapshotLifecyclePolicyMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicy; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicyMetadata; import java.io.IOException; import java.util.Arrays; @@ -47,11 +47,11 @@ public static class Request extends AcknowledgedRequest implements ToXC private String lifecycleId; private SnapshotLifecyclePolicy lifecycle; - Request(String lifecycleId, SnapshotLifecyclePolicy lifecycle) { + public Request(String lifecycleId, SnapshotLifecyclePolicy lifecycle) { this.lifecycleId = lifecycleId; this.lifecycle = lifecycle; } - Request() { } + public Request() { } public String getLifecycleId() { return this.lifecycleId; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleIT.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleIT.java new file mode 100644 index 00000000000000..3d6454618ec17e --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleIT.java @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.snapshotlifecycle; + +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.snapshots.SnapshotInfo; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; +import org.elasticsearch.xpack.core.XPackClientPlugin; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicy; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.DeleteSnapshotLifecycleAction; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.PutSnapshotLifecycleAction; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.startsWith; + +public class SnapshotLifecycleIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Collections.singletonList(LocalStateCompositeXPackPlugin.class); + } + + @Override + protected Collection> transportClientPlugins() { + return Collections.singletonList(XPackClientPlugin.class); + } + + public void testFullPolicySnapshot() throws Exception { + final String IDX = "test"; + createIndex(IDX); + int docCount = randomIntBetween(10, 50); + List indexReqs = new ArrayList<>(); + for (int i = 0; i < docCount; i++) { + indexReqs.add(client().prepareIndex(IDX, "_doc", "" + i).setSource("foo", "bar")); + } + indexRandom(true, indexReqs); + + Path repo = randomRepoPath(); + + // Create a snapshot repo + assertAcked(client().admin().cluster().preparePutRepository("my-repo").setType("fs").setSettings(Settings.builder() + .put("location", repo.toAbsolutePath().toString()))); + + Map snapConfig = new HashMap<>(); + snapConfig.put("indices", Collections.singletonList(IDX)); + SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy("test-policy", "snap", "*/1 * * * * ?", "my-repo", snapConfig); + + client().execute(PutSnapshotLifecycleAction.INSTANCE, new PutSnapshotLifecycleAction.Request("test-policy", policy)).get(); + + // Check that the snapshot was actually taken + assertBusy(() -> { + GetSnapshotsResponse snapResponse = client().admin().cluster().getSnapshots(new GetSnapshotsRequest("my-repo")).get(); + assertThat(snapResponse.getSnapshots().size(), greaterThan(0)); + SnapshotInfo info = snapResponse.getSnapshots().get(0); + assertThat(info.snapshotId().getName(), startsWith("snap-")); + assertThat(info.indices(), equalTo(Collections.singletonList(IDX))); + }); + + client().execute(DeleteSnapshotLifecycleAction.INSTANCE, new DeleteSnapshotLifecycleAction.Request("test-policy")).get(); + } +} diff --git a/x-pack/plugin/ilm/build.gradle b/x-pack/plugin/ilm/build.gradle index 71def8937817c4..6f1ff7668c366d 100644 --- a/x-pack/plugin/ilm/build.gradle +++ b/x-pack/plugin/ilm/build.gradle @@ -1,3 +1,5 @@ +import com.carrotsearch.gradle.junit4.RandomizedTestingTask + evaluationDependsOn(xpackModule('core')) apply plugin: 'elasticsearch.esplugin' diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java index e1cfb7716862a3..1291a1d2ac993e 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java @@ -80,11 +80,12 @@ import org.elasticsearch.xpack.indexlifecycle.action.TransportRetryAction; import org.elasticsearch.xpack.indexlifecycle.action.TransportStartILMAction; import org.elasticsearch.xpack.indexlifecycle.action.TransportStopILMAction; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecycleMetadata; import org.elasticsearch.xpack.snapshotlifecycle.SnapshotLifecycleService; import org.elasticsearch.xpack.snapshotlifecycle.SnapshotLifecycleTask; -import org.elasticsearch.xpack.snapshotlifecycle.action.DeleteSnapshotLifecycleAction; -import org.elasticsearch.xpack.snapshotlifecycle.action.GetSnapshotLifecycleAction; -import org.elasticsearch.xpack.snapshotlifecycle.action.PutSnapshotLifecycleAction; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.DeleteSnapshotLifecycleAction; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.GetSnapshotLifecycleAction; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.PutSnapshotLifecycleAction; import org.elasticsearch.xpack.snapshotlifecycle.action.RestDeleteSnapshotLifecycleAction; import org.elasticsearch.xpack.snapshotlifecycle.action.RestGetSnapshotLifecycleAction; import org.elasticsearch.xpack.snapshotlifecycle.action.RestPutSnapshotLifecycleAction; @@ -153,7 +154,7 @@ public Collection createComponents(Client client, ClusterService cluster indexLifecycleInitialisationService.set(new IndexLifecycleService(settings, client, clusterService, threadPool, getClock(), System::currentTimeMillis, xContentRegistry)); snapshotLifecycleService.set(new SnapshotLifecycleService(settings, - () -> new SnapshotLifecycleTask(client), clusterService, getClock())); + () -> new SnapshotLifecycleTask(client, clusterService), clusterService, getClock())); return Arrays.asList(indexLifecycleInitialisationService.get(), snapshotLifecycleService.get()); } @@ -168,6 +169,8 @@ public List getNa // Custom Metadata new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(IndexLifecycleMetadata.TYPE), parser -> IndexLifecycleMetadata.PARSER.parse(parser, null)), + new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(SnapshotLifecycleMetadata.TYPE), + parser -> SnapshotLifecycleMetadata.PARSER.parse(parser, null)), // Lifecycle Types new NamedXContentRegistry.Entry(LifecycleType.class, new ParseField(TimeseriesLifecycleType.TYPE), (p, c) -> TimeseriesLifecycleType.INSTANCE), diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleService.java index 83c1c05a8f6485..40d84d55450dca 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleService.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleService.java @@ -18,6 +18,9 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.scheduler.CronSchedule; import org.elasticsearch.xpack.core.scheduler.SchedulerEngine; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecycleMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicy; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicyMetadata; import java.io.Closeable; import java.time.Clock; diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleTask.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleTask.java index c8b1db38035e8d..ab69e8978aab00 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleTask.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleTask.java @@ -8,22 +8,88 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; +import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; +import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.scheduler.SchedulerEngine; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecycleMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicyMetadata; +import org.elasticsearch.xpack.indexlifecycle.LifecyclePolicySecurityClient; + +import java.util.Optional; +import java.util.function.BiConsumer; public class SnapshotLifecycleTask implements SchedulerEngine.Listener { private static Logger logger = LogManager.getLogger(SnapshotLifecycleTask.class); private final Client client; + private final ClusterService clusterService; + private final BiConsumer requestFn; + + public SnapshotLifecycleTask(final Client client, final ClusterService clusterService) { + this.client = client; + this.clusterService = clusterService; + this.requestFn = (policyMetadata, request) -> { + final LifecyclePolicySecurityClient clientWithHeaders = new LifecyclePolicySecurityClient(this.client, + ClientHelper.INDEX_LIFECYCLE_ORIGIN, policyMetadata.getHeaders()); + logger.info("triggering periodic snapshot for policy [{}]", policyMetadata.getPolicy().getId()); + clientWithHeaders.admin().cluster().createSnapshot(request, new ActionListener() { + @Override + public void onResponse(CreateSnapshotResponse createSnapshotResponse) { + // TODO: persist this information in cluster state somewhere + logger.info("snapshot response for [{}]: {}", + policyMetadata.getPolicy().getId(), Strings.toString(createSnapshotResponse)); + } + + @Override + public void onFailure(Exception e) { + // TODO: persist the failure information in cluster state somewhere + logger.error("failed to issue create snapshot request for snapshot lifecycle policy " + + policyMetadata.getPolicy().getId(), e); + } + }); + }; + } - public SnapshotLifecycleTask(final Client client) { + // Providing a custom request function is used for unit testing + SnapshotLifecycleTask(final Client client, final ClusterService clusterService, + final BiConsumer requestFn) { this.client = client; + this.clusterService = clusterService; + this.requestFn = requestFn; } @Override public void triggered(SchedulerEngine.Event event) { - logger.info("--> triggered job: {}", event); - // TODO: implement snapshotting the indices from the job + logger.debug("snapshot lifecycle policy task triggered from job [{}]", event.getJobName()); + Optional maybeMetadata = getSnapPolicyMetadata(event.getJobName(), clusterService.state()); + // If we were on JDK 9 and could use ifPresentOrElse this would be simpler. + boolean successful = maybeMetadata.map(policyMetadata -> { + CreateSnapshotRequest request = policyMetadata.getPolicy().toRequest(); + this.requestFn.accept(policyMetadata, request); + return true; + }) + .orElse(false); + + if (successful == false) { + logger.warn("snapshot lifecycle policy for job [{}] no longer exists, snapshot not created", event.getJobName()); + } + } + + /** + * For the given job id, return an optional policy metadata object, if one exists + */ + static Optional getSnapPolicyMetadata(final String jobId, final ClusterState state) { + return Optional.ofNullable((SnapshotLifecycleMetadata) state.metaData().custom(SnapshotLifecycleMetadata.TYPE)) + .map(SnapshotLifecycleMetadata::getSnapshotConfigurations) + .flatMap(configMap -> configMap.values().stream() + .filter(policyMeta -> jobId.equals(SnapshotLifecycleService.getJobId(policyMeta))) + .findFirst()); } } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestDeleteSnapshotLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestDeleteSnapshotLifecycleAction.java index 2a9a291b864128..53f777b340dfe9 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestDeleteSnapshotLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestDeleteSnapshotLifecycleAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.DeleteSnapshotLifecycleAction; public class RestDeleteSnapshotLifecycleAction extends BaseRestHandler { diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestGetSnapshotLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestGetSnapshotLifecycleAction.java index ac4443c7e7ad55..6d7cb4bad012ac 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestGetSnapshotLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestGetSnapshotLifecycleAction.java @@ -13,6 +13,7 @@ import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.GetSnapshotLifecycleAction; public class RestGetSnapshotLifecycleAction extends BaseRestHandler { diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestPutSnapshotLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestPutSnapshotLifecycleAction.java index a0eba8da655d5d..65576f7c3891ba 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestPutSnapshotLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestPutSnapshotLifecycleAction.java @@ -13,6 +13,7 @@ import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.PutSnapshotLifecycleAction; import java.io.IOException; diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/TransportDeleteSnapshotLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/TransportDeleteSnapshotLifecycleAction.java index 2c66cb842a2c17..43b7166a691542 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/TransportDeleteSnapshotLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/TransportDeleteSnapshotLifecycleAction.java @@ -20,8 +20,9 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.snapshotlifecycle.SnapshotLifecycleMetadata; -import org.elasticsearch.xpack.snapshotlifecycle.SnapshotLifecyclePolicyMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecycleMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicyMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.DeleteSnapshotLifecycleAction; import java.util.Map; import java.util.stream.Collectors; diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/TransportGetSnapshotLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/TransportGetSnapshotLifecycleAction.java index 2e7ecf51b6ef5b..30cee15128b19e 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/TransportGetSnapshotLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/TransportGetSnapshotLifecycleAction.java @@ -20,7 +20,8 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.snapshotlifecycle.SnapshotLifecycleMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecycleMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.GetSnapshotLifecycleAction; import java.io.IOException; import java.util.Arrays; diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/TransportPutSnapshotLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/TransportPutSnapshotLifecycleAction.java index ba8b62235a8b0a..01fd69ae9abbd6 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/TransportPutSnapshotLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/TransportPutSnapshotLifecycleAction.java @@ -24,8 +24,9 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy; -import org.elasticsearch.xpack.snapshotlifecycle.SnapshotLifecycleMetadata; -import org.elasticsearch.xpack.snapshotlifecycle.SnapshotLifecyclePolicyMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecycleMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicyMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.PutSnapshotLifecycleAction; import java.io.IOException; import java.time.Instant; diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicyTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicyTests.java index 3ecde4b8e45c6e..c5fedb7403ebdc 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicyTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicyTests.java @@ -9,15 +9,38 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractSerializingTestCase; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicy; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Map; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.startsWith; + public class SnapshotLifecyclePolicyTests extends AbstractSerializingTestCase { private String id; + public void testNameGeneration() { + long time = 1552684146542L; // Fri Mar 15 2019 21:09:06 UTC + SnapshotLifecyclePolicy.ResolverContext context = new SnapshotLifecyclePolicy.ResolverContext(time); + SnapshotLifecyclePolicy p = new SnapshotLifecyclePolicy("id", "name", "1 * * * * ?", "repo", Collections.emptyMap()); + assertThat(p.generateSnapshotName(context), startsWith("name-")); + assertThat(p.generateSnapshotName(context).length(), greaterThan("name-".length())); + + p = new SnapshotLifecyclePolicy("id", "", "1 * * * * ?", "repo", Collections.emptyMap()); + assertThat(p.generateSnapshotName(context), startsWith("name-2019.03.15-")); + assertThat(p.generateSnapshotName(context).length(), greaterThan("name-2019.03.15-".length())); + + p = new SnapshotLifecyclePolicy("id", "", "1 * * * * ?", "repo", Collections.emptyMap()); + assertThat(p.generateSnapshotName(context), startsWith("name-2019.03.01-")); + + p = new SnapshotLifecyclePolicy("id", "", "1 * * * * ?", "repo", Collections.emptyMap()); + assertThat(p.generateSnapshotName(context), startsWith("name-2019-03-15.21:09:00-")); + } + @Override protected SnapshotLifecyclePolicy doParseInstance(XContentParser parser) throws IOException { return SnapshotLifecyclePolicy.parse(parser, id); diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleServiceTests.java index 599fa747d2eec8..8fc6ecdc29742f 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleServiceTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleServiceTests.java @@ -17,6 +17,9 @@ import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.scheduler.SchedulerEngine; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecycleMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicy; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicyMetadata; import org.elasticsearch.xpack.core.watcher.watch.ClockMock; import java.util.ArrayList; @@ -191,7 +194,7 @@ class FakeSnapshotTask extends SnapshotLifecycleTask { private final Consumer onTriggered; FakeSnapshotTask(Consumer onTriggered) { - super(null); + super(null, null); this.onTriggered = onTriggered; } @@ -211,11 +214,11 @@ public ClusterState createState(SnapshotLifecycleMetadata snapMeta) { .build(); } - public SnapshotLifecyclePolicy createPolicy(String id) { + public static SnapshotLifecyclePolicy createPolicy(String id) { return createPolicy(id, randomSchedule()); } - public SnapshotLifecyclePolicy createPolicy(String id, String schedule) { + public static SnapshotLifecyclePolicy createPolicy(String id, String schedule) { Map config = new HashMap<>(); config.put("ignore_unavailable", randomBoolean()); List indices = new ArrayList<>(); @@ -225,7 +228,7 @@ public SnapshotLifecyclePolicy createPolicy(String id, String schedule) { return new SnapshotLifecyclePolicy(id, randomAlphaOfLength(4), schedule, randomAlphaOfLength(4), config); } - private String randomSchedule() { + private static String randomSchedule() { return randomIntBetween(0, 59) + " " + randomIntBetween(0, 59) + " " + randomIntBetween(0, 12) + " * * ?"; diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleTaskTests.java new file mode 100644 index 00000000000000..efc74eabb06de7 --- /dev/null +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleTaskTests.java @@ -0,0 +1,198 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.snapshotlifecycle; + +import org.elasticsearch.Version; +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotAction; +import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; +import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.TriFunction; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.ClusterServiceUtils; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.scheduler.SchedulerEngine; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecycleMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicy; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicyMetadata; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.startsWith; + +public class SnapshotLifecycleTaskTests extends ESTestCase { + + public void testGetSnapMetadata() { + final String id = randomAlphaOfLength(4); + final SnapshotLifecyclePolicyMetadata slpm = makePolicyMeta(id); + final SnapshotLifecycleMetadata meta = new SnapshotLifecycleMetadata(Collections.singletonMap(id, slpm)); + + final ClusterState state = ClusterState.builder(new ClusterName("test")) + .metaData(MetaData.builder() + .putCustom(SnapshotLifecycleMetadata.TYPE, meta) + .build()) + .build(); + + final Optional o = + SnapshotLifecycleTask.getSnapPolicyMetadata(SnapshotLifecycleService.getJobId(slpm), state); + + assertTrue("the policy metadata should be retrieved from the cluster state", o.isPresent()); + assertThat(o.get(), equalTo(slpm)); + + assertFalse(SnapshotLifecycleTask.getSnapPolicyMetadata("bad-jobid", state).isPresent()); + } + + public void testSkipCreatingSnapshotWhenJobDoesNotMatch() { + final String id = randomAlphaOfLength(4); + final SnapshotLifecyclePolicyMetadata slpm = makePolicyMeta(id); + final SnapshotLifecycleMetadata meta = new SnapshotLifecycleMetadata(Collections.singletonMap(id, slpm)); + + final ClusterState state = ClusterState.builder(new ClusterName("test")) + .metaData(MetaData.builder() + .putCustom(SnapshotLifecycleMetadata.TYPE, meta) + .build()) + .build(); + + final ThreadPool threadPool = new TestThreadPool("test"); + try (ClusterService clusterService = ClusterServiceUtils.createClusterService(state, threadPool); + NoOpClient client = new NoOpClient(threadPool)) { + SnapshotLifecycleTask task = new SnapshotLifecycleTask(client, clusterService, (pMeta, req) -> { + fail("should not have tried to take a snapshot"); + }); + + // Trigger the event, but since the job name does not match, it should + // not run the function to create a snapshot + task.triggered(new SchedulerEngine.Event("nonexistent-job", System.currentTimeMillis(), System.currentTimeMillis())); + } + + threadPool.shutdownNow(); + } + + public void testCreateSnapshotOnTrigger() { + final String id = randomAlphaOfLength(4); + final SnapshotLifecyclePolicyMetadata slpm = makePolicyMeta(id); + final SnapshotLifecycleMetadata meta = new SnapshotLifecycleMetadata(Collections.singletonMap(id, slpm)); + + final ClusterState state = ClusterState.builder(new ClusterName("test")) + .metaData(MetaData.builder() + .putCustom(SnapshotLifecycleMetadata.TYPE, meta) + .build()) + .build(); + + final ThreadPool threadPool = new TestThreadPool("test"); + final String createSnapResponse = "{" + + " \"snapshot\" : {" + + " \"snapshot\" : \"snapshot_1\"," + + " \"uuid\" : \"bcP3ClgCSYO_TP7_FCBbBw\"," + + " \"version_id\" : " + Version.CURRENT.id + "," + + " \"version\" : \"" + Version.CURRENT + "\"," + + " \"indices\" : [ ]," + + " \"include_global_state\" : true," + + " \"state\" : \"SUCCESS\"," + + " \"start_time\" : \"2019-03-19T22:19:53.542Z\"," + + " \"start_time_in_millis\" : 1553033993542," + + " \"end_time\" : \"2019-03-19T22:19:53.567Z\"," + + " \"end_time_in_millis\" : 1553033993567," + + " \"duration_in_millis\" : 25," + + " \"failures\" : [ ]," + + " \"shards\" : {" + + " \"total\" : 0," + + " \"failed\" : 0," + + " \"successful\" : 0" + + " }" + + " }" + + "}"; + + final AtomicBoolean called = new AtomicBoolean(false); + try (ClusterService clusterService = ClusterServiceUtils.createClusterService(state, threadPool); + // This verifying client will verify that we correctly invoked + // client.admin().createSnapshot(...) with the appropriate + // request. It also returns a mock real response + VerifyingClient client = new VerifyingClient(threadPool, + (action, request, listener) -> { + assertFalse(called.getAndSet(true)); + assertThat(action, instanceOf(CreateSnapshotAction.class)); + assertThat(request, instanceOf(CreateSnapshotRequest.class)); + + CreateSnapshotRequest req = (CreateSnapshotRequest) request; + + SnapshotLifecyclePolicy policy = slpm.getPolicy(); + assertThat(req.snapshot(), startsWith(policy.getName() + "-")); + assertThat(req.repository(), equalTo(policy.getRepository())); + if (req.indices().length > 0) { + assertThat(Arrays.asList(req.indices()), equalTo(policy.getConfig().get("indices"))); + } + boolean globalState = policy.getConfig().get("include_global_state") == null || + Boolean.parseBoolean((String) policy.getConfig().get("include_global_state")); + assertThat(req.includeGlobalState(), equalTo(globalState)); + + try { + return CreateSnapshotResponse.fromXContent(createParser(JsonXContent.jsonXContent, createSnapResponse)); + } catch (IOException e) { + fail("failed to parse snapshot response"); + return null; + } + })) { + + SnapshotLifecycleTask task = new SnapshotLifecycleTask(client, clusterService); + // Trigger the event with a matching job name for the policy + task.triggered(new SchedulerEngine.Event(SnapshotLifecycleService.getJobId(slpm), + System.currentTimeMillis(), System.currentTimeMillis())); + + assertTrue("snapshot should be triggered once", called.get()); + } + + threadPool.shutdownNow(); + } + + /** + * A client that delegates to a verifying function for action/request/listener + */ + public static class VerifyingClient extends NoOpClient { + + private final TriFunction, ActionRequest, ActionListener, ActionResponse> verifier; + + VerifyingClient(ThreadPool threadPool, + TriFunction, ActionRequest, ActionListener, ActionResponse> verifier) { + super(threadPool); + this.verifier = verifier; + } + + @Override + @SuppressWarnings("unchecked") + protected void doExecute(Action action, + Request request, + ActionListener listener) { + listener.onResponse((Response) verifier.apply(action, request, listener)); + } + } + + private SnapshotLifecyclePolicyMetadata makePolicyMeta(final String id) { + SnapshotLifecyclePolicy policy = SnapshotLifecycleServiceTests.createPolicy(id); + Map headers = new HashMap<>(); + headers.put("X-Opaque-ID", randomAlphaOfLength(4)); + return new SnapshotLifecyclePolicyMetadata(policy, headers, 1, 1); + } +}