From 30fee7a19db704063a14c0f2575a7d0349c62b1b Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Fri, 15 Mar 2019 15:31:38 -0600 Subject: [PATCH] Take a snapshot for the policy when the SLM policy is triggered This commit fills in `SnapshotLifecycleTask` to actually perform the snapshotting when the policy is triggered. Currently there is no handling of the results (other than logging) as that will be added in subsequent work. This also adds unit tests and an integration test that schedules a policy and ensures that a snapshot is correctly taken. Relates to #38461 --- .../create/CreateSnapshotRequest.java | 5 +- .../metadata/IndexNameExpressionResolver.java | 6 +- .../xpack/core/XPackClientPlugin.java | 13 +- .../SnapshotLifecycleMetadata.java | 33 ++- .../SnapshotLifecyclePolicy.java | 60 +++++- .../SnapshotLifecyclePolicyMetadata.java | 6 +- .../action/DeleteSnapshotLifecycleAction.java | 13 +- .../action/GetSnapshotLifecycleAction.java | 10 +- .../action/PutSnapshotLifecycleAction.java | 8 +- .../SnapshotLifecycleIT.java | 80 +++++++ x-pack/plugin/ilm/build.gradle | 2 + .../xpack/indexlifecycle/IndexLifecycle.java | 11 +- .../SnapshotLifecycleService.java | 3 + .../SnapshotLifecycleTask.java | 72 ++++++- .../RestDeleteSnapshotLifecycleAction.java | 1 + .../RestGetSnapshotLifecycleAction.java | 1 + .../RestPutSnapshotLifecycleAction.java | 1 + ...ransportDeleteSnapshotLifecycleAction.java | 5 +- .../TransportGetSnapshotLifecycleAction.java | 3 +- .../TransportPutSnapshotLifecycleAction.java | 5 +- .../SnapshotLifecyclePolicyTests.java | 23 ++ .../SnapshotLifecycleServiceTests.java | 11 +- .../SnapshotLifecycleTaskTests.java | 198 ++++++++++++++++++ 23 files changed, 524 insertions(+), 46 deletions(-) rename x-pack/plugin/{ilm/src/main/java/org/elasticsearch/xpack => core/src/main/java/org/elasticsearch/xpack/core}/snapshotlifecycle/SnapshotLifecycleMetadata.java (71%) rename x-pack/plugin/{ilm/src/main/java/org/elasticsearch/xpack => core/src/main/java/org/elasticsearch/xpack/core}/snapshotlifecycle/SnapshotLifecyclePolicy.java (68%) rename x-pack/plugin/{ilm/src/main/java/org/elasticsearch/xpack => core/src/main/java/org/elasticsearch/xpack/core}/snapshotlifecycle/SnapshotLifecyclePolicyMetadata.java (96%) rename x-pack/plugin/{ilm/src/main/java/org/elasticsearch/xpack => core/src/main/java/org/elasticsearch/xpack/core}/snapshotlifecycle/action/DeleteSnapshotLifecycleAction.java (89%) rename x-pack/plugin/{ilm/src/main/java/org/elasticsearch/xpack => core/src/main/java/org/elasticsearch/xpack/core}/snapshotlifecycle/action/GetSnapshotLifecycleAction.java (95%) rename x-pack/plugin/{ilm/src/main/java/org/elasticsearch/xpack => core/src/main/java/org/elasticsearch/xpack/core}/snapshotlifecycle/action/PutSnapshotLifecycleAction.java (94%) create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleIT.java create mode 100644 x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleTaskTests.java diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java index 8f83da053b2154..8fda5686819f9d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java @@ -34,7 +34,6 @@ import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -391,8 +390,8 @@ public CreateSnapshotRequest source(Map source) { if (name.equals("indices")) { if (entry.getValue() instanceof String) { indices(Strings.splitStringByCommaToArray((String) entry.getValue())); - } else if (entry.getValue() instanceof ArrayList) { - indices((ArrayList) entry.getValue()); + } else if (entry.getValue() instanceof List) { + indices((List) entry.getValue()); } else { throw new IllegalArgumentException("malformed indices section, should be an array of strings"); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 050d97ba54cf04..abea780be58e59 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -541,7 +541,7 @@ boolean isPatternMatchingAllIndices(MetaData metaData, String[] indicesOrAliases return false; } - static final class Context { + public static class Context { private final ClusterState state; private final IndicesOptions options; @@ -561,7 +561,7 @@ static final class Context { this(state, options, startTime, false, false); } - Context(ClusterState state, IndicesOptions options, long startTime, boolean preserveAliases, boolean resolveToWriteIndex) { + protected Context(ClusterState state, IndicesOptions options, long startTime, boolean preserveAliases, boolean resolveToWriteIndex) { this.state = state; this.options = options; this.startTime = startTime; @@ -817,7 +817,7 @@ private static List resolveEmptyOrTrivialWildcard(IndicesOptions options } } - static final class DateMathExpressionResolver implements ExpressionResolver { + public static final class DateMathExpressionResolver implements ExpressionResolver { private static final DateFormatter DEFAULT_DATE_FORMATTER = DateFormatter.forPattern("uuuu.MM.dd"); private static final String EXPRESSION_LEFT_BOUND = "<"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index a745215fa55331..531e553bdfd885 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -57,9 +57,9 @@ import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata; import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction; import org.elasticsearch.xpack.core.indexlifecycle.LifecycleType; -import org.elasticsearch.xpack.core.indexlifecycle.SetPriorityAction; import org.elasticsearch.xpack.core.indexlifecycle.ReadOnlyAction; import org.elasticsearch.xpack.core.indexlifecycle.RolloverAction; +import org.elasticsearch.xpack.core.indexlifecycle.SetPriorityAction; import org.elasticsearch.xpack.core.indexlifecycle.ShrinkAction; import org.elasticsearch.xpack.core.indexlifecycle.TimeseriesLifecycleType; import org.elasticsearch.xpack.core.indexlifecycle.UnfollowAction; @@ -186,6 +186,10 @@ import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchAction; import org.elasticsearch.xpack.core.watcher.transport.actions.service.WatcherServiceAction; import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsAction; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecycleMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.DeleteSnapshotLifecycleAction; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.GetSnapshotLifecycleAction; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.PutSnapshotLifecycleAction; import java.util.ArrayList; import java.util.Arrays; @@ -363,6 +367,10 @@ public List> getClientActions() { RemoveIndexLifecyclePolicyAction.INSTANCE, MoveToStepAction.INSTANCE, RetryAction.INSTANCE, + PutSnapshotLifecycleAction.INSTANCE, + GetSnapshotLifecycleAction.INSTANCE, + DeleteSnapshotLifecycleAction.INSTANCE, + // Freeze TransportFreezeIndexAction.FreezeIndexAction.INSTANCE ); } @@ -431,6 +439,9 @@ public List getNamedWriteables() { new NamedWriteableRegistry.Entry(MetaData.Custom.class, IndexLifecycleMetadata.TYPE, IndexLifecycleMetadata::new), new NamedWriteableRegistry.Entry(NamedDiff.class, IndexLifecycleMetadata.TYPE, IndexLifecycleMetadata.IndexLifecycleMetadataDiff::new), + new NamedWriteableRegistry.Entry(MetaData.Custom.class, SnapshotLifecycleMetadata.TYPE, SnapshotLifecycleMetadata::new), + new NamedWriteableRegistry.Entry(NamedDiff.class, SnapshotLifecycleMetadata.TYPE, + SnapshotLifecycleMetadata.SnapshotLifecycleMetadataDiff::new), // ILM - LifecycleTypes new NamedWriteableRegistry.Entry(LifecycleType.class, TimeseriesLifecycleType.TYPE, (in) -> TimeseriesLifecycleType.INSTANCE), diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecycleMetadata.java similarity index 71% rename from x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleMetadata.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecycleMetadata.java index 3de5a3002bdef8..08df7d2baf9009 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecycleMetadata.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.snapshotlifecycle; +package org.elasticsearch.xpack.core.snapshotlifecycle; import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractDiffable; @@ -12,9 +12,11 @@ import org.elasticsearch.cluster.DiffableUtils; import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.XPackPlugin.XPackMetaDataCustom; @@ -22,8 +24,11 @@ import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.TreeMap; +import java.util.function.Function; +import java.util.stream.Collectors; /** * Custom cluster state metadata that stores all the snapshot lifecycle @@ -32,6 +37,20 @@ public class SnapshotLifecycleMetadata implements XPackMetaDataCustom { public static final String TYPE = "snapshot_lifecycle"; + public static final ParseField POLICIES_FIELD = new ParseField("policies"); + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(TYPE, + a -> new SnapshotLifecycleMetadata( + ((List) a[0]).stream() + .collect(Collectors.toMap(m -> m.getPolicy().getId(), Function.identity())))); + + static { + PARSER.declareNamedObjects(ConstructingObjectParser.constructorArg(), (p, c, n) -> SnapshotLifecyclePolicyMetadata.parse(p, n), + v -> { + throw new IllegalArgumentException("ordered " + POLICIES_FIELD.getPreferredName() + " are not supported"); + }, POLICIES_FIELD); + } private final Map snapshotConfigurations; @@ -75,7 +94,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field("policies", this.snapshotConfigurations); + builder.field(POLICIES_FIELD.getPreferredName(), this.snapshotConfigurations); return builder; } @@ -93,6 +112,12 @@ public static class SnapshotLifecycleMetadataDiff implements NamedDiff newLifecycles = new TreeMap<>( @@ -110,8 +135,8 @@ public void writeTo(StreamOutput out) throws IOException { lifecycles.writeTo(out); } - static Diff readLifecyclePolicyDiffFrom(StreamInput in) throws IOException { - return AbstractDiffable.readDiffFrom(SnapshotLifecyclePolicy::new, in); + static Diff readLifecyclePolicyDiffFrom(StreamInput in) throws IOException { + return AbstractDiffable.readDiffFrom(SnapshotLifecyclePolicyMetadata::new, in); } } } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicy.java similarity index 68% rename from x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicy.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicy.java index 2e5d6495f5b39a..e1837978541a23 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicy.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicy.java @@ -4,13 +4,18 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.snapshotlifecycle; +package org.elasticsearch.xpack.core.snapshotlifecycle; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.AbstractDiffable; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.Diffable; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.Context; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -21,6 +26,9 @@ import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -42,6 +50,8 @@ public class SnapshotLifecyclePolicy extends AbstractDiffable PARSER = @@ -103,8 +113,29 @@ public ValidationException validate() { return null; } + /** + * Since snapshots need to be uniquely named, this method will resolve any date math used in + * the provided name, as well as appending a unique identifier so expressions that may overlap + * still result in unique snapshot names. + */ + public String generateSnapshotName(Context context) { + List candidates = DATE_MATH_RESOLVER.resolve(context, Collections.singletonList(this.name)); + if (candidates.size() != 1) { + throw new IllegalStateException("resolving snapshot name " + this.name + " generated more than one candidate: " + candidates); + } + // TODO: we are breaking the rules of UUIDs by lowercasing this here, find an alternative (snapshot names must be lowercase) + return candidates.get(0) + "-" + UUIDs.randomBase64UUID().toLowerCase(Locale.ROOT); + } + + /** + * Generate a new create snapshot request from this policy. The name of the snapshot is + * generated at this time based on any date math expressions in the "name" field. + */ public CreateSnapshotRequest toRequest() { - throw new UnsupportedOperationException("implement me"); + CreateSnapshotRequest req = new CreateSnapshotRequest(repository, generateSnapshotName(new ResolverContext())); + req.source(configuration); + req.waitForCompletion(false); + return req; } public static SnapshotLifecyclePolicy parse(XContentParser parser, String id) { @@ -157,4 +188,29 @@ public boolean equals(Object obj) { public String toString() { return Strings.toString(this); } + + /** + * This is a context for the DateMathExpressionResolver, which does not require + * {@code IndicesOptions} or {@code ClusterState} since it only uses the start + * time to resolve expressions + */ + public static final class ResolverContext extends Context { + public ResolverContext() { + this(System.currentTimeMillis()); + } + + public ResolverContext(long startTime) { + super(null, null, startTime, false, false); + } + + @Override + public ClusterState getState() { + throw new UnsupportedOperationException("should never be called"); + } + + @Override + public IndicesOptions getOptions() { + throw new UnsupportedOperationException("should never be called"); + } + } } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicyMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicyMetadata.java similarity index 96% rename from x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicyMetadata.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicyMetadata.java index ab52ab84d9944e..1c2e1956707f2a 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicyMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicyMetadata.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.snapshotlifecycle; +package org.elasticsearch.xpack.core.snapshotlifecycle; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.Diffable; @@ -59,6 +59,10 @@ public class SnapshotLifecyclePolicyMetadata extends AbstractDiffable headers, long version, long modifiedDate) { this.policy = policy; this.headers = headers; diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/DeleteSnapshotLifecycleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/action/DeleteSnapshotLifecycleAction.java similarity index 89% rename from x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/DeleteSnapshotLifecycleAction.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/action/DeleteSnapshotLifecycleAction.java index eca85152a11121..8a5295b0d1edab 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/DeleteSnapshotLifecycleAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/action/DeleteSnapshotLifecycleAction.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.snapshotlifecycle.action; +package org.elasticsearch.xpack.core.snapshotlifecycle.action; import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionRequestValidationException; @@ -26,19 +26,18 @@ protected DeleteSnapshotLifecycleAction() { } @Override - public DeleteSnapshotLifecycleAction.Response newResponse() { + public Response newResponse() { return new Response(); } - public static class Request extends AcknowledgedRequest { + public static class Request extends AcknowledgedRequest { private String lifecycleId; - Request(String lifecycleId) { - this.lifecycleId = Objects.requireNonNull(lifecycleId, "id may not be null"); - } + public Request() { } - Request() { + public Request(String lifecycleId) { + this.lifecycleId = Objects.requireNonNull(lifecycleId, "id may not be null"); } public String getLifecycleId() { diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/GetSnapshotLifecycleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/action/GetSnapshotLifecycleAction.java similarity index 95% rename from x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/GetSnapshotLifecycleAction.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/action/GetSnapshotLifecycleAction.java index 7994c4e194b188..5606bf837e2c4f 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/GetSnapshotLifecycleAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/action/GetSnapshotLifecycleAction.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.snapshotlifecycle.action; +package org.elasticsearch.xpack.core.snapshotlifecycle.action; import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionRequestValidationException; @@ -17,8 +17,8 @@ import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.xpack.snapshotlifecycle.SnapshotLifecyclePolicy; -import org.elasticsearch.xpack.snapshotlifecycle.SnapshotLifecyclePolicyMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicy; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicyMetadata; import java.io.IOException; import java.util.Arrays; @@ -47,11 +47,11 @@ public static class Request extends AcknowledgedRequest implements ToXC private String lifecycleId; private SnapshotLifecyclePolicy lifecycle; - Request(String lifecycleId, SnapshotLifecyclePolicy lifecycle) { + public Request(String lifecycleId, SnapshotLifecyclePolicy lifecycle) { this.lifecycleId = lifecycleId; this.lifecycle = lifecycle; } - Request() { } + public Request() { } public String getLifecycleId() { return this.lifecycleId; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleIT.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleIT.java new file mode 100644 index 00000000000000..3d6454618ec17e --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleIT.java @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.snapshotlifecycle; + +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.snapshots.SnapshotInfo; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; +import org.elasticsearch.xpack.core.XPackClientPlugin; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicy; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.DeleteSnapshotLifecycleAction; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.PutSnapshotLifecycleAction; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.startsWith; + +public class SnapshotLifecycleIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Collections.singletonList(LocalStateCompositeXPackPlugin.class); + } + + @Override + protected Collection> transportClientPlugins() { + return Collections.singletonList(XPackClientPlugin.class); + } + + public void testFullPolicySnapshot() throws Exception { + final String IDX = "test"; + createIndex(IDX); + int docCount = randomIntBetween(10, 50); + List indexReqs = new ArrayList<>(); + for (int i = 0; i < docCount; i++) { + indexReqs.add(client().prepareIndex(IDX, "_doc", "" + i).setSource("foo", "bar")); + } + indexRandom(true, indexReqs); + + Path repo = randomRepoPath(); + + // Create a snapshot repo + assertAcked(client().admin().cluster().preparePutRepository("my-repo").setType("fs").setSettings(Settings.builder() + .put("location", repo.toAbsolutePath().toString()))); + + Map snapConfig = new HashMap<>(); + snapConfig.put("indices", Collections.singletonList(IDX)); + SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy("test-policy", "snap", "*/1 * * * * ?", "my-repo", snapConfig); + + client().execute(PutSnapshotLifecycleAction.INSTANCE, new PutSnapshotLifecycleAction.Request("test-policy", policy)).get(); + + // Check that the snapshot was actually taken + assertBusy(() -> { + GetSnapshotsResponse snapResponse = client().admin().cluster().getSnapshots(new GetSnapshotsRequest("my-repo")).get(); + assertThat(snapResponse.getSnapshots().size(), greaterThan(0)); + SnapshotInfo info = snapResponse.getSnapshots().get(0); + assertThat(info.snapshotId().getName(), startsWith("snap-")); + assertThat(info.indices(), equalTo(Collections.singletonList(IDX))); + }); + + client().execute(DeleteSnapshotLifecycleAction.INSTANCE, new DeleteSnapshotLifecycleAction.Request("test-policy")).get(); + } +} diff --git a/x-pack/plugin/ilm/build.gradle b/x-pack/plugin/ilm/build.gradle index 71def8937817c4..6f1ff7668c366d 100644 --- a/x-pack/plugin/ilm/build.gradle +++ b/x-pack/plugin/ilm/build.gradle @@ -1,3 +1,5 @@ +import com.carrotsearch.gradle.junit4.RandomizedTestingTask + evaluationDependsOn(xpackModule('core')) apply plugin: 'elasticsearch.esplugin' diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java index e1cfb7716862a3..1291a1d2ac993e 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java @@ -80,11 +80,12 @@ import org.elasticsearch.xpack.indexlifecycle.action.TransportRetryAction; import org.elasticsearch.xpack.indexlifecycle.action.TransportStartILMAction; import org.elasticsearch.xpack.indexlifecycle.action.TransportStopILMAction; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecycleMetadata; import org.elasticsearch.xpack.snapshotlifecycle.SnapshotLifecycleService; import org.elasticsearch.xpack.snapshotlifecycle.SnapshotLifecycleTask; -import org.elasticsearch.xpack.snapshotlifecycle.action.DeleteSnapshotLifecycleAction; -import org.elasticsearch.xpack.snapshotlifecycle.action.GetSnapshotLifecycleAction; -import org.elasticsearch.xpack.snapshotlifecycle.action.PutSnapshotLifecycleAction; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.DeleteSnapshotLifecycleAction; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.GetSnapshotLifecycleAction; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.PutSnapshotLifecycleAction; import org.elasticsearch.xpack.snapshotlifecycle.action.RestDeleteSnapshotLifecycleAction; import org.elasticsearch.xpack.snapshotlifecycle.action.RestGetSnapshotLifecycleAction; import org.elasticsearch.xpack.snapshotlifecycle.action.RestPutSnapshotLifecycleAction; @@ -153,7 +154,7 @@ public Collection createComponents(Client client, ClusterService cluster indexLifecycleInitialisationService.set(new IndexLifecycleService(settings, client, clusterService, threadPool, getClock(), System::currentTimeMillis, xContentRegistry)); snapshotLifecycleService.set(new SnapshotLifecycleService(settings, - () -> new SnapshotLifecycleTask(client), clusterService, getClock())); + () -> new SnapshotLifecycleTask(client, clusterService), clusterService, getClock())); return Arrays.asList(indexLifecycleInitialisationService.get(), snapshotLifecycleService.get()); } @@ -168,6 +169,8 @@ public List getNa // Custom Metadata new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(IndexLifecycleMetadata.TYPE), parser -> IndexLifecycleMetadata.PARSER.parse(parser, null)), + new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(SnapshotLifecycleMetadata.TYPE), + parser -> SnapshotLifecycleMetadata.PARSER.parse(parser, null)), // Lifecycle Types new NamedXContentRegistry.Entry(LifecycleType.class, new ParseField(TimeseriesLifecycleType.TYPE), (p, c) -> TimeseriesLifecycleType.INSTANCE), diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleService.java index 83c1c05a8f6485..40d84d55450dca 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleService.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleService.java @@ -18,6 +18,9 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.scheduler.CronSchedule; import org.elasticsearch.xpack.core.scheduler.SchedulerEngine; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecycleMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicy; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicyMetadata; import java.io.Closeable; import java.time.Clock; diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleTask.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleTask.java index c8b1db38035e8d..ab69e8978aab00 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleTask.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleTask.java @@ -8,22 +8,88 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; +import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; +import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.scheduler.SchedulerEngine; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecycleMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicyMetadata; +import org.elasticsearch.xpack.indexlifecycle.LifecyclePolicySecurityClient; + +import java.util.Optional; +import java.util.function.BiConsumer; public class SnapshotLifecycleTask implements SchedulerEngine.Listener { private static Logger logger = LogManager.getLogger(SnapshotLifecycleTask.class); private final Client client; + private final ClusterService clusterService; + private final BiConsumer requestFn; + + public SnapshotLifecycleTask(final Client client, final ClusterService clusterService) { + this.client = client; + this.clusterService = clusterService; + this.requestFn = (policyMetadata, request) -> { + final LifecyclePolicySecurityClient clientWithHeaders = new LifecyclePolicySecurityClient(this.client, + ClientHelper.INDEX_LIFECYCLE_ORIGIN, policyMetadata.getHeaders()); + logger.info("triggering periodic snapshot for policy [{}]", policyMetadata.getPolicy().getId()); + clientWithHeaders.admin().cluster().createSnapshot(request, new ActionListener() { + @Override + public void onResponse(CreateSnapshotResponse createSnapshotResponse) { + // TODO: persist this information in cluster state somewhere + logger.info("snapshot response for [{}]: {}", + policyMetadata.getPolicy().getId(), Strings.toString(createSnapshotResponse)); + } + + @Override + public void onFailure(Exception e) { + // TODO: persist the failure information in cluster state somewhere + logger.error("failed to issue create snapshot request for snapshot lifecycle policy " + + policyMetadata.getPolicy().getId(), e); + } + }); + }; + } - public SnapshotLifecycleTask(final Client client) { + // Providing a custom request function is used for unit testing + SnapshotLifecycleTask(final Client client, final ClusterService clusterService, + final BiConsumer requestFn) { this.client = client; + this.clusterService = clusterService; + this.requestFn = requestFn; } @Override public void triggered(SchedulerEngine.Event event) { - logger.info("--> triggered job: {}", event); - // TODO: implement snapshotting the indices from the job + logger.debug("snapshot lifecycle policy task triggered from job [{}]", event.getJobName()); + Optional maybeMetadata = getSnapPolicyMetadata(event.getJobName(), clusterService.state()); + // If we were on JDK 9 and could use ifPresentOrElse this would be simpler. + boolean successful = maybeMetadata.map(policyMetadata -> { + CreateSnapshotRequest request = policyMetadata.getPolicy().toRequest(); + this.requestFn.accept(policyMetadata, request); + return true; + }) + .orElse(false); + + if (successful == false) { + logger.warn("snapshot lifecycle policy for job [{}] no longer exists, snapshot not created", event.getJobName()); + } + } + + /** + * For the given job id, return an optional policy metadata object, if one exists + */ + static Optional getSnapPolicyMetadata(final String jobId, final ClusterState state) { + return Optional.ofNullable((SnapshotLifecycleMetadata) state.metaData().custom(SnapshotLifecycleMetadata.TYPE)) + .map(SnapshotLifecycleMetadata::getSnapshotConfigurations) + .flatMap(configMap -> configMap.values().stream() + .filter(policyMeta -> jobId.equals(SnapshotLifecycleService.getJobId(policyMeta))) + .findFirst()); } } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestDeleteSnapshotLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestDeleteSnapshotLifecycleAction.java index 2a9a291b864128..53f777b340dfe9 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestDeleteSnapshotLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestDeleteSnapshotLifecycleAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.DeleteSnapshotLifecycleAction; public class RestDeleteSnapshotLifecycleAction extends BaseRestHandler { diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestGetSnapshotLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestGetSnapshotLifecycleAction.java index ac4443c7e7ad55..6d7cb4bad012ac 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestGetSnapshotLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestGetSnapshotLifecycleAction.java @@ -13,6 +13,7 @@ import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.GetSnapshotLifecycleAction; public class RestGetSnapshotLifecycleAction extends BaseRestHandler { diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestPutSnapshotLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestPutSnapshotLifecycleAction.java index a0eba8da655d5d..65576f7c3891ba 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestPutSnapshotLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestPutSnapshotLifecycleAction.java @@ -13,6 +13,7 @@ import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.PutSnapshotLifecycleAction; import java.io.IOException; diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/TransportDeleteSnapshotLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/TransportDeleteSnapshotLifecycleAction.java index 2c66cb842a2c17..43b7166a691542 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/TransportDeleteSnapshotLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/TransportDeleteSnapshotLifecycleAction.java @@ -20,8 +20,9 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.snapshotlifecycle.SnapshotLifecycleMetadata; -import org.elasticsearch.xpack.snapshotlifecycle.SnapshotLifecyclePolicyMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecycleMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicyMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.DeleteSnapshotLifecycleAction; import java.util.Map; import java.util.stream.Collectors; diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/TransportGetSnapshotLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/TransportGetSnapshotLifecycleAction.java index 2e7ecf51b6ef5b..30cee15128b19e 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/TransportGetSnapshotLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/TransportGetSnapshotLifecycleAction.java @@ -20,7 +20,8 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.snapshotlifecycle.SnapshotLifecycleMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecycleMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.GetSnapshotLifecycleAction; import java.io.IOException; import java.util.Arrays; diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/TransportPutSnapshotLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/TransportPutSnapshotLifecycleAction.java index ba8b62235a8b0a..01fd69ae9abbd6 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/TransportPutSnapshotLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/TransportPutSnapshotLifecycleAction.java @@ -24,8 +24,9 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy; -import org.elasticsearch.xpack.snapshotlifecycle.SnapshotLifecycleMetadata; -import org.elasticsearch.xpack.snapshotlifecycle.SnapshotLifecyclePolicyMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecycleMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicyMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.action.PutSnapshotLifecycleAction; import java.io.IOException; import java.time.Instant; diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicyTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicyTests.java index 3ecde4b8e45c6e..c5fedb7403ebdc 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicyTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicyTests.java @@ -9,15 +9,38 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractSerializingTestCase; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicy; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Map; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.startsWith; + public class SnapshotLifecyclePolicyTests extends AbstractSerializingTestCase { private String id; + public void testNameGeneration() { + long time = 1552684146542L; // Fri Mar 15 2019 21:09:06 UTC + SnapshotLifecyclePolicy.ResolverContext context = new SnapshotLifecyclePolicy.ResolverContext(time); + SnapshotLifecyclePolicy p = new SnapshotLifecyclePolicy("id", "name", "1 * * * * ?", "repo", Collections.emptyMap()); + assertThat(p.generateSnapshotName(context), startsWith("name-")); + assertThat(p.generateSnapshotName(context).length(), greaterThan("name-".length())); + + p = new SnapshotLifecyclePolicy("id", "", "1 * * * * ?", "repo", Collections.emptyMap()); + assertThat(p.generateSnapshotName(context), startsWith("name-2019.03.15-")); + assertThat(p.generateSnapshotName(context).length(), greaterThan("name-2019.03.15-".length())); + + p = new SnapshotLifecyclePolicy("id", "", "1 * * * * ?", "repo", Collections.emptyMap()); + assertThat(p.generateSnapshotName(context), startsWith("name-2019.03.01-")); + + p = new SnapshotLifecyclePolicy("id", "", "1 * * * * ?", "repo", Collections.emptyMap()); + assertThat(p.generateSnapshotName(context), startsWith("name-2019-03-15.21:09:00-")); + } + @Override protected SnapshotLifecyclePolicy doParseInstance(XContentParser parser) throws IOException { return SnapshotLifecyclePolicy.parse(parser, id); diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleServiceTests.java index 599fa747d2eec8..8fc6ecdc29742f 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleServiceTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleServiceTests.java @@ -17,6 +17,9 @@ import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.scheduler.SchedulerEngine; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecycleMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicy; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicyMetadata; import org.elasticsearch.xpack.core.watcher.watch.ClockMock; import java.util.ArrayList; @@ -191,7 +194,7 @@ class FakeSnapshotTask extends SnapshotLifecycleTask { private final Consumer onTriggered; FakeSnapshotTask(Consumer onTriggered) { - super(null); + super(null, null); this.onTriggered = onTriggered; } @@ -211,11 +214,11 @@ public ClusterState createState(SnapshotLifecycleMetadata snapMeta) { .build(); } - public SnapshotLifecyclePolicy createPolicy(String id) { + public static SnapshotLifecyclePolicy createPolicy(String id) { return createPolicy(id, randomSchedule()); } - public SnapshotLifecyclePolicy createPolicy(String id, String schedule) { + public static SnapshotLifecyclePolicy createPolicy(String id, String schedule) { Map config = new HashMap<>(); config.put("ignore_unavailable", randomBoolean()); List indices = new ArrayList<>(); @@ -225,7 +228,7 @@ public SnapshotLifecyclePolicy createPolicy(String id, String schedule) { return new SnapshotLifecyclePolicy(id, randomAlphaOfLength(4), schedule, randomAlphaOfLength(4), config); } - private String randomSchedule() { + private static String randomSchedule() { return randomIntBetween(0, 59) + " " + randomIntBetween(0, 59) + " " + randomIntBetween(0, 12) + " * * ?"; diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleTaskTests.java new file mode 100644 index 00000000000000..efc74eabb06de7 --- /dev/null +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleTaskTests.java @@ -0,0 +1,198 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.snapshotlifecycle; + +import org.elasticsearch.Version; +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotAction; +import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; +import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.TriFunction; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.ClusterServiceUtils; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.scheduler.SchedulerEngine; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecycleMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicy; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicyMetadata; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.startsWith; + +public class SnapshotLifecycleTaskTests extends ESTestCase { + + public void testGetSnapMetadata() { + final String id = randomAlphaOfLength(4); + final SnapshotLifecyclePolicyMetadata slpm = makePolicyMeta(id); + final SnapshotLifecycleMetadata meta = new SnapshotLifecycleMetadata(Collections.singletonMap(id, slpm)); + + final ClusterState state = ClusterState.builder(new ClusterName("test")) + .metaData(MetaData.builder() + .putCustom(SnapshotLifecycleMetadata.TYPE, meta) + .build()) + .build(); + + final Optional o = + SnapshotLifecycleTask.getSnapPolicyMetadata(SnapshotLifecycleService.getJobId(slpm), state); + + assertTrue("the policy metadata should be retrieved from the cluster state", o.isPresent()); + assertThat(o.get(), equalTo(slpm)); + + assertFalse(SnapshotLifecycleTask.getSnapPolicyMetadata("bad-jobid", state).isPresent()); + } + + public void testSkipCreatingSnapshotWhenJobDoesNotMatch() { + final String id = randomAlphaOfLength(4); + final SnapshotLifecyclePolicyMetadata slpm = makePolicyMeta(id); + final SnapshotLifecycleMetadata meta = new SnapshotLifecycleMetadata(Collections.singletonMap(id, slpm)); + + final ClusterState state = ClusterState.builder(new ClusterName("test")) + .metaData(MetaData.builder() + .putCustom(SnapshotLifecycleMetadata.TYPE, meta) + .build()) + .build(); + + final ThreadPool threadPool = new TestThreadPool("test"); + try (ClusterService clusterService = ClusterServiceUtils.createClusterService(state, threadPool); + NoOpClient client = new NoOpClient(threadPool)) { + SnapshotLifecycleTask task = new SnapshotLifecycleTask(client, clusterService, (pMeta, req) -> { + fail("should not have tried to take a snapshot"); + }); + + // Trigger the event, but since the job name does not match, it should + // not run the function to create a snapshot + task.triggered(new SchedulerEngine.Event("nonexistent-job", System.currentTimeMillis(), System.currentTimeMillis())); + } + + threadPool.shutdownNow(); + } + + public void testCreateSnapshotOnTrigger() { + final String id = randomAlphaOfLength(4); + final SnapshotLifecyclePolicyMetadata slpm = makePolicyMeta(id); + final SnapshotLifecycleMetadata meta = new SnapshotLifecycleMetadata(Collections.singletonMap(id, slpm)); + + final ClusterState state = ClusterState.builder(new ClusterName("test")) + .metaData(MetaData.builder() + .putCustom(SnapshotLifecycleMetadata.TYPE, meta) + .build()) + .build(); + + final ThreadPool threadPool = new TestThreadPool("test"); + final String createSnapResponse = "{" + + " \"snapshot\" : {" + + " \"snapshot\" : \"snapshot_1\"," + + " \"uuid\" : \"bcP3ClgCSYO_TP7_FCBbBw\"," + + " \"version_id\" : " + Version.CURRENT.id + "," + + " \"version\" : \"" + Version.CURRENT + "\"," + + " \"indices\" : [ ]," + + " \"include_global_state\" : true," + + " \"state\" : \"SUCCESS\"," + + " \"start_time\" : \"2019-03-19T22:19:53.542Z\"," + + " \"start_time_in_millis\" : 1553033993542," + + " \"end_time\" : \"2019-03-19T22:19:53.567Z\"," + + " \"end_time_in_millis\" : 1553033993567," + + " \"duration_in_millis\" : 25," + + " \"failures\" : [ ]," + + " \"shards\" : {" + + " \"total\" : 0," + + " \"failed\" : 0," + + " \"successful\" : 0" + + " }" + + " }" + + "}"; + + final AtomicBoolean called = new AtomicBoolean(false); + try (ClusterService clusterService = ClusterServiceUtils.createClusterService(state, threadPool); + // This verifying client will verify that we correctly invoked + // client.admin().createSnapshot(...) with the appropriate + // request. It also returns a mock real response + VerifyingClient client = new VerifyingClient(threadPool, + (action, request, listener) -> { + assertFalse(called.getAndSet(true)); + assertThat(action, instanceOf(CreateSnapshotAction.class)); + assertThat(request, instanceOf(CreateSnapshotRequest.class)); + + CreateSnapshotRequest req = (CreateSnapshotRequest) request; + + SnapshotLifecyclePolicy policy = slpm.getPolicy(); + assertThat(req.snapshot(), startsWith(policy.getName() + "-")); + assertThat(req.repository(), equalTo(policy.getRepository())); + if (req.indices().length > 0) { + assertThat(Arrays.asList(req.indices()), equalTo(policy.getConfig().get("indices"))); + } + boolean globalState = policy.getConfig().get("include_global_state") == null || + Boolean.parseBoolean((String) policy.getConfig().get("include_global_state")); + assertThat(req.includeGlobalState(), equalTo(globalState)); + + try { + return CreateSnapshotResponse.fromXContent(createParser(JsonXContent.jsonXContent, createSnapResponse)); + } catch (IOException e) { + fail("failed to parse snapshot response"); + return null; + } + })) { + + SnapshotLifecycleTask task = new SnapshotLifecycleTask(client, clusterService); + // Trigger the event with a matching job name for the policy + task.triggered(new SchedulerEngine.Event(SnapshotLifecycleService.getJobId(slpm), + System.currentTimeMillis(), System.currentTimeMillis())); + + assertTrue("snapshot should be triggered once", called.get()); + } + + threadPool.shutdownNow(); + } + + /** + * A client that delegates to a verifying function for action/request/listener + */ + public static class VerifyingClient extends NoOpClient { + + private final TriFunction, ActionRequest, ActionListener, ActionResponse> verifier; + + VerifyingClient(ThreadPool threadPool, + TriFunction, ActionRequest, ActionListener, ActionResponse> verifier) { + super(threadPool); + this.verifier = verifier; + } + + @Override + @SuppressWarnings("unchecked") + protected void doExecute(Action action, + Request request, + ActionListener listener) { + listener.onResponse((Response) verifier.apply(action, request, listener)); + } + } + + private SnapshotLifecyclePolicyMetadata makePolicyMeta(final String id) { + SnapshotLifecyclePolicy policy = SnapshotLifecycleServiceTests.createPolicy(id); + Map headers = new HashMap<>(); + headers.put("X-Opaque-ID", randomAlphaOfLength(4)); + return new SnapshotLifecyclePolicyMetadata(policy, headers, 1, 1); + } +}