diff --git a/docs/reference/ilm/apis/explain.asciidoc b/docs/reference/ilm/apis/explain.asciidoc index e63a1db2f5d8a..b8d92debbf309 100644 --- a/docs/reference/ilm/apis/explain.asciidoc +++ b/docs/reference/ilm/apis/explain.asciidoc @@ -122,7 +122,8 @@ that the index is managed and in the `new` phase: "action": "complete" "action_time_millis": 1538475653317, <8> "step": "complete", - "step_time_millis": 1538475653317 <9> + "step_time_millis": 1538475653317, <9> + "actions_order_version": 2 <10> } } } @@ -141,6 +142,9 @@ the index via the `max_age`) <7> When the index entered the current phase <8> When the index entered the current action <9> When the index entered the current step +<10> The version of the actions order this phases is executing. Once an index enters a phase +it will execute the actions in the order they were defined when entering the phase, regardless +of any new order having been made available in the system after a potential upgrade. Once the policy is running on the index, the response includes a `phase_execution` object that shows the definition of the current phase. @@ -182,7 +186,8 @@ phase completes. "version": 3, <2> "modified_date": "2018-10-15T13:21:41.576Z", <3> "modified_date_in_millis": 1539609701576 <4> - } + }, + "actions_order_version": 2 } } } @@ -247,7 +252,8 @@ information for the step that's being performed on the index. "version": 2, "modified_date": "2018-10-15T13:20:02.489Z", "modified_date_in_millis": 1539609602489 - } + }, + "actions_order_version": 2 } } } @@ -308,7 +314,8 @@ the case. "version": 3, "modified_date": "2018-10-15T13:21:41.576Z", "modified_date_in_millis": 1539609701576 - } + }, + "actions_order_version": 2 } } } diff --git a/docs/reference/ilm/error-handling.asciidoc b/docs/reference/ilm/error-handling.asciidoc index 9b9ccdfc7dff7..1b4b6479ce697 100644 --- a/docs/reference/ilm/error-handling.asciidoc +++ b/docs/reference/ilm/error-handling.asciidoc @@ -97,6 +97,7 @@ Which returns the following information: "version" : 1, "modified_date_in_millis" : 1541717264230 } + "actions_order_version": 2 } } } diff --git a/docs/reference/ilm/ilm-tutorial.asciidoc b/docs/reference/ilm/ilm-tutorial.asciidoc index 519eb03b1e7dd..1e0e20b68fa05 100644 --- a/docs/reference/ilm/ilm-tutorial.asciidoc +++ b/docs/reference/ilm/ilm-tutorial.asciidoc @@ -243,8 +243,9 @@ is met. } }, "version": 1, - "modified_date_in_millis": 1539609701576 - } + "modified_date_in_millis": 1539609701576, + }, + "actions_order_version": 2 } } } diff --git a/server/src/main/java/org/elasticsearch/TransportVersion.java b/server/src/main/java/org/elasticsearch/TransportVersion.java index 72978453ff8b1..3471ca522244a 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersion.java +++ b/server/src/main/java/org/elasticsearch/TransportVersion.java @@ -161,9 +161,11 @@ private static TransportVersion registerTransportVersion(int id, String uniqueId public static final TransportVersion V_8_500_037 = registerTransportVersion(8_500_037, "d76a4f22-8878-43e0-acfa-15e452195fa7"); public static final TransportVersion V_8_500_038 = registerTransportVersion(8_500_038, "9ef93580-feae-409f-9989-b49e411ca7a9"); public static final TransportVersion V_8_500_039 = registerTransportVersion(8_500_039, "c23722d7-6139-4cf2-b8a1-600fbd4ec359"); + // Introduced for the actions_order_version in the explain ILM API + public static final TransportVersion V_8_500_040 = registerTransportVersion(8_500_040, "E3C215E9-6F54-465A-A31A-CBC4A65D649B"); private static class CurrentHolder { - private static final TransportVersion CURRENT = findCurrent(V_8_500_039); + private static final TransportVersion CURRENT = findCurrent(V_8_500_040); // finds the pluggable current version, or uses the given fallback private static TransportVersion findCurrent(TransportVersion fallback) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/LifecycleExecutionState.java b/server/src/main/java/org/elasticsearch/cluster/metadata/LifecycleExecutionState.java index 497b0991517d3..dfb4dfb423790 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/LifecycleExecutionState.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/LifecycleExecutionState.java @@ -36,9 +36,16 @@ public record LifecycleExecutionState( String snapshotName, String shrinkIndexName, String snapshotIndexName, - String downsampleIndexName + String downsampleIndexName, + Integer actionsOrderVersion ) { + public LifecycleExecutionState { + if (actionsOrderVersion == null) { + actionsOrderVersion = 1; + } + } + public static final String ILM_CUSTOM_METADATA_KEY = "ilm"; private static final String PHASE = "phase"; @@ -58,6 +65,7 @@ public record LifecycleExecutionState( private static final String SNAPSHOT_INDEX_NAME = "snapshot_index_name"; private static final String SHRINK_INDEX_NAME = "shrink_index_name"; private static final String DOWNSAMPLE_INDEX_NAME = "rollup_index_name"; + private static final String ACTIONS_ORDER_VERSION = "actions_order_version"; public static final LifecycleExecutionState EMPTY_STATE = LifecycleExecutionState.builder().build(); @@ -82,7 +90,8 @@ public static Builder builder(LifecycleExecutionState state) { .setShrinkIndexName(state.shrinkIndexName) .setSnapshotIndexName(state.snapshotIndexName) .setDownsampleIndexName(state.downsampleIndexName) - .setStepTime(state.stepTime); + .setStepTime(state.stepTime) + .setActionsOrderVersion(state.actionsOrderVersion); } public static LifecycleExecutionState fromCustomMetadata(Map customData) { @@ -191,6 +200,10 @@ public static LifecycleExecutionState fromCustomMetadata(Map cus if (downsampleIndexName != null) { builder.setDownsampleIndexName(downsampleIndexName); } + String actionsOrderVersion = customData.get(ACTIONS_ORDER_VERSION); + if (actionsOrderVersion != null) { + builder.setActionsOrderVersion(Integer.parseInt(actionsOrderVersion)); + } return builder.build(); } @@ -253,6 +266,9 @@ public Map asMap() { if (downsampleIndexName != null) { result.put(DOWNSAMPLE_INDEX_NAME, downsampleIndexName); } + if (actionsOrderVersion != null) { + result.put(ACTIONS_ORDER_VERSION, String.valueOf(actionsOrderVersion)); + } return Collections.unmodifiableMap(result); } @@ -274,6 +290,7 @@ public static class Builder { private String shrinkIndexName; private String snapshotIndexName; private String downsampleIndexName; + private Integer actionsOrderVersion; public Builder setPhase(String phase) { this.phase = phase; @@ -360,6 +377,11 @@ public Builder setDownsampleIndexName(String downsampleIndexName) { return this; } + public Builder setActionsOrderVersion(Integer actionsOrderVersion) { + this.actionsOrderVersion = actionsOrderVersion; + return this; + } + public LifecycleExecutionState build() { return new LifecycleExecutionState( phase, @@ -378,7 +400,8 @@ public LifecycleExecutionState build() { snapshotName, shrinkIndexName, snapshotIndexName, - downsampleIndexName + downsampleIndexName, + actionsOrderVersion ); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java index 879db231a99e3..8efa684bb9ec3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java @@ -26,6 +26,8 @@ import java.util.function.Supplier; import java.util.stream.Stream; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.VERSION_ONE; + public class IndexLifecycleExplainResponse implements ToXContentObject, Writeable { private static final ParseField INDEX_FIELD = new ParseField("index"); @@ -54,6 +56,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl private static final ParseField REPOSITORY_NAME = new ParseField("repository_name"); private static final ParseField SHRINK_INDEX_NAME = new ParseField("shrink_index_name"); private static final ParseField SNAPSHOT_NAME = new ParseField("snapshot_name"); + private static ParseField ACTIONS_ORDER_VERSION_FIELD = new ParseField("actions_order_version"); public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "index_lifecycle_explain_response", @@ -76,9 +79,10 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl (String) a[17], (String) a[18], (BytesReference) a[11], - (PhaseExecutionInfo) a[12] + (PhaseExecutionInfo) a[12], // a[13] == "age" // a[20] == "time_since_index_creation" + (Integer) a[21] ) ); static { @@ -111,6 +115,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), SHRINK_INDEX_NAME); PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), INDEX_CREATION_DATE_MILLIS_FIELD); PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), TIME_SINCE_INDEX_CREATION_FIELD); + PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), ACTIONS_ORDER_VERSION_FIELD); } private final String index; @@ -132,6 +137,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl private final String repositoryName; private final String snapshotName; private final String shrinkIndexName; + private final Integer actionsOrderVersion; Supplier nowSupplier = System::currentTimeMillis; // Can be changed for testing @@ -153,7 +159,8 @@ public static IndexLifecycleExplainResponse newManagedIndexResponse( String snapshotName, String shrinkIndexName, BytesReference stepInfo, - PhaseExecutionInfo phaseExecutionInfo + PhaseExecutionInfo phaseExecutionInfo, + Integer actionsOrderVersion ) { return new IndexLifecycleExplainResponse( index, @@ -174,7 +181,8 @@ public static IndexLifecycleExplainResponse newManagedIndexResponse( snapshotName, shrinkIndexName, stepInfo, - phaseExecutionInfo + phaseExecutionInfo, + actionsOrderVersion ); } @@ -198,6 +206,7 @@ public static IndexLifecycleExplainResponse newUnmanagedIndexResponse(String ind null, null, null, + null, null ); } @@ -221,7 +230,8 @@ private IndexLifecycleExplainResponse( String snapshotName, String shrinkIndexName, BytesReference stepInfo, - PhaseExecutionInfo phaseExecutionInfo + PhaseExecutionInfo phaseExecutionInfo, + Integer actionsOrderVersion ) { if (managedByILM) { if (policyName == null) { @@ -283,6 +293,7 @@ private IndexLifecycleExplainResponse( this.repositoryName = repositoryName; this.snapshotName = snapshotName; this.shrinkIndexName = shrinkIndexName; + this.actionsOrderVersion = actionsOrderVersion; } public IndexLifecycleExplainResponse(StreamInput in) throws IOException { @@ -310,6 +321,11 @@ public IndexLifecycleExplainResponse(StreamInput in) throws IOException { } else { indexCreationDate = null; } + if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_500_040)) { + actionsOrderVersion = in.readOptionalInt(); + } else { + actionsOrderVersion = VERSION_ONE; + } } else { policyName = null; lifecycleDate = null; @@ -328,6 +344,7 @@ public IndexLifecycleExplainResponse(StreamInput in) throws IOException { snapshotName = null; shrinkIndexName = null; indexCreationDate = null; + actionsOrderVersion = null; } } @@ -355,6 +372,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) { out.writeOptionalLong(indexCreationDate); } + if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_500_040)) { + out.writeOptionalInt(actionsOrderVersion); + } } } @@ -450,6 +470,10 @@ public String getShrinkIndexName() { return shrinkIndexName; } + public Integer getActionsOrderVersion() { + return actionsOrderVersion; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -514,6 +538,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (phaseExecutionInfo != null) { builder.field(PHASE_EXECUTION_INFO.getPreferredName(), phaseExecutionInfo); } + if (actionsOrderVersion != null) { + builder.field(ACTIONS_ORDER_VERSION_FIELD.getPreferredName(), actionsOrderVersion); + } } builder.endObject(); return builder; @@ -540,7 +567,8 @@ public int hashCode() { snapshotName, shrinkIndexName, stepInfo, - phaseExecutionInfo + phaseExecutionInfo, + actionsOrderVersion ); } @@ -571,7 +599,8 @@ public boolean equals(Object obj) { && Objects.equals(snapshotName, other.snapshotName) && Objects.equals(shrinkIndexName, other.shrinkIndexName) && Objects.equals(stepInfo, other.stepInfo) - && Objects.equals(phaseExecutionInfo, other.phaseExecutionInfo); + && Objects.equals(phaseExecutionInfo, other.phaseExecutionInfo) + && Objects.equals(actionsOrderVersion, other.actionsOrderVersion); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicy.java index 4beebbe971440..f48ad0b48c5c9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicy.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicy.java @@ -206,10 +206,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws * * @param client The Elasticsearch Client to use during execution of {@link AsyncActionStep} * and {@link AsyncWaitStep} steps. + * @param actionsOrderVersion the version of the actions order to use * @param licenseState The license state to use in actions and steps * @return The list of {@link Step} objects in order of their execution. */ - public List toSteps(Client client, XPackLicenseState licenseState) { + public List toSteps(Client client, int actionsOrderVersion, XPackLicenseState licenseState) { List steps = new ArrayList<>(); List orderedPhases = type.getOrderedPhases(phases); ListIterator phaseIterator = orderedPhases.listIterator(orderedPhases.size()); @@ -233,7 +234,7 @@ public List toSteps(Client client, XPackLicenseState licenseState) { } phase = previousPhase; - List orderedActions = type.getOrderedActions(phase); + List orderedActions = type.getOrderedActions(phase, actionsOrderVersion); ListIterator actionIterator = orderedActions.listIterator(orderedActions.size()); // add steps for each action, in reverse while (actionIterator.hasPrevious()) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleType.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleType.java index e926a0821a487..7cd148188b40d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleType.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleType.java @@ -19,7 +19,7 @@ public interface LifecycleType extends NamedWriteable { */ List getOrderedPhases(Map phases); - List getOrderedActions(Phase phase); + List getOrderedActions(Phase phase, int orderVersion); /** * validates whether the specified phases are valid for this @@ -31,4 +31,6 @@ public interface LifecycleType extends NamedWriteable { * if a specific phase or lack of a specific phase is invalid. */ void validate(Collection phases); + + int getLatestActionsOrderVersion(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PhaseCacheManagement.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PhaseCacheManagement.java index 26966195989bb..c84561cd7c328 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PhaseCacheManagement.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PhaseCacheManagement.java @@ -79,6 +79,7 @@ public static void refreshPhaseDefinition( updatedPolicy.getModifiedDate() ); + // we're not updating the actions order version here as we're still executing the same phase (and the step keys are identical) LifecycleExecutionState newExState = LifecycleExecutionState.builder(currentExState) .setPhaseDefinition(Strings.toString(pei, false, false)) .build(); @@ -192,7 +193,9 @@ public static boolean isIndexPhaseDefinitionUpdatable( final Step.StepKey currentStepKey = Step.getCurrentStepKey(executionState); final String currentPhase = currentStepKey.phase(); - final Set newStepKeys = newPolicy.toSteps(client, licenseState) + // reading the same order version of the new policy as if we refresh the cached phase, being the same phase + // as it's currently executing, we won't update the actions order version + final Set newStepKeys = newPolicy.toSteps(client, executionState.actionsOrderVersion(), licenseState) .stream() .map(Step::getKey) .collect(Collectors.toCollection(LinkedHashSet::new)); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleActionsRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleActionsRegistry.java new file mode 100644 index 0000000000000..8d504ecd09da5 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleActionsRegistry.java @@ -0,0 +1,143 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.ilm; + +import org.elasticsearch.common.util.set.Sets; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Stream; + +/** + * Defines a record of the actions that are allowed for the timeseries lifecycle. + * The order of actions is versioned, newer versions must contain all previous defined actions and are + * allowed to make additive changes (we do not support removing actions, but actions to be removed + * must be converted to no-ops) + */ +public final class TimeseriesLifecycleActionsRegistry { + + public static final int VERSION_ONE = 1; + // moves the downsample action after the migrate action in the warm and cold phases + public static final int VERSION_TWO = 2; + public static final int CURRENT_VERSION = VERSION_TWO; + + public static final String HOT_PHASE = "hot"; + public static final String WARM_PHASE = "warm"; + public static final String COLD_PHASE = "cold"; + public static final String FROZEN_PHASE = "frozen"; + public static final String DELETE_PHASE = "delete"; + + public static final Map> ORDERED_VALID_HOT_ACTIONS = Map.of( + VERSION_ONE, + Stream.of( + SetPriorityAction.NAME, + UnfollowAction.NAME, + RolloverAction.NAME, + ReadOnlyAction.NAME, + DownsampleAction.NAME, + ShrinkAction.NAME, + ForceMergeAction.NAME, + SearchableSnapshotAction.NAME + ).filter(Objects::nonNull).toList(), + VERSION_TWO, + Stream.of( + SetPriorityAction.NAME, + UnfollowAction.NAME, + RolloverAction.NAME, + ReadOnlyAction.NAME, + DownsampleAction.NAME, + ShrinkAction.NAME, + ForceMergeAction.NAME, + SearchableSnapshotAction.NAME + ).filter(Objects::nonNull).toList() + ); + + public static final Map> ORDERED_VALID_WARM_ACTIONS = Map.of( + VERSION_ONE, + Stream.of( + SetPriorityAction.NAME, + UnfollowAction.NAME, + ReadOnlyAction.NAME, + DownsampleAction.NAME, + AllocateAction.NAME, + MigrateAction.NAME, + ShrinkAction.NAME, + ForceMergeAction.NAME + ).filter(Objects::nonNull).toList(), + VERSION_TWO, + Stream.of( + SetPriorityAction.NAME, + UnfollowAction.NAME, + ReadOnlyAction.NAME, + AllocateAction.NAME, + MigrateAction.NAME, + DownsampleAction.NAME, + ShrinkAction.NAME, + ForceMergeAction.NAME + ).filter(Objects::nonNull).toList() + ); + + public static final Map> ORDERED_VALID_COLD_ACTIONS = Map.of( + VERSION_ONE, + Stream.of( + SetPriorityAction.NAME, + UnfollowAction.NAME, + ReadOnlyAction.NAME, + DownsampleAction.NAME, + SearchableSnapshotAction.NAME, + AllocateAction.NAME, + MigrateAction.NAME, + FreezeAction.NAME + ).filter(Objects::nonNull).toList(), + VERSION_TWO, + Stream.of( + SetPriorityAction.NAME, + UnfollowAction.NAME, + ReadOnlyAction.NAME, + SearchableSnapshotAction.NAME, + AllocateAction.NAME, + MigrateAction.NAME, + DownsampleAction.NAME, + FreezeAction.NAME + ).filter(Objects::nonNull).toList() + ); + + public static final Map> ORDERED_VALID_FROZEN_ACTIONS = Map.of( + VERSION_ONE, + List.of(UnfollowAction.NAME, SearchableSnapshotAction.NAME), + VERSION_TWO, + List.of(UnfollowAction.NAME, SearchableSnapshotAction.NAME) + ); + public static final Map> ORDERED_VALID_DELETE_ACTIONS = Map.of( + VERSION_ONE, + List.of(WaitForSnapshotAction.NAME, DeleteAction.NAME), + VERSION_TWO, + List.of(WaitForSnapshotAction.NAME, DeleteAction.NAME) + ); + + static final Set VALID_HOT_ACTIONS = Sets.newHashSet(ORDERED_VALID_HOT_ACTIONS.get(CURRENT_VERSION)); + static final Set VALID_WARM_ACTIONS = Sets.newHashSet(ORDERED_VALID_WARM_ACTIONS.get(CURRENT_VERSION)); + static final Set VALID_COLD_ACTIONS = Sets.newHashSet(ORDERED_VALID_COLD_ACTIONS.get(CURRENT_VERSION)); + static final Set VALID_FROZEN_ACTIONS = Sets.newHashSet(ORDERED_VALID_FROZEN_ACTIONS.get(CURRENT_VERSION)); + static final Set VALID_DELETE_ACTIONS = Sets.newHashSet(ORDERED_VALID_DELETE_ACTIONS.get(CURRENT_VERSION)); + + static final Map> ALLOWED_ACTIONS = Map.of( + HOT_PHASE, + VALID_HOT_ACTIONS, + WARM_PHASE, + VALID_WARM_ACTIONS, + COLD_PHASE, + VALID_COLD_ACTIONS, + DELETE_PHASE, + VALID_DELETE_ACTIONS, + FROZEN_PHASE, + VALID_FROZEN_ACTIONS + ); +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java index e378cbb171a38..911dd5443ccc8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java @@ -28,9 +28,15 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; import static java.util.stream.Collectors.toList; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.ALLOWED_ACTIONS; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.CURRENT_VERSION; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.ORDERED_VALID_COLD_ACTIONS; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.ORDERED_VALID_DELETE_ACTIONS; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.ORDERED_VALID_FROZEN_ACTIONS; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.ORDERED_VALID_HOT_ACTIONS; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.ORDERED_VALID_WARM_ACTIONS; /** * Represents the lifecycle of an index from creation to deletion. A @@ -52,58 +58,6 @@ public class TimeseriesLifecycleType implements LifecycleType { static final String DELETE_PHASE = "delete"; public static final List ORDERED_VALID_PHASES = List.of(HOT_PHASE, WARM_PHASE, COLD_PHASE, FROZEN_PHASE, DELETE_PHASE); - public static final List ORDERED_VALID_HOT_ACTIONS = Stream.of( - SetPriorityAction.NAME, - UnfollowAction.NAME, - RolloverAction.NAME, - ReadOnlyAction.NAME, - DownsampleAction.NAME, - ShrinkAction.NAME, - ForceMergeAction.NAME, - SearchableSnapshotAction.NAME - ).filter(Objects::nonNull).toList(); - public static final List ORDERED_VALID_WARM_ACTIONS = Stream.of( - SetPriorityAction.NAME, - UnfollowAction.NAME, - ReadOnlyAction.NAME, - DownsampleAction.NAME, - AllocateAction.NAME, - MigrateAction.NAME, - ShrinkAction.NAME, - ForceMergeAction.NAME - ).filter(Objects::nonNull).toList(); - public static final List ORDERED_VALID_COLD_ACTIONS = Stream.of( - SetPriorityAction.NAME, - UnfollowAction.NAME, - ReadOnlyAction.NAME, - DownsampleAction.NAME, - SearchableSnapshotAction.NAME, - AllocateAction.NAME, - MigrateAction.NAME, - FreezeAction.NAME - ).filter(Objects::nonNull).toList(); - public static final List ORDERED_VALID_FROZEN_ACTIONS = List.of(UnfollowAction.NAME, SearchableSnapshotAction.NAME); - public static final List ORDERED_VALID_DELETE_ACTIONS = List.of(WaitForSnapshotAction.NAME, DeleteAction.NAME); - - static final Set VALID_HOT_ACTIONS = Sets.newHashSet(ORDERED_VALID_HOT_ACTIONS); - static final Set VALID_WARM_ACTIONS = Sets.newHashSet(ORDERED_VALID_WARM_ACTIONS); - static final Set VALID_COLD_ACTIONS = Sets.newHashSet(ORDERED_VALID_COLD_ACTIONS); - static final Set VALID_FROZEN_ACTIONS = Sets.newHashSet(ORDERED_VALID_FROZEN_ACTIONS); - static final Set VALID_DELETE_ACTIONS = Sets.newHashSet(ORDERED_VALID_DELETE_ACTIONS); - - private static final Map> ALLOWED_ACTIONS = Map.of( - HOT_PHASE, - VALID_HOT_ACTIONS, - WARM_PHASE, - VALID_WARM_ACTIONS, - COLD_PHASE, - VALID_COLD_ACTIONS, - DELETE_PHASE, - VALID_DELETE_ACTIONS, - FROZEN_PHASE, - VALID_FROZEN_ACTIONS - ); - static final Set HOT_ACTIONS_THAT_REQUIRE_ROLLOVER = Set.of( ReadOnlyAction.NAME, ShrinkAction.NAME, @@ -177,14 +131,40 @@ public static boolean shouldInjectMigrateStepForPhase(Phase phase) { return true; } - public List getOrderedActions(Phase phase) { + @Override + public List getOrderedActions(Phase phase, int actionsOrderVersion) { + if (actionsOrderVersion > getLatestActionsOrderVersion()) { + throw new IllegalArgumentException( + "invalid actions order requested [" + actionsOrderVersion + "]. Latest version is [" + getLatestActionsOrderVersion() + "]" + ); + } Map actions = phase.getActions(); return switch (phase.getName()) { - case HOT_PHASE -> ORDERED_VALID_HOT_ACTIONS.stream().map(actions::get).filter(Objects::nonNull).collect(toList()); - case WARM_PHASE -> ORDERED_VALID_WARM_ACTIONS.stream().map(actions::get).filter(Objects::nonNull).collect(toList()); - case COLD_PHASE -> ORDERED_VALID_COLD_ACTIONS.stream().map(actions::get).filter(Objects::nonNull).collect(toList()); - case FROZEN_PHASE -> ORDERED_VALID_FROZEN_ACTIONS.stream().map(actions::get).filter(Objects::nonNull).collect(toList()); - case DELETE_PHASE -> ORDERED_VALID_DELETE_ACTIONS.stream().map(actions::get).filter(Objects::nonNull).collect(toList()); + case HOT_PHASE -> ORDERED_VALID_HOT_ACTIONS.get(actionsOrderVersion) + .stream() + .map(actions::get) + .filter(Objects::nonNull) + .collect(toList()); + case WARM_PHASE -> ORDERED_VALID_WARM_ACTIONS.get(actionsOrderVersion) + .stream() + .map(actions::get) + .filter(Objects::nonNull) + .collect(toList()); + case COLD_PHASE -> ORDERED_VALID_COLD_ACTIONS.get(actionsOrderVersion) + .stream() + .map(actions::get) + .filter(Objects::nonNull) + .collect(toList()); + case FROZEN_PHASE -> ORDERED_VALID_FROZEN_ACTIONS.get(actionsOrderVersion) + .stream() + .map(actions::get) + .filter(Objects::nonNull) + .collect(toList()); + case DELETE_PHASE -> ORDERED_VALID_DELETE_ACTIONS.get(actionsOrderVersion) + .stream() + .map(actions::get) + .filter(Objects::nonNull) + .collect(toList()); default -> throw new IllegalArgumentException("lifecycle type [" + TYPE + "] does not support phase [" + phase.getName() + "]"); }; } @@ -254,6 +234,11 @@ && definesAllocationRules((AllocateAction) phase.getActions().get(AllocateAction validateDownsamplingIntervals(phases); } + @Override + public int getLatestActionsOrderVersion() { + return CURRENT_VERSION; + } + static void validateActionsFollowingSearchableSnapshot(Collection phases) { // invalid configurations can occur if searchable_snapshot is defined in the `hot` phase, with subsequent invalid actions // being defined in the warm/cold/frozen phases, or if it is defined in the `cold` phase with subsequent invalid actions diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponseTests.java index a12b4ff75ee39..0c2a71973f916 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponseTests.java @@ -27,6 +27,8 @@ import java.util.Objects; import java.util.function.Supplier; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.CURRENT_VERSION; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.VERSION_ONE; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -73,7 +75,8 @@ private static IndexLifecycleExplainResponse randomManagedIndexExplainResponse() stepNull ? null : randomAlphaOfLength(10), stepNull ? null : randomAlphaOfLength(10), randomBoolean() ? null : new BytesArray(new RandomStepInfo(() -> randomAlphaOfLength(10)).toString()), - randomBoolean() ? null : PhaseExecutionInfoTests.randomPhaseExecutionInfo("") + randomBoolean() ? null : PhaseExecutionInfoTests.randomPhaseExecutionInfo(""), + randomIntBetween(VERSION_ONE, CURRENT_VERSION) ); } @@ -99,7 +102,8 @@ public void testInvalidStepDetails() { randomBoolean() ? null : randomAlphaOfLength(10), randomBoolean() ? null : randomAlphaOfLength(10), randomBoolean() ? null : new BytesArray(new RandomStepInfo(() -> randomAlphaOfLength(10)).toString()), - randomBoolean() ? null : PhaseExecutionInfoTests.randomPhaseExecutionInfo("") + randomBoolean() ? null : PhaseExecutionInfoTests.randomPhaseExecutionInfo(""), + randomIntBetween(VERSION_ONE, CURRENT_VERSION) ) ); assertThat(exception.getMessage(), startsWith("managed index response must have complete step details")); @@ -132,7 +136,8 @@ public void testIndexAges() { null, null, null, - null + null, + randomIntBetween(VERSION_ONE, CURRENT_VERSION) ); assertThat(managedExplainResponse.getLifecycleDate(), is(notNullValue())); Long now = 1_000_000L; @@ -192,8 +197,9 @@ protected IndexLifecycleExplainResponse mutateInstance(IndexLifecycleExplainResp boolean managed = instance.managedByILM(); BytesReference stepInfo = instance.getStepInfo(); PhaseExecutionInfo phaseExecutionInfo = instance.getPhaseExecutionInfo(); + Integer actionsOrderVersion = instance.getActionsOrderVersion(); if (managed) { - switch (between(0, 14)) { + switch (between(0, 15)) { case 0: index = index + randomAlphaOfLengthBetween(1, 5); break; @@ -259,6 +265,9 @@ protected IndexLifecycleExplainResponse mutateInstance(IndexLifecycleExplainResp case 14: shrinkIndexName = randomValueOtherThan(shrinkIndexName, () -> randomAlphaOfLengthBetween(5, 10)); break; + case 15: + actionsOrderVersion = randomValueOtherThan(actionsOrderVersion, () -> randomIntBetween(VERSION_ONE, CURRENT_VERSION)); + break; default: throw new AssertionError("Illegal randomisation branch"); } @@ -280,7 +289,8 @@ protected IndexLifecycleExplainResponse mutateInstance(IndexLifecycleExplainResp snapshotName, shrinkIndexName, stepInfo, - phaseExecutionInfo + phaseExecutionInfo, + actionsOrderVersion ); } else { return switch (between(0, 1)) { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecycleExecutionStateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecycleExecutionStateTests.java index 1758c3729e373..b008fa4e1ee01 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecycleExecutionStateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecycleExecutionStateTests.java @@ -23,8 +23,10 @@ public void testConversion() { } public void testEmptyValuesAreNotSerialized() { + // the actions order version defaults to 1 if missing LifecycleExecutionState empty = LifecycleExecutionState.builder().build(); - assertEquals(new HashMap().entrySet(), empty.asMap().entrySet()); + Map expected = Map.of("actions_order_version", "1"); + assertEquals(expected, empty.asMap()); Map originalMap = createCustomMetadata(); LifecycleExecutionState originalState = LifecycleExecutionState.fromCustomMetadata(originalMap); @@ -135,7 +137,7 @@ public void testGetCurrentStepKey() { private static LifecycleExecutionState mutate(LifecycleExecutionState toMutate) { LifecycleExecutionState.Builder newState = LifecycleExecutionState.builder(toMutate); - switch (randomIntBetween(0, 17)) { + switch (randomIntBetween(0, 18)) { case 0: newState.setPhase(randomValueOtherThan(toMutate.phase(), () -> randomAlphaOfLengthBetween(5, 20))); break; @@ -192,6 +194,9 @@ private static LifecycleExecutionState mutate(LifecycleExecutionState toMutate) newState.setFailedStepRetryCount(randomValueOtherThan(toMutate.failedStepRetryCount(), ESTestCase::randomInt)); break; case 17: + newState.setActionsOrderVersion(randomValueOtherThan(toMutate.actionsOrderVersion(), ESTestCase::randomInt)); + break; + case 18: return LifecycleExecutionState.builder().build(); default: throw new IllegalStateException("unknown randomization branch"); @@ -213,6 +218,7 @@ static Map createCustomMetadata() { long phaseTime = randomLong(); long actionTime = randomLong(); long stepTime = randomLong(); + int actionsOrderVersion = randomNonNegativeInt(); Map customMetadata = new HashMap<>(); customMetadata.put("phase", phase); @@ -232,6 +238,7 @@ static Map createCustomMetadata() { customMetadata.put("rollup_index_name", randomAlphaOfLengthBetween(5, 20)); customMetadata.put("is_auto_retryable_error", String.valueOf(randomBoolean())); customMetadata.put("failed_step_retry_count", String.valueOf(randomInt())); + customMetadata.put("actions_order_version", String.valueOf(actionsOrderVersion)); return customMetadata; } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java index ea1fe24fffea1..8eeb893c6b3aa 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java @@ -31,6 +31,12 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.CURRENT_VERSION; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.VALID_COLD_ACTIONS; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.VALID_DELETE_ACTIONS; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.VALID_FROZEN_ACTIONS; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.VALID_HOT_ACTIONS; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.VALID_WARM_ACTIONS; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.mockito.Mockito.mock; @@ -235,11 +241,11 @@ public static LifecyclePolicy randomTimeseriesLifecyclePolicy(@Nullable String l private static Function> getPhaseToValidActions() { return (phase) -> new HashSet<>(switch (phase) { - case "hot" -> TimeseriesLifecycleType.VALID_HOT_ACTIONS; - case "warm" -> TimeseriesLifecycleType.VALID_WARM_ACTIONS; - case "cold" -> TimeseriesLifecycleType.VALID_COLD_ACTIONS; - case "frozen" -> TimeseriesLifecycleType.VALID_FROZEN_ACTIONS; - case "delete" -> TimeseriesLifecycleType.VALID_DELETE_ACTIONS; + case "hot" -> VALID_HOT_ACTIONS; + case "warm" -> VALID_WARM_ACTIONS; + case "cold" -> VALID_COLD_ACTIONS; + case "frozen" -> VALID_FROZEN_ACTIONS; + case "delete" -> VALID_DELETE_ACTIONS; default -> throw new IllegalArgumentException("invalid phase [" + phase + "]"); }); } @@ -318,7 +324,7 @@ public void testFirstAndLastSteps() { lifecycleName = randomAlphaOfLengthBetween(1, 20); Map phases = new LinkedHashMap<>(); LifecyclePolicy policy = new LifecyclePolicy(TestLifecycleType.INSTANCE, lifecycleName, phases, randomMeta()); - List steps = policy.toSteps(client, null); + List steps = policy.toSteps(client, CURRENT_VERSION, null); assertThat(steps.size(), equalTo(2)); assertThat(steps.get(0), instanceOf(InitializePolicyContextStep.class)); assertThat(steps.get(0).getKey(), equalTo(new StepKey("new", "init", "init"))); @@ -339,7 +345,7 @@ public void testToStepsWithOneStep() { LifecyclePolicy policy = new LifecyclePolicy(TestLifecycleType.INSTANCE, lifecycleName, phases, randomMeta()); StepKey firstStepKey = InitializePolicyContextStep.KEY; StepKey secondStepKey = PhaseCompleteStep.finalStep("new").getKey(); - List steps = policy.toSteps(client, null); + List steps = policy.toSteps(client, CURRENT_VERSION, null); assertThat(steps.size(), equalTo(4)); assertSame(steps.get(0).getKey(), firstStepKey); assertThat(steps.get(0).getNextStepKey(), equalTo(secondStepKey)); @@ -377,7 +383,7 @@ public void testToStepsWithTwoPhases() { phases.put(secondPhase.getName(), secondPhase); LifecyclePolicy policy = new LifecyclePolicy(TestLifecycleType.INSTANCE, lifecycleName, phases, randomMeta()); - List steps = policy.toSteps(client, null); + List steps = policy.toSteps(client, CURRENT_VERSION, null); assertThat(steps.size(), equalTo(7)); assertThat(steps.get(0).getClass(), equalTo(InitializePolicyContextStep.class)); assertThat(steps.get(0).getKey(), equalTo(init.getKey())); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TestLifecycleType.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TestLifecycleType.java index 1731f7700b3ad..a2d72133d050e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TestLifecycleType.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TestLifecycleType.java @@ -14,6 +14,8 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.CURRENT_VERSION; + public class TestLifecycleType implements LifecycleType { public static final TestLifecycleType INSTANCE = new TestLifecycleType(); @@ -34,13 +36,18 @@ public void validate(Collection phases) { // always valid } + @Override + public int getLatestActionsOrderVersion() { + return CURRENT_VERSION; + } + @Override public List getOrderedPhases(Map phases) { return new ArrayList<>(phases.values()); } @Override - public List getOrderedActions(Phase phase) { + public List getOrderedActions(Phase phase, int orderVersion) { return new ArrayList<>(phase.getActions().values()); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java index 47242582d648b..d3f921d992531 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java @@ -23,21 +23,24 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.ORDERED_VALID_COLD_ACTIONS; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.ORDERED_VALID_DELETE_ACTIONS; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.ORDERED_VALID_FROZEN_ACTIONS; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.ORDERED_VALID_HOT_ACTIONS; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.ORDERED_VALID_WARM_ACTIONS; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.VALID_COLD_ACTIONS; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.VALID_DELETE_ACTIONS; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.VALID_FROZEN_ACTIONS; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.VALID_HOT_ACTIONS; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.VALID_WARM_ACTIONS; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.VERSION_ONE; import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT; import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.COLD_PHASE; import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.DELETE_PHASE; import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.FROZEN_PHASE; import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.HOT_PHASE; -import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.ORDERED_VALID_COLD_ACTIONS; -import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.ORDERED_VALID_DELETE_ACTIONS; -import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.ORDERED_VALID_HOT_ACTIONS; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.INSTANCE; import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.ORDERED_VALID_PHASES; -import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.ORDERED_VALID_WARM_ACTIONS; -import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.VALID_COLD_ACTIONS; -import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.VALID_DELETE_ACTIONS; -import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.VALID_FROZEN_ACTIONS; -import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.VALID_HOT_ACTIONS; -import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.VALID_WARM_ACTIONS; import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.WARM_PHASE; import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.validateAllSearchableSnapshotActionsUseSameRepository; import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.validateFrozenPhaseHasSearchableSnapshotAction; @@ -45,7 +48,9 @@ import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; public class TimeseriesLifecycleTypeTests extends ESTestCase { @@ -507,7 +512,10 @@ private boolean isUnfollowInjected(String phaseName, String actionName) { public void testGetOrderedActionsInvalidPhase() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> TimeseriesLifecycleType.INSTANCE.getOrderedActions(new Phase("invalid", TimeValue.ZERO, Collections.emptyMap())) + () -> TimeseriesLifecycleType.INSTANCE.getOrderedActions( + new Phase("invalid", TimeValue.ZERO, Collections.emptyMap()), + INSTANCE.getLatestActionsOrderVersion() + ) ); assertThat(exception.getMessage(), equalTo("lifecycle type [timeseries] does not support phase [invalid]")); } @@ -517,8 +525,9 @@ public void testGetOrderedActionsHot() { .map(this::getTestAction) .collect(Collectors.toMap(LifecycleAction::getWriteableName, Function.identity())); Phase hotPhase = new Phase("hot", TimeValue.ZERO, actions); - List orderedActions = TimeseriesLifecycleType.INSTANCE.getOrderedActions(hotPhase); - assertTrue(isSorted(orderedActions, LifecycleAction::getWriteableName, ORDERED_VALID_HOT_ACTIONS)); + int actionsOrderVersion = INSTANCE.getLatestActionsOrderVersion(); + List orderedActions = TimeseriesLifecycleType.INSTANCE.getOrderedActions(hotPhase, actionsOrderVersion); + assertTrue(isSorted(orderedActions, LifecycleAction::getWriteableName, ORDERED_VALID_HOT_ACTIONS.get(actionsOrderVersion))); assertThat(orderedActions.indexOf(TEST_PRIORITY_ACTION), equalTo(0)); } @@ -527,8 +536,17 @@ public void testGetOrderedActionsWarm() { .map(this::getTestAction) .collect(Collectors.toMap(LifecycleAction::getWriteableName, Function.identity())); Phase warmPhase = new Phase("warm", TimeValue.ZERO, actions); - List orderedActions = TimeseriesLifecycleType.INSTANCE.getOrderedActions(warmPhase); - assertTrue(isSorted(orderedActions, LifecycleAction::getWriteableName, ORDERED_VALID_WARM_ACTIONS)); + List orderedActions = TimeseriesLifecycleType.INSTANCE.getOrderedActions( + warmPhase, + INSTANCE.getLatestActionsOrderVersion() + ); + assertTrue( + isSorted( + orderedActions, + LifecycleAction::getWriteableName, + ORDERED_VALID_WARM_ACTIONS.get(INSTANCE.getLatestActionsOrderVersion()) + ) + ); assertThat(orderedActions.indexOf(TEST_PRIORITY_ACTION), equalTo(0)); } @@ -537,8 +555,17 @@ public void testGetOrderedActionsCold() { .map(this::getTestAction) .collect(Collectors.toMap(LifecycleAction::getWriteableName, Function.identity())); Phase coldPhase = new Phase("cold", TimeValue.ZERO, actions); - List orderedActions = TimeseriesLifecycleType.INSTANCE.getOrderedActions(coldPhase); - assertTrue(isSorted(orderedActions, LifecycleAction::getWriteableName, ORDERED_VALID_COLD_ACTIONS)); + List orderedActions = TimeseriesLifecycleType.INSTANCE.getOrderedActions( + coldPhase, + INSTANCE.getLatestActionsOrderVersion() + ); + assertTrue( + isSorted( + orderedActions, + LifecycleAction::getWriteableName, + ORDERED_VALID_COLD_ACTIONS.get(INSTANCE.getLatestActionsOrderVersion()) + ) + ); assertThat(orderedActions.indexOf(TEST_PRIORITY_ACTION), equalTo(0)); } @@ -547,8 +574,17 @@ public void testGetOrderedActionsDelete() { .map(this::getTestAction) .collect(Collectors.toMap(LifecycleAction::getWriteableName, Function.identity())); Phase deletePhase = new Phase("delete", TimeValue.ZERO, actions); - List orderedActions = TimeseriesLifecycleType.INSTANCE.getOrderedActions(deletePhase); - assertTrue(isSorted(orderedActions, LifecycleAction::getWriteableName, ORDERED_VALID_DELETE_ACTIONS)); + List orderedActions = TimeseriesLifecycleType.INSTANCE.getOrderedActions( + deletePhase, + INSTANCE.getLatestActionsOrderVersion() + ); + assertTrue( + isSorted( + orderedActions, + LifecycleAction::getWriteableName, + ORDERED_VALID_DELETE_ACTIONS.get(INSTANCE.getLatestActionsOrderVersion()) + ) + ); } public void testShouldMigrateDataToTiers() { @@ -817,6 +853,38 @@ public void testValidateFrozenPhaseHasSearchableSnapshot() { } } + public void testTimeseriesRegistryContainsAllActionsVersions() { + for (int version = INSTANCE.getLatestActionsOrderVersion(); version >= VERSION_ONE; version--) { + assertThat(ORDERED_VALID_HOT_ACTIONS.get(version), notNullValue()); + assertThat(ORDERED_VALID_WARM_ACTIONS.get(version), notNullValue()); + assertThat(ORDERED_VALID_COLD_ACTIONS.get(version), notNullValue()); + assertThat(ORDERED_VALID_FROZEN_ACTIONS.get(version), notNullValue()); + assertThat(ORDERED_VALID_DELETE_ACTIONS.get(version), notNullValue()); + } + } + + public void testTimeseriesRegistryDoesntRemoveActionsInFutureVersions() { + assertHigerVersionsContainActionsInLowerVersions(ORDERED_VALID_HOT_ACTIONS); + + assertHigerVersionsContainActionsInLowerVersions(ORDERED_VALID_WARM_ACTIONS); + + assertHigerVersionsContainActionsInLowerVersions(ORDERED_VALID_COLD_ACTIONS); + + assertHigerVersionsContainActionsInLowerVersions(ORDERED_VALID_FROZEN_ACTIONS); + + assertHigerVersionsContainActionsInLowerVersions(ORDERED_VALID_DELETE_ACTIONS); + } + + private static void assertHigerVersionsContainActionsInLowerVersions(Map> versionedActions) { + for (int i = VERSION_ONE; i <= INSTANCE.getLatestActionsOrderVersion() - 1; i++) { + List previousVersionedActions = versionedActions.get(i); + List nextVersionedActions = versionedActions.get(i + 1); + + assertThat(previousVersionedActions.size(), lessThanOrEqualTo(nextVersionedActions.size())); + assertThat(nextVersionedActions, hasItems(previousVersionedActions.toArray(new String[] {}))); + } + } + /** * checks whether an ordered list of objects (usually Phase and LifecycleAction) are found in the same * order as the ordered VALID_PHASES/VALID_HOT_ACTIONS/... lists diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ActionsOrderVersioningTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ActionsOrderVersioningTests.java new file mode 100644 index 0000000000000..984a3e78115fd --- /dev/null +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ActionsOrderVersioningTests.java @@ -0,0 +1,301 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ilm; + +import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainRequest; +import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainResponse; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.rollover.RolloverConditions; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.LifecycleExecutionState; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; +import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.ilm.DownsampleAction; +import org.elasticsearch.xpack.core.ilm.ExplainLifecycleRequest; +import org.elasticsearch.xpack.core.ilm.ExplainLifecycleResponse; +import org.elasticsearch.xpack.core.ilm.ForceMergeAction; +import org.elasticsearch.xpack.core.ilm.IndexLifecycleExplainResponse; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; +import org.elasticsearch.xpack.core.ilm.LifecycleSettings; +import org.elasticsearch.xpack.core.ilm.Phase; +import org.elasticsearch.xpack.core.ilm.PhaseCompleteStep; +import org.elasticsearch.xpack.core.ilm.RolloverAction; +import org.elasticsearch.xpack.core.ilm.WaitForIndexColorStep; +import org.elasticsearch.xpack.core.ilm.WaitForRolloverReadyStep; +import org.elasticsearch.xpack.core.ilm.action.ExplainLifecycleAction; +import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction; +import org.junit.Before; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; +import static org.elasticsearch.cluster.metadata.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY; +import static org.elasticsearch.test.NodeRoles.onlyRole; +import static org.elasticsearch.test.NodeRoles.onlyRoles; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.xpack.core.ilm.DownsampleAction.DOWNSAMPLED_INDEX_PREFIX; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.COLD_PHASE; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.CURRENT_VERSION; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.HOT_PHASE; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.VERSION_ONE; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.WARM_PHASE; +import static org.hamcrest.Matchers.is; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) +public class ActionsOrderVersioningTests extends ESIntegTestCase { + + private String policy; + private String managedIndex; + + @Before + public void refreshDataStreamAndPolicy() { + policy = "policy-" + randomAlphaOfLengthBetween(10, 15).toLowerCase(Locale.ROOT); + managedIndex = "index-" + randomAlphaOfLengthBetween(10, 15).toLowerCase(Locale.ROOT) + "-000001"; + } + + @Override + protected boolean ignoreExternalCluster() { + return true; + } + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(LocalStateCompositeXPackPlugin.class, IndexLifecycle.class); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)); + settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false); + settings.put(XPackSettings.SECURITY_ENABLED.getKey(), false); + settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false); + settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false); + settings.put(LifecycleSettings.LIFECYCLE_POLL_INTERVAL, "1s"); + settings.put(LifecycleSettings.SLM_HISTORY_INDEX_ENABLED_SETTING.getKey(), false); + settings.put(LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED, false); + return settings.build(); + } + + public static Settings hotDataContentNode(final Settings settings) { + return onlyRoles(settings, Set.of(DiscoveryNodeRole.DATA_HOT_NODE_ROLE, DiscoveryNodeRole.DATA_CONTENT_NODE_ROLE)); + } + + public static Settings warmNode(final Settings settings) { + return onlyRole(settings, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); + } + + public static Settings coldNode(final Settings settings) { + return onlyRole(settings, DiscoveryNodeRole.DATA_COLD_NODE_ROLE); + } + + public void testWarmExecutesVersionOneAndColdLatestVersion() throws Exception { + // this test makes sure that if we are in a phase with actions_order_version 1 we will execute + // the actions as ordered in version one. when transitioning to the next phase, we will check the + // lifecycle execution state to make sure we switched to the latest actions_order_version + + // this is not trivial to do in an integration test but the way we achieve is it that we take a managed index + // and update its cluster state to "move" it to the warm phase in the middle of the downsample action whilst also setting the + // actions_order_version to 1 (we chose downsample action because in version 2 it executes at a different point than in version 1) + + // after this lifecycle "teleport" into warm/downsample/wait-for-index-color (note we don't actually execute the downsampling, we + // just create the target index so that the downsample action can complete) we wait until we get to warm/complete/complete and + // assert that the downsample index is located in the warm tier (this means that the version 1 of actions order was indeed + // executed as `migrate` executes after `downsample` in version 1) + + internalCluster().startMasterOnlyNodes(1, Settings.EMPTY); + logger.info("-> starting a data_hot/data_content node"); + internalCluster().startNode(hotDataContentNode(Settings.EMPTY)); + + logger.info("-> starting a warm data node"); + String warmNodeName = internalCluster().startNode(warmNode(Settings.EMPTY)); + + logger.info("-> starting a cold data node"); + String coldNodeName = internalCluster().startNode(coldNode(Settings.EMPTY)); + + RolloverAction rolloverIlmAction = new RolloverAction(RolloverConditions.newBuilder().addMaxIndexDocsCondition(1L).build()); + Phase hotPhase = new Phase(HOT_PHASE, TimeValue.ZERO, Map.of(rolloverIlmAction.getWriteableName(), rolloverIlmAction)); + DownsampleAction warmDownsampleAction = new DownsampleAction(new DateHistogramInterval("1d")); + ForceMergeAction forceMergeAction = new ForceMergeAction(1, null); + Phase warmPhase = new Phase( + WARM_PHASE, + TimeValue.ZERO, + Map.of(warmDownsampleAction.getWriteableName(), warmDownsampleAction, forceMergeAction.getWriteableName(), forceMergeAction) + ); + Phase coldPhase = new Phase(COLD_PHASE, TimeValue.timeValueDays(365), Map.of()); + LifecyclePolicy lifecyclePolicy = new LifecyclePolicy( + policy, + Map.of(HOT_PHASE, hotPhase, WARM_PHASE, warmPhase, COLD_PHASE, coldPhase) + ); + PutLifecycleAction.Request putLifecycleRequest = new PutLifecycleAction.Request(lifecyclePolicy); + assertAcked(client().execute(PutLifecycleAction.INSTANCE, putLifecycleRequest).get()); + + String alias = "aliasName" + randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT); + createIndex(managedIndex, alias, true); + + // allow ILM to pick up the managed index (it'll idle in hot/check-rollover-ready) + assertBusy(() -> { + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); + + IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); + assertThat(indexLifecycleExplainResponse.getPhase(), is(HOT_PHASE)); + assertThat(indexLifecycleExplainResponse.getStep(), is(WaitForRolloverReadyStep.NAME)); + }, 30, TimeUnit.SECONDS); + + String downsampleIndexName = DOWNSAMPLED_INDEX_PREFIX + managedIndex; + // creating another index that'll server as the target index for the downsample action + createIndex(downsampleIndexName, alias, false); + ensureGreen(); + + // oh boy + // manipulating the lifecycle execution state for the managed index to move it into the warm phase, downsample action, + // wait-for-index-colour step, and actions order version ONE + PlainActionFuture.get( + fut -> internalCluster().getCurrentMasterNodeInstance(ClusterService.class) + .submitUnbatchedStateUpdateTask( + "move index to warm/downsample and actions order version ONE", + new ClusterStateUpdateTask() { + + @Override + public ClusterState execute(ClusterState state) { + Metadata.Builder builder = Metadata.builder(state.metadata()); + IndexMetadata originalManagedIndex = state.metadata().index(managedIndex); + LifecycleExecutionState originalIndexLifecycleState = LifecycleExecutionState.builder( + originalManagedIndex.getLifecycleExecutionState() + ) + .setPhase(WARM_PHASE) + .setAction(DownsampleAction.NAME) + .setStep(WaitForIndexColorStep.NAME) + .setDownsampleIndexName(downsampleIndexName) + // NOTE that in this version the downsample action executed _before_ the migrate action (so moving outside + // the downsample action should migrate the index to the warm tier using the migrate action) + .setActionsOrderVersion(VERSION_ONE) + .setPhaseDefinition(String.format(Locale.ROOT, """ + { + "policy" : "%s", + "phase_definition" : { + "min_age" : "0ms", + "actions" : { + "downsample": { + "fixed_interval": "1d" + } + } + }, + "version" : 1, + "modified_date_in_millis" : 1578521007076 + }""", policy)) + .build(); + + IndexMetadata.Builder managedIndexBuilder = IndexMetadata.builder(originalManagedIndex) + .putCustom(ILM_CUSTOM_METADATA_KEY, originalIndexLifecycleState.asMap()); + + builder.put(managedIndexBuilder); + return ClusterState.builder(state).metadata(builder).build(); + } + + @Override + public void onFailure(Exception e) { + logger.error(e.getMessage(), e); + fail("unable to manipulate the cluster state due to [" + e.getMessage() + "]"); + } + + @Override + public void clusterStateProcessed(ClusterState initialState, ClusterState newState) { + fut.onResponse(null); + } + } + ), + 10, + TimeUnit.SECONDS + ); + + // the downsample index should end up in WARM/COMPLETE/COMPLETE + assertBusy(() -> { + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(downsampleIndexName); + ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); + + IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(downsampleIndexName); + assertThat(indexLifecycleExplainResponse.getPhase(), is(WARM_PHASE)); + assertThat(indexLifecycleExplainResponse.getStep(), is(PhaseCompleteStep.NAME)); + + // actions order version must remain ONE until we transition to COLD + assertThat(indexLifecycleExplainResponse.getActionsOrderVersion(), is(VERSION_ONE)); + }, 30, TimeUnit.SECONDS); + + { + // let's check that the migrate action actually did migrate the downsample index to the warm phase (i.e. executed after the + // downsample action, as it should in VERSION_ONE) + ClusterAllocationExplainRequest explainDownsampleIndexShard = new ClusterAllocationExplainRequest().setIndex( + downsampleIndexName + ).setPrimary(true).setShard(0); + ClusterAllocationExplainResponse response = clusterAdmin().allocationExplain(explainDownsampleIndexShard).actionGet(); + assertThat(response.getExplanation().getCurrentNode().getName(), is(warmNodeName)); + } + + // changing the min_age for the COLD phase so we transition and assert that the new phase executes the latest actions order version + lifecyclePolicy = new LifecyclePolicy( + policy, + Map.of(HOT_PHASE, hotPhase, WARM_PHASE, warmPhase, COLD_PHASE, new Phase(COLD_PHASE, TimeValue.ZERO, Map.of())) + ); + putLifecycleRequest = new PutLifecycleAction.Request(lifecyclePolicy); + assertAcked(client().execute(PutLifecycleAction.INSTANCE, putLifecycleRequest).get()); + + assertBusy(() -> { + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(downsampleIndexName); + ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); + + IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(downsampleIndexName); + assertThat(indexLifecycleExplainResponse.getPhase(), is(COLD_PHASE)); + assertThat(indexLifecycleExplainResponse.getStep(), is(PhaseCompleteStep.NAME)); + + // when we transitioned to cold we should've run the latest actions order version + assertThat(indexLifecycleExplainResponse.getActionsOrderVersion(), is(CURRENT_VERSION)); + }, 30, TimeUnit.SECONDS); + + { + ClusterAllocationExplainRequest explainDownsampleIndexShard = new ClusterAllocationExplainRequest().setIndex( + downsampleIndexName + ).setPrimary(true).setShard(0); + ClusterAllocationExplainResponse response = clusterAdmin().allocationExplain(explainDownsampleIndexShard).actionGet(); + assertThat(response.getExplanation().getCurrentNode().getName(), is(coldNodeName)); + } + } + + private void createIndex(String indexName, String alias, boolean isWriteIndex) { + Settings settings = Settings.builder() + .put(indexSettings()) + .put(SETTING_NUMBER_OF_SHARDS, 1) + .put(SETTING_NUMBER_OF_REPLICAS, 0) + .put(LifecycleSettings.LIFECYCLE_NAME, policy) + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) + .build(); + + CreateIndexResponse res = indicesAdmin().prepareCreate(indexName).setAliases(String.format(Locale.ROOT, """ + { + "%s" : { "is_write_index": %b } + }""", alias, isWriteIndex)).setSettings(settings).get(); + assertTrue(res.isAcknowledged()); + } +} diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleRunner.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleRunner.java index efa8e67fee3c8..78a494b6ba6cc 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleRunner.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleRunner.java @@ -112,7 +112,7 @@ static Step getCurrentStep( StepKey currentStepKey = Step.getCurrentStepKey(lifecycleState); logger.trace("[{}] retrieved current step key: {}", indexMetadata.getIndex().getName(), currentStepKey); if (currentStepKey == null) { - return stepRegistry.getFirstStep(policy); + return stepRegistry.getFirstStep(policy, lifecycleState.actionsOrderVersion()); } else { return stepRegistry.getStep(indexMetadata, currentStepKey); } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransition.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransition.java index a87f2d4d2151e..4849b83561ff1 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransition.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransition.java @@ -87,13 +87,14 @@ public static void validateTransition( final Set cachedStepKeys = stepRegistry.parseStepKeysFromPhase( policyName, lifecycleState.phase(), + lifecycleState.actionsOrderVersion(), lifecycleState.phaseDefinition() ); boolean isNewStepCached = cachedStepKeys != null && cachedStepKeys.contains(newStepKey); // Always allow moving to the terminal step or to a step that's present in the cached phase, even if it doesn't exist in the policy if (isNewStepCached == false - && (stepRegistry.stepExists(policyName, newStepKey) == false + && (stepRegistry.stepExists(policyName, lifecycleState.actionsOrderVersion(), newStepKey) == false && newStepKey.equals(TerminalPolicyStep.KEY) == false && newStepKey.equals(PhaseCompleteStep.stepKey(lifecycleState.phase())) == false)) { throw new IllegalArgumentException( @@ -286,6 +287,7 @@ private static LifecycleExecutionState updateExecutionStateToStep( updatedState.setAction(newStep.action()); updatedState.setStep(newStep.name()); updatedState.setStepTime(nowAsMillis); + updatedState.setActionsOrderVersion(existingState.actionsOrderVersion()); // clear any step info or error-related settings from the current step updatedState.setFailedStep(null); @@ -296,10 +298,13 @@ private static LifecycleExecutionState updateExecutionStateToStep( if (currentStep == null || currentStep.phase().equals(newStep.phase()) == false || forcePhaseDefinitionRefresh) { final String newPhaseDefinition; final Phase nextPhase; + LifecyclePolicy policy = policyMetadata.getPolicy(); if ("new".equals(newStep.phase()) || TerminalPolicyStep.KEY.equals(newStep)) { nextPhase = null; + // when starting to manage the index we should use the latest actions order version + updatedState.setActionsOrderVersion(policy.getType().getLatestActionsOrderVersion()); } else { - nextPhase = policyMetadata.getPolicy().getPhases().get(newStep.phase()); + nextPhase = policy.getPhases().get(newStep.phase()); } PhaseExecutionInfo phaseExecutionInfo = new PhaseExecutionInfo( policyMetadata.getName(), @@ -310,6 +315,11 @@ private static LifecycleExecutionState updateExecutionStateToStep( newPhaseDefinition = Strings.toString(phaseExecutionInfo, false, false); updatedState.setPhaseDefinition(newPhaseDefinition); updatedState.setPhaseTime(nowAsMillis); + + // we want to update to the latest actions order version only when we're transitioning to a new phase + if (currentStep != null && currentStep.phase().equals(newStep.phase()) == false) { + updatedState.setActionsOrderVersion(policy.getType().getLatestActionsOrderVersion()); + } } else if (currentStep.phase().equals(InitializePolicyContextStep.INITIALIZATION_PHASE)) { // The "new" phase is the initialization phase, usually the phase // time would be set on phase transition, but since there is no @@ -355,7 +365,7 @@ public static LifecycleExecutionState moveStateToNextActionAndUpdateCachedPhase( return existingState; } - List policySteps = oldPolicy.toSteps(client, licenseState); + List policySteps = oldPolicy.toSteps(client, existingState.actionsOrderVersion(), licenseState); Optional currentStep = policySteps.stream().filter(step -> step.getKey().equals(currentStepKey)).findFirst(); if (currentStep.isPresent() == false) { diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/PolicyStepsRegistry.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/PolicyStepsRegistry.java index a8a6f211c232f..9d2da6619489f 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/PolicyStepsRegistry.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/PolicyStepsRegistry.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.DiffableUtils; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.LifecycleExecutionState; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -50,6 +51,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.VERSION_ONE; + public class PolicyStepsRegistry { private static final Logger logger = LogManager.getLogger(PolicyStepsRegistry.class); @@ -60,9 +63,11 @@ public class PolicyStepsRegistry { // keeps track of existing policies in the cluster state private final SortedMap lifecyclePolicyMap; // keeps track of what the first step in a policy is, the key is policy name - private final Map firstStepMap; + private final Map firstStepMap; // keeps track of a mapping from policy/step-name to respective Step, the key is policy name - private final Map> stepMap; + private final Map> stepMap; + + public record VersionedPolicyKey(int version, String policyName) {} // tracks an index->step cache, where the indexmetadata is also tracked for cache invalidation/eviction purposes. // for a given index, the step can be cached as long as the indexmetadata (and the policy!) hasn't changed. since @@ -75,8 +80,8 @@ public PolicyStepsRegistry(NamedXContentRegistry xContentRegistry, Client client PolicyStepsRegistry( SortedMap lifecyclePolicyMap, - Map firstStepMap, - Map> stepMap, + Map firstStepMap, + Map> stepMap, NamedXContentRegistry xContentRegistry, Client client, XPackLicenseState licenseState @@ -93,11 +98,11 @@ SortedMap getLifecyclePolicyMap() { return lifecyclePolicyMap; } - Map getFirstStepMap() { + Map getFirstStepMap() { return firstStepMap; } - Map> getStepMap() { + Map> getStepMap() { return stepMap; } @@ -131,9 +136,17 @@ public LifecyclePolicyMetadata read(StreamInput in, String key) { ); for (String deletedPolicyName : mapDiff.getDeletes()) { - lifecyclePolicyMap.remove(deletedPolicyName); - firstStepMap.remove(deletedPolicyName); - stepMap.remove(deletedPolicyName); + LifecyclePolicyMetadata policyMetadataToRemove = lifecyclePolicyMap.remove(deletedPolicyName); + if (policyMetadataToRemove != null) { + for (int actionsOrderVersion = policyMetadataToRemove.getPolicy() + .getType() + .getLatestActionsOrderVersion(); actionsOrderVersion >= VERSION_ONE; actionsOrderVersion--) { + // remove all versions for the deleted policy + VersionedPolicyKey key = new VersionedPolicyKey(actionsOrderVersion, policyMetadataToRemove.getName()); + firstStepMap.remove(key); + stepMap.remove(key); + } + } } if (mapDiff.getUpserts().isEmpty() == false) { @@ -145,20 +158,25 @@ public LifecyclePolicyMetadata read(StreamInput in, String key) { policyMetadata.getHeaders() ); lifecyclePolicyMap.put(policyMetadata.getName(), policyMetadata); - List policyAsSteps = policyMetadata.getPolicy().toSteps(policyClient, licenseState); - if (policyAsSteps.isEmpty() == false) { - firstStepMap.put(policyMetadata.getName(), policyAsSteps.get(0)); - final Map stepMapForPolicy = new LinkedHashMap<>(); - for (Step step : policyAsSteps) { - assert ErrorStep.NAME.equals(step.getKey().name()) == false : "unexpected error step in policy"; - stepMapForPolicy.put(step.getKey(), step); + LifecyclePolicy policy = policyMetadata.getPolicy(); + for (int actionsOrderVersion = policy.getType() + .getLatestActionsOrderVersion(); actionsOrderVersion >= VERSION_ONE; actionsOrderVersion--) { + List policyAsSteps = policy.toSteps(policyClient, actionsOrderVersion, licenseState); + if (policyAsSteps.isEmpty() == false) { + VersionedPolicyKey key = new VersionedPolicyKey(actionsOrderVersion, policyMetadata.getName()); + firstStepMap.put(key, policyAsSteps.get(0)); + final Map stepMapForPolicy = new LinkedHashMap<>(); + for (Step step : policyAsSteps) { + assert ErrorStep.NAME.equals(step.getKey().name()) == false : "unexpected error step in policy"; + stepMapForPolicy.put(step.getKey(), step); + } + logger.trace( + "updating cached steps for [{}] policy, new steps: {}", + policyMetadata.getName(), + stepMapForPolicy.keySet() + ); + stepMap.put(key, stepMapForPolicy); } - logger.trace( - "updating cached steps for [{}] policy, new steps: {}", - policyMetadata.getName(), - stepMapForPolicy.keySet() - ); - stepMap.put(policyMetadata.getName(), stepMapForPolicy); } } } @@ -214,7 +232,8 @@ private List getAllStepsForIndex(ClusterState state, Index index) { ClientHelper.INDEX_LIFECYCLE_ORIGIN, policyMetadata.getHeaders() ); - return policyMetadata.getPolicy().toSteps(policyClient, licenseState); + return policyMetadata.getPolicy() + .toSteps(policyClient, indexMetadata.getLifecycleExecutionState().actionsOrderVersion(), licenseState); } /** @@ -262,10 +281,12 @@ public Step.StepKey getFirstStepForPhaseAndAction(ClusterState state, Index inde * Returns null if there's a parsing error. */ @Nullable - public Set parseStepKeysFromPhase(String policy, String currentPhase, String phaseDef) { + public Set parseStepKeysFromPhase(String policy, String currentPhase, int actionsOrderVersion, String phaseDef) { try { String phaseDefNonNull = Objects.requireNonNullElse(phaseDef, InitializePolicyContextStep.INITIALIZATION_PHASE); - return parseStepsFromPhase(policy, currentPhase, phaseDefNonNull).stream().map(Step::getKey).collect(Collectors.toSet()); + return parseStepsFromPhase(policy, currentPhase, actionsOrderVersion, phaseDefNonNull).stream() + .map(Step::getKey) + .collect(Collectors.toSet()); } catch (IOException e) { logger.trace( () -> String.format( @@ -290,7 +311,8 @@ public Set parseStepKeysFromPhase(String policy, String currentPha * (note: this step exists only for BWC reasons as these days we move to the {@code PhaseCompleteStep} when reaching * the end of the phase) */ - private List parseStepsFromPhase(String policy, String currentPhase, String phaseDef) throws IOException { + private List parseStepsFromPhase(String policy, String currentPhase, int actionsOrderVersion, String phaseDef) + throws IOException { final PhaseExecutionInfo phaseExecutionInfo; LifecyclePolicyMetadata policyMetadata = lifecyclePolicyMap.get(policy); if (policyMetadata == null) { @@ -322,7 +344,7 @@ private List parseStepsFromPhase(String policy, String currentPhase, Strin ClientHelper.INDEX_LIFECYCLE_ORIGIN, lifecyclePolicyMap.get(policy).getHeaders() ); - final List steps = policyToExecute.toSteps(policyClient, licenseState); + final List steps = policyToExecute.toSteps(policyClient, actionsOrderVersion, licenseState); // Build a list of steps that correspond with the phase the index is currently in final List phaseSteps; if (steps == null) { @@ -379,14 +401,15 @@ public Step getStep(final IndexMetadata indexMetadata, final Step.StepKey stepKe } // parse phase steps from the phase definition in the index settings + LifecycleExecutionState lifecycleExecutionState = indexMetadata.getLifecycleExecutionState(); final String phaseJson = Objects.requireNonNullElse( - indexMetadata.getLifecycleExecutionState().phaseDefinition(), + lifecycleExecutionState.phaseDefinition(), InitializePolicyContextStep.INITIALIZATION_PHASE ); final List phaseSteps; try { - phaseSteps = parseStepsFromPhase(policyName, phase, phaseJson); + phaseSteps = parseStepsFromPhase(policyName, phase, lifecycleExecutionState.actionsOrderVersion(), phaseJson); } catch (IOException e) { throw new ElasticsearchException("failed to load cached steps for " + stepKey, e); } catch (XContentParseException parseErr) { @@ -416,8 +439,8 @@ public Step getStep(final IndexMetadata indexMetadata, final Step.StepKey stepKe /** * Given a policy and stepkey, return true if a step exists, false otherwise */ - public boolean stepExists(final String policy, final Step.StepKey stepKey) { - Map steps = stepMap.get(policy); + public boolean stepExists(final String policy, int actionsOrderVersion, final Step.StepKey stepKey) { + Map steps = stepMap.get(new VersionedPolicyKey(actionsOrderVersion, policy)); if (steps == null) { return false; } else { @@ -429,8 +452,8 @@ public boolean policyExists(final String policy) { return lifecyclePolicyMap.containsKey(policy); } - public Step getFirstStep(String policy) { - return firstStepMap.get(policy); + public Step getFirstStep(String policy, int actionsOrderVersion) { + return firstStepMap.get(new VersionedPolicyKey(actionsOrderVersion, policy)); } public TimeValue getIndexAgeForPhase(final String policy, final String phase) { diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java index 90438f5a753ba..42a62042b6047 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java @@ -163,7 +163,8 @@ static IndexLifecycleExplainResponse getIndexLifecycleExplainResponse( lifecycleState.snapshotName(), lifecycleState.shrinkIndexName(), stepInfoBytes, - phaseExecutionInfo + phaseExecutionInfo, + lifecycleState.actionsOrderVersion() ); } else { indexResponse = null; diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleRunnerTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleRunnerTests.java index db54e3af543c9..efcb3ff9f15f4 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleRunnerTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleRunnerTests.java @@ -64,6 +64,7 @@ import org.elasticsearch.xpack.core.ilm.Step.StepKey; import org.elasticsearch.xpack.core.ilm.TerminalPolicyStep; import org.elasticsearch.xpack.core.ilm.WaitForRolloverReadyStep; +import org.elasticsearch.xpack.ilm.PolicyStepsRegistry.VersionedPolicyKey; import org.elasticsearch.xpack.ilm.history.ILMHistoryItem; import org.elasticsearch.xpack.ilm.history.ILMHistoryStore; import org.junit.After; @@ -93,6 +94,8 @@ import static org.elasticsearch.cluster.metadata.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.awaitLatch; import static org.elasticsearch.xpack.core.ilm.LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED_SETTING; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.CURRENT_VERSION; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.VERSION_ONE; import static org.elasticsearch.xpack.ilm.LifecyclePolicyTestsUtils.newTestLifecyclePolicy; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -385,6 +388,7 @@ public void testRunStateChangePolicyWithNextStep() throws Exception { "success": true, "state": { "phase": "phase", + "actions_order_version":"1", "action": "action", "step": "next_cluster_state_action_step", "step_time": "%s" @@ -568,6 +572,7 @@ public void testRunStateChangePolicyWithAsyncActionNextStep() throws Exception { "success": true, "state": { "phase": "phase", + "actions_order_version":"1", "action": "action", "step": "async_action_step", "step_time": "0" @@ -777,9 +782,9 @@ public void testGetCurrentStep() { StepKey firstStepKey = new StepKey("phase_1", "action_1", "step_1"); StepKey secondStepKey = new StepKey("phase_1", "action_1", "step_2"); Step firstStep = new MockStep(firstStepKey, secondStepKey); - Map firstStepMap = new HashMap<>(); - firstStepMap.put(policyName, firstStep); - Map> stepMap = new HashMap<>(); + Map firstStepMap = new HashMap<>(); + firstStepMap.put(new VersionedPolicyKey(CURRENT_VERSION, policyName), firstStep); + Map> stepMap = new HashMap<>(); Index index = new Index("test", "uuid"); Step.StepKey MOCK_STEP_KEY = new Step.StepKey("mock", "mock", "mock"); @@ -810,7 +815,7 @@ public void testGetCurrentStep() { // First step is retrieved because there are no settings for the index IndexMetadata indexMetadataWithNoKey = IndexMetadata.builder(index.getName()) .settings(indexSettings) - .putCustom(ILM_CUSTOM_METADATA_KEY, LifecycleExecutionState.builder().build().asMap()) + .putCustom(ILM_CUSTOM_METADATA_KEY, LifecycleExecutionState.builder().setActionsOrderVersion(CURRENT_VERSION).build().asMap()) .build(); Step stepFromNoSettings = IndexLifecycleRunner.getCurrentStep(registry, policy.getName(), indexMetadataWithNoKey); assertEquals(firstStep, stepFromNoSettings); @@ -835,9 +840,12 @@ public void testIsReadyToTransition() { ) ) ); - Map firstStepMap = Collections.singletonMap(policyName, step); + Map firstStepMap = Collections.singletonMap(new VersionedPolicyKey(CURRENT_VERSION, policyName), step); Map policySteps = Collections.singletonMap(step.getKey(), step); - Map> stepMap = Collections.singletonMap(policyName, policySteps); + Map> stepMap = Collections.singletonMap( + new VersionedPolicyKey(CURRENT_VERSION, policyName), + policySteps + ); PolicyStepsRegistry policyStepsRegistry = new PolicyStepsRegistry( lifecyclePolicyMap, firstStepMap, @@ -1209,8 +1217,8 @@ public static class MockPolicyStepsRegistry extends PolicyStepsRegistry { MockPolicyStepsRegistry( SortedMap lifecyclePolicyMap, - Map firstStepMap, - Map> stepMap, + Map firstStepMap, + Map> stepMap, NamedXContentRegistry xContentRegistry, Client client ) { @@ -1241,12 +1249,14 @@ public static MockPolicyStepsRegistry createMultiStepPolicyStepRegistry(String p LifecyclePolicy policy = new LifecyclePolicy(policyName, new HashMap<>()); SortedMap lifecyclePolicyMap = new TreeMap<>(); lifecyclePolicyMap.put(policyName, new LifecyclePolicyMetadata(policy, new HashMap<>(), 1, 1)); - Map firstStepMap = new HashMap<>(); - firstStepMap.put(policyName, steps.get(0)); - Map> stepMap = new HashMap<>(); + Map firstStepMap = new HashMap<>(); + firstStepMap.put(new VersionedPolicyKey(VERSION_ONE, policyName), steps.get(0)); + firstStepMap.put(new VersionedPolicyKey(CURRENT_VERSION, policyName), steps.get(0)); + Map> stepMap = new HashMap<>(); Map policySteps = new HashMap<>(); steps.forEach(step -> policySteps.put(step.getKey(), step)); - stepMap.put(policyName, policySteps); + stepMap.put(new VersionedPolicyKey(VERSION_ONE, policyName), policySteps); + stepMap.put(new VersionedPolicyKey(CURRENT_VERSION, policyName), policySteps); Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); return new MockPolicyStepsRegistry(lifecyclePolicyMap, firstStepMap, stepMap, REGISTRY, client); diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransitionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransitionTests.java index 57c5b9fbb2605..7c5e4550dca48 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransitionTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransitionTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.rollover.RolloverConditions; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -22,6 +23,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -30,6 +32,7 @@ import org.elasticsearch.xpack.core.ilm.DataTierMigrationRoutedStep; import org.elasticsearch.xpack.core.ilm.ErrorStep; import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata; +import org.elasticsearch.xpack.core.ilm.InitializePolicyContextStep; import org.elasticsearch.xpack.core.ilm.LifecycleAction; import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; import org.elasticsearch.xpack.core.ilm.LifecyclePolicyMetadata; @@ -41,11 +44,15 @@ import org.elasticsearch.xpack.core.ilm.OperationMode; import org.elasticsearch.xpack.core.ilm.Phase; import org.elasticsearch.xpack.core.ilm.PhaseCompleteStep; +import org.elasticsearch.xpack.core.ilm.ReadOnlyAction; import org.elasticsearch.xpack.core.ilm.RolloverAction; import org.elasticsearch.xpack.core.ilm.RolloverStep; +import org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction; import org.elasticsearch.xpack.core.ilm.SetPriorityAction; import org.elasticsearch.xpack.core.ilm.Step; import org.elasticsearch.xpack.core.ilm.WaitForRolloverReadyStep; +import org.junit.After; +import org.junit.Before; import java.io.IOException; import java.util.ArrayList; @@ -53,12 +60,18 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; import java.util.function.Function; import java.util.stream.Collectors; import static org.elasticsearch.cluster.metadata.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY; import static org.elasticsearch.xpack.core.ilm.PhaseCacheManagement.eligibleToCheckForRefresh; import static org.elasticsearch.xpack.core.ilm.PhaseCacheManagement.refreshPhaseDefinition; +import static org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction.CONDITIONAL_SKIP_ACTION_STEP; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.COLD_PHASE; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.CURRENT_VERSION; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.HOT_PHASE; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.VERSION_ONE; import static org.elasticsearch.xpack.ilm.IndexLifecycleRunnerTests.createOneStepPolicyStepRegistry; import static org.elasticsearch.xpack.ilm.IndexLifecycleTransition.moveStateToNextActionAndUpdateCachedPhase; import static org.elasticsearch.xpack.ilm.LifecyclePolicyTestsUtils.newTestLifecyclePolicy; @@ -70,6 +83,19 @@ public class IndexLifecycleTransitionTests extends ESTestCase { + private Client client; + + @Before + public void setupClient() { + client = new NoOpClient(getTestName()) { + }; + } + + @After + public void closeClient() { + client.close(); + } + public void testMoveClusterStateToNextStep() { String indexName = "my_index"; LifecyclePolicy policy = randomValueOtherThanMany( @@ -123,6 +149,246 @@ public void testMoveClusterStateToNextStep() { assertClusterStateOnNextStep(clusterState, index, currentStep, nextStep, newClusterState, now); } + public void testActionsOrderVersionManagementOnMoveToNextStep() { + String indexName = "my_index"; + String policyName = "ilm_policy"; + Map hotActions = Map.of( + RolloverAction.NAME, + new RolloverAction(null, null, null, 1L, null, null, null, null, null, null), + ReadOnlyAction.NAME, + new ReadOnlyAction() + ); + Map coldActions = Map.of( + SearchableSnapshotAction.NAME, + new SearchableSnapshotAction(randomAlphaOfLengthBetween(5, 10)) + ); + LifecyclePolicy policy = new LifecyclePolicy( + policyName, + Map.of( + HOT_PHASE, + new Phase(HOT_PHASE, TimeValue.ZERO, hotActions), + COLD_PHASE, + new Phase(COLD_PHASE, TimeValue.ZERO, coldActions) + ) + ); + + List policyMetadatas = Collections.singletonList( + new LifecyclePolicyMetadata(policy, Collections.emptyMap(), randomNonNegativeLong(), randomNonNegativeLong()) + ); + + PolicyStepsRegistry registry = new PolicyStepsRegistry( + new TreeMap<>(), + new HashMap<>(), + new HashMap<>(), + NamedXContentRegistry.EMPTY, + client, + null + ); + registry.update( + new IndexLifecycleMetadata( + Map.of(policyName, new LifecyclePolicyMetadata(policy, Collections.emptyMap(), 2L, 2L)), + OperationMode.RUNNING + ) + ); + long now = randomNonNegativeLong(); + + Step.StepKey waitForRolloverReadyStepKey = new Step.StepKey(HOT_PHASE, RolloverAction.NAME, WaitForRolloverReadyStep.NAME); + Step.StepKey rolloverStepKey = new Step.StepKey(HOT_PHASE, RolloverAction.NAME, RolloverStep.NAME); + + { + // test going from null lifecycle settings to next step + ClusterState clusterState = buildClusterState( + indexName, + Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policy.getName()), + LifecycleExecutionState.builder().build(), // actions version is 1 by default + policyMetadatas + ); + Index index = clusterState.metadata().index(indexName).getIndex(); + ClusterState newClusterState = IndexLifecycleTransition.moveClusterStateToStep( + index, + clusterState, + InitializePolicyContextStep.KEY, + () -> now, + registry, + false + ); + LifecycleExecutionState newLifecycleState = newClusterState.metadata().index(index).getLifecycleExecutionState(); + assertThat(newLifecycleState.phase(), is(InitializePolicyContextStep.KEY.phase())); + assertThat(newLifecycleState.action(), is(InitializePolicyContextStep.KEY.action())); + assertThat(newLifecycleState.step(), is(InitializePolicyContextStep.KEY.name())); + // as we moved from the new phase to the hot phase, the version of the actions order should be the latest + assertThat(newLifecycleState.actionsOrderVersion(), is(CURRENT_VERSION)); + } + + { + // let's move to the next step within the same phase - the actions order version must remain the same (we'll start with an older + // version) + Step waitForRolloverStep = new WaitForRolloverReadyStep( + waitForRolloverReadyStepKey, + rolloverStepKey, + client, + RolloverConditions.newBuilder().addMaxIndexDocsCondition(1L).build() + ); + + LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder(); + lifecycleState.setPhase(waitForRolloverStep.getKey().phase()); + lifecycleState.setAction(waitForRolloverStep.getKey().action()); + lifecycleState.setStep(waitForRolloverStep.getKey().name()); + lifecycleState.setIndexCreationDate(randomNonNegativeLong()); + lifecycleState.setActionsOrderVersion(VERSION_ONE); + + ClusterState currentState = buildClusterState( + indexName, + Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policy.getName()), + lifecycleState.build(), + policyMetadatas + ); + + Index index = currentState.metadata().index(indexName).getIndex(); + ClusterState newClusterState = IndexLifecycleTransition.moveClusterStateToStep( + index, + currentState, + waitForRolloverStep.getNextStepKey(), + () -> now, + registry, + false + ); + // as we didn't switch phases so the actions order version should continue to be ONE + assertThat(newClusterState.metadata().index(index).getLifecycleExecutionState().actionsOrderVersion(), is(VERSION_ONE)); + } + + { + // let's switch from hot to cold (starting in version ONE) + Step.StepKey lastHotKey = PhaseCompleteStep.stepKey(HOT_PHASE); + Step.StepKey coldKey = new Step.StepKey(COLD_PHASE, SearchableSnapshotAction.NAME, CONDITIONAL_SKIP_ACTION_STEP); + + LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder(); + lifecycleState.setPhase(lastHotKey.phase()); + lifecycleState.setAction(lastHotKey.action()); + lifecycleState.setStep(lastHotKey.name()); + lifecycleState.setIndexCreationDate(randomNonNegativeLong()); + lifecycleState.setActionsOrderVersion(VERSION_ONE); + + ClusterState currentState = buildClusterState( + indexName, + Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policy.getName()), + lifecycleState.build(), + policyMetadatas + ); + + Index index = currentState.metadata().index(indexName).getIndex(); + ClusterState newClusterState = IndexLifecycleTransition.moveClusterStateToStep( + index, + currentState, + coldKey, + () -> now, + registry, + false + ); + // as we switched phase we need to be in the latest version now + assertThat(newClusterState.metadata().index(index).getLifecycleExecutionState().actionsOrderVersion(), is(CURRENT_VERSION)); + } + + { + // let's switch from hot to cold (starting in version CURRENT) + Step.StepKey lastHotKey = PhaseCompleteStep.stepKey(HOT_PHASE); + Step.StepKey coldKey = new Step.StepKey(COLD_PHASE, SearchableSnapshotAction.NAME, CONDITIONAL_SKIP_ACTION_STEP); + + LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder(); + lifecycleState.setPhase(lastHotKey.phase()); + lifecycleState.setAction(lastHotKey.action()); + lifecycleState.setStep(lastHotKey.name()); + lifecycleState.setIndexCreationDate(randomNonNegativeLong()); + lifecycleState.setActionsOrderVersion(CURRENT_VERSION); + + ClusterState currentState = buildClusterState( + indexName, + Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policy.getName()), + lifecycleState.build(), + policyMetadatas + ); + + Index index = currentState.metadata().index(indexName).getIndex(); + ClusterState newClusterState = IndexLifecycleTransition.moveClusterStateToStep( + index, + currentState, + coldKey, + () -> now, + registry, + false + ); + assertThat(newClusterState.metadata().index(index).getLifecycleExecutionState().actionsOrderVersion(), is(CURRENT_VERSION)); + } + + { + // only refreshing the cached phase will not bump the actions order version (we start in version ONE) + Step waitForRolloverStep = new WaitForRolloverReadyStep( + waitForRolloverReadyStepKey, + rolloverStepKey, + client, + RolloverConditions.newBuilder().addMaxIndexDocsCondition(1L).build() + ); + + LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder(); + lifecycleState.setPhase(waitForRolloverStep.getKey().phase()); + lifecycleState.setAction(waitForRolloverStep.getKey().action()); + lifecycleState.setStep(waitForRolloverStep.getKey().name()); + lifecycleState.setIndexCreationDate(randomNonNegativeLong()); + lifecycleState.setActionsOrderVersion(VERSION_ONE); + + ClusterState currentState = buildClusterState( + indexName, + Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policy.getName()), + lifecycleState.build(), + policyMetadatas + ); + + Index index = currentState.metadata().index(indexName).getIndex(); + ClusterState newClusterState = IndexLifecycleTransition.moveClusterStateToStep( + index, + currentState, + waitForRolloverStep.getNextStepKey(), + () -> now, + registry, + true + ); + // as we didn't switch phases so the actions order version should continue to be ONE + assertThat(newClusterState.metadata().index(index).getLifecycleExecutionState().actionsOrderVersion(), is(VERSION_ONE)); + } + + { + // refreshing the cached phase whilst also transitioning to a new phase (maybe via a move-to-step API) should fetch the latest + // actions order + Step.StepKey lastHotKey = PhaseCompleteStep.stepKey(HOT_PHASE); + Step.StepKey coldKey = new Step.StepKey(COLD_PHASE, SearchableSnapshotAction.NAME, CONDITIONAL_SKIP_ACTION_STEP); + + LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder(); + lifecycleState.setPhase(lastHotKey.phase()); + lifecycleState.setAction(lastHotKey.action()); + lifecycleState.setStep(lastHotKey.name()); + lifecycleState.setIndexCreationDate(randomNonNegativeLong()); + lifecycleState.setActionsOrderVersion(VERSION_ONE); + + ClusterState currentState = buildClusterState( + indexName, + Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policy.getName()), + lifecycleState.build(), + policyMetadatas + ); + + Index index = currentState.metadata().index(indexName).getIndex(); + ClusterState newClusterState = IndexLifecycleTransition.moveClusterStateToStep( + index, + currentState, + coldKey, + () -> now, + registry, + true + ); + assertThat(newClusterState.metadata().index(index).getLifecycleExecutionState().actionsOrderVersion(), is(CURRENT_VERSION)); + } + } + public void testMoveClusterStateToNextStepSamePhase() { String indexName = "my_index"; LifecyclePolicy policy = randomValueOtherThanMany( diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/LockableLifecycleType.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/LockableLifecycleType.java index f802fa8282f5d..db84fec5856a5 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/LockableLifecycleType.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/LockableLifecycleType.java @@ -16,6 +16,8 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.CURRENT_VERSION; + /** * This {@link LifecycleType} is used for encapsulating test policies * used in integration tests where the underlying {@link LifecycleAction}s are @@ -31,13 +33,18 @@ public List getOrderedPhases(Map phases) { } @Override - public List getOrderedActions(Phase phase) { + public List getOrderedActions(Phase phase, int orderVersion) { return new ArrayList<>(phase.getActions().values()); } @Override public void validate(Collection phases) {} + @Override + public int getLatestActionsOrderVersion() { + return CURRENT_VERSION; + } + @Override public String getWriteableName() { return TYPE; diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/MoveToNextStepUpdateTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/MoveToNextStepUpdateTaskTests.java index 32175ae2fdda7..e9ae534a43b06 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/MoveToNextStepUpdateTaskTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/MoveToNextStepUpdateTaskTests.java @@ -36,6 +36,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.cluster.metadata.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.CURRENT_VERSION; import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -83,7 +84,7 @@ public void setupClusterState() { public void testExecuteSuccessfullyMoved() throws Exception { long now = randomNonNegativeLong(); - List steps = lifecyclePolicy.toSteps(null, null); + List steps = lifecyclePolicy.toSteps(null, CURRENT_VERSION, null); StepKey currentStepKey = steps.get(0).getKey(); StepKey nextStepKey = steps.get(0).getNextStepKey(); @@ -157,7 +158,7 @@ public void testExecuteDifferentPolicy() throws Exception { public void testExecuteSuccessfulMoveWithInvalidNextStep() throws Exception { long now = randomNonNegativeLong(); - List steps = lifecyclePolicy.toSteps(null, null); + List steps = lifecyclePolicy.toSteps(null, CURRENT_VERSION, null); StepKey currentStepKey = steps.get(0).getKey(); StepKey invalidNextStep = new StepKey("next-invalid", "next-invalid", "next-invalid"); @@ -227,7 +228,7 @@ private static class AlwaysExistingStepRegistry extends PolicyStepsRegistry { } @Override - public boolean stepExists(String policy, StepKey stepKey) { + public boolean stepExists(String policy, int actionsOrderVersion, StepKey stepKey) { return true; } } diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/PolicyStepsRegistryTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/PolicyStepsRegistryTests.java index 9317834918916..473314f0b71be 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/PolicyStepsRegistryTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/PolicyStepsRegistryTests.java @@ -43,6 +43,7 @@ import org.elasticsearch.xpack.core.ilm.ShrinkAction; import org.elasticsearch.xpack.core.ilm.ShrinkStep; import org.elasticsearch.xpack.core.ilm.Step; +import org.elasticsearch.xpack.ilm.PolicyStepsRegistry.VersionedPolicyKey; import org.mockito.Mockito; import java.util.Collections; @@ -55,8 +56,11 @@ import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.cluster.metadata.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.CURRENT_VERSION; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleActionsRegistry.VERSION_ONE; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.sameInstance; import static org.mockito.Mockito.mock; @@ -75,18 +79,24 @@ private IndexMetadata emptyMetadata(Index index) { public void testGetFirstStep() { String policyName = randomAlphaOfLengthBetween(2, 10); Step expectedFirstStep = new MockStep(MOCK_STEP_KEY, null); - Map firstStepMap = Collections.singletonMap(policyName, expectedFirstStep); + Map firstStepMap = Collections.singletonMap( + new VersionedPolicyKey(VERSION_ONE, policyName), + expectedFirstStep + ); PolicyStepsRegistry registry = new PolicyStepsRegistry(null, firstStepMap, null, NamedXContentRegistry.EMPTY, null, null); - Step actualFirstStep = registry.getFirstStep(policyName); + Step actualFirstStep = registry.getFirstStep(policyName, VERSION_ONE); assertThat(actualFirstStep, sameInstance(expectedFirstStep)); } public void testGetFirstStepUnknownPolicy() { String policyName = randomAlphaOfLengthBetween(2, 10); Step expectedFirstStep = new MockStep(MOCK_STEP_KEY, null); - Map firstStepMap = Collections.singletonMap(policyName, expectedFirstStep); + Map firstStepMap = Collections.singletonMap( + new VersionedPolicyKey(VERSION_ONE, policyName), + expectedFirstStep + ); PolicyStepsRegistry registry = new PolicyStepsRegistry(null, firstStepMap, null, NamedXContentRegistry.EMPTY, null, null); - Step actualFirstStep = registry.getFirstStep(policyName + "unknown"); + Step actualFirstStep = registry.getFirstStep(policyName + "unknown", VERSION_ONE); assertNull(actualFirstStep); } @@ -186,7 +196,7 @@ public void testUpdateFromNothingToSomethingToNothing() throws Exception { String policyName = randomAlphaOfLength(5); LifecyclePolicy newPolicy = LifecyclePolicyTests.randomTestLifecyclePolicy(policyName); logger.info("--> policy: {}", newPolicy); - List policySteps = newPolicy.toSteps(client, null); + List policySteps = newPolicy.toSteps(client, CURRENT_VERSION, null); Map headers = new HashMap<>(); if (randomBoolean()) { headers.put(randomAlphaOfLength(10), randomAlphaOfLength(10)); @@ -199,6 +209,7 @@ public void testUpdateFromNothingToSomethingToNothing() throws Exception { IndexLifecycleMetadata lifecycleMetadata = new IndexLifecycleMetadata(policyMap, OperationMode.RUNNING); LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder(); lifecycleState.setPhase("new"); + lifecycleState.setActionsOrderVersion(CURRENT_VERSION); Metadata metadata = Metadata.builder() .persistentSettings(settings(Version.CURRENT).build()) .putCustom(IndexLifecycleMetadata.TYPE, lifecycleMetadata) @@ -235,13 +246,18 @@ public void testUpdateFromNothingToSomethingToNothing() throws Exception { // add new policy registry.update(currentState.metadata().custom(IndexLifecycleMetadata.TYPE)); - assertThat(registry.getFirstStep(newPolicy.getName()), equalTo(policySteps.get(0))); + assertThat(registry.getFirstStep(newPolicy.getName(), 1), equalTo(policySteps.get(0))); assertThat(registry.getLifecyclePolicyMap().size(), equalTo(1)); assertNotNull(registry.getLifecyclePolicyMap().get(newPolicy.getName())); assertThat(registry.getLifecyclePolicyMap().get(newPolicy.getName()).getHeaders(), equalTo(headers)); - assertThat(registry.getFirstStepMap().size(), equalTo(1)); - assertThat(registry.getStepMap().size(), equalTo(1)); - Map registeredStepsForPolicy = registry.getStepMap().get(newPolicy.getName()); + assertThat(registry.getFirstStepMap().size(), equalTo(2)); + for (VersionedPolicyKey versionedPolicyKey : registry.getFirstStepMap().keySet()) { + // we track all versions of first steps + assertThat(versionedPolicyKey.policyName(), is(newPolicy.getName())); + } + assertThat(registry.getStepMap().size(), equalTo(2)); + Map registeredStepsForPolicy = registry.getStepMap() + .get(new VersionedPolicyKey(CURRENT_VERSION, newPolicy.getName())); assertThat(registeredStepsForPolicy.size(), equalTo(policySteps.size())); for (Step step : policySteps) { LifecycleExecutionState.Builder newIndexState = LifecycleExecutionState.builder(); @@ -263,8 +279,8 @@ public void testUpdateFromNothingToSomethingToNothing() throws Exception { } Map registryPolicyMap = registry.getLifecyclePolicyMap(); - Map registryFirstStepMap = registry.getFirstStepMap(); - Map> registryStepMap = registry.getStepMap(); + Map registryFirstStepMap = registry.getFirstStepMap(); + Map> registryStepMap = registry.getStepMap(); registry.update(currentState.metadata().custom(IndexLifecycleMetadata.TYPE)); assertThat(registry.getLifecyclePolicyMap(), equalTo(registryPolicyMap)); assertThat(registry.getFirstStepMap(), equalTo(registryFirstStepMap)); @@ -351,7 +367,7 @@ public void testUpdatePolicyButNoPhaseChangeIndexStepsDontChange() throws Except LifecyclePolicy updatedPolicy = new LifecyclePolicy(policyName, phases); logger.info("--> policy: {}", newPolicy); logger.info("--> updated policy: {}", updatedPolicy); - List policySteps = newPolicy.toSteps(client, null); + List policySteps = newPolicy.toSteps(client, CURRENT_VERSION, null); Map headers = new HashMap<>(); if (randomBoolean()) { headers.put(randomAlphaOfLength(10), randomAlphaOfLength(10)); @@ -401,7 +417,8 @@ public void testUpdatePolicyButNoPhaseChangeIndexStepsDontChange() throws Except // add new policy registry.update(currentState.metadata().custom(IndexLifecycleMetadata.TYPE)); - Map registeredStepsForPolicy = registry.getStepMap().get(newPolicy.getName()); + Map registeredStepsForPolicy = registry.getStepMap() + .get(new VersionedPolicyKey(CURRENT_VERSION, newPolicy.getName())); Step shrinkStep = registeredStepsForPolicy.entrySet() .stream() .filter(e -> e.getKey().phase().equals("warm") && e.getKey().name().equals("shrink")) @@ -430,7 +447,7 @@ public void testUpdatePolicyButNoPhaseChangeIndexStepsDontChange() throws Except // Update the policies registry.update(currentState.metadata().custom(IndexLifecycleMetadata.TYPE)); - registeredStepsForPolicy = registry.getStepMap().get(newPolicy.getName()); + registeredStepsForPolicy = registry.getStepMap().get(new VersionedPolicyKey(CURRENT_VERSION, newPolicy.getName())); shrinkStep = registeredStepsForPolicy.entrySet() .stream() .filter(e -> e.getKey().phase().equals("warm") && e.getKey().name().equals("shrink")) diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/history/ILMHistoryItemTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/history/ILMHistoryItemTests.java index ae8d139f3a1ea..b35bf10ce9a7e 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/history/ILMHistoryItemTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/history/ILMHistoryItemTests.java @@ -75,6 +75,7 @@ public void testToXContent() throws IOException { "phase": "phase", "phase_definition": "{}", "action_time": "20", + "actions_order_version":"1", "phase_time": "10", "step_info": "{\\"step_info\\": \\"foo\\"", "action": "action", @@ -99,6 +100,7 @@ public void testToXContent() throws IOException { "failed_step": "step", "phase_definition": "{\\"phase_json\\": \\"eggplant\\"}", "action_time": "20", + "actions_order_version":"1", "is_auto_retryable_error": "true", "failed_step_retry_count": "7", "phase_time": "10",