Speed up ILM cluster task execution (#85405)

docs/changelog/85405.yaml

@@ -0,0 +1,6 @@
+pr: 85405
+summary: Speed up ILM cluster task execution
+area: ILM+SLM
+type: enhancement
+issues:
+ - 82708

server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java

@@ -35,7 +35,7 @@
 
 /**
  * An index abstraction is a reference to one or more concrete indices.
- * An index abstraction has a unique name and encapsulates all the  {@link IndexMetadata} instances it is pointing to.
+ * An index abstraction has a unique name and encapsulates all the {@link Index} instances it is pointing to.
  * Also depending on type it may refer to a single or many concrete indices and may or may not have a write index.
  */
 public interface IndexAbstraction {
@@ -51,7 +51,7 @@ public interface IndexAbstraction {
     String getName();
 
     /**
-     * @return All {@link IndexMetadata} of all concrete indices this index abstraction is referring to.
+     * @return All {@link Index} of all concrete indices this index abstraction is referring to.
      */
     List<Index> getIndices();
 
@@ -150,6 +150,7 @@ class ConcreteIndex implements IndexAbstraction {
         private final DataStream dataStream;
 
         public ConcreteIndex(IndexMetadata indexMetadata, DataStream dataStream) {
+            // note: don't capture a reference to the indexMetadata here
             this.concreteIndexName = indexMetadata.getIndex();
             this.isHidden = indexMetadata.isHidden();
             this.isSystem = indexMetadata.isSystem();
@@ -225,19 +226,20 @@ public int hashCode() {
     class Alias implements IndexAbstraction {
 
         private final String aliasName;
-        private final List<Index> referenceIndexMetadatas;
+        private final List<Index> referenceIndices;
         private final Index writeIndex;
         private final boolean isHidden;
         private final boolean isSystem;
         private final boolean dataStreamAlias;
 
-        public Alias(AliasMetadata aliasMetadata, List<IndexMetadata> indices) {
+        public Alias(AliasMetadata aliasMetadata, List<IndexMetadata> indexMetadatas) {
+            // note: don't capture a reference to any of these indexMetadatas here
             this.aliasName = aliasMetadata.getAlias();
-            this.referenceIndexMetadatas = new ArrayList<>(indices.size());
+            this.referenceIndices = new ArrayList<>(indexMetadatas.size());
             boolean isSystem = true;
             Index widx = null;
-            for (IndexMetadata imd : indices) {
-                this.referenceIndexMetadatas.add(imd.getIndex());
+            for (IndexMetadata imd : indexMetadatas) {
+                this.referenceIndices.add(imd.getIndex());
                 if (Boolean.TRUE.equals(imd.getAliases().get(aliasName).writeIndex())) {
                     if (widx != null) {
                         throw new IllegalStateException("write indices size can only be 0 or 1, but is at least 2");
@@ -247,8 +249,8 @@ public Alias(AliasMetadata aliasMetadata, List<IndexMetadata> indices) {
                 isSystem = isSystem && imd.isSystem();
             }
 
-            if (widx == null && indices.size() == 1 && indices.get(0).getAliases().get(aliasName).writeIndex() == null) {
-                widx = indices.get(0).getIndex();
+            if (widx == null && indexMetadatas.size() == 1 && indexMetadatas.get(0).getAliases().get(aliasName).writeIndex() == null) {
+                widx = indexMetadatas.get(0).getIndex();
             }
             this.writeIndex = widx;
 
@@ -259,7 +261,7 @@ public Alias(AliasMetadata aliasMetadata, List<IndexMetadata> indices) {
 
         public Alias(DataStreamAlias dataStreamAlias, List<Index> indicesOfAllDataStreams, Index writeIndexOfWriteDataStream) {
             this.aliasName = dataStreamAlias.getName();
-            this.referenceIndexMetadatas = indicesOfAllDataStreams;
+            this.referenceIndices = indicesOfAllDataStreams;
             this.writeIndex = writeIndexOfWriteDataStream;
             this.isHidden = false;
             this.isSystem = false;
@@ -277,7 +279,7 @@ public String getName() {
 
         @Override
         public List<Index> getIndices() {
-            return referenceIndexMetadatas;
+            return referenceIndices;
         }
 
         @Nullable
@@ -320,13 +322,13 @@ public boolean equals(Object o) {
                 && isSystem == alias.isSystem
                 && dataStreamAlias == alias.dataStreamAlias
                 && aliasName.equals(alias.aliasName)
-                && referenceIndexMetadatas.equals(alias.referenceIndexMetadatas)
+                && referenceIndices.equals(alias.referenceIndices)
                 && Objects.equals(writeIndex, alias.writeIndex);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(aliasName, referenceIndexMetadatas, writeIndex, isHidden, isSystem, dataStreamAlias);
+            return Objects.hash(aliasName, referenceIndices, writeIndex, isHidden, isSystem, dataStreamAlias);
         }
     }
 

server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java

@@ -75,6 +75,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import static org.elasticsearch.cluster.metadata.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
 import static org.elasticsearch.common.settings.Settings.readSettingsFromStream;
 import static org.elasticsearch.common.settings.Settings.writeSettingsToStream;
 
@@ -299,6 +300,66 @@ public Metadata withIncrementedVersion() {
         );
     }
 
+    /**
+     * Given an index and lifecycle state, returns a metadata where the lifecycle state will be
+     * associated with the given index.
+     *
+     * The passed-in index must already be present in the cluster state, this method cannot
+     * be used to add an index.
+     *
+     * @param index A non-null index
+     * @param lifecycleState A non-null lifecycle execution state
+     * @return a <code>Metadata</code> instance where the index has the provided lifecycle state
+     */
+    public Metadata withLifecycleState(final Index index, final LifecycleExecutionState lifecycleState) {
+        Objects.requireNonNull(index, "index must not be null");
+        Objects.requireNonNull(lifecycleState, "lifecycleState must not be null");
+
+        IndexMetadata indexMetadata = getIndexSafe(index);
+        if (lifecycleState.equals(indexMetadata.getLifecycleExecutionState())) {
+            return this;
+        }
+
+        // build a new index metadata with the version incremented and the new lifecycle state
+        IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(indexMetadata);
+        indexMetadataBuilder.version(indexMetadataBuilder.version() + 1);
+        indexMetadataBuilder.putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.asMap());
+
+        // drop it into the indices
+        final ImmutableOpenMap.Builder<String, IndexMetadata> builder = ImmutableOpenMap.builder(indices);
+        builder.put(index.getName(), indexMetadataBuilder.build());
+
+        // construct a new Metadata object directly rather than using Metadata.builder(this).[...].build().
+        // the Metadata.Builder validation needs to handle the general case where anything at all could
+        // have changed, and hence it is expensive -- since we are changing so little about the metadata
+        // (and at a leaf in the object tree), we can bypass that validation for efficiency's sake
+        return new Metadata(
+            clusterUUID,
+            clusterUUIDCommitted,
+            version,
+            coordinationMetadata,
+            transientSettings,
+            persistentSettings,
+            settings,
+            hashesOfConsistentSettings,
+            totalNumberOfShards,
+            totalOpenIndexShards,
+            builder.build(),
+            aliasedIndices,
+            templates,
+            customs,
+            allIndices,
+            visibleIndices,
+            allOpenIndices,
+            visibleOpenIndices,
+            allClosedIndices,
+            visibleClosedIndices,
+            indicesLookup,
+            mappingsByHash,
+            oldestIndexVersion
+        );
+    }
+
     public long version() {
         return this.version;
     }

server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java

@@ -30,6 +30,7 @@
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.alias.RandomAliasActionsGenerator;
 import org.elasticsearch.index.mapper.MapperService;
@@ -2287,6 +2288,48 @@ public void testMappingDuplication() {
         assertThat(metadata.getMappingsByHash().get(newMapping.getSha256()), nullValue());
     }
 
+    public void testWithLifecycleState() {
+        String indexName = "my-index";
+        String indexUUID = randomAlphaOfLength(10);
+        Metadata metadata1 = Metadata.builder(randomMetadata())
+            .put(
+                IndexMetadata.builder(indexName)
+                    .settings(settings(Version.CURRENT).put(IndexMetadata.SETTING_INDEX_UUID, indexUUID))
+                    .creationDate(randomNonNegativeLong())
+                    .numberOfShards(1)
+                    .numberOfReplicas(0)
+            )
+            .build();
+        IndexMetadata index1 = metadata1.index(indexName);
+        assertThat(metadata1.getIndicesLookup(), notNullValue());
+        assertThat(index1.getLifecycleExecutionState(), sameInstance(LifecycleExecutionState.EMPTY_STATE));
+
+        LifecycleExecutionState state = LifecycleExecutionState.builder().setPhase("phase").setAction("action").setStep("step").build();
+        Metadata metadata2 = metadata1.withLifecycleState(index1.getIndex(), state);
+        IndexMetadata index2 = metadata2.index(indexName);
+
+        // the indices lookups are the same object
+        assertThat(metadata2.getIndicesLookup(), sameInstance(metadata1.getIndicesLookup()));
+
+        // the lifecycle state and version were changed
+        assertThat(index2.getLifecycleExecutionState().asMap(), is(state.asMap()));
+        assertThat(index2.getVersion(), is(index1.getVersion() + 1));
+
+        // but those are the only differences between the two
+        IndexMetadata.Builder builder = IndexMetadata.builder(index2);
+        builder.version(builder.version() - 1);
+        builder.removeCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY);
+        assertThat(index1, equalTo(builder.build()));
+
+        // withLifecycleState returns the same reference if nothing changed
+        Metadata metadata3 = metadata2.withLifecycleState(index2.getIndex(), state);
+        assertThat(metadata3, sameInstance(metadata2));
+
+        // withLifecycleState rejects a nonsense Index
+        String randomUUID = randomValueOtherThan(indexUUID, () -> randomAlphaOfLength(10));
+        expectThrows(IndexNotFoundException.class, () -> metadata1.withLifecycleState(new Index(indexName, randomUUID), state));
+    }
+
     public static Metadata randomMetadata() {
         return randomMetadata(1);
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CopyExecutionStateStep.java

@@ -13,15 +13,12 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.LifecycleExecutionState;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.Index;
 
 import java.util.Objects;
 import java.util.function.BiFunction;
 
-import static org.elasticsearch.cluster.metadata.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
-
 /**
  * Copies the execution state data from one index to another, typically after a
  * new index has been created. As part of the execution state copy it will set the target index
@@ -98,16 +95,17 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
         String action = targetNextStepKey.getAction();
         String step = targetNextStepKey.getName();
 
-        LifecycleExecutionState.Builder relevantTargetCustomData = LifecycleExecutionState.builder(lifecycleState);
+        LifecycleExecutionState.Builder newLifecycleState = LifecycleExecutionState.builder(lifecycleState);
         // Override the phase, action, and step for the target next StepKey
-        relevantTargetCustomData.setPhase(phase);
-        relevantTargetCustomData.setAction(action);
-        relevantTargetCustomData.setStep(step);
-
-        Metadata.Builder newMetadata = Metadata.builder(clusterState.getMetadata())
-            .put(IndexMetadata.builder(targetIndexMetadata).putCustom(ILM_CUSTOM_METADATA_KEY, relevantTargetCustomData.build().asMap()));
-
-        return ClusterState.builder(clusterState).metadata(newMetadata).build();
+        newLifecycleState.setPhase(phase);
+        newLifecycleState.setAction(action);
+        newLifecycleState.setStep(step);
+
+        return LifecycleExecutionStateUtils.newClusterStateWithLifecycleState(
+            clusterState,
+            targetIndexMetadata.getIndex(),
+            newLifecycleState.build()
+        );
     }
 
     @Override

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/GenerateSnapshotNameStep.java

@@ -13,7 +13,6 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.LifecycleExecutionState;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.RepositoriesMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
@@ -25,8 +24,6 @@
 import java.util.Locale;
 import java.util.Objects;
 
-import static org.elasticsearch.cluster.metadata.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
-
 /**
  * Generates a snapshot name for the given index and records it in the index metadata along with the provided snapshot repository.
  * <p>
@@ -78,9 +75,9 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
             );
         }
 
-        LifecycleExecutionState.Builder newCustomData = LifecycleExecutionState.builder(lifecycleState);
-        newCustomData.setSnapshotIndexName(index.getName());
-        newCustomData.setSnapshotRepository(snapshotRepository);
+        LifecycleExecutionState.Builder newLifecycleState = LifecycleExecutionState.builder(lifecycleState);
+        newLifecycleState.setSnapshotIndexName(index.getName());
+        newLifecycleState.setSnapshotRepository(snapshotRepository);
         if (lifecycleState.snapshotName() == null) {
             // generate and validate the snapshotName
             String snapshotNamePrefix = ("<{now/d}-" + index.getName() + "-" + policyName + ">").toLowerCase(Locale.ROOT);
@@ -96,15 +93,14 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
                 throw validationException;
             }
 
-            newCustomData.setSnapshotName(snapshotName);
+            newLifecycleState.setSnapshotName(snapshotName);
         }
 
-        return ClusterState.builder(clusterState)
-            .metadata(
-                Metadata.builder(clusterState.getMetadata())
-                    .put(IndexMetadata.builder(indexMetadata).putCustom(ILM_CUSTOM_METADATA_KEY, newCustomData.build().asMap()))
-            )
-            .build();
+        return LifecycleExecutionStateUtils.newClusterStateWithLifecycleState(
+            clusterState,
+            indexMetadata.getIndex(),
+            newLifecycleState.build()
+        );
     }
 
     @Override

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/GenerateUniqueIndexNameStep.java

@@ -13,7 +13,6 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.LifecycleExecutionState;
 import org.elasticsearch.cluster.metadata.LifecycleExecutionState.Builder;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.core.Nullable;
@@ -25,8 +24,6 @@
 import java.util.function.BiFunction;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.cluster.metadata.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
-
 /**
  * Generates a unique index name prefixing the original index name with the configured
  * prefix, concatenated with a random UUID. The generated index name will be stored in the lifecycle
@@ -77,11 +74,9 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
             return clusterState;
         }
 
-        ClusterState.Builder newClusterStateBuilder = ClusterState.builder(clusterState);
-
         LifecycleExecutionState lifecycleState = indexMetadata.getLifecycleExecutionState();
 
-        Builder newCustomData = LifecycleExecutionState.builder(lifecycleState);
+        Builder newLifecycleState = LifecycleExecutionState.builder(lifecycleState);
         String policyName = indexMetadata.getLifecyclePolicyName();
         String generatedIndexName = generateValidIndexName(prefix, index.getName());
         ActionRequestValidationException validationException = validateGeneratedIndexName(generatedIndexName, clusterState);
@@ -94,12 +89,13 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
             );
             throw validationException;
         }
-        lifecycleStateSetter.apply(generatedIndexName, newCustomData);
+        lifecycleStateSetter.apply(generatedIndexName, newLifecycleState);
 
-        IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(indexMetadata);
-        indexMetadataBuilder.putCustom(ILM_CUSTOM_METADATA_KEY, newCustomData.build().asMap());
-        newClusterStateBuilder.metadata(Metadata.builder(clusterState.getMetadata()).put(indexMetadataBuilder));
-        return newClusterStateBuilder.build();
+        return LifecycleExecutionStateUtils.newClusterStateWithLifecycleState(
+            clusterState,
+            indexMetadata.getIndex(),
+            newLifecycleState.build()
+        );
     }
 
     @Nullable