Cache ILM policy name on IndexMetadata (#83603)

elastic · Feb 8, 2022 · c182b00 · c182b00
1 parent c322ae7
commit c182b00
Show file tree

Hide file tree

Showing 49 changed files with 138 additions and 119 deletions.
diff --git a/docs/changelog/83603.yaml b/docs/changelog/83603.yaml
@@ -0,0 +1,6 @@
+pr: 83603
+summary: Cache ILM policy name on `IndexMetadata`
+area: ILM+SLM
+type: enhancement
+issues:
+ - 83582
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
@@ -547,6 +547,9 @@ public Iterator<Setting<?>> settings() {
 
     private final int shardsPerNodeLimit;
 
+    @Nullable // if an index isn't managed by ilm, it won't have a policy
+    private final String lifecyclePolicyName;
+
     private final LifecycleExecutionState lifecycleExecutionState;
 
     private final AutoExpandReplicas autoExpandReplicas;
@@ -588,6 +591,7 @@ private IndexMetadata(
         final boolean ignoreDiskWatermarks,
         @Nullable final List<String> tierPreference,
         final int shardsPerNodeLimit,
+        final String lifecyclePolicyName,
         final LifecycleExecutionState lifecycleExecutionState,
         final AutoExpandReplicas autoExpandReplicas,
         final boolean isSearchableSnapshot,
@@ -632,6 +636,7 @@ private IndexMetadata(
         this.ignoreDiskWatermarks = ignoreDiskWatermarks;
         this.tierPreference = tierPreference;
         this.shardsPerNodeLimit = shardsPerNodeLimit;
+        this.lifecyclePolicyName = lifecyclePolicyName;
         this.lifecycleExecutionState = lifecycleExecutionState;
         this.autoExpandReplicas = autoExpandReplicas;
         this.isSearchableSnapshot = isSearchableSnapshot;
@@ -677,6 +682,7 @@ IndexMetadata withMappingMetadata(MappingMetadata mapping) {
             this.ignoreDiskWatermarks,
             this.tierPreference,
             this.shardsPerNodeLimit,
+            this.lifecyclePolicyName,
             this.lifecycleExecutionState,
             this.autoExpandReplicas,
             this.isSearchableSnapshot,
@@ -816,6 +822,14 @@ public List<String> getTierPreference() {
         return tierPreference;
     }
 
+    /**
+     * Return the name of the Index Lifecycle Policy associated with this index, or null if it is not managed by ILM.
+     */
+    @Nullable
+    public String getLifecyclePolicyName() {
+        return lifecyclePolicyName;
+    }
+
     public LifecycleExecutionState getLifecycleExecutionState() {
         return lifecycleExecutionState;
     }
@@ -864,6 +878,10 @@ public Index getResizeSourceIndex() {
         Property.PrivateIndex
     );
 
+    // LIFECYCLE_NAME is here an as optimization, see LifecycleSettings.LIFECYCLE_NAME and
+    // LifecycleSettings.LIFECYCLE_NAME_SETTING for the 'real' version
+    public static final String LIFECYCLE_NAME = "index.lifecycle.name";
+
     ImmutableOpenMap<String, DiffableStringMap> getCustomData() {
         return this.customData;
     }
@@ -1699,6 +1717,7 @@ public IndexMetadata build() {
                 DiskThresholdDecider.SETTING_IGNORE_DISK_WATERMARKS.get(settings),
                 tierPreference,
                 ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING.get(settings),
+                settings.get(IndexMetadata.LIFECYCLE_NAME), // n.b. lookup by name to get null-if-not-present semantics
                 lifecycleExecutionState,
                 AutoExpandReplicas.SETTING.get(settings),
                 isSearchableSnapshot,

diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java
@@ -467,6 +467,24 @@ public void testBuildsWithBrokenTierPreference() {
         expectThrows(IllegalArgumentException.class, indexMetadata::getTierPreference);
     }
 
+    public void testLifeCyclePolicyName() {
+        Settings.Builder settings = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 8))
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT);
+
+        IndexMetadata idxMeta1 = IndexMetadata.builder("test").settings(settings).build();
+
+        // null means no policy
+        assertNull(idxMeta1.getLifecyclePolicyName());
+
+        IndexMetadata idxMeta2 = IndexMetadata.builder(idxMeta1)
+            .settings(settings.put(IndexMetadata.LIFECYCLE_NAME, "some_policy").build())
+            .build();
+
+        assertThat(idxMeta2.getLifecyclePolicyName(), equalTo("some_policy"));
+    }
+
     private static Settings indexSettingsWithDataTier(String dataTier) {
         return Settings.builder()
             .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)

diff --git a/...core/src/main/java/org/elasticsearch/xpack/core/ilm/CheckNotDataStreamWriteIndexStep.java b/...core/src/main/java/org/elasticsearch/xpack/core/ilm/CheckNotDataStreamWriteIndexStep.java
@@ -60,7 +60,7 @@ public Result isConditionMet(Index index, ClusterState clusterState) {
             return new Result(false, new Info(errorMessage));
         }
 
-        String policyName = indexMetadata.getSettings().get(LifecycleSettings.LIFECYCLE_NAME);
+        String policyName = indexMetadata.getLifecyclePolicyName();
         IndexAbstraction indexAbstraction = clusterState.metadata().getIndicesLookup().get(indexName);
         assert indexAbstraction != null : "invalid cluster metadata. index [" + indexName + "] was not found";
         IndexAbstraction.DataStream dataStream = indexAbstraction.getParentDataStream();

diff --git a/...lugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CheckTargetShardsCountStep.java b/...lugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CheckTargetShardsCountStep.java
@@ -52,7 +52,7 @@ public Result isConditionMet(Index index, ClusterState clusterState) {
         if (numberOfShards != null) {
             int sourceNumberOfShards = indexMetadata.getNumberOfShards();
             if (sourceNumberOfShards % numberOfShards != 0) {
-                String policyName = indexMetadata.getSettings().get(LifecycleSettings.LIFECYCLE_NAME);
+                String policyName = indexMetadata.getLifecyclePolicyName();
                 String errorMessage = String.format(
                     Locale.ROOT,
                     "lifecycle action of policy [%s] for index [%s] cannot make progress "

diff --git a/...ck/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CleanupShrinkIndexStep.java b/...ck/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CleanupShrinkIndexStep.java
@@ -43,7 +43,7 @@ void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState currentCl
             if (currentClusterState.metadata().index(shrunkenIndexSource) == null) {
                 // if the source index does not exist, we'll skip deleting the
                 // (managed) shrunk index as that will cause data loss
-                String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexMetadata.getSettings());
+                String policyName = indexMetadata.getLifecyclePolicyName();
                 logger.warn(
                     "managed index [{}] as part of policy [{}] is a shrunk index and the source index [{}] does not exist "
                         + "anymore. will skip the [{}] step",

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CleanupSnapshotStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CleanupSnapshotStep.java
@@ -58,7 +58,7 @@ void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState currentCl
                 @Override
                 public void onResponse(AcknowledgedResponse acknowledgedResponse) {
                     if (acknowledgedResponse.isAcknowledged() == false) {
-                        String policyName = indexMetadata.getSettings().get(LifecycleSettings.LIFECYCLE_NAME);
+                        String policyName = indexMetadata.getLifecyclePolicyName();
                         throw new ElasticsearchException(
                             "cleanup snapshot step request for repository ["
                                 + repositoryName
@@ -82,7 +82,7 @@ public void onFailure(Exception e) {
                         listener.onResponse(null);
                     } else {
                         if (e instanceof RepositoryMissingException) {
-                            String policyName = indexMetadata.getSettings().get(LifecycleSettings.LIFECYCLE_NAME);
+                            String policyName = indexMetadata.getLifecyclePolicyName();
                             listener.onFailure(
                                 new IllegalStateException(
                                     "repository ["

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CreateSnapshotStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CreateSnapshotStep.java
@@ -71,7 +71,7 @@ void createSnapshot(IndexMetadata indexMetadata, ActionListener<Boolean> listene
 
         final LifecycleExecutionState lifecycleState = indexMetadata.getLifecycleExecutionState();
 
-        final String policyName = indexMetadata.getSettings().get(LifecycleSettings.LIFECYCLE_NAME);
+        final String policyName = indexMetadata.getLifecyclePolicyName();
         final String snapshotRepository = lifecycleState.snapshotRepository();
         if (Strings.hasText(snapshotRepository) == false) {
             listener.onFailure(

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteStep.java
@@ -32,7 +32,7 @@ public DeleteStep(StepKey key, StepKey nextStepKey, Client client) {
 
     @Override
     public void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState currentState, ActionListener<Void> listener) {
-        String policyName = indexMetadata.getSettings().get(LifecycleSettings.LIFECYCLE_NAME);
+        String policyName = indexMetadata.getLifecyclePolicyName();
         String indexName = indexMetadata.getIndex().getName();
         IndexAbstraction indexAbstraction = currentState.metadata().getIndicesLookup().get(indexName);
         assert indexAbstraction != null : "invalid cluster metadata. index [" + indexName + "] was not found";

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java
@@ -139,7 +139,7 @@ public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey)
                 IndexMetadata indexMetadata = clusterState.metadata().index(index);
                 assert indexMetadata != null : "index " + index.getName() + " must exist in the cluster state";
                 if (indexMetadata.getSettings().get(LifecycleSettings.SNAPSHOT_INDEX_NAME) != null) {
-                    String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexMetadata.getSettings());
+                    String policyName = indexMetadata.getLifecyclePolicyName();
                     logger.warn(
                         "[{}] action is configured for index [{}] in policy [{}] which is mounted as searchable snapshot. "
                             + "Skipping this action",

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeStep.java
@@ -61,7 +61,7 @@ public void performAction(
                 listener.onResponse(null);
             } else {
                 DefaultShardOperationFailedException[] failures = response.getShardFailures();
-                String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexMetadata.getSettings());
+                String policyName = indexMetadata.getLifecyclePolicyName();
                 String errorMessage = String.format(
                     Locale.ROOT,
                     "index [%s] in policy [%s] encountered failures [%s] on step [%s]",

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java
@@ -73,7 +73,7 @@ public List<Step> toSteps(Client client, String phase, StepKey nextStepKey) {
             (index, clusterState) -> {
                 IndexMetadata indexMetadata = clusterState.getMetadata().index(index);
                 assert indexMetadata != null : "index " + index.getName() + " must exist in the cluster state";
-                String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexMetadata.getSettings());
+                String policyName = indexMetadata.getLifecyclePolicyName();
                 if (indexMetadata.getSettings().get(LifecycleSettings.SNAPSHOT_INDEX_NAME) != null) {
                     logger.warn(
                         "[{}] action is configured for index [{}] in policy [{}] which is mounted as searchable snapshot. "

diff --git a/.../plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/GenerateSnapshotNameStep.java b/.../plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/GenerateSnapshotNameStep.java
@@ -59,7 +59,7 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
             return clusterState;
         }
 
-        String policy = indexMetadata.getSettings().get(LifecycleSettings.LIFECYCLE_NAME);
+        String policyName = indexMetadata.getLifecyclePolicyName();
         LifecycleExecutionState lifecycleState = indexMetadata.getLifecycleExecutionState();
 
         // validate that the snapshot repository exists -- because policies are refreshed on later retries, and because
@@ -70,7 +70,7 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
                 "repository ["
                     + snapshotRepository
                     + "] is missing. ["
-                    + policy
+                    + policyName
                     + "] policy for "
                     + "index ["
                     + index.getName()
@@ -83,13 +83,13 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
         newCustomData.setSnapshotRepository(snapshotRepository);
         if (lifecycleState.snapshotName() == null) {
             // generate and validate the snapshotName
-            String snapshotNamePrefix = ("<{now/d}-" + index.getName() + "-" + policy + ">").toLowerCase(Locale.ROOT);
+            String snapshotNamePrefix = ("<{now/d}-" + index.getName() + "-" + policyName + ">").toLowerCase(Locale.ROOT);
             String snapshotName = generateSnapshotName(snapshotNamePrefix);
             ActionRequestValidationException validationException = validateGeneratedSnapshotName(snapshotNamePrefix, snapshotName);
             if (validationException != null) {
                 logger.warn(
                     "unable to generate a snapshot name as part of policy [{}] for index [{}] due to [{}]",
-                    policy,
+                    policyName,
                     index.getName(),
                     validationException.getMessage()
                 );

diff --git a/...ugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/GenerateUniqueIndexNameStep.java b/...ugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/GenerateUniqueIndexNameStep.java
@@ -82,13 +82,13 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
         LifecycleExecutionState lifecycleState = indexMetadata.getLifecycleExecutionState();
 
         Builder newCustomData = LifecycleExecutionState.builder(lifecycleState);
-        String policy = indexMetadata.getSettings().get(LifecycleSettings.LIFECYCLE_NAME);
+        String policyName = indexMetadata.getLifecyclePolicyName();
         String generatedIndexName = generateValidIndexName(prefix, index.getName());
         ActionRequestValidationException validationException = validateGeneratedIndexName(generatedIndexName, clusterState);
         if (validationException != null) {
             logger.warn(
                 "unable to generate a valid index name as part of policy [{}] for index [{}] due to [{}]",
-                policy,
+                policyName,
                 index.getName(),
                 validationException.getMessage()
             );

diff --git a/...ugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/InitializePolicyContextStep.java b/...ugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/InitializePolicyContextStep.java
@@ -59,8 +59,8 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
                     );
             }
         } catch (Exception e) {
-            String policy = indexMetadata.getSettings().get(LifecycleSettings.LIFECYCLE_NAME);
-            throw new InitializePolicyException(policy, index.getName(), e);
+            String policyName = indexMetadata.getLifecyclePolicyName();
+            throw new InitializePolicyException(policyName, index.getName(), e);
         }
 
         ClusterState.Builder newClusterStateBuilder = ClusterState.builder(clusterState);

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicy.java
@@ -290,6 +290,9 @@ public boolean isActionSafe(StepKey stepKey) {
      * @throws IllegalArgumentException if the name is invalid
      */
     public static void validatePolicyName(String policy) {
+        if (Strings.isNullOrEmpty(policy)) {
+            throw new IllegalArgumentException("invalid policy name [" + policy + "]: must not be null or empty");
+        }
         if (policy.contains(",")) {
             throw new IllegalArgumentException("invalid policy name [" + policy + "]: must not contain ','");
         }

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java
@@ -123,7 +123,7 @@ public static ItemUsage calculateUsage(
             .indices()
             .values()
             .stream()
-            .filter(indexMetadata -> policyName.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexMetadata.getSettings())))
+            .filter(indexMetadata -> policyName.equals(indexMetadata.getLifecyclePolicyName()))
             .map(indexMetadata -> indexMetadata.getIndex().getName())
             .collect(Collectors.toList());
 

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleSettings.java
@@ -6,19 +6,18 @@
  */
 package org.elasticsearch.xpack.core.ilm;
 
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xpack.core.scheduler.CronSchedule;
 
-import static org.elasticsearch.common.settings.Setting.timeSetting;
-
 /**
  * Class encapsulating settings related to Index Lifecycle Management X-Pack Plugin
  */
 public class LifecycleSettings {
     public static final String LIFECYCLE_POLL_INTERVAL = "indices.lifecycle.poll_interval";
-    public static final String LIFECYCLE_NAME = "index.lifecycle.name";
+    public static final String LIFECYCLE_NAME = IndexMetadata.LIFECYCLE_NAME;
     public static final String LIFECYCLE_INDEXING_COMPLETE = "index.lifecycle.indexing_complete";
     public static final String LIFECYCLE_ORIGINATION_DATE = "index.lifecycle.origination_date";
     public static final String LIFECYCLE_PARSE_ORIGINATION_DATE = "index.lifecycle.parse_origination_date";
@@ -35,7 +34,7 @@ public class LifecycleSettings {
     // already mounted as a searchable snapshot. Those ILM actions will check if the index has this setting name configured.
     public static final String SNAPSHOT_INDEX_NAME = "index.store.snapshot.index_name";
 
-    public static final Setting<TimeValue> LIFECYCLE_POLL_INTERVAL_SETTING = timeSetting(
+    public static final Setting<TimeValue> LIFECYCLE_POLL_INTERVAL_SETTING = Setting.timeSetting(
         LIFECYCLE_POLL_INTERVAL,
         TimeValue.timeValueMinutes(10),
         TimeValue.timeValueSeconds(1),
@@ -81,7 +80,7 @@ public class LifecycleSettings {
     // This setting configures how much time since step_time should ILM wait for a condition to be met. After the threshold wait time has
     // elapsed ILM will likely stop waiting and go to the next step.
     // Also see {@link org.elasticsearch.xpack.core.ilm.ClusterStateWaitUntilThresholdStep}
-    public static final Setting<TimeValue> LIFECYCLE_STEP_WAIT_TIME_THRESHOLD_SETTING = timeSetting(
+    public static final Setting<TimeValue> LIFECYCLE_STEP_WAIT_TIME_THRESHOLD_SETTING = Setting.timeSetting(
         LIFECYCLE_STEP_WAIT_TIME_THRESHOLD,
         TimeValue.timeValueHours(12),
         TimeValue.timeValueHours(1),
@@ -114,7 +113,7 @@ public class LifecycleSettings {
         Setting.Property.Dynamic,
         Setting.Property.NodeScope
     );
-    public static final Setting<TimeValue> SLM_RETENTION_DURATION_SETTING = timeSetting(
+    public static final Setting<TimeValue> SLM_RETENTION_DURATION_SETTING = Setting.timeSetting(
         SLM_RETENTION_DURATION,
         TimeValue.timeValueHours(1),
         TimeValue.timeValueMillis(500),

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/MigrateAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/MigrateAction.java
@@ -109,7 +109,7 @@ public List<Step> toSteps(Client client, String phase, StepKey nextStepKey) {
 
                     // partially mounted indices will already have data_frozen, and we don't want to change that if they do
                     if (indexMetadata.isPartialSearchableSnapshot()) {
-                        String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexMetadata.getSettings());
+                        String policyName = indexMetadata.getLifecyclePolicyName();
                         logger.debug(
                             "[{}] action in policy [{}] is configured for index [{}] which is a partially mounted index. "
                                 + "skipping this action",

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStep.java
@@ -68,7 +68,7 @@ void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState currentCl
 
         LifecycleExecutionState lifecycleState = indexMetadata.getLifecycleExecutionState();
 
-        String policyName = indexMetadata.getSettings().get(LifecycleSettings.LIFECYCLE_NAME);
+        String policyName = indexMetadata.getLifecyclePolicyName();
         final String snapshotRepository = lifecycleState.snapshotRepository();
         if (Strings.hasText(snapshotRepository) == false) {
             listener.onFailure(

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PhaseCacheManagement.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PhaseCacheManagement.java
@@ -154,7 +154,7 @@ public static boolean updateIndicesForPolicy(
             .indices()
             .values()
             .stream()
-            .filter(meta -> newPolicy.getName().equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(meta.getSettings())))
+            .filter(meta -> newPolicy.getName().equals(meta.getLifecyclePolicyName()))
             .filter(meta -> isIndexPhaseDefinitionUpdatable(xContentRegistry, client, meta, newPolicy.getPolicy(), licenseState))
             .collect(Collectors.toList());