ILM: parse origination date from index name

docs/reference/settings/ilm-settings.asciidoc

@@ -27,6 +27,14 @@ information about rollover, see <<using-policies-rollover>>.
 (<<time-units, time units>>) How often {ilm} checks for indices that meet policy
 criteria. Defaults to `10m`.
 
+`index.lifecycle.parse_origination_date`::
+When configured to `true` the origination date will be parsed from the index
+name. The index format must match the pattern `^.*-{date_format}-\\d+`, where
+the `date_format` is `yyyy.MM.dd` and the trailing digits are optional (an
+index that was rolled over would normally match the full format eg.
+`logs-2016.10.31-000002`). If the index name doesn't match the pattern
+the index creation will fail.
+
 `index.lifecycle.origination_date`::
 The timestamp that will be used to calculate the index age for its phase
 transitions. This allows the users to create an index containing old data and

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleOriginationDateParser.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ilm;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.time.DateFormatter;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.elasticsearch.xpack.core.ilm.LifecycleSettings.LIFECYCLE_ORIGINATION_DATE;
+import static org.elasticsearch.xpack.core.ilm.LifecycleSettings.LIFECYCLE_PARSE_ORIGINATION_DATE;
+
+public class IndexLifecycleOriginationDateParser {
+
+    private static final DateFormatter DATE_FORMATTER = DateFormatter.forPattern("yyyy.MM.dd");
+    private static final String INDEX_NAME_REGEX = "^.*-(\\d{4}.\\d{2}.\\d{2})(-[\\d]+)?$";
+    private static final Pattern INDEX_NAME_PATTERN = Pattern.compile(INDEX_NAME_REGEX);
+
+    /**
+     * Determines if the origination date needs to be parsed from the index name.
+     */
+    public static boolean shouldParseIndexName(Settings indexSettings) {
+        return indexSettings.getAsLong(LIFECYCLE_ORIGINATION_DATE, -1L) == -1L &&
+            indexSettings.getAsBoolean(LIFECYCLE_PARSE_ORIGINATION_DATE, false);
+    }
+
+    /**
+     * Parses the index according to the supported format and extracts the origination date. If the index does not match the expected
+     * format or the date in the index name doesn't match the `yyyy.MM.dd` format it throws an {@link IllegalArgumentException}
+     */
+    public static long parseIndexNameAndExtractDate(String indexName) {
+        Matcher matcher = INDEX_NAME_PATTERN.matcher(indexName);
+        if (matcher.matches()) {
+            String dateAsString = matcher.group(1);
+            try {
+                return DATE_FORMATTER.parseMillis(dateAsString);
+            } catch (ElasticsearchParseException | IllegalArgumentException e) {
+                throw new IllegalArgumentException("index name [" + indexName + "] contains date [" + dateAsString + "] which " +
+                    "couldn't be parsed using the 'yyyy.MM.dd' format", e);
+            }
+        }
+
+        throw new IllegalArgumentException("index name [" + indexName + "] does not match pattern '" + INDEX_NAME_REGEX + "'");
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/InitializePolicyContextStep.java

@@ -10,8 +10,11 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 
+import static org.elasticsearch.xpack.core.ilm.IndexLifecycleOriginationDateParser.parseIndexNameAndExtractDate;
+import static org.elasticsearch.xpack.core.ilm.IndexLifecycleOriginationDateParser.shouldParseIndexName;
 import static org.elasticsearch.xpack.core.ilm.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
 
 /**
@@ -34,19 +37,34 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
             // Index must have been since deleted, ignore it
             return clusterState;
         }
+
         LifecycleExecutionState lifecycleState = LifecycleExecutionState
             .fromIndexMetadata(indexMetaData);
+
         if (lifecycleState.getLifecycleDate() != null) {
             return clusterState;
         }
 
+        IndexMetaData.Builder indexMetadataBuilder = IndexMetaData.builder(indexMetaData);
+        if (shouldParseIndexName(indexMetaData.getSettings())) {
+            long parsedOriginationDate = parseIndexNameAndExtractDate(index.getName());
+            indexMetadataBuilder.settingsVersion(indexMetaData.getSettingsVersion() + 1)
+                .settings(Settings.builder()
+                    .put(indexMetaData.getSettings())
+                    .put(LifecycleSettings.LIFECYCLE_ORIGINATION_DATE, parsedOriginationDate)
+                    .build()
+                );
+        }
+
         ClusterState.Builder newClusterStateBuilder = ClusterState.builder(clusterState);
 
         LifecycleExecutionState.Builder newCustomData = LifecycleExecutionState.builder(lifecycleState);
         newCustomData.setIndexCreationDate(indexMetaData.getCreationDate());
-        newClusterStateBuilder.metaData(MetaData.builder(clusterState.getMetaData()).put(IndexMetaData
-            .builder(indexMetaData)
-            .putCustom(ILM_CUSTOM_METADATA_KEY, newCustomData.build().asMap())));
+        indexMetadataBuilder.putCustom(ILM_CUSTOM_METADATA_KEY, newCustomData.build().asMap());
+
+        newClusterStateBuilder.metaData(
+            MetaData.builder(clusterState.getMetaData()).put(indexMetadataBuilder)
+        );
         return newClusterStateBuilder.build();
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleSettings.java

@@ -18,6 +18,7 @@ public class LifecycleSettings {
     public static final String LIFECYCLE_NAME = "index.lifecycle.name";
     public static final String LIFECYCLE_INDEXING_COMPLETE = "index.lifecycle.indexing_complete";
     public static final String LIFECYCLE_ORIGINATION_DATE = "index.lifecycle.origination_date";
+    public static final String LIFECYCLE_PARSE_ORIGINATION_DATE = "index.lifecycle.parse_origination_date";
 
     public static final String SLM_HISTORY_INDEX_ENABLED = "slm.history_index_enabled";
     public static final String SLM_RETENTION_SCHEDULE = "slm.retention_schedule";
@@ -32,6 +33,8 @@ public class LifecycleSettings {
         Setting.Property.Dynamic, Setting.Property.IndexScope);
     public static final Setting<Long> LIFECYCLE_ORIGINATION_DATE_SETTING =
         Setting.longSetting(LIFECYCLE_ORIGINATION_DATE, -1, -1, Setting.Property.Dynamic, Setting.Property.IndexScope);
+    public static final Setting<Boolean> LIFECYCLE_PARSE_ORIGINATION_DATE_SETTING = Setting.boolSetting(LIFECYCLE_PARSE_ORIGINATION_DATE,
+        false, Setting.Property.Dynamic, Setting.Property.IndexScope);
 
     public static final Setting<Boolean> SLM_HISTORY_INDEX_ENABLED_SETTING = Setting.boolSetting(SLM_HISTORY_INDEX_ENABLED, true,
         Setting.Property.NodeScope);

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleOriginationDateParserTests.java

@@ -0,0 +1,112 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ilm;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Locale;
+import java.util.TimeZone;
+
+import static org.elasticsearch.xpack.core.ilm.IndexLifecycleOriginationDateParser.parseIndexNameAndExtractDate;
+import static org.elasticsearch.xpack.core.ilm.IndexLifecycleOriginationDateParser.shouldParseIndexName;
+import static org.hamcrest.Matchers.is;
+
+public class IndexLifecycleOriginationDateParserTests extends ESTestCase {
+
+    public void testShouldParseIndexNameReturnsFalseWhenOriginationDateIsSet() {
+        Settings settings = Settings.builder()
+            .put(LifecycleSettings.LIFECYCLE_ORIGINATION_DATE, 1L)
+            .build();
+        assertThat(shouldParseIndexName(settings), is(false));
+    }
+
+    public void testShouldParseIndexNameReturnsFalseIfParseOriginationDateIsDisabled() {
+        Settings settings = Settings.builder()
+            .put(LifecycleSettings.LIFECYCLE_PARSE_ORIGINATION_DATE, false)
+            .build();
+        assertThat(shouldParseIndexName(settings), is(false));
+    }
+
+    public void testShouldParseIndexNameReturnsTrueIfParseOriginationDateIsTrueAndOriginationDateIsNotSet() {
+        Settings settings = Settings.builder()
+            .put(LifecycleSettings.LIFECYCLE_PARSE_ORIGINATION_DATE, true)
+            .build();
+        assertThat(shouldParseIndexName(settings), is(true));
+    }
+
+    public void testParseIndexNameThatMatchesExpectedFormat() throws ParseException {
+        SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy.MM.dd", Locale.getDefault());
+        dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+        long expectedDate = dateFormat.parse("2019.09.04").getTime();
+
+        {
+            long parsedDate = parseIndexNameAndExtractDate("indexName-2019.09.04");
+            assertThat("indexName-yyyy.MM.dd is a valid index format", parsedDate, is(expectedDate));
+        }
+
+        {
+            long parsedDate = parseIndexNameAndExtractDate("indexName-2019.09.04-0000001");
+            assertThat("indexName-yyyy.MM.dd-\\d+$ is a valid index format", parsedDate, is(expectedDate));
+        }
+
+        {
+            long parsedDate = parseIndexNameAndExtractDate("indexName-2019.09.04-2019.09.24");
+            long secondDateInIndexName = dateFormat.parse("2019.09.24").getTime();
+            assertThat("indexName-yyyy.MM.dd-yyyy.MM.dd is a valid index format and the second date should be parsed",
+                parsedDate, is(secondDateInIndexName));
+        }
+
+        {
+            long parsedDate = parseIndexNameAndExtractDate("index-2019.09.04-2019.09.24-00002");
+            long secondDateInIndexName = dateFormat.parse("2019.09.24").getTime();
+            assertThat("indexName-yyyy.MM.dd-yyyy.MM.dd-digits is a valid index format and the second date should be parsed",
+                parsedDate, is(secondDateInIndexName));
+        }
+    }
+
+    public void testParseIndexNameThrowsIllegalArgumentExceptionForInvalidIndexFormat() {
+        expectThrows(
+            IllegalArgumentException.class,
+            "plainIndexName does not match the expected pattern",
+            () -> parseIndexNameAndExtractDate("plainIndexName")
+        );
+
+        expectThrows(
+            IllegalArgumentException.class,
+            "indexName--00001 does not match the expected pattern as the origination date is missing",
+            () -> parseIndexNameAndExtractDate("indexName--00001")
+        );
+
+        expectThrows(
+            IllegalArgumentException.class,
+            "indexName-00001 does not match the expected pattern as the origination date is missing",
+            () -> parseIndexNameAndExtractDate("indexName-00001")
+        );
+
+        expectThrows(
+            IllegalArgumentException.class,
+            "indexName_2019.09.04_00001 does not match the expected pattern as _ is not the expected delimiter",
+            () -> parseIndexNameAndExtractDate("indexName_2019.09.04_00001")
+        );
+    }
+
+    public void testParseIndexNameThrowsIllegalArgumentExceptionForInvalidDateFormat() {
+        expectThrows(
+            IllegalArgumentException.class,
+            "indexName-2019.04-00001 does not match the expected pattern as the date does not conform with the yyyy.MM.dd pattern",
+            () -> parseIndexNameAndExtractDate("indexName-2019.04-00001")
+        );
+
+        expectThrows(
+            IllegalArgumentException.class,
+            "java.lang.IllegalArgumentException: failed to parse date field [2019.09.44] with format [yyyy.MM.dd]",
+            () -> parseIndexNameAndExtractDate("index-2019.09.44")
+        );
+    }
+}

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestController;
@@ -141,6 +142,7 @@ public List<Setting<?>> getSettings() {
             LifecycleSettings.LIFECYCLE_POLL_INTERVAL_SETTING,
             LifecycleSettings.LIFECYCLE_NAME_SETTING,
             LifecycleSettings.LIFECYCLE_ORIGINATION_DATE_SETTING,
+            LifecycleSettings.LIFECYCLE_PARSE_ORIGINATION_DATE_SETTING,
             LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE_SETTING,
             RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING,
             LifecycleSettings.SLM_HISTORY_INDEX_ENABLED_SETTING,
@@ -268,6 +270,14 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
         return actions;
     }
 
+    @Override
+    public void onIndexModule(IndexModule indexModule) {
+        if (ilmEnabled) {
+            assert indexLifecycleInitialisationService.get() != null;
+            indexModule.addIndexEventListener(indexLifecycleInitialisationService.get());
+        }
+    }
+
     @Override
     public void close() {
         try {

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleService.java

@@ -22,6 +22,8 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata;
@@ -39,11 +41,14 @@
 import java.util.Set;
 import java.util.function.LongSupplier;
 
+import static org.elasticsearch.xpack.core.ilm.IndexLifecycleOriginationDateParser.parseIndexNameAndExtractDate;
+import static org.elasticsearch.xpack.core.ilm.IndexLifecycleOriginationDateParser.shouldParseIndexName;
+
 /**
  * A service which runs the {@link LifecyclePolicy}s associated with indexes.
  */
 public class IndexLifecycleService
-    implements ClusterStateListener, ClusterStateApplier, SchedulerEngine.Listener, Closeable, LocalNodeMasterListener {
+    implements ClusterStateListener, ClusterStateApplier, SchedulerEngine.Listener, Closeable, LocalNodeMasterListener, IndexEventListener {
     private static final Logger logger = LogManager.getLogger(IndexLifecycleService.class);
     private static final Set<String> IGNORE_STEPS_MAINTENANCE_REQUESTED = Collections.singleton(ShrinkStep.NAME);
     private volatile boolean isMaster = false;
@@ -148,6 +153,13 @@ public String executorName() {
         return ThreadPool.Names.MANAGEMENT;
     }
 
+    @Override
+    public void beforeIndexAddedToCluster(Index index, Settings indexSettings) {
+        if (shouldParseIndexName(indexSettings)) {
+            parseIndexNameAndExtractDate(index.getName());
+        }
+    }
+
     private void updatePollInterval(TimeValue newInterval) {
         this.pollInterval = newInterval;
         maybeScheduleJob();