Add migrate to data tiers API (#74264) (#74625)

This adds the _ilm/migrate_to_data_tiers API to expose the service for
migrating the elasticsearch abstractions (indices, ILM policies and an
optional legacy template to delete) to data tiers routing allocation
(away from custom node attributes)

(cherry picked from commit 636aa7c)
Signed-off-by: Andrei Dan <andrei.dan@elastic.co>

docs/reference/ilm/apis/ilm-api.asciidoc

@@ -1,7 +1,7 @@
 [[index-lifecycle-management-api]]
 == {ilm-cap} APIs
 
-You use the following APIs to set up policies to automatically manage the index lifecycle. 
+You use the following APIs to set up policies to automatically manage the index lifecycle.
 For more information about {ilm} ({ilm-init}), see <<index-lifecycle-management>>.
 
 [discrete]
@@ -28,6 +28,7 @@ For more information about {ilm} ({ilm-init}), see <<index-lifecycle-management>
 * <<ilm-start,Start {ilm-init}>>
 * <<ilm-stop,Stop {ilm-init}>>
 * <<ilm-explain-lifecycle,Explain API>>
+* <<ilm-migrate-to-data-tiers,Migrate to data tiers routing API>>
 
 
 include::put-lifecycle.asciidoc[]
@@ -42,3 +43,4 @@ include::get-status.asciidoc[]
 include::explain.asciidoc[]
 include::start.asciidoc[]
 include::stop.asciidoc[]
+include::migrate-to-data-tiers.asciidoc[]

docs/reference/ilm/apis/migrate-to-data-tiers.asciidoc

@@ -0,0 +1,130 @@
+[role="xpack"]
+[testenv="basic"]
+[[ilm-migrate-to-data-tiers]]
+=== Migrate to data tiers routing API
+++++
+<titleabbrev>Migrate indices and ILM policies to data tiers routing</titleabbrev>
+++++
+
+Switches the indices and ILM policies from using custom node attributes and
+<<shard-allocation-filtering, attribute-based allocation filters>> to using <<data-tiers, data tiers>>, and
+optionally deletes one legacy index template.
+Using node roles enables {ilm-init} to <<data-tier-migration, automatically move the indices>> between
+data tiers.
+
+Migrating away from custom node attributes routing can be manually performed
+as indicated in the <<migrate-index-allocation-filters, Migrate index allocation
+filters to node roles>> page.
+
+This API provides an automated way of executing three out of the four manual steps listed
+in the <<data-tier-migration, migration guide>>:
+
+. <<stop-setting-custom-hot-attribute, Stop setting the custom hot attribute on new indices>>
+. <<remove-custom-allocation-settings, Remove custom allocation settings from existing {ilm-init} policies>>
+. <<set-tier-preference, Replace custom allocation settings from existing indices>> with the corresponding <<data-tier-shard-filtering,tier preference>>
+
+[[ilm-migrate-to-data-tiers-request]]
+==== {api-request-title}
+
+`POST /_ilm/migrate_to_data_tiers`
+
+The API accepts an optional body that allows you to specify:
+
+- The legacy index template name to delete. Defaults to none.
+- The name of the custom node attribute used for the indices and ILM policies allocation filtering.
+Defaults to `data`.
+
+[[ilm-migrate-to-data-tiers-prereqs]]
+==== {api-prereq-title}
+
+* {ilm-init} must be stopped before performing the migration. Use the <<ilm-stop-request, stop ILM API>>
+to stop {ilm-init} and <<ilm-get-status-request, get status API>> to wait until the
+reported operation mode is `STOPPED`.
+
+[[ilm-migrate-to-data-tiers-example]]
+==== {api-examples-title}
+
+The following example migrates the indices and ILM policies away from defining
+custom allocation filtering using the `custom_attribute_name` node attribute, and
+deletes legacy template with name `global-template` if it exists in the system.
+
+////
+[source,console]
+----
+POST _ilm/stop
+
+PUT _template/global-template
+{
+  "index_patterns": ["migrate-to-tiers-*"],
+  "settings": {
+     "index.routing.allocation.require.custom_attribute_name": "hot"
+  }
+}
+
+PUT warm-index-to-migrate-000001
+{
+  "settings": {
+    "index.routing.allocation.require.custom_attribute_name": "warm"
+  }
+}
+
+PUT _ilm/policy/policy_with_allocate_action
+{
+  "policy": {
+    "phases": {
+      "warm": {
+        "actions": {
+          "allocate": {
+            "require": {
+              "custom_attribute_name": "warm"
+            }
+          }
+        }
+      },
+      "delete": {
+        "min_age": "30d",
+        "actions": {
+          "delete": {}
+        }
+      }
+    }
+  }
+}
+----
+// TESTSETUP
+
+[source,console]
+----
+DELETE warm-index-to-migrate-000001
+
+DELETE _ilm/policy/policy_with_allocate_action
+
+POST _ilm/start
+----
+// TEARDOWN
+////
+
+[source,console]
+----------------------------------------------------------------
+POST /_ilm/migrate_to_data_tiers
+{
+  "legacy_template_to_delete": "global-template",
+  "node_attribute": "custom_attribute_name"
+}
+----------------------------------------------------------------
+
+If the request succeeds, a response like the following will be received:
+
+[source,console-result]
+------------------------------------------------------------------------------
+{
+  "removed_legacy_template":"global-template", <1>
+  "migrated_ilm_policies":["policy_with_allocate_action"], <2>
+  "migrated_indices":["warm-index-to-migrate-000001"] <3>
+}
+------------------------------------------------------------------------------
+
+<1> Shows the name of the legacy index template that was deleted. This will be missing
+if no legacy index template was deleted.
+<2> The ILM policies that were updated.
+<3> The indices that were migrated to <<data-tier-shard-filtering,tier preference>> routing.

rest-api-spec/src/main/resources/rest-api-spec/api/ilm.migrate_to_data_tiers.json

@@ -0,0 +1,29 @@
+{
+  "ilm.migrate_to_data_tiers":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-migrate-to-data-tiers.html",
+      "description": "Migrates the indices and ILM policies away from custom node attribute allocation routing to data tiers routing"
+    },
+    "stability":"stable",
+    "visibility":"public",
+    "headers":{
+      "accept": [ "application/json"],
+      "content_type": ["application/json"]
+    },
+    "url":{
+      "paths":[
+        {
+          "path":"/_ilm/migrate_to_data_tiers",
+          "methods":[
+            "POST"
+          ]
+        }
+      ]
+    },
+    "params":{},
+    "body":{
+      "description":"Optionally specify a legacy index template name to delete and optionally specify a node attribute name used for index shard routing (defaults to \"data\")",
+      "required":false
+    }
+  }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersAction.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.xpack.cluster.action;
+
+import org.elasticsearch.action.ActionType;
+
+public class MigrateToDataTiersAction extends ActionType<MigrateToDataTiersResponse> {
+
+    public static final MigrateToDataTiersAction INSTANCE = new MigrateToDataTiersAction();
+    public static final String NAME = "cluster:admin/migrate_to_data_tiers";
+
+    private MigrateToDataTiersAction() {
+        super(NAME, MigrateToDataTiersResponse::new);
+    }
+
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersRequest.java

@@ -0,0 +1,113 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.cluster.action;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ParseField;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.core.Nullable;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class MigrateToDataTiersRequest extends AcknowledgedRequest<MigrateToDataTiersRequest> {
+
+    private static final ParseField LEGACY_TEMPLATE_TO_DELETE = new ParseField("legacy_template_to_delete");
+    private static final ParseField NODE_ATTRIBUTE_NAME = new ParseField("node_attribute");
+
+    @SuppressWarnings("unchecked")
+    public static final ConstructingObjectParser<MigrateToDataTiersRequest, Void> PARSER = new ConstructingObjectParser<>("index_template",
+        false,
+        a -> new MigrateToDataTiersRequest((String) a[0], (String) a[1]));
+
+    static {
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), LEGACY_TEMPLATE_TO_DELETE);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), NODE_ATTRIBUTE_NAME);
+    }
+
+    /**
+     * Represents the name of node attribute used for index shard allocation filtering (usually `data`)
+     */
+    @Nullable
+    private final String nodeAttributeName;
+
+    /**
+     * Represents the name of the legacy (v1) index template to delete.
+     */
+    @Nullable
+    private final String legacyTemplateToDelete;
+
+    public static MigrateToDataTiersRequest parse(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    public MigrateToDataTiersRequest(@Nullable String legacyTemplateToDelete, @Nullable String nodeAttributeName) {
+        this.legacyTemplateToDelete = legacyTemplateToDelete;
+        this.nodeAttributeName = nodeAttributeName;
+    }
+
+    public MigrateToDataTiersRequest() {
+        this(null, null);
+    }
+
+    public MigrateToDataTiersRequest(StreamInput in) throws IOException {
+        super(in);
+        legacyTemplateToDelete = in.readOptionalString();
+        nodeAttributeName = in.readOptionalString();
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeOptionalString(legacyTemplateToDelete);
+        out.writeOptionalString(nodeAttributeName);
+    }
+
+    public String getNodeAttributeName() {
+        return nodeAttributeName;
+    }
+
+    public String getLegacyTemplateToDelete() {
+        return legacyTemplateToDelete;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        MigrateToDataTiersRequest that = (MigrateToDataTiersRequest) o;
+        return Objects.equals(nodeAttributeName, that.nodeAttributeName) && Objects.equals(legacyTemplateToDelete,
+            that.legacyTemplateToDelete);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(nodeAttributeName, legacyTemplateToDelete);
+    }
+
+    @Override
+    public String toString() {
+        return "MigrateToDataTiersRequest{" +
+            "nodeAttributeName='" + nodeAttributeName + '\'' +
+            ", legacyTemplateToDelete='" + legacyTemplateToDelete + '\'' +
+            '}';
+    }
+}