Deprecate indices without soft-deletes (#50502) (#50634)

Soft-deletes will be enabled for all indices in 8.0. Hence, we should
deprecate new indices without soft-deletes in 7.x.

Backport of #50502

docs/reference/index-modules/history-retention.asciidoc

@@ -65,6 +65,9 @@ there>>. {ccr-cap} will not function if soft deletes are disabled.
   configured at index creation and only on indices created on or after 6.5.0.
   The default value is `true`.
 
+  deprecated::[7.6, Creating indices with soft-deletes disabled is
+  deprecated and will be removed in future Elasticsearch versions.]
+
 `index.soft_deletes.retention_lease.period`::
 
   The maximum length of time to retain a shard history retention lease before

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java

@@ -28,6 +28,7 @@
 import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.elasticsearch.common.Booleans;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
@@ -306,7 +307,7 @@ public void testRelocationWithConcurrentIndexing() throws Exception {
         }
     }
 
-    public void testRecoveryWithSoftDeletes() throws Exception {
+    public void testRecovery() throws Exception {
         final String index = "recover_with_soft_deletes";
         if (CLUSTER_TYPE == ClusterType.OLD) {
             Settings.Builder settings = Settings.builder()
@@ -318,8 +319,8 @@ public void testRecoveryWithSoftDeletes() throws Exception {
                 // before timing out
                 .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
                 .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
-            if (getNodeId(v -> v.onOrAfter(Version.V_6_5_0)) != null && randomBoolean()) {
-                settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
+            if (randomBoolean()) {
+                settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
             }
             createIndex(index, settings.build());
             int numDocs = randomInt(10);
@@ -350,7 +351,7 @@ public void testRetentionLeasesEstablishedWhenPromotingPrimary() throws Exceptio
                 .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(1, 2)) // triggers nontrivial promotion
                 .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
                 .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0") // fail faster
-                .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
+                .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
             createIndex(index, settings.build());
             int numDocs = randomInt(10);
             indexDocs(index, 0, numDocs);
@@ -373,7 +374,7 @@ public void testRetentionLeasesEstablishedWhenRelocatingPrimary() throws Excepti
                     .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(0, 1))
                     .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
                     .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0") // fail faster
-                    .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
+                    .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
                 createIndex(index, settings.build());
                 int numDocs = randomInt(10);
                 indexDocs(index, 0, numDocs);
@@ -784,6 +785,26 @@ public void testAutoExpandIndicesDuringRollingUpgrade() throws Exception {
         }
     }
 
+    public void testSoftDeletesDisabledWarning() throws Exception {
+        final String indexName = "test_soft_deletes_disabled_warning";
+        if (CLUSTER_TYPE == ClusterType.OLD) {
+            boolean softDeletesEnabled = true;
+            Settings.Builder settings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1);
+            if (randomBoolean()) {
+                softDeletesEnabled = randomBoolean();
+                settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), softDeletesEnabled);
+            }
+            Request request = new Request("PUT", "/" + indexName);
+            request.setJsonEntity("{\"settings\": " + Strings.toString(settings.build()) + "}");
+            if (softDeletesEnabled == false) {
+                expectSoftDeletesWarning(request, indexName);
+            }
+            client().performRequest(request);
+        }
+        ensureGreen(indexName);
+        indexDocs(indexName, randomInt(100), randomInt(100));
+    }
+
     @SuppressWarnings("unchecked")
     private Map<String, Object> getIndexSettingsAsMap(String index) throws IOException {
         Map<String, Object> indexSettings = getIndexSettings(index);

rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml

@@ -154,3 +154,20 @@
 
   - match: { error.type: "illegal_argument_exception" }
   - match: { error.reason: "The mapping definition cannot be nested under a type [_doc] unless include_type_name is set to true." }
+
+---
+"Create index without soft deletes":
+  - skip:
+      version:  " - 7.5.99"
+      reason:   "indices without soft deletes are deprecated in 7.6"
+      features: "warnings"
+
+  - do:
+      warnings:
+        - Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions.
+          Please do not specify value for setting [index.soft_deletes.enabled] of index [test_index].
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            soft_deletes.enabled: false

rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml

@@ -1,11 +1,19 @@
 ---
 "Translog retention without soft_deletes":
+  - skip:
+      version:  " - 7.5.99"
+      reason:   "indices without soft deletes are deprecated in 7.6"
+      features: "warnings"
+
   - do:
       indices.create:
         index: test
         body:
           settings:
             soft_deletes.enabled: false
+      warnings:
+        - Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions.
+          Please do not specify value for setting [index.soft_deletes.enabled] of index [test].
   - do:
       cluster.health:
         wait_for_no_initializing_shards: true
@@ -132,15 +140,20 @@
 ---
 "Translog stats on closed indices without soft-deletes":
   - skip:
-      version: " - 7.2.99"
-      reason:  "closed indices have translog stats starting version 7.3.0"
+      version: " - 7.5.99"
+      reason:  "indices without soft deletes are deprecated in 7.6"
+      features: "warnings"
 
   - do:
       indices.create:
         index: test
         body:
           settings:
             soft_deletes.enabled: false
+      warnings:
+        - Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions.
+          Please do not specify value for setting [index.soft_deletes.enabled] of index [test].
+
   - do:
       cluster.health:
         wait_for_no_initializing_shards: true

server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java

@@ -53,6 +53,7 @@
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -63,6 +64,7 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
@@ -104,6 +106,7 @@
  */
 public class MetaDataCreateIndexService {
     private static final Logger logger = LogManager.getLogger(MetaDataCreateIndexService.class);
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(logger);
 
     public static final int MAX_INDEX_NAME_BYTES = 255;
 
@@ -455,6 +458,11 @@ static Settings aggregateIndexSettings(ClusterState currentState, CreateIndexClu
          * that will be used to create this index.
          */
         MetaDataCreateIndexService.checkShardLimit(indexSettings, currentState);
+        if (indexSettings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) == false) {
+            DEPRECATION_LOGGER.deprecatedAndMaybeLog("soft_deletes_disabled",
+                "Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. " +
+                "Please do not specify value for setting [index.soft_deletes.enabled] of index [" + request.index() + "].");
+        }
         return indexSettings;
     }
 

server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java

@@ -858,6 +858,21 @@ public void testGetIndexNumberOfRoutingShardsYieldsSourceNumberOfShards() {
         assertThat(targetRoutingNumberOfShards, is(6));
     }
 
+    public void testSoftDeletesDisabledDeprecation() {
+        request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test");
+        request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), false).build());
+        aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Collections.emptyList(), Collections.emptyMap(),
+            null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
+        assertWarnings("Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. "
+            + "Please do not specify value for setting [index.soft_deletes.enabled] of index [test].");
+        request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test");
+        if (randomBoolean()) {
+            request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), true).build());
+        }
+        aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Collections.emptyList(), Collections.emptyMap(),
+            null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
+    }
+
     private IndexTemplateMetaData addMatchingTemplate(Consumer<IndexTemplateMetaData.Builder> configurator) {
         IndexTemplateMetaData.Builder builder = templateMetaDataBuilder("template1", "te*");
         configurator.accept(builder);

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -52,6 +52,7 @@
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.internal.io.IOUtils;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.seqno.ReplicationTracker;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.snapshots.SnapshotState;
@@ -969,23 +970,27 @@ protected static void ensureNoInitializingShards() throws IOException {
     }
 
     protected static void createIndex(String name, Settings settings) throws IOException {
-        Request request = new Request("PUT", "/" + name);
-        request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings) + "}");
-        client().performRequest(request);
+        createIndex(name, settings, null);
     }
 
     protected static void createIndex(String name, Settings settings, String mapping) throws IOException {
-        Request request = new Request("PUT", "/" + name);
-        request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings)
-                + ", \"mappings\" : {" + mapping + "} }");
-        client().performRequest(request);
+        createIndex(name, settings, mapping, null);
     }
 
     protected static void createIndex(String name, Settings settings, String mapping, String aliases) throws IOException {
         Request request = new Request("PUT", "/" + name);
-        request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings)
-            + ", \"mappings\" : {" + mapping + "}"
-            + ", \"aliases\": {" + aliases + "} }");
+        String entity = "{\"settings\": " + Strings.toString(settings);
+        if (mapping != null) {
+            entity += ",\"mappings\" : {" + mapping + "}";
+        }
+        if (aliases != null) {
+            entity += ",\"aliases\": {" + aliases + "}";
+        }
+        entity += "}";
+        if (settings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) == false) {
+            expectSoftDeletesWarning(request, name);
+        }
+        request.setJsonEntity(entity);
         client().performRequest(request);
     }
 
@@ -1004,6 +1009,20 @@ private static void updateIndexSettings(String index, Settings settings) throws
         client().performRequest(request);
     }
 
+    protected static void expectSoftDeletesWarning(Request request, String indexName) {
+        final List<String> expectedWarnings = Collections.singletonList(
+            "Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. " +
+            "Please do not specify value for setting [index.soft_deletes.enabled] of index [" + indexName + "].");
+        final Builder requestOptions = RequestOptions.DEFAULT.toBuilder();
+        if (nodeVersions.stream().allMatch(version -> version.onOrAfter(Version.V_7_6_0))) {
+            requestOptions.setWarningsHandler(warnings -> warnings.equals(expectedWarnings) == false);
+            request.setOptions(requestOptions);
+        } else if (nodeVersions.stream().anyMatch(version -> version.onOrAfter(Version.V_7_6_0))) {
+            requestOptions.setWarningsHandler(warnings -> warnings.isEmpty() == false && warnings.equals(expectedWarnings) == false);
+            request.setOptions(requestOptions);
+        }
+    }
+
     protected static Map<String, Object> getIndexSettings(String index) throws IOException {
         Request request = new Request("GET", "/" + index + "/_settings");
         request.addParameter("flat_settings", "true");