Fail index creation using custom data path (#76792)

Custom per-index data paths were deprecated in 7.14. This commit blocks
creation of indices in 8.0 from configuring custom data paths.

relates #73168

docs/reference/migration/migrate_8_0/indices.asciidoc

@@ -116,4 +116,18 @@ index setting].
 Accept the new behaviour, or specify `?wait_for_active_shards=0` to preserve
 the old behaviour if needed.
 ====
+
+.The setting `index.data_path` is not longer allowed on new indices.
+[%collapsible]
+====
+*Details* +
+Creating an index relative to the node level `path.shared_data` setting was
+previously used with shadow replicas prior to their removal in 6.0. The
+per-index data path in `index.data_path` was deprecated in 7.14.0. In 8.0,
+creating new indices with `index.data_path` is no longer supported.
+
+*Impact* +
+Discontinue use of the `index.data_path` setting. Creating new indices with
+this setting will return an error.
+====
 //end::notable-breaking-changes[]

server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java

@@ -66,6 +66,7 @@
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.InternalSettingsPlugin;
+import org.elasticsearch.test.VersionUtils;
 import org.junit.Assert;
 
 import java.io.IOException;
@@ -116,6 +117,11 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
         return pluginList(InternalSettingsPlugin.class);
     }
 
+    @Override
+    protected boolean forbidPrivateIndexSettings() {
+        return false;
+    }
+
     public void testLockTryingToDelete() throws Exception {
         createIndex("test");
         ensureGreen();
@@ -214,8 +220,11 @@ public void testIndexDirIsDeletedWhenShardRemoved() throws Exception {
         Environment env = getInstanceFromNode(Environment.class);
         Path idxPath = env.sharedDataFile().resolve(randomAlphaOfLength(10));
         logger.info("--> idxPath: [{}]", idxPath);
+        Version createdVersion =
+            VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(Version.V_8_0_0));
         Settings idxSettings = Settings.builder()
             .put(IndexMetadata.SETTING_DATA_PATH, idxPath)
+            .put(IndexMetadata.SETTING_VERSION_CREATED, createdVersion)
             .build();
         createIndex("test", idxSettings);
         ensureGreen("test");
@@ -253,8 +262,14 @@ public void testIndexCanChangeCustomDataPath() throws Exception {
         final Path sharedDataPath = getInstanceFromNode(Environment.class).sharedDataFile().resolve(randomAsciiLettersOfLength(10));
         final Path indexDataPath = sharedDataPath.resolve("start-" + randomAsciiLettersOfLength(10));
 
-        logger.info("--> creating index [{}] with data_path [{}]", index, indexDataPath);
-        createIndex(index, Settings.builder().put(IndexMetadata.SETTING_DATA_PATH, indexDataPath.toAbsolutePath().toString()).build());
+        logger.info("--> creating legacy index [{}] with data_path [{}]", index, indexDataPath);
+        Version createdVersion =
+            VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(Version.V_8_0_0));
+        Settings settings = Settings.builder()
+            .put(IndexMetadata.SETTING_DATA_PATH, indexDataPath.toAbsolutePath().toString())
+            .put(IndexMetadata.SETTING_VERSION_CREATED, createdVersion)
+            .build();
+        createIndex(index, settings);
         client().prepareIndex(index).setId("1").setSource("foo", "bar").setRefreshPolicy(IMMEDIATE).get();
         ensureGreen(index);
 

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java

@@ -48,7 +48,6 @@
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexModule;
@@ -69,7 +68,6 @@
 
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
-import java.nio.file.Path;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -796,6 +794,7 @@ static Settings aggregateIndexSettings(ClusterState currentState, CreateIndexClu
         validateSoftDeleteSettings(indexSettings);
         validateTranslogRetentionSettings(indexSettings);
         validateStoreTypeSetting(indexSettings);
+        validateNoCustomPath(indexSettings);
         return indexSettings;
     }
 
@@ -1047,7 +1046,7 @@ public void validateIndexSettings(String indexName, final Settings settings, fin
     }
 
     List<String> getIndexSettingsValidationErrors(final Settings settings, final boolean forbidPrivateIndexSettings) {
-        List<String> validationErrors = validateIndexCustomPath(settings, env.sharedDataFile());
+        List<String> validationErrors = new ArrayList<>();
         if (forbidPrivateIndexSettings) {
             validationErrors.addAll(validatePrivateSettingsNotExplicitlySet(settings, indexScopedSettings));
         }
@@ -1068,27 +1067,16 @@ private static List<String> validatePrivateSettingsNotExplicitlySet(Settings set
     }
 
     /**
-     * Validates that the configured index data path (if any) is a sub-path of the configured shared data path (if any)
+     * Validates an index data path is not specified.
      *
      * @param settings the index configured settings
-     * @param sharedDataPath the configured `path.shared_data` (if any)
-     * @return a list containing validaton errors or an empty list if there aren't any errors
      */
-    private static List<String> validateIndexCustomPath(Settings settings, @Nullable Path sharedDataPath) {
-        String customPath = IndexMetadata.INDEX_DATA_PATH_SETTING.get(settings);
-        List<String> validationErrors = new ArrayList<>();
-        if (Strings.isEmpty(customPath) == false) {
-            if (sharedDataPath == null) {
-                validationErrors.add("path.shared_data must be set in order to use custom data paths");
-            } else {
-                Path resolvedPath = PathUtils.get(new Path[]{sharedDataPath}, customPath);
-                if (resolvedPath == null) {
-                    validationErrors.add("custom path [" + customPath +
-                        "] is not a sub-path of path.shared_data [" + sharedDataPath + "]");
-                }
-            }
+    static void validateNoCustomPath(Settings settings) {
+        if (IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(Version.V_8_0_0) &&
+            IndexMetadata.INDEX_DATA_PATH_SETTING.exists(settings)) {
+            throw new IllegalArgumentException("per-index custom data path using setting ["
+                + IndexMetadata.INDEX_DATA_PATH_SETTING.getKey() + "] is no longer supported on new indices");
         }
-        return validationErrors;
     }
 
     /**

server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java

@@ -34,10 +34,10 @@
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexNotFoundException;
@@ -91,9 +91,9 @@
 import static org.elasticsearch.cluster.metadata.MetadataCreateIndexService.getIndexNumberOfRoutingShards;
 import static org.elasticsearch.cluster.metadata.MetadataCreateIndexService.parseV1Mappings;
 import static org.elasticsearch.cluster.metadata.MetadataCreateIndexService.resolveAndValidateAliases;
-
 import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING;
 import static org.elasticsearch.indices.ShardLimitValidatorTests.createTestShardLimitService;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasKey;
@@ -281,6 +281,18 @@ public void testValidateSplitIndex() {
             Settings.builder().put("index.number_of_shards", targetShards).build());
     }
 
+    public void testValidateNoCustomPath() {
+        Settings indexSettings = Settings.builder()
+            .put(SETTING_VERSION_CREATED, Version.V_8_0_0)
+            .put(IndexMetadata.INDEX_DATA_PATH_SETTING.getKey(), "some/path")
+            .build();
+        var e = expectThrows(IllegalArgumentException.class,
+            () -> MetadataCreateIndexService.validateNoCustomPath(indexSettings));
+        assertThat(e.getMessage(), containsString("per-index custom data path"));
+
+        MetadataCreateIndexService.validateNoCustomPath(Settings.EMPTY);
+    }
+
     public void testPrepareResizeIndexSettings() {
         final List<Version> versions = Arrays.asList(VersionUtils.randomVersion(random()), VersionUtils.randomVersion(random()));
         versions.sort(Comparator.comparingLong(l -> l.id));

test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java

@@ -686,12 +686,6 @@ public Settings indexSettings() {
         if (numberOfReplicas >= 0) {
             builder.put(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas).build();
         }
-        // 30% of the time
-        if (randomInt(9) < 3) {
-            final String dataPath = randomAlphaOfLength(10);
-            logger.info("using custom data_path for index: [{}]", dataPath);
-            builder.put(IndexMetadata.SETTING_DATA_PATH, dataPath);
-        }
         // always default delayed allocation to 0 to make sure we have tests are not delayed
         builder.put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), 0);
         if (randomBoolean()) {

x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java

@@ -234,12 +234,11 @@ protected void checkSoftDeletesNotEagerlyLoaded(String restoredIndexName) {
 
     protected void assertShardFolders(String indexName, boolean snapshotDirectory) throws IOException {
         final Index restoredIndex = resolveIndex(indexName);
-        final String customDataPath = resolveCustomDataPath(indexName);
         final ShardId shardId = new ShardId(restoredIndex, 0);
         boolean shardFolderFound = false;
         for (String node : internalCluster().getNodeNames()) {
             final NodeEnvironment service = internalCluster().getInstance(NodeEnvironment.class, node);
-            final ShardPath shardPath = ShardPath.loadShardPath(logger, service, shardId, customDataPath);
+            final ShardPath shardPath = ShardPath.loadShardPath(logger, service, shardId, null);
             if (shardPath != null && Files.exists(shardPath.getDataPath())) {
                 shardFolderFound = true;
                 assertEquals(snapshotDirectory, Files.notExists(shardPath.resolveIndex()));