From 2979332c72537a78caf1c293ca3f60a1ce41ca1f Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Thu, 9 Mar 2023 18:52:17 +0530 Subject: [PATCH 1/5] Added pagination query param for delete path --- .../apache/hadoop/fs/azurebfs/AbfsConfiguration.java | 8 ++++++++ .../fs/azurebfs/constants/ConfigurationKeys.java | 4 ++++ .../azurebfs/constants/FileSystemConfigurations.java | 1 + .../fs/azurebfs/constants/HttpQueryParams.java | 1 + .../hadoop/fs/azurebfs/services/AbfsClient.java | 12 ++++++++++-- 5 files changed, 24 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 80f803d80dab0..ecb615cbc712c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -328,6 +328,10 @@ public class AbfsConfiguration{ FS_AZURE_ENABLE_ABFS_LIST_ITERATOR, DefaultValue = DEFAULT_ENABLE_ABFS_LIST_ITERATOR) private boolean enableAbfsListIterator; + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = + FS_AZURE_ENABLE_PAGINATED_DELETE, DefaultValue = DEFAULT_ENABLE_PAGINATED_DELETE) + private boolean enablePaginatedDelete; + public AbfsConfiguration(final Configuration rawConfig, String accountName) throws IllegalAccessException, InvalidConfigurationValueException, IOException { this.rawConfig = ProviderUtils.excludeIncompatibleCredentialProviders( @@ -702,6 +706,10 @@ public boolean isEnabledMkdirOverwrite() { return mkdirOverwrite; } + public boolean isEnabledPaginatedDelete() { + return enablePaginatedDelete; + } + public String getAppendBlobDirs() { return this.azureAppendBlobDirs; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index a59f76b6d0fe0..a2011f64d214c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -189,6 +189,10 @@ public final class ConfigurationKeys { public static final String FS_AZURE_SKIP_SUPER_USER_REPLACEMENT = "fs.azure.identity.transformer.skip.superuser.replacement"; public static final String AZURE_KEY_ACCOUNT_KEYPROVIDER = "fs.azure.account.keyprovider"; public static final String AZURE_KEY_ACCOUNT_SHELLKEYPROVIDER_SCRIPT = "fs.azure.shellkeyprovider.script"; + /** + * Specify whether paginated behavior is to be expected or not in delete path. + */ + public static final String FS_AZURE_ENABLE_PAGINATED_DELETE = "fs.azure.enable.paginated.delete"; /** * Enable or disable readahead buffer in AbfsInputStream. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 9994d9f5207f3..21bb2e7edffa3 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -114,6 +114,7 @@ public final class FileSystemConfigurations { public static final String DEFAULT_VALUE_UNKNOWN = "UNKNOWN"; public static final boolean DEFAULT_DELETE_CONSIDERED_IDEMPOTENT = true; + public static final boolean DEFAULT_ENABLE_PAGINATED_DELETE = false; public static final int DEFAULT_CLOCK_SKEW_WITH_SERVER_IN_MS = 5 * 60 * 1000; // 5 mins public static final int STREAM_ID_LEN = 12; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpQueryParams.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpQueryParams.java index e9bb95cad21cd..7bdc420220b72 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpQueryParams.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpQueryParams.java @@ -30,6 +30,7 @@ public final class HttpQueryParams { public static final String QUERY_PARAM_DIRECTORY = "directory"; public static final String QUERY_PARAM_CONTINUATION = "continuation"; public static final String QUERY_PARAM_RECURSIVE = "recursive"; + public static final String QUERY_PARAM_PAGINATED = "paginated"; public static final String QUERY_PARAM_MAXRESULTS = "maxResults"; public static final String QUERY_PARAM_ACTION = "action"; public static final String QUERY_FS_ACTION = "fsAction"; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 25562660ae231..d5c70c35dcda7 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -38,6 +38,7 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; import org.apache.hadoop.fs.store.LogExactlyOnce; import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.base.Strings; @@ -72,8 +73,7 @@ import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.RENAME_PATH_ATTEMPTS; import static org.apache.hadoop.fs.azurebfs.AzureBlobFileSystemStore.extractEtagHeader; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.*; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_DELETE_CONSIDERED_IDEMPOTENT; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.SERVER_SIDE_ENCRYPTION_ALGORITHM; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.*; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.HTTPS_SCHEME; import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.*; import static org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams.*; @@ -867,6 +867,14 @@ public AbfsRestOperation deletePath(final String path, final boolean recursive, final List requestHeaders = createDefaultHeaders(); final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); + + boolean enablePagination = abfsConfiguration.getBoolean( + ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, + DEFAULT_ENABLE_PAGINATED_DELETE + ); + if (enablePagination) { + abfsUriQueryBuilder.addQuery(QUERY_PARAM_PAGINATED, String.valueOf(enablePagination)); + } abfsUriQueryBuilder.addQuery(QUERY_PARAM_RECURSIVE, String.valueOf(recursive)); abfsUriQueryBuilder.addQuery(QUERY_PARAM_CONTINUATION, continuation); String operation = recursive ? SASTokenProvider.DELETE_RECURSIVE_OPERATION : SASTokenProvider.DELETE_OPERATION; From cc80b778d97602ac42a164344aab92e4cbcfbd8d Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Wed, 15 Mar 2023 15:03:16 +0530 Subject: [PATCH 2/5] Adding tests for paginated delete --- .../fs/azurebfs/services/TestAbfsClient.java | 8 +- .../services/TestPaginatedDelete.java | 89 +++++++++++++++++++ 2 files changed, 93 insertions(+), 4 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestPaginatedDelete.java diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java index 08eb3adc92697..3495ef66a5e1c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java @@ -341,10 +341,10 @@ public static AbfsClient getMockAbfsClient(AbfsClient baseAbfsClientInstance, return client; } - private static AbfsClient setAbfsClientField( - final AbfsClient client, - final String fieldName, - Object fieldObject) throws Exception { + static AbfsClient setAbfsClientField( + final AbfsClient client, + final String fieldName, + Object fieldObject) throws Exception { Field field = AbfsClient.class.getDeclaredField(fieldName); field.setAccessible(true); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestPaginatedDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestPaginatedDelete.java new file mode 100644 index 0000000000000..bd7074dfe880b --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestPaginatedDelete.java @@ -0,0 +1,89 @@ +package org.apache.hadoop.fs.azurebfs.services; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; +import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; +import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem; +import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants; +import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; +import org.assertj.core.api.Assertions; + +import java.io.IOException; +import java.net.HttpURLConnection; + +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ACCOUNT_NAME; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT; + +public class TestPaginatedDelete extends AbstractAbfsIntegrationTest { + public TestPaginatedDelete() throws Exception { + loadConfiguredFileSystem(); + boolean isHnsEnabled = this.getConfiguration().getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false); + Assume.assumeTrue(isHnsEnabled); + } + + @Test + public void testVersionForPagination() throws Exception { + createLargeDir(); + AbfsConfiguration abfsConfig = getConfiguration(); // update to retrieve fixed test configs + AzureBlobFileSystem fs = getFileSystem(); // update to retrieve fixed test configs + AbfsClient client = fs.getAbfsStore().getClient(); + client = TestAbfsClient.setAbfsClientField(client, "xMsVersion", "2021-12-02"); + + abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, true); + updateTestConfiguration(); + + // delete should fail with bad request as version does not support pagination + String path = "/LargeDir"; + AbfsRestOperation resultOp = client.deletePath(path, true, null, getTestTracingContext(fs, false)); + int statusCode = resultOp.getResult().getStatusCode(); + assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, statusCode); + + abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, false); + updateTestConfiguration(); + + resultOp = client.deletePath(path, true, null, getTestTracingContext(fs, false)); + statusCode = resultOp.getResult().getStatusCode(); + assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, statusCode); + } + + + @Test + public void testDefaultBehaviorWithoutPagination() { + + } + + @Before + public void createLargeDir() throws IOException { + AzureBlobFileSystem fs = getFileSystem(); + String rootPath = "/largeDir"; + String firstFilePath = rootPath + "/placeholderFile"; + fs.create(new Path(firstFilePath)); + + for (int i = 1; i <= 2000; i++) { + String dirPath = "/dir" + String.valueOf(i); + String filePath = rootPath + dirPath + "/file" + String.valueOf(i); + fs.create(new Path(filePath)); + } + + } + + private void fixTestConfiguration() { + + } + + private void updateTestConfiguration() { + + } + + private String getSmallDirPath() { + return "abc"; + } + + private String getLargeDirPath() { + return "abc"; + } + +} From 4ee8a1bb34e949f9bf7023bfff409353dfc8b8ca Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Mon, 20 Mar 2023 11:24:59 +0530 Subject: [PATCH 3/5] Default config value and test changes --- .../fs/azurebfs/constants/FileSystemConfigurations.java | 2 +- .../hadoop/fs/azurebfs/services/TestPaginatedDelete.java | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 21bb2e7edffa3..3f3089a286eb5 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -114,7 +114,7 @@ public final class FileSystemConfigurations { public static final String DEFAULT_VALUE_UNKNOWN = "UNKNOWN"; public static final boolean DEFAULT_DELETE_CONSIDERED_IDEMPOTENT = true; - public static final boolean DEFAULT_ENABLE_PAGINATED_DELETE = false; + public static final boolean DEFAULT_ENABLE_PAGINATED_DELETE = true; public static final int DEFAULT_CLOCK_SKEW_WITH_SERVER_IN_MS = 5 * 60 * 1000; // 5 mins public static final int STREAM_ID_LEN = 12; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestPaginatedDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestPaginatedDelete.java index bd7074dfe880b..e3be49bc4f24e 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestPaginatedDelete.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestPaginatedDelete.java @@ -30,13 +30,13 @@ public void testVersionForPagination() throws Exception { AbfsConfiguration abfsConfig = getConfiguration(); // update to retrieve fixed test configs AzureBlobFileSystem fs = getFileSystem(); // update to retrieve fixed test configs AbfsClient client = fs.getAbfsStore().getClient(); - client = TestAbfsClient.setAbfsClientField(client, "xMsVersion", "2021-12-02"); + // client = TestAbfsClient.setAbfsClientField(client, "xMsVersion", "2021-12-02"); abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, true); updateTestConfiguration(); // delete should fail with bad request as version does not support pagination - String path = "/LargeDir"; + String path = "/largeDir"; AbfsRestOperation resultOp = client.deletePath(path, true, null, getTestTracingContext(fs, false)); int statusCode = resultOp.getResult().getStatusCode(); assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, statusCode); @@ -55,7 +55,6 @@ public void testDefaultBehaviorWithoutPagination() { } - @Before public void createLargeDir() throws IOException { AzureBlobFileSystem fs = getFileSystem(); String rootPath = "/largeDir"; @@ -63,7 +62,7 @@ public void createLargeDir() throws IOException { fs.create(new Path(firstFilePath)); for (int i = 1; i <= 2000; i++) { - String dirPath = "/dir" + String.valueOf(i); + String dirPath = "/dirLevel1" + String.valueOf(i) + "/dirLevel2" + String.valueOf(i); String filePath = rootPath + dirPath + "/file" + String.valueOf(i); fs.create(new Path(filePath)); } From 6418bc1efdab055e67a5cad055194e5aab25cc7e Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Fri, 31 Mar 2023 13:48:36 +0530 Subject: [PATCH 4/5] Adding users in test --- .../constants/TestConfigurationKeys.java | 5 + .../services/TestPaginatedDelete.java | 242 ++++++++++++++++-- 2 files changed, 219 insertions(+), 28 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java index 9e40f22d231b0..2914fbbba323d 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java @@ -43,10 +43,15 @@ public final class TestConfigurationKeys { public static final String FS_AZURE_BLOB_FS_CLIENT_SECRET = "fs.azure.account.oauth2.client.secret"; public static final String FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID = "fs.azure.account.test.oauth2.client.id"; + + public static final String FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID_2 = "fs.azure.account.test2.oauth2.client.id"; public static final String FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET = "fs.azure.account.test.oauth2.client.secret"; + public static final String FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET_2 = "fs.azure.account.test2.oauth2.client.secret"; public static final String FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_GUID = "fs.azure.check.access.testuser.guid"; + public static final String FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_2_GUID = "fs.azure.check.access.testuser2.guid"; + public static final String MOCK_SASTOKENPROVIDER_FAIL_INIT = "mock.sastokenprovider.fail.init"; public static final String MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN = "mock.sastokenprovider.return.empty.sasToken"; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestPaginatedDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestPaginatedDelete.java index e3be49bc4f24e..7571fae993918 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestPaginatedDelete.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestPaginatedDelete.java @@ -1,11 +1,24 @@ package org.apache.hadoop.fs.azurebfs.services; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem; import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants; import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; +import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException; +import org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider; +import org.apache.hadoop.fs.azurebfs.utils.AclTestHelpers; +import org.apache.hadoop.fs.azurebfs.utils.TracingContext; +import org.apache.hadoop.fs.permission.AclEntry; +import org.apache.hadoop.fs.permission.AclEntryScope; +import org.apache.hadoop.fs.permission.AclEntryType; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.util.Lists; import org.junit.Assume; import org.junit.Before; import org.junit.Test; @@ -13,76 +26,249 @@ import java.io.IOException; import java.net.HttpURLConnection; +import java.util.List; -import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ACCOUNT_NAME; -import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.*; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.*; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; public class TestPaginatedDelete extends AbstractAbfsIntegrationTest { + + private long knownPageSize; + private FileSystem superUserFs; + private AzureBlobFileSystem firstTestUserFs; + private FileSystem secondTestUserFs; + private String firstTestUserGuid; + private String secondTestUserGuid; + public TestPaginatedDelete() throws Exception { + super.setup(); loadConfiguredFileSystem(); boolean isHnsEnabled = this.getConfiguration().getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false); Assume.assumeTrue(isHnsEnabled); + this.knownPageSize = 1024; // value set based on current DC limit on tenant + this.superUserFs = getFileSystem(); + this.firstTestUserGuid = getConfiguration() + .get(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_GUID); + this.secondTestUserGuid = getConfiguration() + .get(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_2_GUID); + setFirstTestUserFsAuth(); + setSecondTestUserFsAuth(); } + private void setFirstTestUserFsAuth() throws IOException { + if (this.firstTestUserFs != null) { + return; + } + checkIfConfigIsSet(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT + + "." + getAccountName()); + Configuration conf = getRawConfiguration(); + setTestFsConf(FS_AZURE_BLOB_FS_CLIENT_ID, + FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID); + setTestFsConf(FS_AZURE_BLOB_FS_CLIENT_SECRET, + FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET); + conf.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.OAuth.name()); + conf.set(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + "." + + getAccountName(), ClientCredsTokenProvider.class.getName()); + conf.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, + false); + this.firstTestUserFs = (AzureBlobFileSystem) FileSystem.newInstance(getRawConfiguration()); + } + + private void setSecondTestUserFsAuth() throws IOException { + if (this.secondTestUserFs != null) { + return; + } + checkIfConfigIsSet(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT + + "." + getAccountName()); + Configuration conf = getRawConfiguration(); + setTestFsConf(FS_AZURE_BLOB_FS_CLIENT_ID, + FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID_2); + setTestFsConf(FS_AZURE_BLOB_FS_CLIENT_SECRET, + FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET_2); + conf.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.OAuth.name()); + conf.set(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + "." + + getAccountName(), ClientCredsTokenProvider.class.getName()); + conf.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, + false); + this.secondTestUserFs = FileSystem.newInstance(getRawConfiguration()); + } + + private void setTestFsConf(final String fsConfKey, + final String testFsConfKey) { + final String confKeyWithAccountName = fsConfKey + "." + getAccountName(); + final String confValue = getConfiguration() + .getString(testFsConfKey, ""); + getRawConfiguration().set(confKeyWithAccountName, confValue); + } + + private void setDefaultAclOnRoot(Path parentDir, String uid) + throws IOException { + List aclSpec = Lists.newArrayList(AclTestHelpers + .aclEntry(AclEntryScope.ACCESS, AclEntryType.USER, uid, FsAction.ALL), + AclTestHelpers.aclEntry(AclEntryScope.DEFAULT, AclEntryType.USER, uid, + FsAction.ALL)); + this.superUserFs.modifyAclEntries(parentDir, aclSpec); + } + + @Test + public void testNormalPaginationBehavior() { + + } @Test public void testVersionForPagination() throws Exception { - createLargeDir(); + Path smallDirPath = createSmallDir(); + // setting defaultAcl on root for the first User + setDefaultAclOnRoot(smallDirPath, this.firstTestUserGuid); + + AbfsConfiguration abfsConfig = getConfiguration(); // update to retrieve fixed test configs + AbfsClient client = this.firstTestUserFs.getAbfsStore().getClient(); + client.deletePath(smallDirPath.toString(), true, null, getTestTracingContext(this.firstTestUserFs, false)); +// client = TestAbfsClient.setAbfsClientField(client, "xMsVersion", "2021-12-02"); +// AbfsClient finalClient = client; +// +// String path = "/smallDir"; +// finalClient.deletePath(path, true, null, getTestTracingContext(fs, false)); +// // delete should fail with bad request as version does not support pagination +// abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, true); +// AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> +// finalClient.deletePath(path, true, null, getTestTracingContext(fs, false)) +// ); +// assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, e.getStatusCode()); +// +// // delete should fail again irrespective of pagination parameter value +// abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, false); +// e = intercept(AbfsRestOperationException.class, () -> +// finalClient.deletePath(path, true, null, getTestTracingContext(fs, false)) +// ); +// assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, e.getStatusCode()); + } + + @Test + public void testInvalidPaginationTrueRecursiveFalse() throws Exception { + Path smallDirPath = createSmallDir(); + // setting defaultAcl on root for the first User + setDefaultAclOnRoot(smallDirPath, this.firstTestUserGuid); AbfsConfiguration abfsConfig = getConfiguration(); // update to retrieve fixed test configs AzureBlobFileSystem fs = getFileSystem(); // update to retrieve fixed test configs AbfsClient client = fs.getAbfsStore().getClient(); - // client = TestAbfsClient.setAbfsClientField(client, "xMsVersion", "2021-12-02"); abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, true); - updateTestConfiguration(); - // delete should fail with bad request as version does not support pagination - String path = "/largeDir"; - AbfsRestOperation resultOp = client.deletePath(path, true, null, getTestTracingContext(fs, false)); - int statusCode = resultOp.getResult().getStatusCode(); - assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, statusCode); + // delete should fail with bad request as recursive will be set to false + // but pagination parameter is set to true + String path = smallDirPath.toString(); + AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> + client.deletePath(path, false, null, getTestTracingContext(fs, false))); + assertEquals(HttpURLConnection.HTTP_CONFLICT, e.getStatusCode()); + } + + @Test + public void testInvalidPaginationFalseCtNotNull() throws Exception { + Path largeDirPath = createLargeDir(); + // setting defaultAcl on root for the first User + setDefaultAclOnRoot(largeDirPath, this.firstTestUserGuid); + AbfsConfiguration abfsConfig = getConfiguration(); // update to retrieve fixed test configs + AzureBlobFileSystem fs = getFileSystem(); // update to retrieve fixed test configs + AbfsClient client = fs.getAbfsStore().getClient(); + TracingContext testTracingContext = getTestTracingContext(fs, true); + abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, true); + + String path = largeDirPath.toString(); + // issuing delete once on large dir to get valid ct + AbfsRestOperation resultOp = client.deletePath(path, true, null, testTracingContext); + String continuationToken = getContinuationToken(resultOp.getResult()); + + // setting pagination param to false abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, false); - updateTestConfiguration(); + AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> + client.deletePath(path, true, continuationToken, getTestTracingContext(fs, false)) + ); + assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, e.getStatusCode()); + } - resultOp = client.deletePath(path, true, null, getTestTracingContext(fs, false)); - statusCode = resultOp.getResult().getStatusCode(); - assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, statusCode); + @Test + public void testPaginationFailureWithChangedPath() throws Exception { + Path largeDirPath = createLargeDir(); + Path smallDirPath = createSmallDir(); + // setting defaultAcl on small directory root for the first User + setDefaultAclOnRoot(smallDirPath, this.firstTestUserGuid); + // setting defaultAcl on large directory root for the first User + setDefaultAclOnRoot(largeDirPath, this.firstTestUserGuid); + AzureBlobFileSystem fs = getFileSystem(); + AbfsConfiguration abfsConfig = getConfiguration(); // update to retrieve fixed test configs + AbfsClient client = fs.getAbfsStore().getClient(); + TracingContext testTracingContext = getTestTracingContext(fs, true); + + abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, true); + + String path1 = "/largeDir"; + // issuing first delete on large directory to obtain + // non-null continuation token + AbfsRestOperation resultOp = client.deletePath(path1, true, null, testTracingContext); + String continuationToken = getContinuationToken(resultOp.getResult()); + + // issuing second delete on a different path + // should fail due to mismatch of continuation token and current delete path + String path2 = "/smallDir"; + AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> + client.deletePath(path2, true, continuationToken, getTestTracingContext(fs, false)) + ); + assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, e.getStatusCode()); } + @Test + public void testPaginationFailureWithChangedUser() throws Exception { + setFirstTestUserFsAuth(); + } + + // test correct no. of resources get deleted based on page size + // total no. of resources/no. of loops in delete = page size set + + private String getContinuationToken(AbfsHttpOperation resultOp) { + String continuation = resultOp.getResponseHeader(HttpHeaderConfigurations.X_MS_CONTINUATION); + return continuation; + } @Test public void testDefaultBehaviorWithoutPagination() { } - public void createLargeDir() throws IOException { + private Path createLargeDir() throws IOException { AzureBlobFileSystem fs = getFileSystem(); String rootPath = "/largeDir"; String firstFilePath = rootPath + "/placeholderFile"; fs.create(new Path(firstFilePath)); - for (int i = 1; i <= 2000; i++) { + for (int i = 1; i <= 10000; i++) { String dirPath = "/dirLevel1" + String.valueOf(i) + "/dirLevel2" + String.valueOf(i); String filePath = rootPath + dirPath + "/file" + String.valueOf(i); fs.create(new Path(filePath)); } - - } - - private void fixTestConfiguration() { - + return new Path(rootPath); } - private void updateTestConfiguration() { - - } + private Path createSmallDir() throws IOException { + String rootPath = "/smallDir"; + String firstFilePath = rootPath + "/placeholderFile"; + this.superUserFs.create(new Path(firstFilePath)); - private String getSmallDirPath() { - return "abc"; + for (int i = 1; i <= 5; i++) { + String dirPath = "/dirLevel1-" + i + "/dirLevel2-" + i; + String filePath = rootPath + dirPath + "/file-" + i; + this.superUserFs.create(new Path(filePath)); + } + return new Path(rootPath); } - private String getLargeDirPath() { - return "abc"; + private void checkIfConfigIsSet(String configKey){ + AbfsConfiguration conf = getConfiguration(); + String value = conf.get(configKey); + Assume.assumeTrue(configKey + " config is mandatory for the test to run", + value != null && value.trim().length() > 1); } } From 6f8ebab991723e99a65c423de95fdf61a05447cd Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Tue, 16 May 2023 17:41:32 +0530 Subject: [PATCH 5/5] Paginated Delete driver side short tests --- .../constants/FileSystemConfigurations.java | 2 +- .../fs/azurebfs/services/AbfsClient.java | 8 +- .../services/ITestAbfsPaginatedDelete.java | 300 ++++++++++++++++++ .../services/TestPaginatedDelete.java | 274 ---------------- 4 files changed, 305 insertions(+), 279 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsPaginatedDelete.java delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestPaginatedDelete.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 3f3089a286eb5..21bb2e7edffa3 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -114,7 +114,7 @@ public final class FileSystemConfigurations { public static final String DEFAULT_VALUE_UNKNOWN = "UNKNOWN"; public static final boolean DEFAULT_DELETE_CONSIDERED_IDEMPOTENT = true; - public static final boolean DEFAULT_ENABLE_PAGINATED_DELETE = true; + public static final boolean DEFAULT_ENABLE_PAGINATED_DELETE = false; public static final int DEFAULT_CLOCK_SKEW_WITH_SERVER_IN_MS = 5 * 60 * 1000; // 5 mins public static final int STREAM_ID_LEN = 12; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index d5c70c35dcda7..6e80ff3b6e1b4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -87,7 +87,7 @@ public class AbfsClient implements Closeable { private final URL baseUrl; private final SharedKeyCredentials sharedKeyCredentials; - private final String xMsVersion = "2019-12-12"; + private String xMsVersion = "2019-12-12"; private final ExponentialRetryPolicy retryPolicy; private final String filesystem; private final AbfsConfiguration abfsConfiguration; @@ -872,9 +872,9 @@ public AbfsRestOperation deletePath(final String path, final boolean recursive, ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, DEFAULT_ENABLE_PAGINATED_DELETE ); - if (enablePagination) { - abfsUriQueryBuilder.addQuery(QUERY_PARAM_PAGINATED, String.valueOf(enablePagination)); - } + + abfsUriQueryBuilder.addQuery(QUERY_PARAM_PAGINATED, String.valueOf(enablePagination)); + abfsUriQueryBuilder.addQuery(QUERY_PARAM_RECURSIVE, String.valueOf(recursive)); abfsUriQueryBuilder.addQuery(QUERY_PARAM_CONTINUATION, continuation); String operation = recursive ? SASTokenProvider.DELETE_RECURSIVE_OPERATION : SASTokenProvider.DELETE_OPERATION; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsPaginatedDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsPaginatedDelete.java new file mode 100644 index 0000000000000..f542c9f90cf03 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsPaginatedDelete.java @@ -0,0 +1,300 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.services; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; +import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; +import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem; +import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; +import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; +import org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider; +import org.apache.hadoop.fs.azurebfs.utils.AclTestHelpers; +import org.apache.hadoop.fs.azurebfs.utils.TracingContext; +import org.apache.hadoop.fs.permission.AclEntry; +import org.apache.hadoop.fs.permission.AclEntryScope; +import org.apache.hadoop.fs.permission.AclEntryType; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.util.Lists; +import org.junit.Assume; +import org.junit.Test; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.util.List; + +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_GUID; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CLIENT_ID; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CLIENT_SECRET; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; + +public class ITestAbfsPaginatedDelete extends AbstractAbfsIntegrationTest { + + private AzureBlobFileSystem superUserFs; + private AzureBlobFileSystem firstTestUserFs; + private String firstTestUserGuid; + + private boolean isHnsEnabled; + public ITestAbfsPaginatedDelete() throws Exception { + } + + @Override + public void setup() throws Exception { + isHnsEnabled = this.getConfiguration().getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false); + loadConfiguredFileSystem(); + super.setup(); + this.superUserFs = getFileSystem(); + this.firstTestUserGuid = getConfiguration() + .get(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_GUID); + + if(isHnsEnabled) { + // setting up ACL permissions for test user + setFirstTestUserFsAuth(); + setDefaultAclOnRoot(this.firstTestUserGuid); + } + + } + + @Test + public void testFnsDeleteWithPaginationTrue() throws Exception { + Assume.assumeFalse(isHnsEnabled); + Path smallDirPath = createSmallDir(); + + AbfsClient client = getFileSystem().getAbfsStore().getClient(); + AbfsClient finalClient = TestAbfsClient.setAbfsClientField(client, "xMsVersion", "2023-08-03"); + AbfsConfiguration abfsConfig = finalClient.getAbfsConfiguration(); + abfsConfig.setBoolean(FS_AZURE_ENABLE_PAGINATED_DELETE, true); + + TracingContext testTracingContext = getTestTracingContext(this.firstTestUserFs, true); + finalClient.deletePath(smallDirPath.toString(), true, null, testTracingContext); + + AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> + finalClient.getPathStatus(smallDirPath.toString(), false, testTracingContext)); + assertEquals(HttpURLConnection.HTTP_NOT_FOUND, e.getStatusCode()); + } + + @Test + public void testFnsDeleteWithPaginationFalse() throws Exception { + Assume.assumeFalse(isHnsEnabled); + Path smallDirPath = createSmallDir(); + + AbfsClient client = getFileSystem().getAbfsStore().getClient(); + AbfsClient finalClient = TestAbfsClient.setAbfsClientField(client, "xMsVersion", "2023-08-03"); + AbfsConfiguration abfsConfig = finalClient.getAbfsConfiguration(); + abfsConfig.setBoolean(FS_AZURE_ENABLE_PAGINATED_DELETE, true); + + TracingContext testTracingContext = getTestTracingContext(this.firstTestUserFs, true); + finalClient.deletePath(smallDirPath.toString(), true, null, testTracingContext); + + AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> + finalClient.getPathStatus(smallDirPath.toString(), false, testTracingContext)); + assertEquals(HttpURLConnection.HTTP_NOT_FOUND, e.getStatusCode()); + } + + @Test + public void testVersionForPagination() throws Exception { + Assume.assumeTrue(isHnsEnabled); + Path smallDirPath = createSmallDir(); + AbfsClient client = this.firstTestUserFs.getAbfsStore().getClient(); + AbfsConfiguration abfsConfig = client.getAbfsConfiguration(); + + // delete should fail with bad request as version does not support pagination + abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, true); + AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> + client.deletePath(smallDirPath.toString(), true, null, getTestTracingContext(this.firstTestUserFs, false)) + ); + assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, e.getStatusCode()); + } + + @Test + public void testInvalidPaginationTrueRecursiveFalse() throws Exception { + Assume.assumeTrue(isHnsEnabled); + Path smallDirPath = createSmallDir(); + + AbfsClient client = this.firstTestUserFs.getAbfsStore().getClient(); + AbfsClient finalClient = TestAbfsClient.setAbfsClientField(client, "xMsVersion", "2023-08-03"); + AbfsConfiguration abfsConfig = finalClient.getAbfsConfiguration(); + abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, true); + + // delete should fail with HTTP as recursive will be set to false + // but pagination parameter is set to true + String path = smallDirPath.toString(); + AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> + finalClient.deletePath(path, false, null, getTestTracingContext(this.firstTestUserFs, false))); + assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, e.getStatusCode()); + } + + @Test + public void testInvalidPaginationFalseRandomCt() throws Exception { + Assume.assumeTrue(isHnsEnabled); + Path smallDirPath = createSmallDir(); + + AbfsClient client = this.firstTestUserFs.getAbfsStore().getClient(); + AbfsClient finalClient = TestAbfsClient.setAbfsClientField(client, "xMsVersion", "2023-08-03"); + AbfsConfiguration abfsConfig = finalClient.getAbfsConfiguration(); + TracingContext testTracingContext = getTestTracingContext(this.firstTestUserFs, true); + + abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, false); + String ct = "randomToken12345"; + AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> + finalClient.deletePath(smallDirPath.toString(), true, ct, getTestTracingContext(this.firstTestUserFs, false)) + ); + + assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, e.getStatusCode()); + } + + @Test + public void testInvalidRecursiveFalseRandomCt() throws Exception { + Assume.assumeTrue(isHnsEnabled); + Path smallDirPath = createSmallDir(); + + AbfsClient client = this.firstTestUserFs.getAbfsStore().getClient(); + AbfsClient finalClient = TestAbfsClient.setAbfsClientField(client, "xMsVersion", "2023-08-03"); + AbfsConfiguration abfsConfig = finalClient.getAbfsConfiguration(); + TracingContext testTracingContext = getTestTracingContext(this.firstTestUserFs, true); + + abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, true); + String ct = "randomToken12345"; + AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> + finalClient.deletePath(smallDirPath.toString(), false, ct, getTestTracingContext(this.firstTestUserFs, false)) + ); + + assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, e.getStatusCode()); + } + + @Test + public void testInvalidRecursiveFalsePaginationTrue() throws Exception { + Assume.assumeTrue(isHnsEnabled); + Path smallDirPath = createSmallDir(); + + AbfsClient client = this.firstTestUserFs.getAbfsStore().getClient(); + AbfsClient finalClient = TestAbfsClient.setAbfsClientField(client, "xMsVersion", "2023-08-03"); + AbfsConfiguration abfsConfig = finalClient.getAbfsConfiguration(); + TracingContext testTracingContext = getTestTracingContext(this.firstTestUserFs, true); + + abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, true); + AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> + finalClient.deletePath(smallDirPath.toString(), false, null, getTestTracingContext(this.firstTestUserFs, false)) + ); + } + + @Test + public void testValidRecursiveTruePaginationFalse() throws Exception { + Assume.assumeTrue(isHnsEnabled); + Path smallDirPath = createSmallDir(); + + AbfsClient client = this.firstTestUserFs.getAbfsStore().getClient(); + AbfsClient finalClient = TestAbfsClient.setAbfsClientField(client, "xMsVersion", "2023-08-03"); + AbfsConfiguration abfsConfig = finalClient.getAbfsConfiguration(); + TracingContext testTracingContext = getTestTracingContext(this.firstTestUserFs, true); + + abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, false); + finalClient.deletePath(smallDirPath.toString(), true, null, testTracingContext); + + AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> + finalClient.getPathStatus(smallDirPath.toString(), false, testTracingContext)); + assertEquals(HttpURLConnection.HTTP_NOT_FOUND, e.getStatusCode()); + } + + private void setFirstTestUserFsAuth() throws IOException { + if (this.firstTestUserFs != null) { + return; + } + checkIfConfigIsSet(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT + + "." + getAccountName()); + Configuration conf = getRawConfiguration(); + setTestFsConf(FS_AZURE_BLOB_FS_CLIENT_ID, FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID); + setTestFsConf(FS_AZURE_BLOB_FS_CLIENT_SECRET, + FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET); + conf.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.OAuth.name()); + conf.set(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + "." + + getAccountName(), ClientCredsTokenProvider.class.getName()); + conf.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, + false); + this.firstTestUserFs = (AzureBlobFileSystem) FileSystem.newInstance(getRawConfiguration()); + } + + + private void setTestFsConf(final String fsConfKey, + final String testFsConfKey) { + final String confKeyWithAccountName = fsConfKey + "." + getAccountName(); + final String confValue = getConfiguration() + .getString(testFsConfKey, ""); + getRawConfiguration().set(confKeyWithAccountName, confValue); + } + + private void setDefaultAclOnRoot(String uid) + throws IOException { + List aclSpec = Lists.newArrayList(AclTestHelpers + .aclEntry(AclEntryScope.ACCESS, AclEntryType.USER, uid, FsAction.ALL), + AclTestHelpers.aclEntry(AclEntryScope.DEFAULT, AclEntryType.USER, uid, FsAction.ALL)); + this.superUserFs.modifyAclEntries(new Path("/"), aclSpec); + } + + private String getContinuationToken(AbfsHttpOperation resultOp) { + String continuation = resultOp.getResponseHeader(HttpHeaderConfigurations.X_MS_CONTINUATION); + return continuation; + } + + + private Path createLargeDir() throws IOException { + AzureBlobFileSystem fs = getFileSystem(); + String rootPath = "/largeDir"; + String firstFilePath = rootPath + "/placeholderFile"; + fs.create(new Path(firstFilePath)); + + for (int i = 1; i <= 515; i++) { + String dirPath = "/dirLevel1" + String.valueOf(i) + "/dirLevel2" + String.valueOf(i); + String filePath = rootPath + dirPath + "/file" + String.valueOf(i); + fs.create(new Path(filePath)); + } + return new Path(rootPath); + } + + private Path createSmallDir() throws IOException { + String rootPath = "/smallDir"; + String firstFilePath = rootPath + "/placeholderFile"; + this.superUserFs.create(new Path(firstFilePath)); + + for (int i = 1; i <= 2; i++) { + String dirPath = "/dirLevel1-" + i + "/dirLevel2-" + i; + String filePath = rootPath + dirPath + "/file-" + i; + this.superUserFs.create(new Path(filePath)); + } + return new Path(rootPath); + } + + private void checkIfConfigIsSet(String configKey){ + AbfsConfiguration conf = getConfiguration(); + String value = conf.get(configKey); + Assume.assumeTrue(configKey + " config is mandatory for the test to run", + value != null && value.trim().length() > 1); + } +} diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestPaginatedDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestPaginatedDelete.java deleted file mode 100644 index 7571fae993918..0000000000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestPaginatedDelete.java +++ /dev/null @@ -1,274 +0,0 @@ -package org.apache.hadoop.fs.azurebfs.services; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; -import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; -import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem; -import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants; -import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; -import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations; -import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; -import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException; -import org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider; -import org.apache.hadoop.fs.azurebfs.utils.AclTestHelpers; -import org.apache.hadoop.fs.azurebfs.utils.TracingContext; -import org.apache.hadoop.fs.permission.AclEntry; -import org.apache.hadoop.fs.permission.AclEntryScope; -import org.apache.hadoop.fs.permission.AclEntryType; -import org.apache.hadoop.fs.permission.FsAction; -import org.apache.hadoop.util.Lists; -import org.junit.Assume; -import org.junit.Before; -import org.junit.Test; -import org.assertj.core.api.Assertions; - -import java.io.IOException; -import java.net.HttpURLConnection; -import java.util.List; - -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.*; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION; -import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.*; -import static org.apache.hadoop.test.LambdaTestUtils.intercept; - -public class TestPaginatedDelete extends AbstractAbfsIntegrationTest { - - private long knownPageSize; - private FileSystem superUserFs; - private AzureBlobFileSystem firstTestUserFs; - private FileSystem secondTestUserFs; - private String firstTestUserGuid; - private String secondTestUserGuid; - - public TestPaginatedDelete() throws Exception { - super.setup(); - loadConfiguredFileSystem(); - boolean isHnsEnabled = this.getConfiguration().getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false); - Assume.assumeTrue(isHnsEnabled); - this.knownPageSize = 1024; // value set based on current DC limit on tenant - this.superUserFs = getFileSystem(); - this.firstTestUserGuid = getConfiguration() - .get(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_GUID); - this.secondTestUserGuid = getConfiguration() - .get(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_2_GUID); - setFirstTestUserFsAuth(); - setSecondTestUserFsAuth(); - } - - private void setFirstTestUserFsAuth() throws IOException { - if (this.firstTestUserFs != null) { - return; - } - checkIfConfigIsSet(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT - + "." + getAccountName()); - Configuration conf = getRawConfiguration(); - setTestFsConf(FS_AZURE_BLOB_FS_CLIENT_ID, - FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID); - setTestFsConf(FS_AZURE_BLOB_FS_CLIENT_SECRET, - FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET); - conf.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.OAuth.name()); - conf.set(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + "." - + getAccountName(), ClientCredsTokenProvider.class.getName()); - conf.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, - false); - this.firstTestUserFs = (AzureBlobFileSystem) FileSystem.newInstance(getRawConfiguration()); - } - - private void setSecondTestUserFsAuth() throws IOException { - if (this.secondTestUserFs != null) { - return; - } - checkIfConfigIsSet(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT - + "." + getAccountName()); - Configuration conf = getRawConfiguration(); - setTestFsConf(FS_AZURE_BLOB_FS_CLIENT_ID, - FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID_2); - setTestFsConf(FS_AZURE_BLOB_FS_CLIENT_SECRET, - FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET_2); - conf.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.OAuth.name()); - conf.set(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + "." - + getAccountName(), ClientCredsTokenProvider.class.getName()); - conf.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, - false); - this.secondTestUserFs = FileSystem.newInstance(getRawConfiguration()); - } - - private void setTestFsConf(final String fsConfKey, - final String testFsConfKey) { - final String confKeyWithAccountName = fsConfKey + "." + getAccountName(); - final String confValue = getConfiguration() - .getString(testFsConfKey, ""); - getRawConfiguration().set(confKeyWithAccountName, confValue); - } - - private void setDefaultAclOnRoot(Path parentDir, String uid) - throws IOException { - List aclSpec = Lists.newArrayList(AclTestHelpers - .aclEntry(AclEntryScope.ACCESS, AclEntryType.USER, uid, FsAction.ALL), - AclTestHelpers.aclEntry(AclEntryScope.DEFAULT, AclEntryType.USER, uid, - FsAction.ALL)); - this.superUserFs.modifyAclEntries(parentDir, aclSpec); - } - - @Test - public void testNormalPaginationBehavior() { - - } - @Test - public void testVersionForPagination() throws Exception { - Path smallDirPath = createSmallDir(); - // setting defaultAcl on root for the first User - setDefaultAclOnRoot(smallDirPath, this.firstTestUserGuid); - - AbfsConfiguration abfsConfig = getConfiguration(); // update to retrieve fixed test configs - AbfsClient client = this.firstTestUserFs.getAbfsStore().getClient(); - client.deletePath(smallDirPath.toString(), true, null, getTestTracingContext(this.firstTestUserFs, false)); -// client = TestAbfsClient.setAbfsClientField(client, "xMsVersion", "2021-12-02"); -// AbfsClient finalClient = client; -// -// String path = "/smallDir"; -// finalClient.deletePath(path, true, null, getTestTracingContext(fs, false)); -// // delete should fail with bad request as version does not support pagination -// abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, true); -// AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> -// finalClient.deletePath(path, true, null, getTestTracingContext(fs, false)) -// ); -// assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, e.getStatusCode()); -// -// // delete should fail again irrespective of pagination parameter value -// abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, false); -// e = intercept(AbfsRestOperationException.class, () -> -// finalClient.deletePath(path, true, null, getTestTracingContext(fs, false)) -// ); -// assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, e.getStatusCode()); - } - - @Test - public void testInvalidPaginationTrueRecursiveFalse() throws Exception { - Path smallDirPath = createSmallDir(); - // setting defaultAcl on root for the first User - setDefaultAclOnRoot(smallDirPath, this.firstTestUserGuid); - AbfsConfiguration abfsConfig = getConfiguration(); // update to retrieve fixed test configs - AzureBlobFileSystem fs = getFileSystem(); // update to retrieve fixed test configs - AbfsClient client = fs.getAbfsStore().getClient(); - - abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, true); - - // delete should fail with bad request as recursive will be set to false - // but pagination parameter is set to true - String path = smallDirPath.toString(); - AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> - client.deletePath(path, false, null, getTestTracingContext(fs, false))); - assertEquals(HttpURLConnection.HTTP_CONFLICT, e.getStatusCode()); - } - - @Test - public void testInvalidPaginationFalseCtNotNull() throws Exception { - Path largeDirPath = createLargeDir(); - // setting defaultAcl on root for the first User - setDefaultAclOnRoot(largeDirPath, this.firstTestUserGuid); - AbfsConfiguration abfsConfig = getConfiguration(); // update to retrieve fixed test configs - AzureBlobFileSystem fs = getFileSystem(); // update to retrieve fixed test configs - AbfsClient client = fs.getAbfsStore().getClient(); - TracingContext testTracingContext = getTestTracingContext(fs, true); - - abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, true); - - String path = largeDirPath.toString(); - // issuing delete once on large dir to get valid ct - AbfsRestOperation resultOp = client.deletePath(path, true, null, testTracingContext); - String continuationToken = getContinuationToken(resultOp.getResult()); - - // setting pagination param to false - abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, false); - AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> - client.deletePath(path, true, continuationToken, getTestTracingContext(fs, false)) - ); - assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, e.getStatusCode()); - } - - @Test - public void testPaginationFailureWithChangedPath() throws Exception { - Path largeDirPath = createLargeDir(); - Path smallDirPath = createSmallDir(); - // setting defaultAcl on small directory root for the first User - setDefaultAclOnRoot(smallDirPath, this.firstTestUserGuid); - // setting defaultAcl on large directory root for the first User - setDefaultAclOnRoot(largeDirPath, this.firstTestUserGuid); - AzureBlobFileSystem fs = getFileSystem(); - AbfsConfiguration abfsConfig = getConfiguration(); // update to retrieve fixed test configs - AbfsClient client = fs.getAbfsStore().getClient(); - TracingContext testTracingContext = getTestTracingContext(fs, true); - - abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, true); - - String path1 = "/largeDir"; - // issuing first delete on large directory to obtain - // non-null continuation token - AbfsRestOperation resultOp = client.deletePath(path1, true, null, testTracingContext); - String continuationToken = getContinuationToken(resultOp.getResult()); - - // issuing second delete on a different path - // should fail due to mismatch of continuation token and current delete path - String path2 = "/smallDir"; - AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> - client.deletePath(path2, true, continuationToken, getTestTracingContext(fs, false)) - ); - assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, e.getStatusCode()); - } - - @Test - public void testPaginationFailureWithChangedUser() throws Exception { - setFirstTestUserFsAuth(); - } - - // test correct no. of resources get deleted based on page size - // total no. of resources/no. of loops in delete = page size set - - private String getContinuationToken(AbfsHttpOperation resultOp) { - String continuation = resultOp.getResponseHeader(HttpHeaderConfigurations.X_MS_CONTINUATION); - return continuation; - } - - @Test - public void testDefaultBehaviorWithoutPagination() { - - } - - private Path createLargeDir() throws IOException { - AzureBlobFileSystem fs = getFileSystem(); - String rootPath = "/largeDir"; - String firstFilePath = rootPath + "/placeholderFile"; - fs.create(new Path(firstFilePath)); - - for (int i = 1; i <= 10000; i++) { - String dirPath = "/dirLevel1" + String.valueOf(i) + "/dirLevel2" + String.valueOf(i); - String filePath = rootPath + dirPath + "/file" + String.valueOf(i); - fs.create(new Path(filePath)); - } - return new Path(rootPath); - } - - private Path createSmallDir() throws IOException { - String rootPath = "/smallDir"; - String firstFilePath = rootPath + "/placeholderFile"; - this.superUserFs.create(new Path(firstFilePath)); - - for (int i = 1; i <= 5; i++) { - String dirPath = "/dirLevel1-" + i + "/dirLevel2-" + i; - String filePath = rootPath + dirPath + "/file-" + i; - this.superUserFs.create(new Path(filePath)); - } - return new Path(rootPath); - } - - private void checkIfConfigIsSet(String configKey){ - AbfsConfiguration conf = getConfiguration(); - String value = conf.get(configKey); - Assume.assumeTrue(configKey + " config is mandatory for the test to run", - value != null && value.trim().length() > 1); - } - -}