diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 80f803d80dab0..ecb615cbc712c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -328,6 +328,10 @@ public class AbfsConfiguration{ FS_AZURE_ENABLE_ABFS_LIST_ITERATOR, DefaultValue = DEFAULT_ENABLE_ABFS_LIST_ITERATOR) private boolean enableAbfsListIterator; + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = + FS_AZURE_ENABLE_PAGINATED_DELETE, DefaultValue = DEFAULT_ENABLE_PAGINATED_DELETE) + private boolean enablePaginatedDelete; + public AbfsConfiguration(final Configuration rawConfig, String accountName) throws IllegalAccessException, InvalidConfigurationValueException, IOException { this.rawConfig = ProviderUtils.excludeIncompatibleCredentialProviders( @@ -702,6 +706,10 @@ public boolean isEnabledMkdirOverwrite() { return mkdirOverwrite; } + public boolean isEnabledPaginatedDelete() { + return enablePaginatedDelete; + } + public String getAppendBlobDirs() { return this.azureAppendBlobDirs; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index a59f76b6d0fe0..a2011f64d214c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -189,6 +189,10 @@ public final class ConfigurationKeys { public static final String FS_AZURE_SKIP_SUPER_USER_REPLACEMENT = "fs.azure.identity.transformer.skip.superuser.replacement"; public static final String AZURE_KEY_ACCOUNT_KEYPROVIDER = "fs.azure.account.keyprovider"; public static final String AZURE_KEY_ACCOUNT_SHELLKEYPROVIDER_SCRIPT = "fs.azure.shellkeyprovider.script"; + /** + * Specify whether paginated behavior is to be expected or not in delete path. + */ + public static final String FS_AZURE_ENABLE_PAGINATED_DELETE = "fs.azure.enable.paginated.delete"; /** * Enable or disable readahead buffer in AbfsInputStream. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 9994d9f5207f3..21bb2e7edffa3 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -114,6 +114,7 @@ public final class FileSystemConfigurations { public static final String DEFAULT_VALUE_UNKNOWN = "UNKNOWN"; public static final boolean DEFAULT_DELETE_CONSIDERED_IDEMPOTENT = true; + public static final boolean DEFAULT_ENABLE_PAGINATED_DELETE = false; public static final int DEFAULT_CLOCK_SKEW_WITH_SERVER_IN_MS = 5 * 60 * 1000; // 5 mins public static final int STREAM_ID_LEN = 12; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpQueryParams.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpQueryParams.java index e9bb95cad21cd..7bdc420220b72 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpQueryParams.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpQueryParams.java @@ -30,6 +30,7 @@ public final class HttpQueryParams { public static final String QUERY_PARAM_DIRECTORY = "directory"; public static final String QUERY_PARAM_CONTINUATION = "continuation"; public static final String QUERY_PARAM_RECURSIVE = "recursive"; + public static final String QUERY_PARAM_PAGINATED = "paginated"; public static final String QUERY_PARAM_MAXRESULTS = "maxResults"; public static final String QUERY_PARAM_ACTION = "action"; public static final String QUERY_FS_ACTION = "fsAction"; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 25562660ae231..6e80ff3b6e1b4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -38,6 +38,7 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; import org.apache.hadoop.fs.store.LogExactlyOnce; import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.base.Strings; @@ -72,8 +73,7 @@ import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.RENAME_PATH_ATTEMPTS; import static org.apache.hadoop.fs.azurebfs.AzureBlobFileSystemStore.extractEtagHeader; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.*; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_DELETE_CONSIDERED_IDEMPOTENT; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.SERVER_SIDE_ENCRYPTION_ALGORITHM; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.*; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.HTTPS_SCHEME; import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.*; import static org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams.*; @@ -87,7 +87,7 @@ public class AbfsClient implements Closeable { private final URL baseUrl; private final SharedKeyCredentials sharedKeyCredentials; - private final String xMsVersion = "2019-12-12"; + private String xMsVersion = "2019-12-12"; private final ExponentialRetryPolicy retryPolicy; private final String filesystem; private final AbfsConfiguration abfsConfiguration; @@ -867,6 +867,14 @@ public AbfsRestOperation deletePath(final String path, final boolean recursive, final List requestHeaders = createDefaultHeaders(); final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); + + boolean enablePagination = abfsConfiguration.getBoolean( + ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, + DEFAULT_ENABLE_PAGINATED_DELETE + ); + + abfsUriQueryBuilder.addQuery(QUERY_PARAM_PAGINATED, String.valueOf(enablePagination)); + abfsUriQueryBuilder.addQuery(QUERY_PARAM_RECURSIVE, String.valueOf(recursive)); abfsUriQueryBuilder.addQuery(QUERY_PARAM_CONTINUATION, continuation); String operation = recursive ? SASTokenProvider.DELETE_RECURSIVE_OPERATION : SASTokenProvider.DELETE_OPERATION; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java index 9e40f22d231b0..2914fbbba323d 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java @@ -43,10 +43,15 @@ public final class TestConfigurationKeys { public static final String FS_AZURE_BLOB_FS_CLIENT_SECRET = "fs.azure.account.oauth2.client.secret"; public static final String FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID = "fs.azure.account.test.oauth2.client.id"; + + public static final String FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID_2 = "fs.azure.account.test2.oauth2.client.id"; public static final String FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET = "fs.azure.account.test.oauth2.client.secret"; + public static final String FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET_2 = "fs.azure.account.test2.oauth2.client.secret"; public static final String FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_GUID = "fs.azure.check.access.testuser.guid"; + public static final String FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_2_GUID = "fs.azure.check.access.testuser2.guid"; + public static final String MOCK_SASTOKENPROVIDER_FAIL_INIT = "mock.sastokenprovider.fail.init"; public static final String MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN = "mock.sastokenprovider.return.empty.sasToken"; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsPaginatedDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsPaginatedDelete.java new file mode 100644 index 0000000000000..f542c9f90cf03 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsPaginatedDelete.java @@ -0,0 +1,300 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.services; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; +import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; +import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem; +import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; +import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; +import org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider; +import org.apache.hadoop.fs.azurebfs.utils.AclTestHelpers; +import org.apache.hadoop.fs.azurebfs.utils.TracingContext; +import org.apache.hadoop.fs.permission.AclEntry; +import org.apache.hadoop.fs.permission.AclEntryScope; +import org.apache.hadoop.fs.permission.AclEntryType; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.util.Lists; +import org.junit.Assume; +import org.junit.Test; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.util.List; + +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_GUID; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CLIENT_ID; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CLIENT_SECRET; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; + +public class ITestAbfsPaginatedDelete extends AbstractAbfsIntegrationTest { + + private AzureBlobFileSystem superUserFs; + private AzureBlobFileSystem firstTestUserFs; + private String firstTestUserGuid; + + private boolean isHnsEnabled; + public ITestAbfsPaginatedDelete() throws Exception { + } + + @Override + public void setup() throws Exception { + isHnsEnabled = this.getConfiguration().getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false); + loadConfiguredFileSystem(); + super.setup(); + this.superUserFs = getFileSystem(); + this.firstTestUserGuid = getConfiguration() + .get(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_GUID); + + if(isHnsEnabled) { + // setting up ACL permissions for test user + setFirstTestUserFsAuth(); + setDefaultAclOnRoot(this.firstTestUserGuid); + } + + } + + @Test + public void testFnsDeleteWithPaginationTrue() throws Exception { + Assume.assumeFalse(isHnsEnabled); + Path smallDirPath = createSmallDir(); + + AbfsClient client = getFileSystem().getAbfsStore().getClient(); + AbfsClient finalClient = TestAbfsClient.setAbfsClientField(client, "xMsVersion", "2023-08-03"); + AbfsConfiguration abfsConfig = finalClient.getAbfsConfiguration(); + abfsConfig.setBoolean(FS_AZURE_ENABLE_PAGINATED_DELETE, true); + + TracingContext testTracingContext = getTestTracingContext(this.firstTestUserFs, true); + finalClient.deletePath(smallDirPath.toString(), true, null, testTracingContext); + + AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> + finalClient.getPathStatus(smallDirPath.toString(), false, testTracingContext)); + assertEquals(HttpURLConnection.HTTP_NOT_FOUND, e.getStatusCode()); + } + + @Test + public void testFnsDeleteWithPaginationFalse() throws Exception { + Assume.assumeFalse(isHnsEnabled); + Path smallDirPath = createSmallDir(); + + AbfsClient client = getFileSystem().getAbfsStore().getClient(); + AbfsClient finalClient = TestAbfsClient.setAbfsClientField(client, "xMsVersion", "2023-08-03"); + AbfsConfiguration abfsConfig = finalClient.getAbfsConfiguration(); + abfsConfig.setBoolean(FS_AZURE_ENABLE_PAGINATED_DELETE, true); + + TracingContext testTracingContext = getTestTracingContext(this.firstTestUserFs, true); + finalClient.deletePath(smallDirPath.toString(), true, null, testTracingContext); + + AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> + finalClient.getPathStatus(smallDirPath.toString(), false, testTracingContext)); + assertEquals(HttpURLConnection.HTTP_NOT_FOUND, e.getStatusCode()); + } + + @Test + public void testVersionForPagination() throws Exception { + Assume.assumeTrue(isHnsEnabled); + Path smallDirPath = createSmallDir(); + AbfsClient client = this.firstTestUserFs.getAbfsStore().getClient(); + AbfsConfiguration abfsConfig = client.getAbfsConfiguration(); + + // delete should fail with bad request as version does not support pagination + abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, true); + AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> + client.deletePath(smallDirPath.toString(), true, null, getTestTracingContext(this.firstTestUserFs, false)) + ); + assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, e.getStatusCode()); + } + + @Test + public void testInvalidPaginationTrueRecursiveFalse() throws Exception { + Assume.assumeTrue(isHnsEnabled); + Path smallDirPath = createSmallDir(); + + AbfsClient client = this.firstTestUserFs.getAbfsStore().getClient(); + AbfsClient finalClient = TestAbfsClient.setAbfsClientField(client, "xMsVersion", "2023-08-03"); + AbfsConfiguration abfsConfig = finalClient.getAbfsConfiguration(); + abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, true); + + // delete should fail with HTTP as recursive will be set to false + // but pagination parameter is set to true + String path = smallDirPath.toString(); + AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> + finalClient.deletePath(path, false, null, getTestTracingContext(this.firstTestUserFs, false))); + assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, e.getStatusCode()); + } + + @Test + public void testInvalidPaginationFalseRandomCt() throws Exception { + Assume.assumeTrue(isHnsEnabled); + Path smallDirPath = createSmallDir(); + + AbfsClient client = this.firstTestUserFs.getAbfsStore().getClient(); + AbfsClient finalClient = TestAbfsClient.setAbfsClientField(client, "xMsVersion", "2023-08-03"); + AbfsConfiguration abfsConfig = finalClient.getAbfsConfiguration(); + TracingContext testTracingContext = getTestTracingContext(this.firstTestUserFs, true); + + abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, false); + String ct = "randomToken12345"; + AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> + finalClient.deletePath(smallDirPath.toString(), true, ct, getTestTracingContext(this.firstTestUserFs, false)) + ); + + assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, e.getStatusCode()); + } + + @Test + public void testInvalidRecursiveFalseRandomCt() throws Exception { + Assume.assumeTrue(isHnsEnabled); + Path smallDirPath = createSmallDir(); + + AbfsClient client = this.firstTestUserFs.getAbfsStore().getClient(); + AbfsClient finalClient = TestAbfsClient.setAbfsClientField(client, "xMsVersion", "2023-08-03"); + AbfsConfiguration abfsConfig = finalClient.getAbfsConfiguration(); + TracingContext testTracingContext = getTestTracingContext(this.firstTestUserFs, true); + + abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, true); + String ct = "randomToken12345"; + AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> + finalClient.deletePath(smallDirPath.toString(), false, ct, getTestTracingContext(this.firstTestUserFs, false)) + ); + + assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, e.getStatusCode()); + } + + @Test + public void testInvalidRecursiveFalsePaginationTrue() throws Exception { + Assume.assumeTrue(isHnsEnabled); + Path smallDirPath = createSmallDir(); + + AbfsClient client = this.firstTestUserFs.getAbfsStore().getClient(); + AbfsClient finalClient = TestAbfsClient.setAbfsClientField(client, "xMsVersion", "2023-08-03"); + AbfsConfiguration abfsConfig = finalClient.getAbfsConfiguration(); + TracingContext testTracingContext = getTestTracingContext(this.firstTestUserFs, true); + + abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, true); + AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> + finalClient.deletePath(smallDirPath.toString(), false, null, getTestTracingContext(this.firstTestUserFs, false)) + ); + } + + @Test + public void testValidRecursiveTruePaginationFalse() throws Exception { + Assume.assumeTrue(isHnsEnabled); + Path smallDirPath = createSmallDir(); + + AbfsClient client = this.firstTestUserFs.getAbfsStore().getClient(); + AbfsClient finalClient = TestAbfsClient.setAbfsClientField(client, "xMsVersion", "2023-08-03"); + AbfsConfiguration abfsConfig = finalClient.getAbfsConfiguration(); + TracingContext testTracingContext = getTestTracingContext(this.firstTestUserFs, true); + + abfsConfig.setBoolean(ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE, false); + finalClient.deletePath(smallDirPath.toString(), true, null, testTracingContext); + + AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> + finalClient.getPathStatus(smallDirPath.toString(), false, testTracingContext)); + assertEquals(HttpURLConnection.HTTP_NOT_FOUND, e.getStatusCode()); + } + + private void setFirstTestUserFsAuth() throws IOException { + if (this.firstTestUserFs != null) { + return; + } + checkIfConfigIsSet(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT + + "." + getAccountName()); + Configuration conf = getRawConfiguration(); + setTestFsConf(FS_AZURE_BLOB_FS_CLIENT_ID, FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID); + setTestFsConf(FS_AZURE_BLOB_FS_CLIENT_SECRET, + FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET); + conf.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.OAuth.name()); + conf.set(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + "." + + getAccountName(), ClientCredsTokenProvider.class.getName()); + conf.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, + false); + this.firstTestUserFs = (AzureBlobFileSystem) FileSystem.newInstance(getRawConfiguration()); + } + + + private void setTestFsConf(final String fsConfKey, + final String testFsConfKey) { + final String confKeyWithAccountName = fsConfKey + "." + getAccountName(); + final String confValue = getConfiguration() + .getString(testFsConfKey, ""); + getRawConfiguration().set(confKeyWithAccountName, confValue); + } + + private void setDefaultAclOnRoot(String uid) + throws IOException { + List aclSpec = Lists.newArrayList(AclTestHelpers + .aclEntry(AclEntryScope.ACCESS, AclEntryType.USER, uid, FsAction.ALL), + AclTestHelpers.aclEntry(AclEntryScope.DEFAULT, AclEntryType.USER, uid, FsAction.ALL)); + this.superUserFs.modifyAclEntries(new Path("/"), aclSpec); + } + + private String getContinuationToken(AbfsHttpOperation resultOp) { + String continuation = resultOp.getResponseHeader(HttpHeaderConfigurations.X_MS_CONTINUATION); + return continuation; + } + + + private Path createLargeDir() throws IOException { + AzureBlobFileSystem fs = getFileSystem(); + String rootPath = "/largeDir"; + String firstFilePath = rootPath + "/placeholderFile"; + fs.create(new Path(firstFilePath)); + + for (int i = 1; i <= 515; i++) { + String dirPath = "/dirLevel1" + String.valueOf(i) + "/dirLevel2" + String.valueOf(i); + String filePath = rootPath + dirPath + "/file" + String.valueOf(i); + fs.create(new Path(filePath)); + } + return new Path(rootPath); + } + + private Path createSmallDir() throws IOException { + String rootPath = "/smallDir"; + String firstFilePath = rootPath + "/placeholderFile"; + this.superUserFs.create(new Path(firstFilePath)); + + for (int i = 1; i <= 2; i++) { + String dirPath = "/dirLevel1-" + i + "/dirLevel2-" + i; + String filePath = rootPath + dirPath + "/file-" + i; + this.superUserFs.create(new Path(filePath)); + } + return new Path(rootPath); + } + + private void checkIfConfigIsSet(String configKey){ + AbfsConfiguration conf = getConfiguration(); + String value = conf.get(configKey); + Assume.assumeTrue(configKey + " config is mandatory for the test to run", + value != null && value.trim().length() > 1); + } +} diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java index 08eb3adc92697..3495ef66a5e1c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java @@ -341,10 +341,10 @@ public static AbfsClient getMockAbfsClient(AbfsClient baseAbfsClientInstance, return client; } - private static AbfsClient setAbfsClientField( - final AbfsClient client, - final String fieldName, - Object fieldObject) throws Exception { + static AbfsClient setAbfsClientField( + final AbfsClient client, + final String fieldName, + Object fieldObject) throws Exception { Field field = AbfsClient.class.getDeclaredField(fieldName); field.setAccessible(true);