Skip to content

[ABFS] HADOOP-18656: Paginated Delete Driver Testing #5661

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 5 commits into from
Closed

[ABFS] HADOOP-18656: Paginated Delete Driver Testing #5661

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2979332
Added pagination query param for delete path
sreeb-msft Mar 9, 2023
cc80b77
Adding tests for paginated delete
sreeb-msft Mar 15, 2023
4ee8a1b
Default config value and test changes
sreeb-msft Mar 20, 2023
6418bc1
Adding users in test
sreeb-msft Mar 31, 2023
6f8ebab
Paginated Delete driver side short tests
sreeb-msft May 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -328,6 +328,10 @@ public class AbfsConfiguration{
FS_AZURE_ENABLE_ABFS_LIST_ITERATOR, DefaultValue = DEFAULT_ENABLE_ABFS_LIST_ITERATOR)
private boolean enableAbfsListIterator;

@BooleanConfigurationValidatorAnnotation(ConfigurationKey =
FS_AZURE_ENABLE_PAGINATED_DELETE, DefaultValue = DEFAULT_ENABLE_PAGINATED_DELETE)
private boolean enablePaginatedDelete;

public AbfsConfiguration(final Configuration rawConfig, String accountName)
throws IllegalAccessException, InvalidConfigurationValueException, IOException {
this.rawConfig = ProviderUtils.excludeIncompatibleCredentialProviders(
Expand Down Expand Up @@ -702,6 +706,10 @@ public boolean isEnabledMkdirOverwrite() {
return mkdirOverwrite;
}

public boolean isEnabledPaginatedDelete() {
return enablePaginatedDelete;
}

public String getAppendBlobDirs() {
return this.azureAppendBlobDirs;
}
Expand Down
4 changes: 4 additions & 0 deletions ...hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,10 @@ public final class ConfigurationKeys {
public static final String FS_AZURE_SKIP_SUPER_USER_REPLACEMENT = "fs.azure.identity.transformer.skip.superuser.replacement";
public static final String AZURE_KEY_ACCOUNT_KEYPROVIDER = "fs.azure.account.keyprovider";
public static final String AZURE_KEY_ACCOUNT_SHELLKEYPROVIDER_SCRIPT = "fs.azure.shellkeyprovider.script";
/**
* Specify whether paginated behavior is to be expected or not in delete path.
*/
public static final String FS_AZURE_ENABLE_PAGINATED_DELETE = "fs.azure.enable.paginated.delete";

/**
* Enable or disable readahead buffer in AbfsInputStream.
Expand Down
1 change: 1 addition & 0 deletions ...azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ public final class FileSystemConfigurations {
public static final String DEFAULT_VALUE_UNKNOWN = "UNKNOWN";

public static final boolean DEFAULT_DELETE_CONSIDERED_IDEMPOTENT = true;
public static final boolean DEFAULT_ENABLE_PAGINATED_DELETE = false;
public static final int DEFAULT_CLOCK_SKEW_WITH_SERVER_IN_MS = 5 * 60 * 1000; // 5 mins

public static final int STREAM_ID_LEN = 12;
Expand Down
1 change: 1 addition & 0 deletions ...s/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpQueryParams.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ public final class HttpQueryParams {
public static final String QUERY_PARAM_DIRECTORY = "directory";
public static final String QUERY_PARAM_CONTINUATION = "continuation";
public static final String QUERY_PARAM_RECURSIVE = "recursive";
public static final String QUERY_PARAM_PAGINATED = "paginated";
public static final String QUERY_PARAM_MAXRESULTS = "maxResults";
public static final String QUERY_PARAM_ACTION = "action";
public static final String QUERY_FS_ACTION = "fsAction";
Expand Down
14 changes: 11 additions & 3 deletions ...p-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import java.util.concurrent.TimeUnit;

import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys;
import org.apache.hadoop.fs.store.LogExactlyOnce;
import org.apache.hadoop.util.Preconditions;
import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
Expand Down Expand Up @@ -72,8 +73,7 @@
import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.RENAME_PATH_ATTEMPTS;
import static org.apache.hadoop.fs.azurebfs.AzureBlobFileSystemStore.extractEtagHeader;
import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.*;
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_DELETE_CONSIDERED_IDEMPOTENT;
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.SERVER_SIDE_ENCRYPTION_ALGORITHM;
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.*;
Copy link
Contributor

@saxenapranav saxenapranav May 17, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

wildcard import.

import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.HTTPS_SCHEME;
import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.*;
import static org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams.*;
Expand All @@ -87,7 +87,7 @@ public class AbfsClient implements Closeable {

private final URL baseUrl;
private final SharedKeyCredentials sharedKeyCredentials;
private final String xMsVersion = "2019-12-12";
private String xMsVersion = "2019-12-12";
Copy link
Contributor

@saxenapranav saxenapranav May 17, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Dont we need to switch it to "2023-08-03"?

Should we switch only for deletePath and not for others?

private final ExponentialRetryPolicy retryPolicy;
private final String filesystem;
private final AbfsConfiguration abfsConfiguration;
Expand Down Expand Up @@ -867,6 +867,14 @@ public AbfsRestOperation deletePath(final String path, final boolean recursive,
final List<AbfsHttpHeader> requestHeaders = createDefaultHeaders();

final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder();

boolean enablePagination = abfsConfiguration.getBoolean(
ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE,
DEFAULT_ENABLE_PAGINATED_DELETE
);

Comment on lines +871 to +875
Copy link
Contributor

@saxenapranav saxenapranav May 17, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

lets use abfsConfiguration.isEnabledPaginatedDelete

abfsUriQueryBuilder.addQuery(QUERY_PARAM_PAGINATED, String.valueOf(enablePagination));

abfsUriQueryBuilder.addQuery(QUERY_PARAM_RECURSIVE, String.valueOf(recursive));
abfsUriQueryBuilder.addQuery(QUERY_PARAM_CONTINUATION, continuation);
String operation = recursive ? SASTokenProvider.DELETE_RECURSIVE_OPERATION : SASTokenProvider.DELETE_OPERATION;
Expand Down
5 changes: 5 additions & 0 deletions ...op-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,15 @@ public final class TestConfigurationKeys {
public static final String FS_AZURE_BLOB_FS_CLIENT_SECRET = "fs.azure.account.oauth2.client.secret";

public static final String FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID = "fs.azure.account.test.oauth2.client.id";

public static final String FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID_2 = "fs.azure.account.test2.oauth2.client.id";
public static final String FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET = "fs.azure.account.test.oauth2.client.secret";
public static final String FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET_2 = "fs.azure.account.test2.oauth2.client.secret";

public static final String FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_GUID = "fs.azure.check.access.testuser.guid";

public static final String FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_2_GUID = "fs.azure.check.access.testuser2.guid";

public static final String MOCK_SASTOKENPROVIDER_FAIL_INIT = "mock.sastokenprovider.fail.init";
public static final String MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN = "mock.sastokenprovider.return.empty.sasToken";

Expand Down
Loading