Skip to content

HADOOP-17385. ITestS3ADeleteCost.testDirMarkersFileCreation failure #2473

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Nov 26, 2020
Merged

HADOOP-17385. ITestS3ADeleteCost.testDirMarkersFileCreation failure #2473

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
54c1cd3
HADOOP-17385. ITestS3ADeleteCost.testDirMarkersFileCreation failure
steveloughran Nov 18, 2020
5da48dd
HADOOP-17385. fix import
steveloughran Nov 19, 2020
57f30e3
HADOOP-17385. ITestS3ADeleteCost -fix nits.
steveloughran Nov 23, 2020
36524ff
HADOOP-17385. Add and use explicit S3AFileSystem.initializeClass()
steveloughran Nov 25, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4950,6 +4950,15 @@ public MarkerToolOperations createMarkerToolOperations() {
return new MarkerToolOperationsImpl(operationCallbacks);
}

/**
* This is purely for testing, as it force initializes all static
* initializers. See HADOOP-17385 for details.
*/
@InterfaceAudience.Private
public static void initializeClass() {
LOG.debug("Initialize S3A class");
}

/**
* The implementation of context accessors.
*/
Expand Down
4 changes: 4 additions & 0 deletions hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.S3ATestUtils;

/**
Expand Down Expand Up @@ -53,6 +54,9 @@ public S3AContract(Configuration conf) {
*/
public S3AContract(Configuration conf, boolean addContractResource) {
super(conf);
// Force deprecated key load through the
// static initializers. See: HADOOP-17385
S3AFileSystem.initializeClass();
//insert the base features
if (addContractResource) {
addConfResource(CONTRACT_XML);
Expand Down
3 changes: 3 additions & 0 deletions hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,9 @@ public void setup() throws Exception {
// filesystems which add default configuration resources to do it before
// our tests start adding/removing options. See HADOOP-16626.
FileSystem.getLocal(new Configuration());
// Force deprecated key load through the
// static initializers. See: HADOOP-17385
S3AFileSystem.initializeClass();
super.setup();
}

Expand Down
9 changes: 9 additions & 0 deletions ...ls/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.Statistic;
import org.apache.hadoop.fs.s3a.Tristate;
import org.apache.hadoop.fs.s3a.impl.DirectoryPolicy;
import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum;

import static org.apache.hadoop.fs.s3a.Constants.*;
Expand Down Expand Up @@ -141,6 +142,14 @@ public void setup() throws Exception {

isDeleting = !isKeeping;

// check that the FS has the expected state
DirectoryPolicy markerPolicy = fs.getDirectoryMarkerPolicy();
Assertions.assertThat(markerPolicy.getMarkerPolicy())
.describedAs("Marker policy for filesystem %s", fs)
.isEqualTo(isKeepingMarkers()
? DirectoryPolicy.MarkerPolicy.Keep
: DirectoryPolicy.MarkerPolicy.Delete);

// insert new metrics so as to keep the list sorted
costValidator = OperationCostValidator.builder(getFileSystem())
.withMetrics(
Expand Down
1 change: 1 addition & 0 deletions ...ols/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,7 @@ public void testDirMarkersFileCreation() throws Throwable {

verifyMetrics(() -> {
file(new Path(srcDir, "source.txt"));
LOG.info("Metrics: {}\n{}", getMetricSummary(), getFileSystem());
return "after touch(fs, srcFilePath) " + getMetricSummary();
},
with(DIRECTORIES_CREATED, 0),
Expand Down