Skip to content
Permalink
Browse files
HBASE-25900 Hadoop 3.2 and 3.3 support (#25)
Build on top of SteveL's original changes. Added indirection so
that the correct S3ClientFactory can be included at test-time. Provides
s3a.xml contracts for each version to reflect what actually works. Skips
the new tests which we know would fail on the older <Hadoop3.3 releases.


Co-authored-by: Steve Loughran <stevel@apache.org>
Co-authored-by: Peter Somogyi <psomogyi@cloudera.com>
Signed-off-by: Peter Somogyi <psomogyi@cloudera.com>
  • Loading branch information
3 people committed Oct 5, 2021
1 parent 01d975d commit 16693581c356c695f91bbe45a28bcb62789ca4c0
Showing 16 changed files with 772 additions and 191 deletions.
@@ -0,0 +1,63 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hbase.filesystem</groupId>
<artifactId>hbase-filesystem</artifactId>
<version>1.0.0-alpha2-SNAPSHOT</version>
</parent>
<artifactId>hadoop-testutils</artifactId>
<name>Common test utilities across Hadoop versions</name>
<dependencies>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>${commons-lang3.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<!-- Banned import in HBase -->
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-bundle</artifactId>
<version>${aws-java-sdk.version}</version>
</dependency>
<dependency>
<groupId>org.apache.yetus</groupId>
<artifactId>audience-annotations</artifactId>
<version>${audience-annotations.version}</version>
</dependency>
</dependencies>
</project>
@@ -15,14 +15,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.hadoop.hbase.oss;

import com.amazonaws.AmazonServiceException;
import com.amazonaws.SdkClientException;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.services.s3.AbstractAmazonS3;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.CopyObjectRequest;
import com.amazonaws.services.s3.model.CopyObjectResult;
@@ -43,84 +40,26 @@
import java.io.File;
import java.io.InputStream;
import java.io.IOException;
import java.net.URI;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.s3a.S3ClientFactory;
import org.apache.hadoop.fs.s3a.s3guard.LocalMetadataStore;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static org.apache.hadoop.hbase.oss.Constants.*;
import static org.apache.hadoop.fs.s3a.Constants.*;

@InterfaceAudience.Private
@InterfaceStability.Unstable
public class EmbeddedS3 {

public static boolean usingEmbeddedS3 = false;

private static final String BUCKET = "embedded";

public static void conditionalStart(Configuration conf) {
if (StringUtils.isEmpty(conf.get(S3_METADATA_STORE_IMPL))) {
conf.set(S3_METADATA_STORE_IMPL, LocalMetadataStore.class.getName());
}

boolean notConfigured = StringUtils.isEmpty(conf.get(DATA_URI));
if (notConfigured) {
usingEmbeddedS3 = true;
conf.set(S3_CLIENT_FACTORY_IMPL,
EmbeddedS3ClientFactory.class.getName());
conf.set(DATA_URI, "s3a://" + BUCKET);
} else {
usingEmbeddedS3 = false;
}
}

/**
* Replaces the default S3ClientFactory to inject an EmbeddedAmazonS3
* instance. This is currently a private API in Hadoop, but is the same method
* used by S3Guard's inconsistency-injection tests. The method signature
* defined in the interface varies depending on the Hadoop version.
*
* Due to compatibility purposes for both hadoop 2 and 3 main versions,
* we are omitting "@override" annotation from overridden methods.
*/
public static class EmbeddedS3ClientFactory implements S3ClientFactory {

public AmazonS3 createS3Client(URI name) {
AmazonS3 s3 = new EmbeddedAmazonS3();
s3.createBucket(BUCKET);
return s3;
}

public AmazonS3 createS3Client(URI name,
String bucket,
AWSCredentialsProvider credentialSet,
String userAgentSuffix) {
AmazonS3 s3 = new EmbeddedAmazonS3();
s3.createBucket(bucket);
return s3;
}

public AmazonS3 createS3Client(URI name,
String bucket,
AWSCredentialsProvider credentialSet) {
return createS3Client(name);
}
}
public static final String BUCKET = "embedded";

/**
* Emulates an S3-connected client. This is the bare minimum implementation
@@ -172,7 +111,7 @@ public S3ObjectInputStream getObjectContent() {
}
}

private Map<String, EmbeddedS3Object> bucket = new HashMap<>();
private Map<String, EmbeddedS3Object> bucket = new ConcurrentHashMap<>();

private void simulateServerSideCopy() {
try {
@@ -0,0 +1,56 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hbase.filesystem</groupId>
<artifactId>hbase-filesystem</artifactId>
<version>1.0.0-alpha2-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>hadoop3-2-testutils</artifactId>
<name>Test utility classes for Hadoop 3.1.x and 3.2.x</name>
<packaging>jar</packaging>

<dependencies>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hbase.filesystem</groupId>
<artifactId>hadoop-testutils</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-bundle</artifactId>
<version>${aws-java-sdk.version}</version>
</dependency>
<dependency>
<groupId>org.apache.yetus</groupId>
<artifactId>audience-annotations</artifactId>
<version>${audience-annotations.version}</version>
</dependency>
</dependencies>
</project>
@@ -0,0 +1,50 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.oss;

import java.io.IOException;
import java.net.URI;

import org.apache.hadoop.fs.s3a.S3ClientFactory;
import org.apache.hadoop.hbase.oss.EmbeddedS3.EmbeddedAmazonS3;

import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;

public class Hadoop32EmbeddedS3ClientFactory implements S3ClientFactory {
public AmazonS3 createS3Client(URI name) {
AmazonS3 s3 = new EmbeddedAmazonS3();
s3.createBucket(EmbeddedS3.BUCKET);
return s3;
}

public AmazonS3 createS3Client(URI name,
String bucket,
AWSCredentialsProvider credentialSet,
String userAgentSuffix) {
AmazonS3 s3 = new EmbeddedAmazonS3();
s3.createBucket(bucket);
return s3;
}

public AmazonS3 createS3Client(URI name,
String bucket,
AWSCredentialsProvider credentialSet) {
return createS3Client(name);
}
}
@@ -0,0 +1,56 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hbase.filesystem</groupId>
<artifactId>hbase-filesystem</artifactId>
<version>1.0.0-alpha2-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>hadoop3-3-testutils</artifactId>
<name>Test utility classes for Hadoop 3.3.1 and beyond</name>
<packaging>jar</packaging>

<dependencies>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hbase.filesystem</groupId>
<artifactId>hadoop-testutils</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-bundle</artifactId>
<version>${aws-java-sdk.version}</version>
</dependency>
<dependency>
<groupId>org.apache.yetus</groupId>
<artifactId>audience-annotations</artifactId>
<version>${audience-annotations.version}</version>
</dependency>
</dependencies>
</project>

0 comments on commit 1669358

Please sign in to comment.