HBASE-25900 Hadoop 3.2 and 3.3 support (#25)

Build on top of SteveL's original changes. Added indirection so
that the correct S3ClientFactory can be included at test-time. Provides
s3a.xml contracts for each version to reflect what actually works. Skips
the new tests which we know would fail on the older <Hadoop3.3 releases.


Co-authored-by: Steve Loughran <stevel@apache.org>
Co-authored-by: Peter Somogyi <psomogyi@cloudera.com>
Signed-off-by: Peter Somogyi <psomogyi@cloudera.com>

hadoop-testutils/pom.xml

@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hbase.filesystem</groupId>
+    <artifactId>hbase-filesystem</artifactId>
+    <version>1.0.0-alpha2-SNAPSHOT</version>
+  </parent>
+  <artifactId>hadoop-testutils</artifactId>
+  <name>Common test utilities across Hadoop versions</name>
+   <dependencies>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+      <version>${commons-lang3.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <version>${slf4j.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <version>${hadoop.version}</version>
+      <exclusions>
+        <exclusion>
+          <!-- Banned import in HBase -->
+          <groupId>com.google.code.findbugs</groupId>
+          <artifactId>jsr305</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-aws</artifactId>
+      <version>${hadoop.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.amazonaws</groupId>
+      <artifactId>aws-java-sdk-bundle</artifactId>
+      <version>${aws-java-sdk.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.yetus</groupId>
+      <artifactId>audience-annotations</artifactId>
+      <version>${audience-annotations.version}</version>
+    </dependency>
+  </dependencies>
+</project>

hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/EmbeddedS3.java → hadoop-testutils/src/main/java/org/apache/hadoop/hbase/oss/EmbeddedS3.java

@@ -15,14 +15,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.hadoop.hbase.oss;
 
 import com.amazonaws.AmazonServiceException;
 import com.amazonaws.SdkClientException;
-import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.services.s3.AbstractAmazonS3;
-import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.model.Bucket;
 import com.amazonaws.services.s3.model.CopyObjectRequest;
 import com.amazonaws.services.s3.model.CopyObjectResult;
@@ -43,84 +40,26 @@
 import java.io.File;
 import java.io.InputStream;
 import java.io.IOException;
-import java.net.URI;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.s3a.S3ClientFactory;
-import org.apache.hadoop.fs.s3a.s3guard.LocalMetadataStore;
 import org.apache.yetus.audience.InterfaceAudience;
 import org.apache.yetus.audience.InterfaceStability;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import static org.apache.hadoop.hbase.oss.Constants.*;
-import static org.apache.hadoop.fs.s3a.Constants.*;
-
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 public class EmbeddedS3 {
 
-  public static boolean usingEmbeddedS3 = false;
-
-  private static final String BUCKET = "embedded";
-
-  public static void conditionalStart(Configuration conf) {
-    if (StringUtils.isEmpty(conf.get(S3_METADATA_STORE_IMPL))) {
-      conf.set(S3_METADATA_STORE_IMPL, LocalMetadataStore.class.getName());
-    }
-
-    boolean notConfigured = StringUtils.isEmpty(conf.get(DATA_URI));
-    if (notConfigured) {
-      usingEmbeddedS3 = true;
-      conf.set(S3_CLIENT_FACTORY_IMPL,
-            EmbeddedS3ClientFactory.class.getName());
-      conf.set(DATA_URI, "s3a://" + BUCKET);
-    } else {
-      usingEmbeddedS3 = false;
-    }
-  }
-
-  /**
-   * Replaces the default S3ClientFactory to inject an EmbeddedAmazonS3
-   * instance. This is currently a private API in Hadoop, but is the same method
-   * used by S3Guard's inconsistency-injection tests. The method signature
-   * defined in the interface varies depending on the Hadoop version.
-   *
-   * Due to compatibility purposes for both hadoop 2 and 3 main versions,
-   * we are omitting "@override" annotation from overridden methods.
-   */
-  public static class EmbeddedS3ClientFactory implements S3ClientFactory {
-
-    public AmazonS3 createS3Client(URI name) {
-      AmazonS3 s3 = new EmbeddedAmazonS3();
-      s3.createBucket(BUCKET);
-      return s3;
-    }
-
-    public AmazonS3 createS3Client(URI name,
-        String bucket,
-        AWSCredentialsProvider credentialSet,
-        String userAgentSuffix) {
-      AmazonS3 s3 = new EmbeddedAmazonS3();
-      s3.createBucket(bucket);
-      return s3;
-    }
-
-    public AmazonS3 createS3Client(URI name,
-        String bucket,
-        AWSCredentialsProvider credentialSet) {
-      return createS3Client(name);
-    }
-  }
+  public static final String BUCKET = "embedded";
 
   /**
    * Emulates an S3-connected client. This is the bare minimum implementation
@@ -172,7 +111,7 @@ public S3ObjectInputStream getObjectContent() {
       }
     }
 
-    private Map<String, EmbeddedS3Object> bucket = new HashMap<>();
+    private Map<String, EmbeddedS3Object> bucket = new ConcurrentHashMap<>();
 
     private void simulateServerSideCopy() {
       try {

hadoop3-2-testutils/pom.xml

@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hbase.filesystem</groupId>
+    <artifactId>hbase-filesystem</artifactId>
+    <version>1.0.0-alpha2-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+  <artifactId>hadoop3-2-testutils</artifactId>
+  <name>Test utility classes for Hadoop 3.1.x and 3.2.x</name>
+  <packaging>jar</packaging>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <version>${slf4j.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hbase.filesystem</groupId>
+      <artifactId>hadoop-testutils</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-aws</artifactId>
+      <version>${hadoop.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.amazonaws</groupId>
+      <artifactId>aws-java-sdk-bundle</artifactId>
+      <version>${aws-java-sdk.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.yetus</groupId>
+      <artifactId>audience-annotations</artifactId>
+      <version>${audience-annotations.version}</version>
+    </dependency>
+  </dependencies>
+</project>

hadoop3-2-testutils/src/main/java/org/apache/hadoop/hbase/oss/Hadoop32EmbeddedS3ClientFactory.java

@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.oss;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.hadoop.fs.s3a.S3ClientFactory;
+import org.apache.hadoop.hbase.oss.EmbeddedS3.EmbeddedAmazonS3;
+
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.services.s3.AmazonS3;
+
+public class Hadoop32EmbeddedS3ClientFactory implements S3ClientFactory {
+  public AmazonS3 createS3Client(URI name) {
+    AmazonS3 s3 = new EmbeddedAmazonS3();
+    s3.createBucket(EmbeddedS3.BUCKET);
+    return s3;
+  }
+
+  public AmazonS3 createS3Client(URI name,
+      String bucket,
+      AWSCredentialsProvider credentialSet,
+      String userAgentSuffix) {
+    AmazonS3 s3 = new EmbeddedAmazonS3();
+    s3.createBucket(bucket);
+    return s3;
+  }
+
+  public AmazonS3 createS3Client(URI name,
+      String bucket,
+      AWSCredentialsProvider credentialSet) {
+    return createS3Client(name);
+  }
+}

hadoop3-3-testutils/pom.xml

@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hbase.filesystem</groupId>
+    <artifactId>hbase-filesystem</artifactId>
+    <version>1.0.0-alpha2-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+  <artifactId>hadoop3-3-testutils</artifactId>
+  <name>Test utility classes for Hadoop 3.3.1 and beyond</name>
+  <packaging>jar</packaging>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <version>${slf4j.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hbase.filesystem</groupId>
+      <artifactId>hadoop-testutils</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-aws</artifactId>
+      <version>${hadoop.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.amazonaws</groupId>
+      <artifactId>aws-java-sdk-bundle</artifactId>
+      <version>${aws-java-sdk.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.yetus</groupId>
+      <artifactId>audience-annotations</artifactId>
+      <version>${audience-annotations.version}</version>
+    </dependency>
+  </dependencies>
+</project>