HBASE-27042. Remove S3Guard awareness from HBoss

* Cut HBoss test code which would explicitly disable S3Guard
  -obsolete and breaks the build on forthcoming hadoop releases.
* Also cut all references in docs and XML.
* EmbeddedS3 to implement deleteObject() and so support
  AWS SDKs 1.11.1026+, which expects this (hadoop 3.3.2+).
* Maven test runs to provide full stack traces on failures,
  so runtime failures can be debugged and fixed.

Change-Id: Iab10cfc6780cea064ea116efc1c177332d780514
Signed-off-by: Josh Elser <elserj@apache.org>

hadoop-testutils/src/main/java/org/apache/hadoop/hbase/oss/EmbeddedS3.java

@@ -23,6 +23,7 @@
 import com.amazonaws.services.s3.model.Bucket;
 import com.amazonaws.services.s3.model.CopyObjectRequest;
 import com.amazonaws.services.s3.model.CopyObjectResult;
+import com.amazonaws.services.s3.model.DeleteObjectRequest;
 import com.amazonaws.services.s3.model.DeleteObjectsRequest;
 import com.amazonaws.services.s3.model.DeleteObjectsResult;
 import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
@@ -145,6 +146,11 @@ public void deleteObject(String bucketName, String key) {
       bucket.remove(key);
     }
 
+    public void deleteObject(DeleteObjectRequest request) {
+      LOG.debug("deleteObject: {}", request);
+      bucket.remove(request.getKey());
+    }
+
     public DeleteObjectsResult deleteObjects(DeleteObjectsRequest request) {
       for (DeleteObjectsRequest.KeyVersion keyVersion : request.getKeys()) {
         String key = keyVersion.getKey();

hbase-oss/README.md

@@ -74,19 +74,17 @@ You may also want to configure:
 
 An implementation based on Amazon's DynamoDB lock library was considered but
 was not completed due to the lack of an efficient way to traverse the tree and
-discover locks on child nodes. The benefit is that S3Guard is required for s3a
-use and as such there's a dependency on DynamoDB anyway.
+discover locks on child nodes.
 
 ## Storage Implementations
 
 Currently HBOSS is primarily designed for and exclusively tested with Hadoop's
-s3a client against Amazon S3. *S3Guard must be enabled, which is available in
-Hadoop 2.9.0, 3.0.0, and higher*.
+s3a client against Amazon S3.
 
-Both the use of S3Guard and Zookeeper for locking (i.e. Zookeeper) have
+The use of Zookeeper for locking (i.e. Zookeeper) have
 implications for other clients that are not configured to share the same
-metadata store and Zookeeper ensemble. Ideally, all clients should be have the
-same configuration in these respects. Read-only clients may not share these
+Zookeeper ensemble. Ideally, all clients should have the
+same configuration. Read-only clients may not share these
 resources with the HBase processes, but they will not have the added safety
 provided by these features. Clients that do not share these resources and modify
 data can compromise the correctness of HBase.

hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/TestUtils.java

@@ -19,7 +19,6 @@
 package org.apache.hadoop.hbase.oss;
 
 import static org.apache.hadoop.fs.s3a.Constants.S3_CLIENT_FACTORY_IMPL;
-import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
 import static org.apache.hadoop.hbase.oss.Constants.DATA_URI;
 
 import java.net.URI;
@@ -29,7 +28,6 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.s3guard.LocalMetadataStore;
 import org.apache.hadoop.hbase.oss.sync.EmbeddedZK;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.yetus.audience.InterfaceAudience;
@@ -69,9 +67,6 @@ public String getIdentifier() {
   public static boolean usingEmbeddedS3 = false;
 
   public static void conditionalStart(Configuration conf) {
-    if (StringUtils.isEmpty(conf.get(S3_METADATA_STORE_IMPL))) {
-      conf.set(S3_METADATA_STORE_IMPL, LocalMetadataStore.class.getName());
-    }
 
     boolean notConfigured = StringUtils.isEmpty(conf.get(DATA_URI));
     if (notConfigured) {

hbase-oss/src/test/resources/core-site.xml

@@ -42,18 +42,6 @@
         </property>
       </configuration>
 
-    S3Guard will also be configured to the LocalMetadataStore implementation
-    by default. You may wish to configure DynamoDBMetadataStore for integration
-    or if S3Guard is not needed (S3-compatible appliance, for instance), you
-    can disable it by configuring NullMetadataStore explicitly (see below).
-  -->
-
-  <!--
-  <property>
-    <name>fs.s3a.metadatastore.impl</name>
-    <value>org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore</value>
-    <description>Uncomment to disable S3Guard entirely</description>
-  </property>
   -->
 
   <include xmlns="http://www.w3.org/2001/XInclude" href="auth-keys.xml">

pom.xml

@@ -106,6 +106,7 @@
           <artifactId>maven-surefire-plugin</artifactId>
           <version>2.22.1</version>
           <configuration>
+            <trimStackTrace>false</trimStackTrace>
             <redirectTestOutputToFile>true</redirectTestOutputToFile>
             <reuseForks>false</reuseForks>
             <systemProperties>