Revert "HADOOP-14556. S3A to support Delegation Tokens."

This reverts commit d715233.
apache · Jan 8, 2019 · 7f78397 · 7f78397
1 parent 4894115
commit 7f78397
Show file tree

Hide file tree

Showing 103 changed files with 765 additions and 11,345 deletions.
diff --git a/...p-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/...p-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
@@ -396,11 +396,8 @@ public void checkPath(Path path) {
       thatPort = this.getUriDefaultPort();
     }
     if (thisPort != thatPort) {
-      throw new InvalidPathException("Wrong FS: " + path
-          + " and port=" + thatPort
-          + ", expected: "
-          + this.getUri()
-          + " with port=" + thisPort);
+      throw new InvalidPathException("Wrong FS: " + path + ", expected: "
+          + this.getUri());
     }
   }
 

diff --git a/...op-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageStatistics.java b/...op-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageStatistics.java
@@ -18,7 +18,6 @@
 package org.apache.hadoop.fs;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
 
 import java.util.Iterator;
 
@@ -38,13 +37,9 @@ public abstract class StorageStatistics {
    *
    * When adding new common statistic name constants, please make them unique.
    * By convention, they are implicitly unique:
-   * <ul>
-   *   <li>the name of the constants are uppercase, words separated by
-   *   underscores.</li>
-   *   <li>the value of the constants are lowercase of the constant names.</li>
-   * </ul>
+   *  - the name of the constants are uppercase, words separated by underscores.
+   *  - the value of the constants are lowercase of the constant names.
    */
-  @InterfaceStability.Evolving
   public interface CommonStatisticNames {
     // The following names are for file system operation invocations
     String OP_APPEND = "op_append";
@@ -54,7 +49,6 @@ public interface CommonStatisticNames {
     String OP_DELETE = "op_delete";
     String OP_EXISTS = "op_exists";
     String OP_GET_CONTENT_SUMMARY = "op_get_content_summary";
-    String OP_GET_DELEGATION_TOKEN = "op_get_delegation_token";
     String OP_GET_FILE_CHECKSUM = "op_get_file_checksum";
     String OP_GET_FILE_STATUS = "op_get_file_status";
     String OP_GET_STATUS = "op_get_status";

diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1024,33 +1024,19 @@
 
 <property>
   <name>fs.s3a.aws.credentials.provider</name>
-  <value>
-    org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider,
-    org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,
-    com.amazonaws.auth.EnvironmentVariableCredentialsProvider,
-    org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider
-  </value>
   <description>
     Comma-separated class names of credential provider classes which implement
     com.amazonaws.auth.AWSCredentialsProvider.
 
-    When S3A delegation tokens are not enabled, this list will be used
-    to directly authenticate with S3 and DynamoDB services.
-    When S3A Delegation tokens are enabled, depending upon the delegation
-    token binding it may be used
-    to communicate wih the STS endpoint to request session/role
-    credentials.
-
     These are loaded and queried in sequence for a valid set of credentials.
     Each listed class must implement one of the following means of
     construction, which are attempted in order:
-    * a public constructor accepting java.net.URI and
+    1. a public constructor accepting java.net.URI and
         org.apache.hadoop.conf.Configuration,
-    * a public constructor accepting org.apache.hadoop.conf.Configuration,
-    * a public static method named getInstance that accepts no
+    2. a public static method named getInstance that accepts no
        arguments and returns an instance of
        com.amazonaws.auth.AWSCredentialsProvider, or
-    * a public default constructor.
+    3. a public default constructor.
 
     Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows
     anonymous access to a publicly accessible S3 bucket without any credentials.
@@ -1060,15 +1046,13 @@
 
     If unspecified, then the default list of credential provider classes,
     queried in sequence, is:
-    * org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider: looks
-       for session login secrets in the Hadoop configuration.
-    * org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider:
+    1. org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider:
        Uses the values of fs.s3a.access.key and fs.s3a.secret.key.
-    * com.amazonaws.auth.EnvironmentVariableCredentialsProvider: supports
+    2. com.amazonaws.auth.EnvironmentVariableCredentialsProvider: supports
         configuration of AWS access key ID and secret access key in
         environment variables named AWS_ACCESS_KEY_ID and
         AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK.
-    * com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use
+    3. com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use
         of instance profile credentials if running in an EC2 VM.
   </description>
 </property>
@@ -1124,7 +1108,7 @@
   <value>30m</value>
   <description>
     Duration of assumed roles before a refresh is attempted.
-    Used when session tokens are requested.
+    Only used if AssumedRoleCredentialProvider is the AWS credential provider.
     Range: 15m to 1h
   </description>
 </property>
@@ -1136,20 +1120,17 @@
     AWS Security Token Service Endpoint.
     If unset, uses the default endpoint.
     Only used if AssumedRoleCredentialProvider is the AWS credential provider.
-    Used by the AssumedRoleCredentialProvider and in Session and Role delegation
-    tokens.
   </description>
 </property>
 
 <property>
   <name>fs.s3a.assumed.role.sts.endpoint.region</name>
-  <value></value>
+  <value>us-west-1</value>
   <description>
     AWS Security Token Service Endpoint's region;
     Needed if fs.s3a.assumed.role.sts.endpoint points to an endpoint
     other than the default one and the v4 signature is used.
-    Used by the AssumedRoleCredentialProvider and in Session and Role delegation
-    tokens.
+    Only used if AssumedRoleCredentialProvider is the AWS credential provider.
   </description>
 </property>
 
@@ -1164,29 +1145,6 @@
   </description>
 </property>
 
-<property>
-  <name>fs.s3a.delegation.tokens.enabled</name>
-  <value>false</value>
-  <description></description>
-</property>
-
-<property>
-  <name>fs.s3a.delegation.token.binding</name>
-  <value></value>
-  <description>
-    The name of a class to provide delegation tokens support in S3A.
-    If unset: delegation token support is disabled.
-
-    Note: for job submission to actually collect these tokens,
-    Kerberos must be enabled.
-
-    Options are:
-    org.apache.hadoop.fs.s3a.auth.delegation.SessionTokenBinding
-    org.apache.hadoop.fs.s3a.auth.delegation.FullCredentialsTokenBinding
-    and org.apache.hadoop.fs.s3a.auth.delegation.RoleTokenBinding
-  </description>
-</property>
-
 <property>
   <name>fs.s3a.connection.maximum</name>
   <value>15</value>

diff --git a/...common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java b/...common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java
@@ -20,7 +20,6 @@
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.List;
 
 import org.apache.hadoop.fs.FileStatus;
@@ -520,8 +519,7 @@ private FileStatus[] verifyListStatus(int expected,
       Path path,
       PathFilter filter) throws IOException {
     FileStatus[] result = getFileSystem().listStatus(path, filter);
-    assertEquals("length of listStatus(" + path + ", " + filter + " ) " +
-        Arrays.toString(result),
+    assertEquals("length of listStatus(" + path + ", " + filter + " )",
         expected, result.length);
     return result;
   }

diff --git a/...op-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/LambdaTestUtils.java b/...op-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/LambdaTestUtils.java
@@ -23,11 +23,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Time;
 
-import java.io.IOException;
-import java.security.PrivilegedExceptionAction;
 import java.util.Optional;
 import java.util.concurrent.Callable;
 import java.util.concurrent.TimeoutException;
@@ -648,48 +645,6 @@ public static void eval(VoidCallable closure) {
     }
   }
 
-  /**
-   * Evaluate a closure and return the result, after verifying that it is
-   * not null.
-   * @param message message to use in assertion text if the result is null
-   * @param eval closure to evaluate
-   * @param <T> type of response
-   * @return the evaluated result
-   * @throws Exception on any problem
-   */
-  public static<T> T notNull(String message, Callable<T> eval)
-      throws Exception {
-    T t = eval.call();
-    Assert.assertNotNull(message, t);
-    return t;
-  }
-
-  /**
-   * Execute a closure as the given user.
-   * @param user user to invoke the closure as
-   * @param eval closure to evaluate
-   * @param <T> return type
-   * @return the result of calling the closure under the identity of the user.
-   * @throws IOException IO failure
-   * @throws InterruptedException interrupted operation.
-   */
-  public static<T> T doAs(UserGroupInformation user, Callable<T> eval)
-      throws IOException, InterruptedException {
-    return user.doAs(new PrivilegedOperation<>(eval));
-  }
-
-  /**
-   * Execute a closure as the given user.
-   * @param user user to invoke the closure as
-   * @param eval closure to evaluate
-   * @throws IOException IO failure
-   * @throws InterruptedException interrupted operation.
-   */
-  public static void doAs(UserGroupInformation user, VoidCallable eval)
-      throws IOException, InterruptedException {
-    user.doAs(new PrivilegedVoidOperation(eval));
-  }
-
   /**
    * Returns {@code TimeoutException} on a timeout. If
    * there was a inner class passed in, includes it as the
@@ -857,50 +812,4 @@ public Void call() throws Exception {
     }
   }
 
-  /**
-   * A lambda-invoker for doAs use; invokes the callable provided
-   * in the constructor.
-   * @param <T> return type.
-   */
-  public static class PrivilegedOperation<T>
-      implements PrivilegedExceptionAction<T> {
-
-    private final Callable<T> callable;
-
-    /**
-     * Constructor.
-     * @param callable a non-null callable/closure.
-     */
-    public PrivilegedOperation(final Callable<T> callable) {
-      this.callable = Preconditions.checkNotNull(callable);
-    }
-
-    @Override
-    public T run() throws Exception {
-      return callable.call();
-    }
-  }
-
-  /**
-   * VoidCaller variant of {@link PrivilegedOperation}: converts
-   * a void-returning closure to an action which {@code doAs} can call.
-   */
-  public static class PrivilegedVoidOperation
-      implements PrivilegedExceptionAction<Void> {
-
-    private final Callable<Void> callable;
-
-    /**
-     * Constructor.
-     * @param callable a non-null callable/closure.
-     */
-    public PrivilegedVoidOperation(final VoidCallable callable) {
-      this.callable = new VoidCaller(callable);
-    }
-
-    @Override
-    public Void run() throws Exception {
-      return callable.call();
-    }
-  }
 }
diff --git a/...ce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java b/...ce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
@@ -42,8 +42,6 @@
 import org.apache.hadoop.mapreduce.util.ConfigUtil;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.records.ReservationId;
-
-import com.google.common.annotations.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -1531,10 +1529,7 @@ public static Map<String, Boolean> getArchiveSharedCacheUploadPolicies(
     return getSharedCacheUploadPolicies(conf, false);
   }
 
-  /** Only for mocking via unit tests. */
-  @Private
-  @VisibleForTesting
-  synchronized void connect()
+  private synchronized void connect()
           throws IOException, InterruptedException, ClassNotFoundException {
     if (cluster == null) {
       cluster = 
@@ -1554,8 +1549,7 @@ boolean isConnected() {
 
   /** Only for mocking via unit tests. */
   @Private
-  @VisibleForTesting
-  JobSubmitter getJobSubmitter(FileSystem fs,
+  public JobSubmitter getJobSubmitter(FileSystem fs, 
       ClientProtocol submitClient) throws IOException {
     return new JobSubmitter(fs, submitClient);
   }

diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
@@ -1469,11 +1469,6 @@
           <artifactId>snakeyaml</artifactId>
           <version>${snakeyaml.version}</version>
         </dependency>
-        <dependency>
-          <groupId>org.hamcrest</groupId>
-          <artifactId>hamcrest-library</artifactId>
-          <version>1.3</version>
-        </dependency>
         <dependency>
           <groupId>org.assertj</groupId>
           <artifactId>assertj-core</artifactId>

diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
@@ -490,10 +490,6 @@
       <artifactId>hadoop-minikdc</artifactId>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.hamcrest</groupId>
-      <artifactId>hamcrest-library</artifactId>
-    </dependency>
     <!-- Used to create SSL certs for a secure Keystore -->
     <dependency>
       <groupId>org.bouncycastle</groupId>