Skip to content

Commit

Permalink
Merge branch 'apache:trunk' into MAPREDUCE-7385
Browse files Browse the repository at this point in the history
  • Loading branch information
slfan1989 committed Jul 18, 2022
2 parents 2ca28f0 + a55ace7 commit 64fd58d
Show file tree
Hide file tree
Showing 559 changed files with 17,009 additions and 4,635 deletions.
58 changes: 38 additions & 20 deletions LICENSE-binary
Original file line number Diff line number Diff line change
Expand Up @@ -218,20 +218,20 @@ com.aliyun.oss:aliyun-sdk-oss:3.13.2
com.amazonaws:aws-java-sdk-bundle:1.11.901
com.cedarsoftware:java-util:1.9.0
com.cedarsoftware:json-io:2.5.1
com.fasterxml.jackson.core:jackson-annotations:2.13.2
com.fasterxml.jackson.core:jackson-core:2.13.2
com.fasterxml.jackson.core:jackson-databind:2.13.2.2
com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.13.2
com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.13.2
com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.13.2
com.fasterxml.jackson.core:jackson-annotations:2.12.7
com.fasterxml.jackson.core:jackson-core:2.12.7
com.fasterxml.jackson.core:jackson-databind:2.12.7
com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.12.7
com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.12.7
com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.12.7
com.fasterxml.uuid:java-uuid-generator:3.1.4
com.fasterxml.woodstox:woodstox-core:5.3.0
com.github.davidmoten:rxjava-extras:0.8.0.17
com.github.stephenc.jcip:jcip-annotations:1.0-1
com.google:guice:4.0
com.google:guice-servlet:4.0
com.google.api.grpc:proto-google-common-protos:1.0.0
com.google.code.gson:2.2.4
com.google.code.gson:2.9.0
com.google.errorprone:error_prone_annotations:2.2.0
com.google.j2objc:j2objc-annotations:1.1
com.google.json-simple:json-simple:1.1.1
Expand Down Expand Up @@ -261,17 +261,36 @@ io.grpc:grpc-protobuf:1.26.0
io.grpc:grpc-protobuf-lite:1.26.0
io.grpc:grpc-stub:1.26.0
io.netty:netty:3.10.6.Final
io.netty:netty-all:4.1.42.Final
io.netty:netty-buffer:4.1.27.Final
io.netty:netty-codec:4.1.27.Final
io.netty:netty-codec-http:4.1.27.Final
io.netty:netty-codec-http2:4.1.27.Final
io.netty:netty-codec-socks:4.1.27.Final
io.netty:netty-common:4.1.27.Final
io.netty:netty-handler:4.1.27.Final
io.netty:netty-handler-proxy:4.1.27.Final
io.netty:netty-resolver:4.1.27.Final
io.netty:netty-transport:4.1.27.Final
io.netty:netty-all:4.1.77.Final
io.netty:netty-buffer:4.1.77.Final
io.netty:netty-codec:4.1.77.Final
io.netty:netty-codec-dns:4.1.77.Final
io.netty:netty-codec-haproxy:4.1.77.Final
io.netty:netty-codec-http:4.1.77.Final
io.netty:netty-codec-http2:4.1.77.Final
io.netty:netty-codec-memcache:4.1.77.Final
io.netty:netty-codec-mqtt:4.1.77.Final
io.netty:netty-codec-redis:4.1.77.Final
io.netty:netty-codec-smtp:4.1.77.Final
io.netty:netty-codec-socks:4.1.77.Final
io.netty:netty-codec-stomp:4.1.77.Final
io.netty:netty-codec-xml:4.1.77.Final
io.netty:netty-common:4.1.77.Final
io.netty:netty-handler:4.1.77.Final
io.netty:netty-handler-proxy:4.1.77.Final
io.netty:netty-resolver:4.1.77.Final
io.netty:netty-resolver-dns:4.1.77.Final
io.netty:netty-transport:4.1.77.Final
io.netty:netty-transport-rxtx:4.1.77.Final
io.netty:netty-transport-sctp:4.1.77.Final
io.netty:netty-transport-udt:4.1.77.Final
io.netty:netty-transport-classes-epoll:4.1.77.Final
io.netty:netty-transport-native-unix-common:4.1.77.Final
io.netty:netty-transport-classes-kqueue:4.1.77.Final
io.netty:netty-resolver-dns-classes-macos:4.1.77.Final
io.netty:netty-transport-native-epoll:4.1.77.Final
io.netty:netty-transport-native-kqueue:4.1.77.Final
io.netty:netty-resolver-dns-native-macos:4.1.77.Final
io.opencensus:opencensus-api:0.12.3
io.opencensus:opencensus-contrib-grpc-metrics:0.12.3
io.reactivex:rxjava:1.3.8
Expand Down Expand Up @@ -416,7 +435,7 @@ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanage
bootstrap v3.3.6
broccoli-asset-rev v2.4.2
broccoli-funnel v1.0.1
datatables v1.10.8
datatables v1.10.19
em-helpers v0.5.13
em-table v0.1.6
ember v2.2.0
Expand Down Expand Up @@ -491,7 +510,6 @@ javax.annotation:javax.annotation-api:1.3.2
javax.servlet:javax.servlet-api:3.1.0
javax.servlet.jsp:jsp-api:2.1
javax.websocket:javax.websocket-api:1.0
javax.ws.rs:javax.ws.rs-api:2.1.1
javax.ws.rs:jsr311-api:1.1.1
javax.xml.bind:jaxb-api:2.2.11

Expand Down
2 changes: 1 addition & 1 deletion dev-support/Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ pipeline {

options {
buildDiscarder(logRotator(numToKeepStr: '5'))
timeout (time: 24, unit: 'HOURS')
timeout (time: 48, unit: 'HOURS')
timestamps()
checkoutToSubdirectory('src')
}
Expand Down
7 changes: 6 additions & 1 deletion hadoop-client-modules/hadoop-client-minicluster/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -704,7 +704,6 @@
<exclude>org.bouncycastle:*</exclude>
<!-- Leave snappy that includes native methods which cannot be relocated. -->
<exclude>org.xerial.snappy:*</exclude>
<exclude>javax.ws.rs:javax.ws.rs-api</exclude>
</excludes>
</artifactSet>
<filters>
Expand Down Expand Up @@ -757,6 +756,12 @@
<exclude>META-INF/versions/11/module-info.class</exclude>
</excludes>
</filter>
<filter>
<artifact>com.google.code.gson:gson</artifact>
<excludes>
<exclude>META-INF/versions/9/module-info.class</exclude>
</excludes>
</filter>

<!-- Mockito tries to include its own unrelocated copy of hamcrest. :( -->
<filter>
Expand Down
8 changes: 7 additions & 1 deletion hadoop-client-modules/hadoop-client-runtime/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,6 @@
<exclude>org.bouncycastle:*</exclude>
<!-- Leave snappy that includes native methods which cannot be relocated. -->
<exclude>org.xerial.snappy:*</exclude>
<exclude>javax.ws.rs:javax.ws.rs-api</exclude>
</excludes>
</artifactSet>
<filters>
Expand Down Expand Up @@ -249,6 +248,13 @@
<exclude>META-INF/versions/11/module-info.class</exclude>
</excludes>
</filter>
<filter>
<artifact>com.google.code.gson:gson</artifact>
<excludes>
<exclude>META-INF/versions/9/module-info.class</exclude>
</excludes>
</filter>

</filters>
<relocations>
<relocation>
Expand Down
2 changes: 1 addition & 1 deletion hadoop-cloud-storage-project/hadoop-cos/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@
<dependency>
<groupId>com.qcloud</groupId>
<artifactId>cos_api-bundle</artifactId>
<version>5.6.19</version>
<version>5.6.69</version>
<scope>compile</scope>
</dependency>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
package org.apache.hadoop.security.authentication.util;

import java.io.ByteArrayInputStream;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.security.PublicKey;
import java.security.cert.CertificateException;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
/**
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License. See accompanying LICENSE file.
*/
package org.apache.hadoop.security.authentication.util;

import java.util.HashMap;
import java.util.Map;
import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.Configuration;


/**
* Creates a programmatic version of a jaas.conf file. This can be used
* instead of writing a jaas.conf file and setting the system property,
* "java.security.auth.login.config", to point to that file. It is meant to be
* used for connecting to ZooKeeper.
*/
public class JaasConfiguration extends Configuration {

private final javax.security.auth.login.Configuration baseConfig =
javax.security.auth.login.Configuration.getConfiguration();
private final AppConfigurationEntry[] entry;
private final String entryName;

/**
* Add an entry to the jaas configuration with the passed in name,
* principal, and keytab. The other necessary options will be set for you.
*
* @param entryName The name of the entry (e.g. "Client")
* @param principal The principal of the user
* @param keytab The location of the keytab
*/
public JaasConfiguration(String entryName, String principal, String keytab) {
this.entryName = entryName;
Map<String, String> options = new HashMap<>();
options.put("keyTab", keytab);
options.put("principal", principal);
options.put("useKeyTab", "true");
options.put("storeKey", "true");
options.put("useTicketCache", "false");
options.put("refreshKrb5Config", "true");
String jaasEnvVar = System.getenv("HADOOP_JAAS_DEBUG");
if ("true".equalsIgnoreCase(jaasEnvVar)) {
options.put("debug", "true");
}
entry = new AppConfigurationEntry[]{
new AppConfigurationEntry(getKrb5LoginModuleName(),
AppConfigurationEntry.LoginModuleControlFlag.REQUIRED,
options)};
}

@Override
public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
return (entryName.equals(name)) ? entry : ((baseConfig != null)
? baseConfig.getAppConfigurationEntry(name) : null);
}

private String getKrb5LoginModuleName() {
String krb5LoginModuleName;
if (System.getProperty("java.vendor").contains("IBM")) {
krb5LoginModuleName = "com.ibm.security.auth.module.Krb5LoginModule";
} else {
krb5LoginModuleName = "com.sun.security.auth.module.Krb5LoginModule";
}
return krb5LoginModuleName;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,7 @@ public static final String getServicePrincipal(String service,
*/
static final String[] getPrincipalNames(String keytabFileName) throws IOException {
Keytab keytab = Keytab.loadKeytab(new File(keytabFileName));
Set<String> principals = new HashSet<String>();
Set<String> principals = new HashSet<>();
List<PrincipalName> entries = keytab.getPrincipals();
for (PrincipalName entry : entries) {
principals.add(entry.getName().replace("\\", "/"));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,9 @@
import java.nio.ByteBuffer;
import java.security.SecureRandom;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Random;
import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.Configuration;
import javax.servlet.ServletContext;
import org.apache.curator.RetryPolicy;
Expand Down Expand Up @@ -429,62 +426,4 @@ public List<ACL> getAclForPath(String path) {
return saslACL;
}
}

/**
* Creates a programmatic version of a jaas.conf file. This can be used
* instead of writing a jaas.conf file and setting the system property,
* "java.security.auth.login.config", to point to that file. It is meant to be
* used for connecting to ZooKeeper.
*/
@InterfaceAudience.Private
public static class JaasConfiguration extends Configuration {

private final javax.security.auth.login.Configuration baseConfig =
javax.security.auth.login.Configuration.getConfiguration();
private static AppConfigurationEntry[] entry;
private String entryName;

/**
* Add an entry to the jaas configuration with the passed in name,
* principal, and keytab. The other necessary options will be set for you.
*
* @param entryName The name of the entry (e.g. "Client")
* @param principal The principal of the user
* @param keytab The location of the keytab
*/
public JaasConfiguration(String entryName, String principal, String keytab) {
this.entryName = entryName;
Map<String, String> options = new HashMap<String, String>();
options.put("keyTab", keytab);
options.put("principal", principal);
options.put("useKeyTab", "true");
options.put("storeKey", "true");
options.put("useTicketCache", "false");
options.put("refreshKrb5Config", "true");
String jaasEnvVar = System.getenv("HADOOP_JAAS_DEBUG");
if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar)) {
options.put("debug", "true");
}
entry = new AppConfigurationEntry[]{
new AppConfigurationEntry(getKrb5LoginModuleName(),
AppConfigurationEntry.LoginModuleControlFlag.REQUIRED,
options)};
}

@Override
public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
return (entryName.equals(name)) ? entry : ((baseConfig != null)
? baseConfig.getAppConfigurationEntry(name) : null);
}

private String getKrb5LoginModuleName() {
String krb5LoginModuleName;
if (System.getProperty("java.vendor").contains("IBM")) {
krb5LoginModuleName = "com.ibm.security.auth.module.Krb5LoginModule";
} else {
krb5LoginModuleName = "com.sun.security.auth.module.Krb5LoginModule";
}
return krb5LoginModuleName;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -108,9 +108,9 @@ public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
public static <T> T doAs(String principal, final Callable<T> callable) throws Exception {
LoginContext loginContext = null;
try {
Set<Principal> principals = new HashSet<Principal>();
Set<Principal> principals = new HashSet<>();
principals.add(new KerberosPrincipal(KerberosTestUtils.getClientPrincipal()));
Subject subject = new Subject(false, principals, new HashSet<Object>(), new HashSet<Object>());
Subject subject = new Subject(false, principals, new HashSet<>(), new HashSet<>());
loginContext = new LoginContext("", subject, null, new KerberosConfiguration(principal));
loginContext.login();
subject = loginContext.getSubject();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
import javax.security.sasl.AuthenticationException;

import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang.reflect.FieldUtils;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.hadoop.minikdc.KerberosSecurityTestcase;
import org.apache.hadoop.security.authentication.KerberosTestUtils;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
import java.util.List;
import java.util.ArrayList;
import java.util.Properties;
import java.util.Vector;
import java.util.Date;

import javax.servlet.ServletException;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
*/
package org.apache.hadoop.security.authentication.server;

import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authentication.client.PseudoAuthenticator;
import org.junit.Assert;
import org.junit.Test;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
import javax.servlet.ServletContext;

import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ public void test() throws Exception {
krb5LoginModuleName = "com.sun.security.auth.module.Krb5LoginModule";
}

ZKSignerSecretProvider.JaasConfiguration jConf =
new ZKSignerSecretProvider.JaasConfiguration("foo", "foo/localhost",
JaasConfiguration jConf =
new JaasConfiguration("foo", "foo/localhost",
"/some/location/foo.keytab");
AppConfigurationEntry[] entries = jConf.getAppConfigurationEntry("bar");
Assert.assertNull(entries);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -774,7 +774,7 @@ private void updatePropertiesWithDeprecatedKeys(
private void handleDeprecation() {
LOG.debug("Handling deprecation for all properties in config...");
DeprecationContext deprecations = deprecationContext.get();
Set<Object> keys = new HashSet<Object>();
Set<Object> keys = new HashSet<>();
keys.addAll(getProps().keySet());
for (Object item: keys) {
LOG.debug("Handling deprecation for " + (String)item);
Expand Down

0 comments on commit 64fd58d

Please sign in to comment.