From fe15cdf7f1214e7e76d3102c5eca5d0818a4631f Mon Sep 17 00:00:00 2001 From: Gabor Bota Date: Thu, 29 Nov 2018 16:27:55 +0100 Subject: [PATCH 1/2] HADOOP-16210. Update guava to 27.0-jre in hadoop-project trunk --- .../java/org/apache/hadoop/security/Groups.java | 2 +- .../federation/resolver/order/LocalResolver.java | 2 +- .../hdfs/qjournal/client/IPCLoggerChannel.java | 2 +- .../hadoop/hdfs/qjournal/client/QuorumCall.java | 3 ++- .../datanode/checker/DatasetVolumeChecker.java | 13 +++++++------ .../datanode/checker/ThrottledAsyncChecker.java | 2 +- .../checker/TestThrottledAsyncCheckerTimeout.java | 11 +++++++---- .../hadoop/mapred/LocatedFileStatusFetcher.java | 9 ++++++--- hadoop-project/pom.xml | 2 +- .../hadoop/yarn/service/utils/ZookeeperUtils.java | 2 +- 10 files changed, 28 insertions(+), 20 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java index 63ec9a5d29e95..b29278bd20751 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java @@ -375,7 +375,7 @@ public void onFailure(Throwable t) { backgroundRefreshException.incrementAndGet(); backgroundRefreshRunning.decrementAndGet(); } - }); + }, MoreExecutors.directExecutor()); return listenableFuture; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/LocalResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/LocalResolver.java index a774677c5bbaf..58a8ed278b3dc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/LocalResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/LocalResolver.java @@ -196,7 +196,7 @@ private Map getNamenodesSubcluster( try { String nsId = nn.getNameserviceId(); String rpcAddress = nn.getRpcAddress(); - String hostname = HostAndPort.fromString(rpcAddress).getHostText(); + String hostname = HostAndPort.fromString(rpcAddress).getHost(); ret.put(hostname, nsId); if (hostname.equals(localHostname)) { ret.put(localIp, nsId); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java index d462ef684ade0..324747618aeb4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java @@ -447,7 +447,7 @@ public void onFailure(Throwable t) { public void onSuccess(Void t) { unreserveQueueSpace(data.length); } - }); + }, MoreExecutors.directExecutor()); } } return ret; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java index dee74e6fcfd5e..ef32eb11c355a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java @@ -22,6 +22,7 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeUnit; +import com.google.common.util.concurrent.MoreExecutors; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.util.StopWatch; import org.apache.hadoop.util.Timer; @@ -80,7 +81,7 @@ public void onFailure(Throwable t) { public void onSuccess(RESULT res) { qr.addResult(e.getKey(), res); } - }); + }, MoreExecutors.directExecutor()); } return qr; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java index 5c590f6081e32..7c13ed0ea5acf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java @@ -24,6 +24,7 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -224,12 +225,12 @@ public Set checkAllVolumes( Futures.addCallback(olf.get(), new ResultHandler(reference, healthyVolumes, failedVolumes, numVolumes, new Callback() { - @Override - public void call(Set ignored1, - Set ignored2) { - latch.countDown(); - } - })); + @Override + public void call(Set ignored1, + Set ignored2) { + latch.countDown(); + } + }), MoreExecutors.directExecutor()); } else { IOUtils.cleanup(null, reference); if (numVolumes.decrementAndGet() == 0) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/ThrottledAsyncChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/ThrottledAsyncChecker.java index bb1ed469696d7..88444539337de 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/ThrottledAsyncChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/ThrottledAsyncChecker.java @@ -182,7 +182,7 @@ public void onFailure(@Nonnull Throwable t) { t, timer.monotonicNow())); } } - }); + }, MoreExecutors.directExecutor()); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestThrottledAsyncCheckerTimeout.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestThrottledAsyncCheckerTimeout.java index 91a793b7d2d1d..926747d457315 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestThrottledAsyncCheckerTimeout.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestThrottledAsyncCheckerTimeout.java @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import org.apache.hadoop.util.FakeTimer; import org.junit.Before; import org.junit.Rule; @@ -101,7 +102,7 @@ public void onFailure(Throwable t) { numCallbackInvocationsFailure.incrementAndGet(); callbackResult.set(true); } - }); + }, MoreExecutors.directExecutor()); while (!callbackResult.get()) { // Wait for the callback @@ -133,7 +134,8 @@ public void testDiskCheckTimeoutInvokesOneCallbackOnly() throws Exception { .schedule(target, true); assertTrue(olf1.isPresent()); - Futures.addCallback(olf1.get(), futureCallback); + Futures.addCallback(olf1.get(), futureCallback, + MoreExecutors.directExecutor()); // Verify that timeout results in only 1 onFailure call and 0 onSuccess // calls. @@ -149,7 +151,8 @@ public void testDiskCheckTimeoutInvokesOneCallbackOnly() throws Exception { .schedule(target, true); assertTrue(olf2.isPresent()); - Futures.addCallback(olf2.get(), futureCallback); + Futures.addCallback(olf2.get(), futureCallback, + MoreExecutors.directExecutor()); // Verify that normal check (dummy) results in only 1 onSuccess call. // Number of times onFailure is invoked should remain the same i.e. 1. @@ -187,7 +190,7 @@ public void onFailure(Throwable t) { throwable[0] = t; callbackResult.set(true); } - }); + }, MoreExecutors.directExecutor()); while (!callbackResult.get()) { // Wait for the callback diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LocatedFileStatusFetcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LocatedFileStatusFetcher.java index 1b1025e9da9ff..3869c493a06a8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LocatedFileStatusFetcher.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LocatedFileStatusFetcher.java @@ -120,7 +120,8 @@ public Iterable getFileStatuses() throws InterruptedException, runningTasks.incrementAndGet(); ListenableFuture future = exec .submit(new ProcessInitialInputPathCallable(p, conf, inputFilter)); - Futures.addCallback(future, processInitialInputPathCallback); + Futures.addCallback(future, processInitialInputPathCallback, + MoreExecutors.directExecutor()); } runningTasks.decrementAndGet(); @@ -267,7 +268,8 @@ public void onSuccess(ProcessInputDirCallable.Result result) { ListenableFuture future = exec .submit(new ProcessInputDirCallable(result.fs, fileStatus, recursive, inputFilter)); - Futures.addCallback(future, processInputDirCallback); + Futures.addCallback(future, processInputDirCallback, + MoreExecutors.directExecutor()); } } decrementRunningAndCheckCompletion(); @@ -353,7 +355,8 @@ public void onSuccess(ProcessInitialInputPathCallable.Result result) { ListenableFuture future = exec .submit(new ProcessInputDirCallable(result.fs, matched, recursive, inputFilter)); - Futures.addCallback(future, processInputDirCallback); + Futures.addCallback(future, processInputDirCallback, + MoreExecutors.directExecutor()); } } decrementRunningAndCheckCompletion(); diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 7bd012df7bf84..aac03156d6d8c 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -92,7 +92,7 @@ 3.1.0-RC1 2.1.7 - 11.0.2 + 27.0-jre 4.0 2.9.9 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java index c6e85252593be..60eb9b4019f96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java @@ -87,7 +87,7 @@ public static List splitToHostsAndPorts(String hostPortQuorumList) public static String buildHostsOnlyList(List hostAndPorts) { StringBuilder sb = new StringBuilder(); for (HostAndPort hostAndPort : hostAndPorts) { - sb.append(hostAndPort.getHostText()).append(","); + sb.append(hostAndPort.getHost()).append(","); } if (sb.length() > 0) { sb.delete(sb.length() - 1, sb.length()); From 75bd0de22f0b1f258c802376596ce058806a1c23 Mon Sep 17 00:00:00 2001 From: Gabor Bota Date: Fri, 29 Mar 2019 17:55:17 +0100 Subject: [PATCH 2/2] trigger yetus on hadoop-tools --- hadoop-tools/pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml index 655b5a5d5c03b..2a1edf12d00e1 100644 --- a/hadoop-tools/pom.xml +++ b/hadoop-tools/pom.xml @@ -68,3 +68,5 @@ + +