From b188b9281e3eb52e9bad1cff0b01db79ce7ea4d3 Mon Sep 17 00:00:00 2001 From: Laszlo Vegh Date: Thu, 23 Jun 2022 13:34:51 +0200 Subject: [PATCH 1/2] HIVE-26352: Tez queue access check fails with GSS Exception on Compaction. --- .../hadoop/hive/ql/exec/tez/YarnQueueHelper.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/YarnQueueHelper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/YarnQueueHelper.java index 14a956008994..2c8ba9091b20 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/YarnQueueHelper.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/YarnQueueHelper.java @@ -23,10 +23,12 @@ import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URL; +import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Arrays; import org.apache.commons.io.IOUtils; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.security.UserGroupInformation; @@ -92,6 +94,20 @@ public YarnQueueHelper(HiveConf conf) { } public void checkQueueAccess( + String queueName, String userName) throws IOException, InterruptedException { + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + ugi.doAs((PrivilegedExceptionAction) () -> { + checkQueueAccessInternal(queueName, userName); + return null; + }); + try { + FileSystem.closeAllForUGI(ugi); + } catch (IOException exception) { + LOG.error("Could not clean up file-system handles for UGI: " + ugi, exception); + } + } + + private void checkQueueAccessInternal( String queueName, String userName) throws IOException, HiveException { String urlSuffix = String.format(PERMISSION_PATH, queueName, userName); // TODO: if we ever use this endpoint for anything else, refactor cycling into a separate class. From bce635f511c2169543531b52dde66b4bbace2970 Mon Sep 17 00:00:00 2001 From: Laszlo Vegh Date: Fri, 24 Jun 2022 11:22:00 +0200 Subject: [PATCH 2/2] HIVE-26352: address review comments --- .../hive/ql/exec/tez/YarnQueueHelper.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/YarnQueueHelper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/YarnQueueHelper.java index 2c8ba9091b20..88322556c0b7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/YarnQueueHelper.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/YarnQueueHelper.java @@ -96,14 +96,17 @@ public YarnQueueHelper(HiveConf conf) { public void checkQueueAccess( String queueName, String userName) throws IOException, InterruptedException { UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - ugi.doAs((PrivilegedExceptionAction) () -> { - checkQueueAccessInternal(queueName, userName); - return null; - }); try { - FileSystem.closeAllForUGI(ugi); - } catch (IOException exception) { - LOG.error("Could not clean up file-system handles for UGI: " + ugi, exception); + ugi.doAs((PrivilegedExceptionAction) () -> { + checkQueueAccessInternal(queueName, userName); + return null; + }); + } finally { + try { + FileSystem.closeAllForUGI(ugi); + } catch (IOException exception) { + LOG.error("Could not clean up file-system handles for UGI: " + ugi, exception); + } } }