From 95dda6acadf7ed2f36d906ea63e86f72011f6a78 Mon Sep 17 00:00:00 2001 From: senmiaoliu Date: Mon, 26 Feb 2024 17:49:24 +0800 Subject: [PATCH 1/3] --ignore-launch-engine --- .../apache/hive/beeline/KyuubiBeeLine.java | 15 ++++++++++++++ .../apache/hive/beeline/KyuubiCommands.java | 5 +++++ .../hive/beeline/KyuubiBeeLineTest.java | 20 +++++++++++++++++++ .../kyuubi/jdbc/hive/KyuubiConnection.java | 6 +++++- 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiBeeLine.java b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiBeeLine.java index 5aad29fd789..8ef5454fa7d 100644 --- a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiBeeLine.java +++ b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiBeeLine.java @@ -60,7 +60,9 @@ public class KyuubiBeeLine extends BeeLine { private static final int ERRNO_OTHER = 2; private static final String PYTHON_MODE_PREFIX = "--python-mode"; + private static final String IGNORE_LAUNCH_ENGINE_PREFIX = "--ignore-launch-engine"; private boolean pythonMode = false; + private boolean ignoreLaunchEngine = false; public KyuubiBeeLine() { this(true); @@ -90,6 +92,8 @@ void usage() { super.usage(); output("Usage: java " + KyuubiBeeLine.class.getCanonicalName()); output(" --python-mode Execute python code/script."); + output("Usage: java " + KyuubiBeeLine.class.getCanonicalName()); + output(" --ignore-launch-engine Ignore launch engine log."); } public boolean isPythonMode() { @@ -101,6 +105,15 @@ public void setPythonMode(boolean pythonMode) { this.pythonMode = pythonMode; } + public boolean isIgnoreLaunchEngine() { + return ignoreLaunchEngine; + } + + // Visible for testing + public void setIgnoreLaunchEngine(boolean ignoreLaunchEngine) { + this.ignoreLaunchEngine = ignoreLaunchEngine; + } + /** Starts the program. */ public static void main(String[] args) throws IOException { mainWithInputRedirection(args, null); @@ -169,6 +182,8 @@ int initArgs(String[] args) { protected void processOption(String arg, ListIterator iter) throws ParseException { if (PYTHON_MODE_PREFIX.equals(arg)) { pythonMode = true; + } else if (IGNORE_LAUNCH_ENGINE_PREFIX.equals(arg)) { + ignoreLaunchEngine = true; } else { super.processOption(arg, iter); } diff --git a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiCommands.java b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiCommands.java index fcfee49edb0..d98a8b1343c 100644 --- a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiCommands.java +++ b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiCommands.java @@ -18,6 +18,7 @@ package org.apache.hive.beeline; import static org.apache.kyuubi.jdbc.hive.JdbcConnectionParams.*; +import static org.apache.kyuubi.jdbc.hive.KyuubiConnection.IGNORE_LAUNCH_ENGINE_PROPERTY; import com.google.common.annotations.VisibleForTesting; import java.io.*; @@ -450,6 +451,10 @@ public boolean connect(Properties props) throws IOException { AUTH_PASSWD, "javax.jdo.option.ConnectionPassword", "ConnectionPassword", }); + if (beeLine.isIgnoreLaunchEngine()) { + props.put(IGNORE_LAUNCH_ENGINE_PROPERTY, "true"); + } + if (url == null || url.length() == 0) { return beeLine.error("Property \"url\" is required"); } diff --git a/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/KyuubiBeeLineTest.java b/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/KyuubiBeeLineTest.java index 9c7aec35a42..278a53872a7 100644 --- a/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/KyuubiBeeLineTest.java +++ b/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/KyuubiBeeLineTest.java @@ -88,6 +88,26 @@ public void testKyuubiBeeLinePythonMode() { kyuubiBeeLine.setPythonMode(false); } + @Test + public void testKyuubiBeelineIgnoreLaunchEngine() { + KyuubiBeeLine kyuubiBeeLine = new KyuubiBeeLine(); + String[] args1 = {"-u", "badUrl", "--ignore-launch-engine"}; + kyuubiBeeLine.initArgs(args1); + assertTrue(kyuubiBeeLine.isIgnoreLaunchEngine()); + kyuubiBeeLine.setIgnoreLaunchEngine(false); + + String[] args2 = {"--ignore-launch-engine", "-f", "test.sql"}; + kyuubiBeeLine.initArgs(args2); + assertTrue(kyuubiBeeLine.isIgnoreLaunchEngine()); + assert kyuubiBeeLine.getOpts().getScriptFile().equals("test.sql"); + kyuubiBeeLine.setIgnoreLaunchEngine(false); + + String[] args3 = {"-u", "badUrl"}; + kyuubiBeeLine.initArgs(args3); + assertTrue(!kyuubiBeeLine.isIgnoreLaunchEngine()); + kyuubiBeeLine.setIgnoreLaunchEngine(false); + } + @Test public void testKyuubiBeelineComment() { KyuubiBeeLine kyuubiBeeLine = new KyuubiBeeLine(); diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java index 87872581c9c..cd5d97feb8b 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java @@ -78,6 +78,7 @@ public class KyuubiConnection implements SQLConnection, KyuubiLoggable { public static final Logger LOG = LoggerFactory.getLogger(KyuubiConnection.class.getName()); public static final String BEELINE_MODE_PROPERTY = "BEELINE_MODE"; + public static final String IGNORE_LAUNCH_ENGINE_PROPERTY = "IGNORE_LAUNCH_ENGINE_LOG"; public static final String HS2_PROXY_USER = "hive.server2.proxy.user"; public static int DEFAULT_ENGINE_LOG_THREAD_TIMEOUT = 10 * 1000; @@ -112,6 +113,8 @@ public class KyuubiConnection implements SQLConnection, KyuubiLoggable { private boolean isBeeLineMode; + private boolean ignoreLaunchEngineLog; + /** Get all direct HiveServer2 URLs from a ZooKeeper based HiveServer2 URL */ public static List getAllUrls(String zookeeperBasedHS2Url) throws Exception { @@ -127,6 +130,7 @@ public static List getAllUrls(String zookeeperBasedHS2Url) public KyuubiConnection(String uri, Properties info) throws SQLException { isBeeLineMode = Boolean.parseBoolean(info.getProperty(BEELINE_MODE_PROPERTY)); + ignoreLaunchEngineLog = Boolean.parseBoolean(info.getProperty(IGNORE_LAUNCH_ENGINE_PROPERTY)); try { connParams = Utils.parseURL(uri, info); } catch (ZooKeeperHiveClientException e) { @@ -262,7 +266,7 @@ public List getExecLog() throws SQLException, ClosedOrCancelledException "Method getExecLog() failed. The " + "connection has been closed."); } - if (launchEngineOpHandle == null) { + if (launchEngineOpHandle == null || ignoreLaunchEngineLog) { return Collections.emptyList(); } From fd9c92536b287fff3bde7ffb5aafa867f1c3fcaa Mon Sep 17 00:00:00 2001 From: senmiaoliu Date: Wed, 28 Feb 2024 15:50:27 +0800 Subject: [PATCH 2/3] respect beeline verbose --- .../main/java/org/apache/hive/beeline/KyuubiBeeLine.java | 4 ++++ .../java/org/apache/hive/beeline/KyuubiBeeLineTest.java | 9 ++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiBeeLine.java b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiBeeLine.java index 8ef5454fa7d..706fce467aa 100644 --- a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiBeeLine.java +++ b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiBeeLine.java @@ -191,6 +191,10 @@ protected void processOption(String arg, ListIterator iter) throws ParseExceptio }; cl = beelineParser.parse(options, args); + if (getOpts().getVerbose()) { + ignoreLaunchEngine = false; + } + connSuccessful = DynMethods.builder("connectUsingArgs") .hiddenImpl(BeeLine.class, BeelineParser.class, CommandLine.class) diff --git a/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/KyuubiBeeLineTest.java b/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/KyuubiBeeLineTest.java index 278a53872a7..9cdc3340c32 100644 --- a/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/KyuubiBeeLineTest.java +++ b/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/KyuubiBeeLineTest.java @@ -19,6 +19,7 @@ package org.apache.hive.beeline; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.IOException; @@ -104,7 +105,13 @@ public void testKyuubiBeelineIgnoreLaunchEngine() { String[] args3 = {"-u", "badUrl"}; kyuubiBeeLine.initArgs(args3); - assertTrue(!kyuubiBeeLine.isIgnoreLaunchEngine()); + assertFalse(kyuubiBeeLine.isIgnoreLaunchEngine()); + kyuubiBeeLine.setIgnoreLaunchEngine(false); + + String[] args4 = {"--ignore-launch-engine", "--verbose", "-f", "test.sql"}; + kyuubiBeeLine.initArgs(args4); + assertFalse(kyuubiBeeLine.isIgnoreLaunchEngine()); + assert kyuubiBeeLine.getOpts().getScriptFile().equals("test.sql"); kyuubiBeeLine.setIgnoreLaunchEngine(false); } From a9c08adf39e0582680acfd6ac3f458d580f5be31 Mon Sep 17 00:00:00 2001 From: senmiaoliu Date: Mon, 11 Mar 2024 10:12:24 +0800 Subject: [PATCH 3/3] skip create thread engine-launch-log --- .../main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java index cd5d97feb8b..46b41777b9c 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java @@ -289,7 +289,7 @@ public List getExecLog() throws SQLException, ClosedOrCancelledException } private void showLaunchEngineLog() { - if (launchEngineOpHandle != null) { + if (launchEngineOpHandle != null && !ignoreLaunchEngineLog) { LOG.info("Starting to get launch engine log."); engineLogThread = new Thread("engine-launch-log") {