From 5f90193536d6cb0f0a7fb71ae667146ecd57edc1 Mon Sep 17 00:00:00 2001 From: Anishek Agarwal Date: Mon, 6 Mar 2017 15:15:30 +0530 Subject: [PATCH] HIVE-16115: Stop printing progress info from operation logs with beeline progress bar also fixing the issue where the session hive configuration was not read correctly for hive.server2.in.place.progress using the execution mode for operation logs to make sure that the text representation of the progress information is not displayed. --- itests/hive-unit/pom.xml | 12 - .../hive/beeline/TestBeeLineWithArgs.java | 313 +++++++++++------- .../exec/tez/monitoring/RenderStrategy.java | 22 +- .../apache/hive/service/cli/CLIService.java | 8 +- 4 files changed, 218 insertions(+), 137 deletions(-) diff --git a/itests/hive-unit/pom.xml b/itests/hive-unit/pom.xml index 6a190d1c26f6..789192bebdac 100644 --- a/itests/hive-unit/pom.xml +++ b/itests/hive-unit/pom.xml @@ -136,12 +136,6 @@ ${project.version} test - - org.apache.hive - hive-jdbc - ${project.version} - test - org.apache.hive hive-metastore @@ -381,12 +375,6 @@ ${hadoop.version} true - - org.apache.hadoop - hadoop-yarn-api - ${hadoop.version} - test - org.apache.curator curator-test diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java index 42ef280ddb38..650c4b711dcc 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.ByteArrayOutputStream; @@ -34,12 +35,17 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.google.common.base.Function; +import com.google.common.collect.Lists; +import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; @@ -76,6 +82,7 @@ private List getBaseArgs(String jdbcUrl) { argList.add(userName); return argList; } + /** * Start up a local Hive Server 2 for these tests */ @@ -119,7 +126,7 @@ private static void createTable() throws ClassNotFoundException, SQLException { try { stmt.execute("drop table " + tableName); } catch (Exception ex) { - fail(ex.toString()); + fail(ex.toString() + " " + ExceptionUtils.getStackTrace(ex)); } // create table @@ -150,7 +157,8 @@ public static void postTests() { * @return The stderr and stdout from running the script * @throws Throwable */ - private String testCommandLineScript(List argList, InputStream inputStream, OutStream streamType) + private static String testCommandLineScript(List argList, InputStream inputStream, + OutStream streamType) throws Throwable { BeeLine beeLine = new BeeLine(); ByteArrayOutputStream os = new ByteArrayOutputStream(); @@ -177,7 +185,7 @@ private String testCommandLineScript(List argList, InputStream inputStre * Attempt to execute a simple script file with the -f and -i option to * BeeLine to test for presence of an expected pattern in the output (stdout * or stderr), fail if not found. Print PASSED or FAILED - * + * * @param expectedRegex * Text to look for in command output (stdout) * @param shouldMatch @@ -185,9 +193,11 @@ private String testCommandLineScript(List argList, InputStream inputStre * @throws Exception * on command execution error */ - private void testScriptFile(String scriptText, String expectedRegex, - boolean shouldMatch, List argList) throws Throwable { - testScriptFile(scriptText, expectedRegex, shouldMatch, argList, true, true, OutStream.OUT); + private void testScriptFile(String scriptText, List argList, String expectedRegex, + boolean shouldMatch) throws Throwable { + testScriptFile(scriptText, argList, OutStream.OUT, + Collections.singletonList(new Tuple<>(expectedRegex, shouldMatch)) + ); } /** @@ -195,33 +205,40 @@ private void testScriptFile(String scriptText, String expectedRegex, * to BeeLine to test for presence of an expected pattern * in the output (stdout or stderr), fail if not found. * Print PASSED or FAILED - * @param expectedRegex Text to look for in command output (stdout) - * @param shouldMatch true if the pattern should be found, false if it should not * @param argList arguments * @param outType output stream type + * @param expectedRegex Text to look for in command output (stdout) + * @param shouldMatch true if the pattern should be found, false if it should not * @throws Throwable */ - private void testScriptFile(String scriptText, String expectedRegex, - boolean shouldMatch, List argList, OutStream outType) throws Throwable { - testScriptFile(scriptText, expectedRegex, shouldMatch, argList, true, true, outType); + private void testScriptFile(String scriptText, List argList, OutStream outType, + String expectedRegex, boolean shouldMatch) throws Throwable { + testScriptFile(scriptText, argList, outType, + Collections.singletonList(new Tuple<>(expectedRegex, shouldMatch)) + ); } - + + private void testScriptFile(String scriptText, List argList, OutStream streamType, + List> expectedMatches) throws Throwable { + testScriptFile(scriptText, argList, streamType, expectedMatches, + Arrays.asList(Modes.values())); + } + /** * Attempt to execute a simple script file with the -f or -i option * to BeeLine (or both) to test for presence of an expected pattern * in the output (stdout or stderr), fail if not found. * Print PASSED or FAILED - * @param expectedRegex Text to look for in command output/error - * @param shouldMatch true if the pattern should be found, false if it should not - * @param testScript Whether we should test -f - * @param testInit Whether we should test -i + * @param scriptText script to test the output for + * @param argList arguments to be passed to the script file to execute and produce output * @param streamType Whether match should be done against STDERR or STDOUT + * @param expectedMatches List of Tuple's defining the pattern to match and result of matching + * @param modes testing modes we have to run the script as * @throws Exception on command execution error */ - private void testScriptFile(String scriptText, String expectedRegex, - boolean shouldMatch, List argList, - boolean testScript, boolean testInit, OutStream streamType) throws Throwable { - + private void testScriptFile(String scriptText, List argList, + OutStream streamType, List> expectedMatches, List modes) + throws Throwable { // Put the script content in a temp file File scriptFile = File.createTempFile(this.getClass().getSimpleName(), "temp"); System.out.println("script file is " + scriptFile.getAbsolutePath()); @@ -230,43 +247,59 @@ private void testScriptFile(String scriptText, String expectedRegex, os.print(scriptText); os.close(); - Pattern expectedPattern = Pattern.compile(".*" + expectedRegex + ".*", Pattern.DOTALL); - if (testScript) { - List copy = new ArrayList(argList); - copy.add("-f"); - copy.add(scriptFile.getAbsolutePath()); - - String output = testCommandLineScript(copy, null, streamType); - - Matcher m = expectedPattern.matcher(output); - boolean matches = m.matches(); - if (shouldMatch != matches) { - //failed - fail("Output" + output + " should" + (shouldMatch ? "" : " not") + - " contain " + expectedRegex); + List> patternsToBeMatched = Lists.transform(expectedMatches, + new Function, Tuple>() { + @Override + public Tuple apply(Tuple tuple) { + return new Tuple<>( + Pattern.compile(".*" + tuple.pattern + ".*", Pattern.DOTALL), + tuple.shouldMatch + ); + } + }); + + for (Modes mode : modes) { + String output = mode.output(scriptFile, argList, streamType); + for (Tuple patternToMatch : patternsToBeMatched) { + Matcher m = patternToMatch.pattern.matcher(output); + boolean matches = m.matches(); + if (patternToMatch.shouldMatch != matches) { + //failed + fail("Output" + output + " should" + (patternToMatch.shouldMatch ? "" : " not") + + " contain " + patternToMatch.pattern.pattern()); + } } } + scriptFile.delete(); + } - // Not all scripts can be used as init scripts, so we parameterize. - // (scripts that test !connect, for eg., since -i runs after connects) - // So, we keep this optional. Most tests should leave this as true, however. - if (testInit) { - List copy = new ArrayList(argList); - copy.add("-i"); - copy.add(scriptFile.getAbsolutePath()); - - String output = testCommandLineScript(copy, new StringBufferInputStream("!quit\n"), streamType); - Matcher m = expectedPattern.matcher(output); - boolean matches = m.matches(); - if (shouldMatch != matches) { - //failed - fail("Output" + output + " should" + (shouldMatch ? "" : " not") + - " contain " + expectedRegex); + /* + We are testing for both type of modes always so not passing that as a parameter for now + */ + enum Modes { + INIT { + @Override + String output(File scriptFile, List argList, OutStream streamType) throws Throwable { + List copy = new ArrayList<>(argList); + copy.add("-i"); + copy.add(scriptFile.getAbsolutePath()); + return testCommandLineScript(copy, new StringBufferInputStream("!quit\n"), streamType); } - } - scriptFile.delete(); + }, SCRIPT { + @Override + String output(File scriptFile, List argList, OutStream streamType) throws Throwable { + List copy = new ArrayList<>(argList); + copy.add("-f"); + copy.add(scriptFile.getAbsolutePath()); + return testCommandLineScript(copy, null, streamType); + } + }; + + abstract String output(File scriptFile, List argList, OutStream streamType) + throws Throwable; } + /** * Attempt to execute the enclosed query with the -e option to BeeLine * Test for presence of an expected pattern @@ -302,7 +335,7 @@ public void testWhitespaceBeforeCommentScriptFile() throws Throwable { final String SCRIPT_TEXT = " -- comment has spaces and tabs before it\n # comment has spaces and tabs before it\n"; final String EXPECTED_PATTERN = "cannot recognize input near ''"; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, false, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, false); } /** @@ -316,7 +349,7 @@ public void testPositiveScriptFile() throws Throwable { final String SCRIPT_TEXT = "show databases;\n"; final String EXPECTED_PATTERN = " default "; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); - testScriptFile( SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -328,7 +361,7 @@ public void testLastLineCmdInScriptFile() throws Throwable { final String SCRIPT_TEXT = "show databases;\nshow tables;"; final String EXPECTED_PATTERN = " testbeelinetable1 "; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); - testScriptFile( SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -344,7 +377,7 @@ public void testBeelineHiveVariable() throws Throwable { argList.add("DUMMY_TBL=dummy"); final String SCRIPT_TEXT = "create table ${DUMMY_TBL} (d int);\nshow tables;\n drop table ${DUMMY_TBL};"; final String EXPECTED_PATTERN = "dummy"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } @Test @@ -355,7 +388,7 @@ public void testBeelineHiveConfVariable() throws Throwable { final String SCRIPT_TEXT = "create table ${hiveconf:test.hive.table.name} (d int);\nshow tables;\n" + " drop table ${hiveconf:test.hive.table.name};\n"; final String EXPECTED_PATTERN = "dummy"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -384,7 +417,7 @@ public void testBeelineMultiHiveVariable() throws Throwable { + "(${hiveconf:COLUMN_NAME} ${hiveconf:COLUMN_TYPE});" + "\nshow tables;\n drop ${OBJECT} ${TABLE_NAME};\n"; final String EXPECTED_PATTERN = "dummy2"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -397,7 +430,7 @@ public void testBreakOnErrorScriptFile() throws Throwable { List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); final String SCRIPT_TEXT = "select * from abcdefg01;\nshow databases;\n"; final String EXPECTED_PATTERN = " default "; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, false, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, false); } @Test @@ -406,7 +439,7 @@ public void testTabInScriptFile() throws Throwable { final String SCRIPT_TEXT = "CREATE\tTABLE IF NOT EXISTS testTabInScriptFile\n(id\tint);\nSHOW TABLES;" + "\ndrop table testTabInScriptFile"; final String EXPECTED_PATTERN = "testTabInScriptFile"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } @Test @@ -414,7 +447,11 @@ public void testBeelineShellCommand() throws Throwable { List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); final String SCRIPT_TEXT = "!sh echo \"hello world.\" > hw.txt\n!sh cat hw.txt\n!rm hw.txt"; final String EXPECTED_PATTERN = "hello world"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + + testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT, + Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true)), + Collections.singletonList(Modes.SCRIPT) + ); } /** @@ -426,7 +463,7 @@ public void testNullDefault() throws Throwable { final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + "select null from " + tableName + " limit 1 ;\n"; final String EXPECTED_PATTERN = "NULL"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL())); + testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), EXPECTED_PATTERN, true); } /** @@ -438,14 +475,14 @@ public void testNullNonEmpty() throws Throwable { final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + "!set nullemptystring false\n select null from " + tableName + " limit 1 ;\n"; final String EXPECTED_PATTERN = "NULL"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL())); + testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), EXPECTED_PATTERN, true); } @Test public void testGetVariableValue() throws Throwable { final String SCRIPT_TEXT = "set env:TERM;"; final String EXPECTED_PATTERN = "env:TERM"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL())); + testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), EXPECTED_PATTERN, true); } /** @@ -457,13 +494,13 @@ public void testGetVariableValue() throws Throwable { @Test public void testNullEmpty() throws Throwable { final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + - "!set nullemptystring true\n select 'abc',null,'def' from " + tableName + " limit 1 ;\n"; + "!set nullemptystring true\n select 'abc',null,'def' from " + tableName + " limit 1 ;\n"; final String EXPECTED_PATTERN = "abc,,def"; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=csv2"); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -477,7 +514,7 @@ public void testDSVOutput() throws Throwable { argList.add("--delimiterForDSV=;"); final String EXPECTED_PATTERN = "1;NULL;defg;ab\"c;1.0"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -490,7 +527,7 @@ public void testTSV2Output() throws Throwable { argList.add("--outputformat=tsv2"); final String EXPECTED_PATTERN = "1\tNULL\tdefg\tab\"c\t1.0"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -503,7 +540,7 @@ public void testTSVOutput() throws Throwable { argList.add("--outputformat=tsv"); final String EXPECTED_PATTERN = "'1'\t'NULL'\t'defg'\t'ab\"c\'\t'1.0'"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -517,7 +554,7 @@ public void testTSV2OutputWithDoubleQuotes() throws Throwable { System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV,"false"); final String EXPECTED_PATTERN = "1\tNULL\tdefg\t\"ab\"\"c\"\t\"\"\"aa\"\"\"\t1.0"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true"); } @@ -532,7 +569,7 @@ public void testTSVOutputWithDoubleQuotes() throws Throwable { System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false"); final String EXPECTED_PATTERN = "'1'\t'NULL'\t'defg'\t'ab\"c'\t'\"aa\"'\t'1.0'"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true"); } @@ -547,7 +584,7 @@ public void testCSV2OutputWithDoubleQuotes() throws Throwable { System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false"); final String EXPECTED_PATTERN = "1,NULL,defg,\"ab\"\"c\",\"\"\"aa\"\"\",1.0"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true"); } @@ -562,7 +599,7 @@ public void testCSVOutputWithDoubleQuotes() throws Throwable { System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false"); final String EXPECTED_PATTERN = "'1','NULL','defg','ab\"c','\"aa\"','1.0'"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true"); } @@ -578,7 +615,7 @@ public void testDSVOutputWithDoubleQuotes() throws Throwable { System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false"); final String EXPECTED_PATTERN = "1;NULL;defg;\"ab\"\"c\";\"\"\"aa\"\"\";1.0"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true"); } @@ -593,7 +630,7 @@ public void testTSVOutputDeprecation() throws Throwable { argList.add("--outputformat=tsv"); final String EXPECTED_PATTERN = "Format tsv is deprecated, please use tsv2"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, OutStream.ERR); + testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, EXPECTED_PATTERN, true); } /** @@ -607,7 +644,8 @@ public void testCSVOutputDeprecation() throws Throwable { argList.add("--outputformat=csv"); final String EXPECTED_PATTERN = "Format csv is deprecated, please use csv2"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, true, true, OutStream.ERR); + testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, + Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true))); } /** @@ -619,10 +657,9 @@ public void testCSVOutput() throws Throwable { List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=csv"); final String EXPECTED_PATTERN = "'1','NULL','defg','ab\"c\','1.0'"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } - private String getFormatTestQuery() { return "set hive.support.concurrency = false;\n" + "select 1, null, 'defg', 'ab\"c', 1.0D from " + tableName + " limit 1 ;\n"; @@ -642,14 +679,14 @@ private String getFormatTestQueryForEableQuotes() { @Test public void testNullEmptyCmdArg() throws Throwable { final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + - "select 'abc',null,'def' from " + tableName + " limit 1 ;\n"; + "select 'abc',null,'def' from " + tableName + " limit 1 ;\n"; final String EXPECTED_PATTERN = "'abc','','def'"; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--nullemptystring=true"); argList.add("--outputformat=csv"); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -694,12 +731,12 @@ public void testNPE() throws UnsupportedEncodingException { beeLine.runCommands( new String[] {"!typeinfo"} ); String output = os.toString("UTF8"); Assert.assertFalse( output.contains("java.lang.NullPointerException") ); - Assert.assertTrue( output.contains("No current connection") ); + assertTrue(output.contains("No current connection")); beeLine.runCommands( new String[] {"!nativesql"} ); output = os.toString("UTF8"); Assert.assertFalse( output.contains("java.lang.NullPointerException") ); - Assert.assertTrue( output.contains("No current connection") ); + assertTrue(output.contains("No current connection")); System.out.println(">>> PASSED " + "testNPE" ); } @@ -709,21 +746,21 @@ public void testHiveVarSubstitution() throws Throwable { List argList = getBaseArgs(miniHS2.getBaseJdbcURL() + "#D_TBL=dummy_t"); final String SCRIPT_TEXT = "create table ${D_TBL} (d int);\nshow tables;\ndrop table ${D_TBL};\n"; final String EXPECTED_PATTERN = "dummy_t"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } @Test public void testEmbeddedBeelineConnection() throws Throwable{ String embeddedJdbcURL = Utils.URL_PREFIX+"/Default"; List argList = getBaseArgs(embeddedJdbcURL); - argList.add("--hivevar"); + argList.add("--hivevar"); argList.add("DUMMY_TBL=embedded_table"); // Set to non-zk lock manager to avoid trying to connect to zookeeper final String SCRIPT_TEXT = "set hive.lock.manager=org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager;\n" + - "create table ${DUMMY_TBL} (d int);\nshow tables;\n drop table ${DUMMY_TBL};\n"; + "create table ${DUMMY_TBL} (d int);\nshow tables;\n drop table ${DUMMY_TBL};\n"; final String EXPECTED_PATTERN = "embedded_table"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -732,12 +769,19 @@ public void testEmbeddedBeelineConnection() throws Throwable{ */ @Test public void testQueryProgress() throws Throwable { - final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + - "select count(*) from " + tableName + ";\n"; + final String SCRIPT_TEXT = + "set hive.support.concurrency = false;\n" + + "set hive.server2.logging.operation.level=execution;\n" + + "select count(*) from " + tableName + ";\n"; // Check for part of log message as well as part of progress information - final String EXPECTED_PATTERN = "Number of reducers determined to be.*ELAPSED TIME"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL()), - OutStream.ERR); + final String EXPECTED_PATTERN = "ELAPSED TIME"; + final String UNEXPECTED_PATTERN = "(?=Reducer 2\\:).*(?=Map 1\\:)"; + testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR, + Arrays.asList( + new Tuple<>(EXPECTED_PATTERN, true), + new Tuple<>(UNEXPECTED_PATTERN, false) + ) + ); } /** @@ -757,8 +801,9 @@ public void testQueryProgressParallel() throws Throwable { "select count(*) from " + tableName + ";\n"; // Check for part of log message as well as part of progress information final String EXPECTED_PATTERN = "Number of reducers determined to be."; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL()), - OutStream.ERR); + testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR, + EXPECTED_PATTERN, true + ); } /** @@ -771,7 +816,23 @@ public void testQueryProgressHidden() throws Throwable { "!set silent true\n" + "select count(*) from " + tableName + ";\n"; final String EXPECTED_PATTERN = "Executing command"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, false, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR); + testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR, + EXPECTED_PATTERN, false); + } + + @Test + public void testQueryProgressWithHiveServer2ProgressBarDisabled() + throws Throwable { + final String SCRIPT_TEXT = + "set hive.support.concurrency = false;\nset hive.server2.in.place.progress=false;\n" + + "select count(*) from " + tableName + ";\n"; + // Check for part of log message as well as part of progress information + final String EXPECTED_PATTERN = "(?=Reducer 2\\:).*(?=Map 1\\:)"; + testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR, + Arrays.asList( + new Tuple<>(EXPECTED_PATTERN, true), + new Tuple<>("ELAPSED TIME", false)) + ); } @Test @@ -780,10 +841,10 @@ public void testMultiCommandsInOneline() throws Throwable { +"(key int);show tables; --multicommands in one line"; final String EXPECTED_PATTERN = " multicmdtbl "; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); final String SCRIPT_TEXT_DROP = "drop table multiCmdTbl;show tables;"; - testScriptFile(SCRIPT_TEXT_DROP, EXPECTED_PATTERN, false, argList); + testScriptFile(SCRIPT_TEXT_DROP, argList, EXPECTED_PATTERN, false); } @Test @@ -804,10 +865,10 @@ public void testOneCommandInMultiLines() throws Throwable { + "(key int);show tables; --one command in multiple lines"; final String EXPECTED_PATTERN = " multicmdtbl "; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); final String SCRIPT_TEXT_DROP = "drop table\nmultiCmdTbl;show tables;"; - testScriptFile(SCRIPT_TEXT_DROP, EXPECTED_PATTERN, false, argList); + testScriptFile(SCRIPT_TEXT_DROP, argList, EXPECTED_PATTERN, false); } @Test @@ -817,10 +878,10 @@ public void testEscapeSemiColonInQueries() throws Throwable { + " TERMINATED BY '\\n';show tables; --one command in multiple lines"; final String EXPECTED_PATTERN = " multicmdtbl "; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); final String SCRIPT_TEXT_DROP = "drop table\nmultiCmdTbl;show tables;"; - testScriptFile(SCRIPT_TEXT_DROP, EXPECTED_PATTERN, false, argList); + testScriptFile(SCRIPT_TEXT_DROP, argList, EXPECTED_PATTERN, false); } @Test @@ -847,7 +908,7 @@ public void testEmbeddedBeelineOutputs() throws Throwable{ + "set a=1;\nselect count(*) from embeddedBeelineOutputs;\n" + "drop table embeddedBeelineOutputs;\n"; final String EXPECTED_PATTERN = "Stage-1 map ="; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, OutStream.ERR); + testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, EXPECTED_PATTERN, true); } @Test @@ -855,7 +916,7 @@ public void testConnectionUrlWithSemiColon() throws Throwable{ List argList = getBaseArgs(miniHS2.getJdbcURL("default", "sess_var_list?var1=value1")); final String SCRIPT_TEXT = "set var1"; final String EXPECTED_PATTERN = "var1=value1"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -865,7 +926,7 @@ public void testConnectionUrlWithSemiColon() throws Throwable{ @Test public void testBeelineConnectEnvVar() throws Throwable { final String jdbcUrl = miniHS2.getBaseJdbcURL(); - List argList = new ArrayList(); + List argList = new ArrayList<>(); argList.add("-u"); argList.add("blue"); argList.add("-d"); @@ -892,7 +953,9 @@ public String get(String envVar) { }; BeeLineOpts.setEnv(newEnv); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, true, false, OutStream.OUT); + testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT, + Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true)), + Collections.singletonList(Modes.SCRIPT)); } /** @@ -904,11 +967,13 @@ public void testBeelineReconnect() throws Throwable { List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); final String SCRIPT_TEXT = "!close\n" + - "!reconnect\n\n\n" + - "create table reconnecttest (d int);\nshow tables;\ndrop table reconnecttest;\n"; + "!reconnect\n\n\n" + + "create table reconnecttest (d int);\nshow tables;\ndrop table reconnecttest;\n"; final String EXPECTED_PATTERN = "reconnecttest"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, true, false, OutStream.OUT); + testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT, + Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true)), + Collections.singletonList(Modes.SCRIPT)); } @@ -921,14 +986,14 @@ public void testBeelineReconnect() throws Throwable { @Test public void testConnectionWithURLParams() throws Throwable { final String EXPECTED_PATTERN = " hivetest "; - List argList = new ArrayList(); + List argList = new ArrayList<>(); argList.add("-d"); argList.add(BeeLine.BEELINE_DEFAULT_JDBC_DRIVER); argList.add("-u"); argList.add(miniHS2.getBaseJdbcURL() + ";user=hivetest;password=hive"); String SCRIPT_TEXT = "select current_user();"; - testScriptFile( SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -937,10 +1002,10 @@ public void testConnectionWithURLParams() throws Throwable { @Test public void testQueryNonEscapedSemiColon() throws Throwable { String SCRIPT_TEXT = "drop table if exists nonEscapedSemiColon;create table nonEscapedSemiColon " - + "(key int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ';';show tables;"; + + "(key int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ';';show tables;"; final String EXPECTED_PATTERN = " nonEscapedSemiColon "; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } @Test @@ -949,7 +1014,7 @@ public void testSelectQueryWithNonEscapedSemiColon() throws Throwable { final String EXPECTED_PATTERN = ";\t';'\t\";\"\t';\t;'\t\";\t;\""; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=tsv2"); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -961,21 +1026,23 @@ public void testSelectQueryWithNonEscapedSemiColon() throws Throwable { @Test public void testShowDbInPrompt() throws Throwable { final String EXPECTED_PATTERN = " \\(default\\)>"; - List argList = new ArrayList(); + List argList = new ArrayList<>(); argList.add("--showDbInPrompt"); argList.add("-u"); argList.add(miniHS2.getBaseJdbcURL() + ";user=hivetest;password=hive"); String SCRIPT_TEXT = "select current_user();"; - testScriptFile( SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } @Test public void testBeelineShellCommandWithoutConn() throws Throwable { - List argList = new ArrayList(); + List argList = new ArrayList<>(); final String SCRIPT_TEXT = "!sh echo hello world"; final String EXPECTED_PATTERN = "hello world"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, true, false, OutStream.OUT); + testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT, + Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true)), + Collections.singletonList(Modes.SCRIPT)); } /** @@ -985,11 +1052,21 @@ public void testBeelineShellCommandWithoutConn() throws Throwable { @Test public void testBeelineWithForce() throws Throwable { final String SCRIPT_TEXT = "drop table does_not_exist;\ncreate table incomplete_syntax(a, string, );\n " - + "drop table if exists new_table;\n create table new_table(foo int, bar string);\n " - + "desc new_table;\n"; + + "drop table if exists new_table;\n create table new_table(foo int, bar string);\n " + + "desc new_table;\n"; final String EXPECTED_PATTERN = "2 rows selected"; List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--force"); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, OutStream.ERR); + testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, EXPECTED_PATTERN, true); + } + + private static class Tuple { + final K pattern; + final boolean shouldMatch; + + Tuple(K pattern, boolean shouldMatch) { + this.pattern = pattern; + this.shouldMatch = shouldMatch; + } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java index bb9a5e7dd27b..2535b1085616 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java @@ -2,10 +2,13 @@ import org.apache.hadoop.hive.common.log.InPlaceUpdate; import org.apache.hadoop.hive.common.log.ProgressMonitor; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.tez.dag.api.client.DAGStatus; import org.apache.tez.dag.api.client.Progress; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.StringWriter; import java.util.Map; @@ -48,10 +51,17 @@ private boolean showReport(String report) { || System.currentTimeMillis() >= lastPrintTime + PRINT_INTERVAL; } + /* + This is used to print the progress information as pure text , a sample is as below: + Map 1: 0/1 Reducer 2: 0/1 + Map 1: 0(+1)/1 Reducer 2: 0/1 + Map 1: 1/1 Reducer 2: 0(+1)/1 + Map 1: 1/1 Reducer 2: 1/1 + */ private String getReport(Map progressMap) { StringWriter reportBuffer = new StringWriter(); - SortedSet keys = new TreeSet(progressMap.keySet()); + SortedSet keys = new TreeSet<>(progressMap.keySet()); for (String s : keys) { Progress progress = progressMap.get(s); final int complete = progress.getSucceededTaskCount(); @@ -109,7 +119,9 @@ private String getReport(Map progressMap) { * same information to beeline client when requested. */ static class LogToFileFunction extends BaseUpdateFunction { - + private static final Logger LOGGER = LoggerFactory.getLogger(LogToFileFunction.class); + private boolean hiveServer2InPlaceProgressEnabled = + SessionState.get().getConf().getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_INPLACE_PROGRESS); LogToFileFunction(TezJobMonitor monitor) { super(monitor); } @@ -121,7 +133,11 @@ public void renderProgress(ProgressMonitor progressMonitor) { @Override public void renderReport(String report) { - monitor.console.printInfo(report); + if (hiveServer2InPlaceProgressEnabled) { + LOGGER.info(report); + } else { + monitor.console.printInfo(report); + } } } diff --git a/service/src/java/org/apache/hive/service/cli/CLIService.java b/service/src/java/org/apache/hive/service/cli/CLIService.java index 5bb4ee0c3ea5..689b948a87a9 100644 --- a/service/src/java/org/apache/hive/service/cli/CLIService.java +++ b/service/src/java/org/apache/hive/service/cli/CLIService.java @@ -442,8 +442,8 @@ public OperationStatus getOperationStatus(OperationHandle opHandle, boolean getP * we block for a duration determined by a step function, before we return * However, if the background operation is complete, we return immediately. */ + HiveConf conf = operation.getParentSession().getHiveConf(); if (operation.shouldRunAsync()) { - HiveConf conf = operation.getParentSession().getHiveConf(); long maxTimeout = HiveConf.getTimeVar(conf, HiveConf.ConfVars.HIVE_SERVER2_LONG_POLLING_TIMEOUT, TimeUnit.MILLISECONDS); @@ -472,7 +472,7 @@ public OperationStatus getOperationStatus(OperationHandle opHandle, boolean getP } OperationStatus opStatus = operation.getStatus(); LOG.debug(opHandle + ": getOperationStatus()"); - opStatus.setJobProgressUpdate(progressUpdateLog(getProgressUpdate, operation)); + opStatus.setJobProgressUpdate(progressUpdateLog(getProgressUpdate, operation, conf)); return opStatus; } @@ -482,8 +482,8 @@ public HiveConf getSessionConf(SessionHandle sessionHandle) } private static final long PROGRESS_MAX_WAIT_NS = 30 * 1000000000l; - private JobProgressUpdate progressUpdateLog(boolean isProgressLogRequested, Operation operation) { - if (!isProgressLogRequested || !ServiceUtils.canProvideProgressLog(hiveConf) + private JobProgressUpdate progressUpdateLog(boolean isProgressLogRequested, Operation operation, HiveConf conf) { + if (!isProgressLogRequested || !ServiceUtils.canProvideProgressLog(conf) || !OperationType.EXECUTE_STATEMENT.equals(operation.getType())) { return new JobProgressUpdate(ProgressMonitor.NULL); }