From 3951aa7053e3b33a0288eac879506845b03a1ca7 Mon Sep 17 00:00:00 2001 From: Tinkoff DWH Date: Tue, 16 May 2017 12:29:56 +0500 Subject: [PATCH 1/4] [ZEPPELIN-2554] sql parser fix (backslash) --- .../apache/zeppelin/jdbc/JDBCInterpreter.java | 21 ++++++++++--------- .../zeppelin/jdbc/JDBCInterpreterTest.java | 14 +++++++++++++ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java index b75d8b85fdc..df6da189caa 100644 --- a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java +++ b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java @@ -499,7 +499,7 @@ protected ArrayList splitSqlQueries(String sql) { StringBuilder query = new StringBuilder(); char character; - Boolean antiSlash = false; + Boolean backslash = false; Boolean multiLineComment = false; Boolean singleLineComment = false; Boolean quoteString = false; @@ -508,6 +508,12 @@ protected ArrayList splitSqlQueries(String sql) { for (int item = 0; item < sql.length(); item++) { character = sql.charAt(item); + if (backslash) { + query.append(character); + backslash = false; + continue; + } + if ((singleLineComment && (character == '\n' || item == sql.length() - 1)) || (multiLineComment && character == '/' && sql.charAt(item - 1) == '*')) { singleLineComment = false; @@ -523,13 +529,11 @@ protected ArrayList splitSqlQueries(String sql) { } if (character == '\\') { - antiSlash = true; + backslash = true; } if (character == '\'') { - if (antiSlash) { - antiSlash = false; - } else if (quoteString) { + if (quoteString) { quoteString = false; } else if (!doubleQuoteString) { quoteString = true; @@ -537,9 +541,7 @@ protected ArrayList splitSqlQueries(String sql) { } if (character == '"') { - if (antiSlash) { - antiSlash = false; - } else if (doubleQuoteString) { + if (doubleQuoteString) { doubleQuoteString = false; } else if (!quoteString) { doubleQuoteString = true; @@ -559,7 +561,7 @@ protected ArrayList splitSqlQueries(String sql) { } } - if (character == ';' && !antiSlash && !quoteString && !doubleQuoteString) { + if (character == ';' && !backslash && !quoteString && !doubleQuoteString) { queries.add(StringUtils.trim(query.toString())); query = new StringBuilder(); } else if (item == sql.length() - 1) { @@ -597,7 +599,6 @@ private InterpreterResult executeSql(String propertyKey, String sql, String user = interpreterContext.getAuthenticationInfo().getUser(); InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS); - try { connection = getConnection(propertyKey, interpreterContext); if (connection == null) { diff --git a/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java b/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java index 7c2eef39b07..886bc28edc5 100644 --- a/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java +++ b/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java @@ -188,6 +188,20 @@ public void testSplitSqlQuery() throws SQLException, IOException { assertEquals("select * from test_table WHERE ID = ';'", multipleSqlArray.get(3)); } + @Test + public void testSqlQueryWithBackslash() throws SQLException, IOException { + String sqlQuery = "select '\\n', ';';" + + "select replace('A\\;B', '\\', 'text')"; + + Properties properties = new Properties(); + JDBCInterpreter t = new JDBCInterpreter(properties); + t.open(); + ArrayList multipleSqlArray = t.splitSqlQueries(sqlQuery); + assertEquals(2, multipleSqlArray.size()); + assertEquals("select '\\n', ';'", multipleSqlArray.get(0)); + assertEquals("select replace('A\\;B', '\\', 'text')", multipleSqlArray.get(1)); + } + @Test public void testSelectMultipleQuries() throws SQLException, IOException { Properties properties = new Properties(); From af508f1ca4fbd6765175d01a454b54893054f3bc Mon Sep 17 00:00:00 2001 From: Tinkoff DWH Date: Tue, 16 May 2017 14:19:51 +0500 Subject: [PATCH 2/4] [ZEPPELIN-2554] fix tests --- .../java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java b/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java index 886bc28edc5..45a3fb80c2f 100644 --- a/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java +++ b/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java @@ -180,7 +180,7 @@ public void testSplitSqlQuery() throws SQLException, IOException { Properties properties = new Properties(); JDBCInterpreter t = new JDBCInterpreter(properties); t.open(); - ArrayList multipleSqlArray = t.splitSqlQueries(sqlQuery); + List multipleSqlArray = t.splitSqlQueries(sqlQuery); assertEquals(4, multipleSqlArray.size()); assertEquals("insert into test_table(id, name) values ('a', ';\"')", multipleSqlArray.get(0)); assertEquals("select * from test_table", multipleSqlArray.get(1)); @@ -196,7 +196,7 @@ public void testSqlQueryWithBackslash() throws SQLException, IOException { Properties properties = new Properties(); JDBCInterpreter t = new JDBCInterpreter(properties); t.open(); - ArrayList multipleSqlArray = t.splitSqlQueries(sqlQuery); + List multipleSqlArray = t.splitSqlQueries(sqlQuery); assertEquals(2, multipleSqlArray.size()); assertEquals("select '\\n', ';'", multipleSqlArray.get(0)); assertEquals("select replace('A\\;B', '\\', 'text')", multipleSqlArray.get(1)); From e8be7b36dd717f2e7a81a235fae6c01e4fb82fa4 Mon Sep 17 00:00:00 2001 From: Tinkoff DWH Date: Mon, 22 May 2017 10:04:44 +0500 Subject: [PATCH 3/4] [ZEPPELIN-2554] fix parsing backslash and single quote, tests --- .../apache/zeppelin/jdbc/JDBCInterpreter.java | 15 +----- .../zeppelin/jdbc/JDBCInterpreterTest.java | 48 +++++++++++++++---- 2 files changed, 40 insertions(+), 23 deletions(-) diff --git a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java index df6da189caa..e2530b8a45d 100644 --- a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java +++ b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java @@ -499,7 +499,6 @@ protected ArrayList splitSqlQueries(String sql) { StringBuilder query = new StringBuilder(); char character; - Boolean backslash = false; Boolean multiLineComment = false; Boolean singleLineComment = false; Boolean quoteString = false; @@ -508,12 +507,6 @@ protected ArrayList splitSqlQueries(String sql) { for (int item = 0; item < sql.length(); item++) { character = sql.charAt(item); - if (backslash) { - query.append(character); - backslash = false; - continue; - } - if ((singleLineComment && (character == '\n' || item == sql.length() - 1)) || (multiLineComment && character == '/' && sql.charAt(item - 1) == '*')) { singleLineComment = false; @@ -528,10 +521,6 @@ protected ArrayList splitSqlQueries(String sql) { continue; } - if (character == '\\') { - backslash = true; - } - if (character == '\'') { if (quoteString) { quoteString = false; @@ -541,7 +530,7 @@ protected ArrayList splitSqlQueries(String sql) { } if (character == '"') { - if (doubleQuoteString) { + if (doubleQuoteString && item > 0) { doubleQuoteString = false; } else if (!quoteString) { doubleQuoteString = true; @@ -561,7 +550,7 @@ protected ArrayList splitSqlQueries(String sql) { } } - if (character == ';' && !backslash && !quoteString && !doubleQuoteString) { + if (character == ';' && !quoteString && !doubleQuoteString) { queries.add(StringUtils.trim(query.toString())); query = new StringBuilder(); } else if (item == sql.length() - 1) { diff --git a/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java b/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java index 45a3fb80c2f..7b56dd5e5d9 100644 --- a/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java +++ b/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java @@ -27,8 +27,11 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.sql.*; -import java.util.ArrayList; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; import java.util.List; import java.util.Properties; @@ -175,31 +178,56 @@ public void testSplitSqlQuery() throws SQLException, IOException { String sqlQuery = "insert into test_table(id, name) values ('a', ';\"');" + "select * from test_table;" + "select * from test_table WHERE ID = \";'\";" + - "select * from test_table WHERE ID = ';'"; + "select * from test_table WHERE ID = ';';" + + "select '\n', ';';" + + "select replace('A\\;B', '\\', 'text');" + + "select '\\', ';';" + + "select '''', ';'"; Properties properties = new Properties(); JDBCInterpreter t = new JDBCInterpreter(properties); t.open(); List multipleSqlArray = t.splitSqlQueries(sqlQuery); - assertEquals(4, multipleSqlArray.size()); + assertEquals(8, multipleSqlArray.size()); assertEquals("insert into test_table(id, name) values ('a', ';\"')", multipleSqlArray.get(0)); assertEquals("select * from test_table", multipleSqlArray.get(1)); assertEquals("select * from test_table WHERE ID = \";'\"", multipleSqlArray.get(2)); assertEquals("select * from test_table WHERE ID = ';'", multipleSqlArray.get(3)); + assertEquals("select '\n', ';'", multipleSqlArray.get(4)); + assertEquals("select replace('A\\;B', '\\', 'text')", multipleSqlArray.get(5)); + assertEquals("select '\\', ';'", multipleSqlArray.get(6)); + assertEquals("select '''', ';'", multipleSqlArray.get(7)); } @Test - public void testSqlQueryWithBackslash() throws SQLException, IOException { + public void testQueryWithEsсapedCharacters() throws SQLException, IOException { String sqlQuery = "select '\\n', ';';" + - "select replace('A\\;B', '\\', 'text')"; + "select replace('A\\;B', '\\', 'text');" + + "select '\\', ';';" + + "select '''', ';'"; Properties properties = new Properties(); + properties.setProperty("common.max_count", "1000"); + properties.setProperty("common.max_retry", "3"); + properties.setProperty("default.driver", "org.h2.Driver"); + properties.setProperty("default.url", getJdbcConnection()); + properties.setProperty("default.user", ""); + properties.setProperty("default.password", ""); JDBCInterpreter t = new JDBCInterpreter(properties); t.open(); - List multipleSqlArray = t.splitSqlQueries(sqlQuery); - assertEquals(2, multipleSqlArray.size()); - assertEquals("select '\\n', ';'", multipleSqlArray.get(0)); - assertEquals("select replace('A\\;B', '\\', 'text')", multipleSqlArray.get(1)); + + InterpreterResult interpreterResult = t.interpret(sqlQuery, interpreterContext); + + assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code()); + assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType()); + assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(1).getType()); + assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(2).getType()); + assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(3).getType()); + assertEquals("'\\n'\t';'\n\\n\t;\n", interpreterResult.message().get(0).getData()); + assertEquals("'Atext;B'\nAtext;B\n", interpreterResult.message().get(1).getData()); + assertEquals("'\\'\t';'\n\\\t;\n", interpreterResult.message().get(2).getData()); + assertEquals("''''\t';'\n'\t;\n", interpreterResult.message().get(3).getData()); + } @Test From b28ebbef6f9be43b9e27740eeb2ce1e02f77d9b3 Mon Sep 17 00:00:00 2001 From: Tinkoff DWH Date: Wed, 24 May 2017 09:44:13 +0500 Subject: [PATCH 4/4] [ZEPPELIN-2554] added parameter key.splitQueries --- docs/interpreter/jdbc.md | 5 ++++ .../apache/zeppelin/jdbc/JDBCInterpreter.java | 21 ++++++++++++--- .../main/resources/interpreter-setting.json | 6 +++++ .../zeppelin/jdbc/JDBCInterpreterTest.java | 27 ++++++++++++++++++- 4 files changed, 55 insertions(+), 4 deletions(-) diff --git a/docs/interpreter/jdbc.md b/docs/interpreter/jdbc.md index b7ac45ae442..0dfa32f147c 100644 --- a/docs/interpreter/jdbc.md +++ b/docs/interpreter/jdbc.md @@ -128,6 +128,11 @@ The JDBC interpreter properties are defined by default like below. Сomma separated schema (schema = catalog = database) filters to get metadata for completions. Supports '%' symbol is equivalent to any set of characters. (ex. prod_v_%,public%,info) + + default.splitQueries + false + Each query is executed apart and returns the result + If you want to connect other databases such as `Mysql`, `Redshift` and `Hive`, you need to edit the property values. diff --git a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java index e2530b8a45d..a5a0aeb09e1 100644 --- a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java +++ b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java @@ -23,6 +23,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -101,6 +102,7 @@ public class JDBCInterpreter extends Interpreter { static final String PASSWORD_KEY = "password"; static final String PRECODE_KEY = "precode"; static final String COMPLETER_SCHEMA_FILTERS_KEY = "completer.schemaFilters"; + static final String SPLIT_QURIES_KEY = "splitQueries"; static final String JDBC_JCEKS_FILE = "jceks.file"; static final String JDBC_JCEKS_CREDENTIAL_KEY = "jceks.credentialKey"; static final String PRECODE_KEY_TEMPLATE = "%s.precode"; @@ -587,6 +589,12 @@ private InterpreterResult executeSql(String propertyKey, String sql, String paragraphId = interpreterContext.getParagraphId(); String user = interpreterContext.getAuthenticationInfo().getUser(); + boolean splitQuery = false; + String splitQueryProperty = getProperty(String.format("%s.%s", propertyKey, SPLIT_QURIES_KEY)); + if (StringUtils.isNotBlank(splitQueryProperty) && splitQueryProperty.equalsIgnoreCase("true")) { + splitQuery = true; + } + InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS); try { connection = getConnection(propertyKey, interpreterContext); @@ -594,9 +602,16 @@ private InterpreterResult executeSql(String propertyKey, String sql, return new InterpreterResult(Code.ERROR, "Prefix not found."); } - ArrayList multipleSqlArray = splitSqlQueries(sql); - for (int i = 0; i < multipleSqlArray.size(); i++) { - String sqlToExecute = multipleSqlArray.get(i); + + List sqlArray; + if (splitQuery) { + sqlArray = splitSqlQueries(sql); + } else { + sqlArray = Arrays.asList(sql); + } + + for (int i = 0; i < sqlArray.size(); i++) { + String sqlToExecute = sqlArray.get(i); statement = connection.createStatement(); if (statement == null) { return new InterpreterResult(Code.ERROR, "Prefix not found."); diff --git a/jdbc/src/main/resources/interpreter-setting.json b/jdbc/src/main/resources/interpreter-setting.json index fb8b8b2a23d..139821a0897 100644 --- a/jdbc/src/main/resources/interpreter-setting.json +++ b/jdbc/src/main/resources/interpreter-setting.json @@ -40,6 +40,12 @@ "defaultValue": "", "description": "SQL which executes while opening connection" }, + "default.splitQueries": { + "envName": null, + "propertyName": "default.splitQueries", + "defaultValue": "false", + "description": "Each query is executed apart and returns the result" + }, "common.max_count": { "envName": null, "propertyName": "common.max_count", diff --git a/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java b/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java index 7b56dd5e5d9..bfe1ee8ee19 100644 --- a/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java +++ b/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java @@ -213,6 +213,7 @@ public void testSplitSqlQuery() throws SQLException, IOException { properties.setProperty("default.url", getJdbcConnection()); properties.setProperty("default.user", ""); properties.setProperty("default.password", ""); + properties.setProperty("default.splitQueries", "true"); JDBCInterpreter t = new JDBCInterpreter(properties); t.open(); @@ -231,7 +232,7 @@ public void testSplitSqlQuery() throws SQLException, IOException { } @Test - public void testSelectMultipleQuries() throws SQLException, IOException { + public void testSelectMultipleQueries() throws SQLException, IOException { Properties properties = new Properties(); properties.setProperty("common.max_count", "1000"); properties.setProperty("common.max_retry", "3"); @@ -239,6 +240,7 @@ public void testSelectMultipleQuries() throws SQLException, IOException { properties.setProperty("default.url", getJdbcConnection()); properties.setProperty("default.user", ""); properties.setProperty("default.password", ""); + properties.setProperty("default.splitQueries", "true"); JDBCInterpreter t = new JDBCInterpreter(properties); t.open(); @@ -255,6 +257,28 @@ public void testSelectMultipleQuries() throws SQLException, IOException { assertEquals("ID\tNAME\n", interpreterResult.message().get(1).getData()); } + @Test + public void testDefaultSplitQuries() throws SQLException, IOException { + Properties properties = new Properties(); + properties.setProperty("common.max_count", "1000"); + properties.setProperty("common.max_retry", "3"); + properties.setProperty("default.driver", "org.h2.Driver"); + properties.setProperty("default.url", getJdbcConnection()); + properties.setProperty("default.user", ""); + properties.setProperty("default.password", ""); + JDBCInterpreter t = new JDBCInterpreter(properties); + t.open(); + + String sqlQuery = "select * from test_table;" + + "select * from test_table WHERE ID = ';';"; + InterpreterResult interpreterResult = t.interpret(sqlQuery, interpreterContext); + assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code()); + assertEquals(1, interpreterResult.message().size()); + + assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType()); + assertEquals("ID\tNAME\na\ta_name\nb\tb_name\nc\tnull\n", interpreterResult.message().get(0).getData()); + } + @Test public void testSelectQueryWithNull() throws SQLException, IOException { Properties properties = new Properties(); @@ -507,6 +531,7 @@ public void testExcludingComments() throws SQLException, IOException { properties.setProperty("default.url", getJdbcConnection()); properties.setProperty("default.user", ""); properties.setProperty("default.password", ""); + properties.setProperty("default.splitQueries", "true"); JDBCInterpreter t = new JDBCInterpreter(properties); t.open();