From 20a8deccf1b6e0bda571507bffb542bd1e98c629 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 13 May 2014 12:01:00 +0900 Subject: [PATCH] TAJO-824 --- .../org/apache/tajo/cli/ParsedResult.java | 3 +- .../org/apache/tajo/cli/SimpleParser.java | 25 ++- .../java/org/apache/tajo/cli/TajoCli.java | 22 +-- .../org/apache/tajo/cli/TestSimpleParser.java | 151 ++++++++++++++++-- 4 files changed, 173 insertions(+), 28 deletions(-) diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/ParsedResult.java b/tajo-client/src/main/java/org/apache/tajo/cli/ParsedResult.java index 001adedcdf..232aaac981 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/ParsedResult.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/ParsedResult.java @@ -22,7 +22,8 @@ public class ParsedResult { public static enum StatementType { META, - STATEMENT + SQL_STATEMENT, + JSON_STATEMENT, } private final StatementType type; diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/SimpleParser.java b/tajo-client/src/main/java/org/apache/tajo/cli/SimpleParser.java index afb8a59489..af0ea099d4 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/SimpleParser.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/SimpleParser.java @@ -22,7 +22,8 @@ import java.util.List; import static org.apache.tajo.cli.ParsedResult.StatementType.META; -import static org.apache.tajo.cli.ParsedResult.StatementType.STATEMENT; +import static org.apache.tajo.cli.ParsedResult.StatementType.SQL_STATEMENT; +import static org.apache.tajo.cli.ParsedResult.StatementType.JSON_STATEMENT; /** * This is a parser used in tsql to parse multiple SQL lines into SQL statements. @@ -56,6 +57,8 @@ public static enum ParsingState { public static final ParsingState START_STATE = ParsingState.TOK_START; + private boolean isJsonStatement = false; + /** *

State Machine

* All whitespace are ignored in all cases except for @@ -314,13 +317,25 @@ private boolean isNewLine(char character) { } private boolean isStatementStart(char character) { - return state == ParsingState.TOK_START && (Character.isLetterOrDigit(character)); + if (state == ParsingState.TOK_START) { + isJsonStatement = isJsonStatementStart(character); + return isJsonStatement ? true : isSqlStatementStart(character); + } + return false; } private boolean isStatementContinue() { return state == ParsingState.WITHIN_QUOTE || state == ParsingState.STATEMENT; } + private boolean isSqlStatementStart(char character) { + return Character.isLetterOrDigit(character); + } + + private boolean isJsonStatementStart(char character) { + return character == '{'; + } + /** * process all parsed statements so far and return a list of parsed results. * @@ -349,7 +364,11 @@ private List doProcessEndOfStatement(boolean endOfFile) throws Inv parsedResults.add(new ParsedResult(META, rawStatement, historyStatement)); state = ParsingState.TOK_START; } else if (state == ParsingState.STATEMENT_EOS) { - parsedResults.add(new ParsedResult(STATEMENT, rawStatement, historyStatement)); + if (isJsonStatement) { + parsedResults.add(new ParsedResult(JSON_STATEMENT, rawStatement, historyStatement)); + } else { + parsedResults.add(new ParsedResult(SQL_STATEMENT, rawStatement, historyStatement)); + } } else { throw new InvalidStatementException("ERROR: " + errorMessage); } diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java index e0ca62aaee..ba3974f558 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java @@ -42,7 +42,8 @@ import java.util.TreeMap; import static org.apache.tajo.cli.ParsedResult.StatementType.META; -import static org.apache.tajo.cli.ParsedResult.StatementType.STATEMENT; +import static org.apache.tajo.cli.ParsedResult.StatementType.SQL_STATEMENT; +import static org.apache.tajo.cli.ParsedResult.StatementType.JSON_STATEMENT; import static org.apache.tajo.cli.SimpleParser.ParsingState; public class TajoCli { @@ -301,18 +302,15 @@ public int runShell() throws Exception { continue; } - if (line.startsWith("{")) { - executeJsonQuery(line); - } else { - List parsedResults = parser.parseLines(line); + List parsedResults = parser.parseLines(line); - if (parsedResults.size() > 0) { - for (ParsedResult parsed : parsedResults) { - history.addStatement(parsed.getHistoryStatement() + (parsed.getType() == STATEMENT ? ";" : "")); - } - executeParsedResults(parsedResults); - currentPrompt = updatePrompt(parser.getState()); + if (parsedResults.size() > 0) { + for (ParsedResult parsed : parsedResults) { + history.addStatement(parsed.getHistoryStatement() + + ((parsed.getType() == SQL_STATEMENT || parsed.getType() == JSON_STATEMENT) ? ";" : "")); } + executeParsedResults(parsedResults); + currentPrompt = updatePrompt(parser.getState()); } } return code; @@ -322,6 +320,8 @@ private void executeParsedResults(Collection parsedResults) throws for (ParsedResult parsedResult : parsedResults) { if (parsedResult.getType() == META) { executeMetaCommand(parsedResult.getStatement()); + } else if (parsedResult.getType() == JSON_STATEMENT) { + executeJsonQuery(parsedResult.getStatement()); } else { executeQuery(parsedResult.getStatement()); } diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/TestSimpleParser.java b/tajo-core/src/test/java/org/apache/tajo/cli/TestSimpleParser.java index 5b5057bd4b..3cda6608fc 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/TestSimpleParser.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/TestSimpleParser.java @@ -18,6 +18,7 @@ package org.apache.tajo.cli; +import org.apache.tajo.cli.ParsedResult.StatementType; import org.junit.Test; import java.util.List; @@ -76,46 +77,46 @@ public final void testMetaCommands() throws InvalidStatementException { public final void testParseScript() throws InvalidStatementException { List res1 = SimpleParser.parseScript("select * from test;"); assertEquals(1, res1.size()); - assertEquals(ParsedResult.StatementType.STATEMENT, res1.get(0).getType()); + assertEquals(ParsedResult.StatementType.SQL_STATEMENT, res1.get(0).getType()); assertEquals("select * from test", res1.get(0).getStatement()); assertEquals("select * from test", res1.get(0).getHistoryStatement()); List res2 = SimpleParser.parseScript("select * from test;"); assertEquals(1, res2.size()); - assertEquals(ParsedResult.StatementType.STATEMENT, res2.get(0).getType()); + assertEquals(ParsedResult.StatementType.SQL_STATEMENT, res2.get(0).getType()); assertEquals("select * from test", res2.get(0).getStatement()); assertEquals("select * from test", res2.get(0).getHistoryStatement()); List res3 = SimpleParser.parseScript("select * from test1;select * from test2;"); assertEquals(2, res3.size()); - assertEquals(ParsedResult.StatementType.STATEMENT, res3.get(0).getType()); + assertEquals(ParsedResult.StatementType.SQL_STATEMENT, res3.get(0).getType()); assertEquals("select * from test1", res3.get(0).getStatement()); assertEquals("select * from test1", res3.get(0).getHistoryStatement()); - assertEquals(ParsedResult.StatementType.STATEMENT, res3.get(1).getType()); + assertEquals(ParsedResult.StatementType.SQL_STATEMENT, res3.get(1).getType()); assertEquals("select * from test2", res3.get(1).getStatement()); assertEquals("select * from test2", res3.get(1).getHistoryStatement()); List res4 = SimpleParser.parseScript("\t\t\n\rselect * from \ntest1;select * from test2\n;"); assertEquals(2, res4.size()); - assertEquals(ParsedResult.StatementType.STATEMENT, res4.get(0).getType()); + assertEquals(ParsedResult.StatementType.SQL_STATEMENT, res4.get(0).getType()); assertEquals("select * from \ntest1", res4.get(0).getStatement()); assertEquals("select * from test1", res4.get(0).getHistoryStatement()); - assertEquals(ParsedResult.StatementType.STATEMENT, res4.get(1).getType()); + assertEquals(ParsedResult.StatementType.SQL_STATEMENT, res4.get(1).getType()); assertEquals("select * from test2", res4.get(1).getStatement()); assertEquals("select * from test2", res4.get(1).getHistoryStatement()); List res5 = SimpleParser.parseScript("\t\t\n\rselect * from \ntest1;\\d test;select * from test2;\n\nselect 1;"); assertEquals(4, res5.size()); - assertEquals(ParsedResult.StatementType.STATEMENT, res5.get(0).getType()); + assertEquals(ParsedResult.StatementType.SQL_STATEMENT, res5.get(0).getType()); assertEquals("select * from \ntest1", res5.get(0).getStatement()); assertEquals("select * from test1", res5.get(0).getHistoryStatement()); assertEquals(ParsedResult.StatementType.META, res5.get(1).getType()); assertEquals("\\d test", res5.get(1).getStatement()); - assertEquals(ParsedResult.StatementType.STATEMENT, res5.get(2).getType()); + assertEquals(ParsedResult.StatementType.SQL_STATEMENT, res5.get(2).getType()); assertEquals("select * from test2", res5.get(2).getStatement()); assertEquals("select * from test2", res5.get(2).getHistoryStatement()); - assertEquals(ParsedResult.StatementType.STATEMENT, res5.get(3).getType()); + assertEquals(ParsedResult.StatementType.SQL_STATEMENT, res5.get(3).getType()); assertEquals("select 1", res5.get(3).getStatement()); assertEquals("select 1", res5.get(3).getHistoryStatement()); @@ -136,12 +137,74 @@ public final void testParseScript() throws InvalidStatementException { assertEquals(ParsedResult.StatementType.META, res8.get(0).getType()); assertEquals("\\d test", res8.get(0).getStatement()); assertEquals("\\d test", res8.get(0).getHistoryStatement()); - assertEquals(ParsedResult.StatementType.STATEMENT, res8.get(1).getType()); + assertEquals(ParsedResult.StatementType.SQL_STATEMENT, res8.get(1).getType()); assertEquals("select * \n--from test1;\nfrom test2", res8.get(1).getStatement()); assertEquals("select * from test2", res8.get(1).getHistoryStatement()); assertEquals(ParsedResult.StatementType.META, res8.get(2).getType()); assertEquals("\\d test2", res8.get(2).getStatement()); assertEquals("\\d test2", res8.get(2).getHistoryStatement()); + + String jsonStmt = "{\n" + + " \"IsDistinct\": false,\n" + + " \"Projections\": [\n" + + " {\n" + + " \"Expr\": {\n" + + " \"OpType\": \"Asterisk\"\n" + + " },\n" + + " \"OpType\": \"Target\"\n" + + " }\n" + + " ],\n" + + " \"Expr\": {\n" + + " \"Relations\": [\n" + + " {\n" + + " \"TableName\": \"test\",\n" + + " \"OpType\": \"Relation\"\n" + + " }\n" + + " ],\n" + + " \"OpType\": \"RelationList\"\n" + + " },\n" + + " \"OpType\": \"Projection\"\n" + + "}"; + String historyStatement = "{ \"IsDistinct\": false, \"Projections\": [ { \"Expr\": { " + + "\"OpType\": \"Asterisk\" }, \"OpType\": \"Target\" } ], \"Expr\": { \"Relations\": [" + + " { \"TableName\": \"test\", \"OpType\": \"Relation\" } ], \"OpType\": " + + "\"RelationList\" }, \"OpType\": \"Projection\" }"; + List res9 = SimpleParser.parseScript(jsonStmt + ";"); + assertEquals(1, res9.size()); + assertEquals(StatementType.JSON_STATEMENT, res9.get(0).getType()); + assertEquals(jsonStmt, res9.get(0).getStatement()); + assertEquals(historyStatement, res9.get(0).getHistoryStatement()); + + String metaStmt = "\\d test\n"; + String jsonStmt2 = "{\n" + + " \"IsDistinct\": false,\n" + + " \"Projections\": [\n" + + " {\n" + + " \"Expr\": {\n" + + " \"OpType\": \"Asterisk\"\n" + + " },\n" + + " \"OpType\": \"Target\"\n" + + " }\n" + + " ],\n" + + " \"Expr\": {\n" + + "--this is the comment test;\n" + + " \"Relations\": [\n" + + " {\n" + + " \"TableName\": \"test\",\n" + + " \"OpType\": \"Relation\"\n" + + " }\n" + + " ],\n" + + " \"OpType\": \"RelationList\"\n" + + " },\n" + + " \"OpType\": \"Projection\"\n" + + "}"; + List res10 = SimpleParser.parseScript(metaStmt + jsonStmt2 + ";"); + assertEquals(2, res10.size()); + assertEquals(StatementType.META, res10.get(0).getType()); + assertEquals("\\d test", res10.get(0).getStatement()); + assertEquals(StatementType.JSON_STATEMENT, res10.get(1).getType()); + assertEquals(jsonStmt2, res10.get(1).getStatement()); + assertEquals(historyStatement, res10.get(1).getHistoryStatement()); } @Test @@ -192,25 +255,55 @@ public final void testParseLines() throws InvalidStatementException { assertEquals(1, res1.size()); assertEquals("select * from test1 where col1 = '123'", res1.get(0).getHistoryStatement()); assertEquals("select * from \ntest1 --select * from test2;\nwhere col1 = '123'", res1.get(0).getStatement()); + + String jsonStmt = "{\n" + + " \"IsDistinct\": false,\n" + + " \"Projections\": [\n" + + " {\n" + + " \"Expr\": {\n" + + " \"OpType\": \"Asterisk\"\n" + + " },\n" + + " \"OpType\": \"Target\"\n" + + " }\n" + + " ],\n" + + " \"Expr\": {\n" + + " \"Relations\": [\n" + + " {\n" + + " \"TableName\": \"test\",\n" + + " \"OpType\": \"Relation\"\n" + + " }\n" + + " ],\n" + + " \"OpType\": \"RelationList\"\n" + + " },\n" + + " \"OpType\": \"Projection\"\n" + + "}"; + String historyStatement = "{ \"IsDistinct\": false, \"Projections\": [ { \"Expr\": { " + + "\"OpType\": \"Asterisk\" }, \"OpType\": \"Target\" } ], \"Expr\": { \"Relations\": [" + + " { \"TableName\": \"test\", \"OpType\": \"Relation\" } ], \"OpType\": " + + "\"RelationList\" }, \"OpType\": \"Projection\" }"; + res1 = simpleParser.parseLines(jsonStmt + ";"); + assertEquals(1, res1.size()); + assertEquals(jsonStmt, res1.get(0).getStatement()); + assertEquals(historyStatement, res1.get(0).getHistoryStatement()); } @Test public final void testQuoted() throws InvalidStatementException { List res1 = SimpleParser.parseScript("select '\n;' from test;"); assertEquals(1, res1.size()); - assertEquals(ParsedResult.StatementType.STATEMENT, res1.get(0).getType()); + assertEquals(ParsedResult.StatementType.SQL_STATEMENT, res1.get(0).getType()); assertEquals("select '\n;' from test", res1.get(0).getHistoryStatement()); assertEquals("select '\n;' from test", res1.get(0).getStatement()); List res2 = SimpleParser.parseScript("select 'abc\nbbc\nddf' from test;"); assertEquals(1, res2.size()); - assertEquals(ParsedResult.StatementType.STATEMENT, res2.get(0).getType()); + assertEquals(ParsedResult.StatementType.SQL_STATEMENT, res2.get(0).getType()); assertEquals("select 'abc\nbbc\nddf' from test", res2.get(0).getHistoryStatement()); assertEquals("select 'abc\nbbc\nddf' from test", res2.get(0).getStatement()); List res3 = SimpleParser.parseScript("select '--test', \n'--test2' from test"); assertEquals(1, res3.size()); - assertEquals(ParsedResult.StatementType.STATEMENT, res3.get(0).getType()); + assertEquals(ParsedResult.StatementType.SQL_STATEMENT, res3.get(0).getType()); assertEquals("select '--test', '--test2' from test", res3.get(0).getHistoryStatement()); assertEquals("select '--test', \n'--test2' from test", res3.get(0).getStatement()); @@ -220,6 +313,38 @@ public final void testQuoted() throws InvalidStatementException { } catch (InvalidStatementException is) { assertTrue(true); } + + String jsonStmt = "{\n" + + " \"IsDistinct\": false,\n" + + " \"Projections\": [\n" + + " {\n" + + " \"Expr\": {\n" + + " \"Value\": '\\n;',\n" + + " \"ValueType\": \"String\",\n" + + " \"OpType\": \"Literal\"\n" + + " },\n" + + " \"OpType\": \"Target\"\n" + + " }\n" + + " ],\n" + + " \"Expr\": {\n" + + " \"Relations\": [\n" + + " {\n" + + " \"TableName\": \"test\",\n" + + " \"OpType\": \"Relation\"\n" + + " }\n" + + " ],\n" + + " \"OpType\": \"RelationList\"\n" + + " },\n" + + " \"OpType\": \"Projection\"\n" + + "}"; + String historyStmt = "{ \"IsDistinct\": false, \"Projections\": [ { \"Expr\": { \"Value\": " + + "'\\n;', \"ValueType\": \"String\", \"OpType\": \"Literal\" }, \"OpType\": " + + "\"Target\" } ], \"Expr\": { \"Relations\": [ { \"TableName\": \"test\", " + + "\"OpType\": \"Relation\" } ], \"OpType\": \"RelationList\" }, \"OpType\": \"Projection\" }"; + List res4 = SimpleParser.parseScript(jsonStmt); + assertEquals(1, res4.size()); + assertEquals(ParsedResult.StatementType.JSON_STATEMENT, res4.get(0).getType()); + assertEquals(historyStmt, res4.get(0).getHistoryStatement()); } @Test