From 021c872fbfedf90e5eb579dfe109590242fa1bae Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 9 Dec 2014 10:00:24 +0900 Subject: [PATCH 1/8] Add session statement to parser. --- .../java/org/apache/tajo/algebra/OpType.java | 3 + .../org/apache/tajo/algebra/SetSession.java | 68 +++++++++++++++++++ .../org/apache/tajo/catalog/CatalogUtil.java | 2 +- .../org/apache/tajo/engine/parser/SQLLexer.g4 | 4 +- .../apache/tajo/engine/parser/SQLParser.g4 | 12 +++- .../tajo/engine/parser/SQLAnalyzer.java | 39 +++++++++++ .../tajo/engine/parser/TestSQLAnalyzer.java | 60 ++++++++++++++++ .../queries/TestSQLAnalyzer/setcatalog1.sql | 1 + .../queries/TestSQLAnalyzer/setcatalog2.sql | 1 + .../queries/TestSQLAnalyzer/setsession1.sql | 1 + .../queries/TestSQLAnalyzer/setsession2.sql | 1 + .../queries/TestSQLAnalyzer/setsession3.sql | 1 + .../queries/TestSQLAnalyzer/setsession4.sql | 1 + .../queries/TestSQLAnalyzer/setsession5.sql | 1 + .../queries/TestSQLAnalyzer/setsession6.sql | 1 + .../queries/TestSQLAnalyzer/setsession7.sql | 1 + .../queries/TestSQLAnalyzer/settimezone1.sql | 1 + .../queries/TestSQLAnalyzer/settimezone2.sql | 1 + .../queries/TestSQLAnalyzer/settimezone3.sql | 1 + .../TestSQLAnalyzer/setcatalog1.result | 5 ++ .../TestSQLAnalyzer/setcatalog2.result | 5 ++ .../TestSQLAnalyzer/setsession1.result | 5 ++ .../TestSQLAnalyzer/setsession2.result | 5 ++ .../TestSQLAnalyzer/setsession3.result | 5 ++ .../TestSQLAnalyzer/setsession4.result | 5 ++ .../TestSQLAnalyzer/setsession5.result | 5 ++ .../TestSQLAnalyzer/setsession6.result | 5 ++ .../TestSQLAnalyzer/setsession7.result | 5 ++ .../TestSQLAnalyzer/settimezone1.result | 5 ++ .../TestSQLAnalyzer/settimezone2.result | 5 ++ .../TestSQLAnalyzer/settimezone3.result | 4 ++ 31 files changed, 256 insertions(+), 3 deletions(-) create mode 100644 tajo-algebra/src/main/java/org/apache/tajo/algebra/SetSession.java create mode 100644 tajo-core/src/test/resources/queries/TestSQLAnalyzer/setcatalog1.sql create mode 100644 tajo-core/src/test/resources/queries/TestSQLAnalyzer/setcatalog2.sql create mode 100644 tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession1.sql create mode 100644 tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession2.sql create mode 100644 tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession3.sql create mode 100644 tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession4.sql create mode 100644 tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession5.sql create mode 100644 tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession6.sql create mode 100644 tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession7.sql create mode 100644 tajo-core/src/test/resources/queries/TestSQLAnalyzer/settimezone1.sql create mode 100644 tajo-core/src/test/resources/queries/TestSQLAnalyzer/settimezone2.sql create mode 100644 tajo-core/src/test/resources/queries/TestSQLAnalyzer/settimezone3.sql create mode 100644 tajo-core/src/test/resources/results/TestSQLAnalyzer/setcatalog1.result create mode 100644 tajo-core/src/test/resources/results/TestSQLAnalyzer/setcatalog2.result create mode 100644 tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession1.result create mode 100644 tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession2.result create mode 100644 tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession3.result create mode 100644 tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession4.result create mode 100644 tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession5.result create mode 100644 tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession6.result create mode 100644 tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession7.result create mode 100644 tajo-core/src/test/resources/results/TestSQLAnalyzer/settimezone1.result create mode 100644 tajo-core/src/test/resources/results/TestSQLAnalyzer/settimezone2.result create mode 100644 tajo-core/src/test/resources/results/TestSQLAnalyzer/settimezone3.result diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java index 19c4ab5b2b..3e7d2779dc 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java @@ -24,6 +24,9 @@ public enum OpType { + // session statements + SetSession(SetSession.class), + // relational operators Projection(Projection.class), Limit(Limit.class), diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/SetSession.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/SetSession.java new file mode 100644 index 0000000000..c2ea92c226 --- /dev/null +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/SetSession.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.algebra; + + +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.gson.annotations.Expose; +import com.google.gson.annotations.SerializedName; + +public class SetSession extends Expr { + @Expose @SerializedName("name") + private String name; + @Expose @SerializedName("value") + private String value; + + public SetSession(String name, String value) { + super(OpType.SetSession); + + this.name = name; + this.value = value; + } + + public boolean isDefault() { + return value == null; + } + + public String getName() { + return name; + } + + public String getValue() { + return value; + } + + public int hashCode() { + return Objects.hashCode(name, value); + } + + boolean equalsTo(Expr expr) { + SetSession another = (SetSession) expr; + return name.equals(another.name) && value.equals(another.value); + } + + @Override + public Object clone() throws CloneNotSupportedException { + SetSession setOperation = (SetSession) super.clone(); + setOperation.name = name; + setOperation.value = value; + return setOperation; + } +} diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index f2d9b9cad6..39ac82eeee 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -699,7 +699,7 @@ public static void closeQuietly(Statement stmt, ResultSet res) { "AS", "ALL", "AND", "ANY", "ASYMMETRIC", "ASC", "BOTH", "CASE", "CAST", "CREATE", "CROSS", "CURRENT_DATE", "CURRENT_TIME", "CURRENT_TIMESTAMP", - "DESC", "DISTINCT", + "DEFAULT", "DESC", "DISTINCT", "END", "ELSE", "EXCEPT", "FALSE", "FULL", "FROM", "GROUP", diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 index 4bdbc3d876..62dffd15ee 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 @@ -132,7 +132,7 @@ CURRENT_DATE: C U R R E N T UNDERLINE D A T E; CURRENT_TIME: C U R R E N T UNDERLINE T I M E; CURRENT_TIMESTAMP: C U R R E N T UNDERLINE T I M E S T A M P; - +DEFAULT : D E F A U L T; DESC : D E S C; DISTINCT : D I S T I N C T; @@ -202,6 +202,7 @@ ALTER : A L T E R; BETWEEN : B E T W E E N; BY : B Y; +CATALOG : C A T A L O G; CENTURY : C E N T U R Y; CHARACTER : C H A R A C T E R; COLLECT : C O L L E C T; @@ -288,6 +289,7 @@ ROW_NUMBER : R O W UNDERLINE N U M B E R; RENAME : R E N A M E; SECOND : S E C O N D; +SESSION : S E S S I O N; SET : S E T; SIMILAR : S I M I L A R; STDDEV_POP : S T D D E V UNDERLINE P O P; diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 index beba248be9..f36ffafa20 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 @@ -43,12 +43,20 @@ explain_clause ; statement - : data_statement + : session_statement + | data_statement | data_change_statement | schema_statement | index_statement ; +session_statement + : SET CATALOG dbname = identifier + | SET TIME ZONE (Character_String_Literal | signed_numerical_literal | DEFAULT) + | SET (SESSION)? name=identifier (TO | EQUAL)? + (Character_String_Literal | signed_numerical_literal | boolean_literal | DEFAULT) + ; + data_statement : query_expression ; @@ -221,6 +229,7 @@ nonreserved_keywords | ALTER | BETWEEN | BY + | CATALOG | CENTURY | CHARACTER | COALESCE @@ -291,6 +300,7 @@ nonreserved_keywords | ROW_NUMBER | SECOND | SET + | SESSION | SIMILAR | STDDEV_POP | STDDEV_SAMP diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index 03b10c9909..ca04301562 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@ -24,6 +24,7 @@ import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.misc.NotNull; import org.antlr.v4.runtime.tree.TerminalNode; +import org.apache.tajo.SessionVars; import org.apache.tajo.algebra.*; import org.apache.tajo.algebra.Aggregation.GroupType; import org.apache.tajo.algebra.LiteralValue.LiteralType; @@ -81,6 +82,44 @@ public Expr visitSql(SqlContext ctx) { } } + public Expr visitSession_statement(@NotNull SQLParser.Session_statementContext ctx) { + + if (checkIfExist(ctx.CATALOG())) { + + return new SetSession(SessionVars.CURRENT_DATABASE.name(), ctx.dbname.getText()); + + + } else if (checkIfExist(ctx.name)) { + String value; + if (checkIfExist(ctx.boolean_literal())) { + value = ctx.boolean_literal().getText(); + } else if (checkIfExist(ctx.Character_String_Literal())) { + value = stripQuote(ctx.Character_String_Literal().getText()); + } else if (checkIfExist(ctx.signed_numerical_literal())) { + value = ctx.signed_numerical_literal().getText(); + } else { + value = null; + } + return new SetSession(ctx.name.getText(), value); + + + } else if (checkIfExist(ctx.TIME()) && checkIfExist(ctx.ZONE())) { + + String value; + if (checkIfExist(ctx.Character_String_Literal())) { + value = stripQuote(ctx.Character_String_Literal().getText()); + } else if (checkIfExist(ctx.signed_numerical_literal())) { + value = ctx.signed_numerical_literal().getText(); + } else { + value = null; + } + return new SetSession(SessionVars.TZ.name(), value); + + } else { + throw new SQLSyntaxError("Unsupported session statement"); + } + } + @Override public Expr visitNon_join_query_expression(SQLParser.Non_join_query_expressionContext ctx) { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java index 23314dd295..272f718f11 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java @@ -532,4 +532,64 @@ public void windowFunction8() throws IOException { public void windowFunction9() throws IOException { assertParseResult("window9.sql", "window9.result"); } + + @Test + public void testSetCatalog1() throws IOException { + assertParseResult("setcatalog1.sql", "setcatalog1.result"); + } + + @Test + public void testSetCatalog2() throws IOException { + assertParseResult("setcatalog2.sql", "setcatalog2.result"); + } + + @Test + public void testTimezone1() throws IOException { + assertParseResult("settimezone1.sql", "settimezone1.result"); + } + + @Test + public void testTimezone2() throws IOException { + assertParseResult("settimezone2.sql", "settimezone2.result"); + } + + @Test + public void testTimezone3() throws IOException { + assertParseResult("settimezone3.sql", "settimezone3.result"); + } + + @Test + public void testSetSession1() throws IOException { + assertParseResult("setsession1.sql", "setsession1.result"); + } + + @Test + public void testSetSession2() throws IOException { + assertParseResult("setsession2.sql", "setsession2.result"); + } + + @Test + public void testSetSession3() throws IOException { + assertParseResult("setsession3.sql", "setsession3.result"); + } + + @Test + public void testSetSession4() throws IOException { + assertParseResult("setsession4.sql", "setsession4.result"); + } + + @Test + public void testSetSession5() throws IOException { + assertParseResult("setsession5.sql", "setsession5.result"); + } + + @Test + public void testSetSession6() throws IOException { + assertParseResult("setsession6.sql", "setsession6.result"); + } + + @Test + public void testSetSession7() throws IOException { + assertParseResult("setsession7.sql", "setsession7.result"); + } } diff --git a/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setcatalog1.sql b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setcatalog1.sql new file mode 100644 index 0000000000..345531c14d --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setcatalog1.sql @@ -0,0 +1 @@ +SET CATALOG tajo; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setcatalog2.sql b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setcatalog2.sql new file mode 100644 index 0000000000..d2ef28302b --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setcatalog2.sql @@ -0,0 +1 @@ +SET CATALOG "Mixed Letter"; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession1.sql b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession1.sql new file mode 100644 index 0000000000..8efe69628d --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession1.sql @@ -0,0 +1 @@ +SET SESSION ENABLE_SEQSCAN TO true; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession2.sql b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession2.sql new file mode 100644 index 0000000000..458f1e5322 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession2.sql @@ -0,0 +1 @@ +SET SESSION ENABLE_SEQSCAN TO false; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession3.sql b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession3.sql new file mode 100644 index 0000000000..432d1234e1 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession3.sql @@ -0,0 +1 @@ +SET SESSION EXTSORT_BUFFER_SIZE TO 100; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession4.sql b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession4.sql new file mode 100644 index 0000000000..a363702874 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession4.sql @@ -0,0 +1 @@ +SET SESSION EXTSORT_BUFFER_SIZE TO 50.7; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession5.sql b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession5.sql new file mode 100644 index 0000000000..c9fcfc5049 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession5.sql @@ -0,0 +1 @@ +SET SESSION EXTSORT_BUFFER_SIZE TO 'ABCD'; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession6.sql b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession6.sql new file mode 100644 index 0000000000..957232bdca --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession6.sql @@ -0,0 +1 @@ +SET SESSION EXTSORT_BUFFER_SIZE 'ABCD'; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession7.sql b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession7.sql new file mode 100644 index 0000000000..179428e417 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/setsession7.sql @@ -0,0 +1 @@ +SET SESSION EXTSORT_BUFFER_SIZE = 'ABCD'; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestSQLAnalyzer/settimezone1.sql b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/settimezone1.sql new file mode 100644 index 0000000000..a86d2550c0 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/settimezone1.sql @@ -0,0 +1 @@ +SET TIME ZONE 'PDT'; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestSQLAnalyzer/settimezone2.sql b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/settimezone2.sql new file mode 100644 index 0000000000..85dae0fc27 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/settimezone2.sql @@ -0,0 +1 @@ +SET TIME ZONE -7; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestSQLAnalyzer/settimezone3.sql b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/settimezone3.sql new file mode 100644 index 0000000000..ef56b12157 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/settimezone3.sql @@ -0,0 +1 @@ +SET TIME ZONE DEFAULT; \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestSQLAnalyzer/setcatalog1.result b/tajo-core/src/test/resources/results/TestSQLAnalyzer/setcatalog1.result new file mode 100644 index 0000000000..cb24cdb5a1 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSQLAnalyzer/setcatalog1.result @@ -0,0 +1,5 @@ +{ + "name": "CURRENT_DATABASE", + "value": "tajo", + "OpType": "SetSession" +} \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestSQLAnalyzer/setcatalog2.result b/tajo-core/src/test/resources/results/TestSQLAnalyzer/setcatalog2.result new file mode 100644 index 0000000000..a4f0df2208 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSQLAnalyzer/setcatalog2.result @@ -0,0 +1,5 @@ +{ + "name": "CURRENT_DATABASE", + "value": "Mixed Letter", + "OpType": "SetSession" +} \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession1.result b/tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession1.result new file mode 100644 index 0000000000..ffd27dd20b --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession1.result @@ -0,0 +1,5 @@ +{ + "name": "enable_seqscan", + "value": "true", + "OpType": "SetSession" +} \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession2.result b/tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession2.result new file mode 100644 index 0000000000..7809a2e2bb --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession2.result @@ -0,0 +1,5 @@ +{ + "name": "enable_seqscan", + "value": "false", + "OpType": "SetSession" +} \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession3.result b/tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession3.result new file mode 100644 index 0000000000..9a3675581b --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession3.result @@ -0,0 +1,5 @@ +{ + "name": "extsort_buffer_size", + "value": "100", + "OpType": "SetSession" +} \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession4.result b/tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession4.result new file mode 100644 index 0000000000..007a56327d --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession4.result @@ -0,0 +1,5 @@ +{ + "name": "extsort_buffer_size", + "value": "50.7", + "OpType": "SetSession" +} \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession5.result b/tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession5.result new file mode 100644 index 0000000000..8e61229c31 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession5.result @@ -0,0 +1,5 @@ +{ + "name": "extsort_buffer_size", + "value": "ABCD", + "OpType": "SetSession" +} \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession6.result b/tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession6.result new file mode 100644 index 0000000000..8e61229c31 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession6.result @@ -0,0 +1,5 @@ +{ + "name": "extsort_buffer_size", + "value": "ABCD", + "OpType": "SetSession" +} \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession7.result b/tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession7.result new file mode 100644 index 0000000000..8e61229c31 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSQLAnalyzer/setsession7.result @@ -0,0 +1,5 @@ +{ + "name": "extsort_buffer_size", + "value": "ABCD", + "OpType": "SetSession" +} \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestSQLAnalyzer/settimezone1.result b/tajo-core/src/test/resources/results/TestSQLAnalyzer/settimezone1.result new file mode 100644 index 0000000000..366e447d12 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSQLAnalyzer/settimezone1.result @@ -0,0 +1,5 @@ +{ + "name": "TZ", + "value": "PDT", + "OpType": "SetSession" +} \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestSQLAnalyzer/settimezone2.result b/tajo-core/src/test/resources/results/TestSQLAnalyzer/settimezone2.result new file mode 100644 index 0000000000..7653373e5a --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSQLAnalyzer/settimezone2.result @@ -0,0 +1,5 @@ +{ + "name": "TZ", + "value": "-7", + "OpType": "SetSession" +} \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestSQLAnalyzer/settimezone3.result b/tajo-core/src/test/resources/results/TestSQLAnalyzer/settimezone3.result new file mode 100644 index 0000000000..8bb188ec15 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSQLAnalyzer/settimezone3.result @@ -0,0 +1,4 @@ +{ + "name": "TZ", + "OpType": "SetSession" +} \ No newline at end of file From 56b3016a579896e542a44289e59cc7cb769f8880 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 9 Dec 2014 18:09:27 +0900 Subject: [PATCH 2/8] TAJO-1238: Add SET SESSION and RESET statement. --- .../org/apache/tajo/catalog/CatalogUtil.java | 6 ++ .../tajo/cli/tsql/commands/SetCommand.java | 7 +- .../org/apache/tajo/client/QueryClient.java | 4 +- .../apache/tajo/client/QueryClientImpl.java | 10 ++- .../apache/tajo/client/SessionConnection.java | 75 ++++++++++++----- tajo-client/src/main/proto/ClientProtos.proto | 15 ++-- .../main/proto/TajoMasterClientProtocol.proto | 2 +- .../org/apache/tajo/util/KeyValueSet.java | 5 ++ .../java/org/apache/tajo/util/ProtoUtil.java | 13 +++ .../org/apache/tajo/engine/parser/SQLLexer.g4 | 3 +- .../apache/tajo/engine/parser/SQLParser.g4 | 4 +- .../org/apache/tajo/master/GlobalEngine.java | 37 ++++++++- .../tajo/master/TajoMasterClientService.java | 31 +++++-- .../engine/query/TestSetSessionQuery.java | 81 +++++++++++++++++++ .../tajo/plan/LogicalPlanPreprocessor.java | 7 ++ .../org/apache/tajo/plan/LogicalPlanner.java | 11 +++ .../tajo/plan/algebra/AlgebraVisitor.java | 3 + .../tajo/plan/algebra/BaseAlgebraVisitor.java | 8 ++ .../apache/tajo/plan/logical/NodeType.java | 2 + .../tajo/plan/logical/SetSessionNode.java | 68 ++++++++++++++++ .../apache/tajo/plan/util/PlannerUtil.java | 10 +++ .../plan/verifier/PreLogicalPlanVerifier.java | 20 +++++ .../plan/visitor/BasicLogicalPlanVisitor.java | 9 +++ .../tajo/plan/visitor/LogicalPlanVisitor.java | 4 + 24 files changed, 389 insertions(+), 46 deletions(-) create mode 100644 tajo-core/src/test/java/org/apache/tajo/engine/query/TestSetSessionQuery.java create mode 100644 tajo-plan/src/main/java/org/apache/tajo/plan/logical/SetSessionNode.java diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 39ac82eeee..274033d034 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -18,6 +18,7 @@ package org.apache.tajo.catalog; +import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import org.apache.hadoop.fs.Path; import org.apache.tajo.DataTypeUtil; @@ -51,6 +52,11 @@ public class CatalogUtil { + public static boolean isValidIdentifier(String identifier) { + Preconditions.checkNotNull(identifier); + return identifier.length() > 0 && Character.isAlphabetic(identifier.charAt(0)); + } + /** * Normalize an identifier. Normalization means a translation from a identifier to be a refined identifier name. * diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/tsql/commands/SetCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/tsql/commands/SetCommand.java index 21c4be51b5..e7d2fe530f 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/tsql/commands/SetCommand.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/tsql/commands/SetCommand.java @@ -55,14 +55,9 @@ public void set(String key, String val) throws ServiceException { SessionVars sessionVar = null; if (SessionVars.exists(key)) { // if the variable is one of the session variables - sessionVar = SessionVars.get(key); // is it cli-side variable? - if (sessionVar.getMode() == VariableMode.CLI_SIDE_VAR) { - context.setCliSideVar(key, val); - } else { - updateSessionVariable(key, val); - } + updateSessionVariable(key, val); if (SessionVars.isDeprecated(key)) { context.getOutput().println("Warning: deprecated to directly use config key in TajoConf.ConfVars. " + diff --git a/tajo-client/src/main/java/org/apache/tajo/client/QueryClient.java b/tajo-client/src/main/java/org/apache/tajo/client/QueryClient.java index 32ef97d9d9..73a4d359e3 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/QueryClient.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/QueryClient.java @@ -63,9 +63,9 @@ public interface QueryClient extends Closeable { public Boolean selectDatabase(final String databaseName) throws ServiceException; - public Boolean updateSessionVariables(final Map variables) throws ServiceException; + public Map updateSessionVariables(final Map variables) throws ServiceException; - public Boolean unsetSessionVariables(final List variables) throws ServiceException; + public Map unsetSessionVariables(final List variables) throws ServiceException; public String getSessionVariable(final String varname) throws ServiceException; diff --git a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java index 1cee51572b..6809fda2dc 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java @@ -140,12 +140,12 @@ public Boolean selectDatabase(String databaseName) throws ServiceException { } @Override - public Boolean updateSessionVariables(Map variables) throws ServiceException { + public Map updateSessionVariables(Map variables) throws ServiceException { return connection.updateSessionVariables(variables); } @Override - public Boolean unsetSessionVariables(List variables) throws ServiceException { + public Map unsetSessionVariables(List variables) throws ServiceException { return connection.unsetSessionVariables(variables); } @@ -214,7 +214,11 @@ public ResultSet executeQueryAndGetResult(String sql) throws ServiceException, I ClientProtos.SubmitQueryResponse response = executeQuery(sql); if (response.getResultCode() == ClientProtos.ResultCode.ERROR) { - throw new ServiceException(response.getErrorTrace()); + if (response.hasErrorMessage()) { + throw new ServiceException(response.getErrorMessage()); + } else if (response.hasErrorTrace()) { + throw new ServiceException(response.getErrorTrace()); + } } QueryId queryId = new QueryId(response.getQueryId()); diff --git a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java index 922984f4e5..dcec1a3e98 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java @@ -28,6 +28,7 @@ import org.apache.tajo.annotation.Nullable; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.ipc.ClientProtos; +import org.apache.tajo.ipc.ClientProtos.ResultCode; import org.apache.tajo.ipc.TajoMasterClientProtocol; import org.apache.tajo.rpc.NettyClientBase; import org.apache.tajo.rpc.RpcConnectionPool; @@ -35,12 +36,14 @@ import org.apache.tajo.util.HAServiceUtil; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.NetUtils; +import org.apache.tajo.util.ProtoUtil; import org.jboss.netty.channel.ConnectTimeoutException; import java.io.Closeable; import java.io.IOException; import java.net.InetSocketAddress; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -71,7 +74,7 @@ public class SessionConnection implements Closeable { private AtomicBoolean closed = new AtomicBoolean(false); /** session variable cache */ - Map clientSideSessionVars = new ConcurrentHashMap(); + private final Map sessionVarsCache = new HashMap(); public SessionConnection(TajoConf conf) throws IOException { @@ -111,7 +114,7 @@ public SessionConnection(TajoConf conf, InetSocketAddress addr, @Nullable String } public Map getClientSideSessionVars() { - return Collections.unmodifiableMap(clientSideSessionVars); + return Collections.unmodifiableMap(sessionVarsCache); } public T getStub(QueryId queryId, Class protocolClass, boolean asyncMode) throws NoSuchMethodException, @@ -180,10 +183,11 @@ public String call(NettyClientBase client) throws ServiceException { }.withRetries(); } - public Boolean updateSessionVariables(final Map variables) throws ServiceException { - return new ServerCallable(connPool, getTajoMasterAddr(), TajoMasterClientProtocol.class, false, true) { + public Map updateSessionVariables(final Map variables) throws ServiceException { + return new ServerCallable>(connPool, getTajoMasterAddr(), + TajoMasterClientProtocol.class, false, true) { - public Boolean call(NettyClientBase client) throws ServiceException { + public Map call(NettyClientBase client) throws ServiceException { checkSessionAndGet(client); // keep client-side session variables @@ -192,7 +196,7 @@ public Boolean call(NettyClientBase client) throws ServiceException { if (SessionVars.exists(entry.getKey())) { SessionVars configKey = SessionVars.get(key); if (configKey.getMode() == SessionVars.VariableMode.CLI_SIDE_VAR) { - clientSideSessionVars.put(key, entry.getValue()); + sessionVarsCache.put(key, entry.getValue()); } } } @@ -204,21 +208,32 @@ public Boolean call(NettyClientBase client) throws ServiceException { .setSessionId(sessionId) .setSetVariables(keyValueSet.getProto()).build(); - return tajoMasterService.updateSessionVariables(null, request).getValue(); + ClientProtos.SessionUpdateResponse response = tajoMasterService.updateSessionVariables(null, request); + if (response.getResultCode() == ResultCode.OK) { + Map updated = ProtoUtil.convertToMap(response.getVariables()); + + synchronized (sessionVarsCache) { + sessionVarsCache.clear(); + sessionVarsCache.putAll(updated); + } + return Collections.unmodifiableMap(sessionVarsCache); + } else { + throw new ServiceException(response.getMessage()); + } } }.withRetries(); } - public Boolean unsetSessionVariables(final List variables) throws ServiceException { - return new ServerCallable(connPool, getTajoMasterAddr(), TajoMasterClientProtocol.class, false, true) { + public Map unsetSessionVariables(final List variables) throws ServiceException { + return new ServerCallable>(connPool, getTajoMasterAddr(), TajoMasterClientProtocol.class, false, true) { - public Boolean call(NettyClientBase client) throws ServiceException { + public Map call(NettyClientBase client) throws ServiceException { checkSessionAndGet(client); // Remove matched session vars for (String key : variables) { - if (clientSideSessionVars.containsKey(key)) { - clientSideSessionVars.remove(key); + if (sessionVarsCache.containsKey(key)) { + sessionVarsCache.remove(key); } } @@ -226,19 +241,39 @@ public Boolean call(NettyClientBase client) throws ServiceException { ClientProtos.UpdateSessionVariableRequest request = ClientProtos.UpdateSessionVariableRequest.newBuilder() .setSessionId(sessionId) .addAllUnsetVariables(variables).build(); - return tajoMasterService.updateSessionVariables(null, request).getValue(); + + ClientProtos.SessionUpdateResponse response = tajoMasterService.updateSessionVariables(null, request); + if (response.getResultCode() == ResultCode.OK) { + Map updated = ProtoUtil.convertToMap(response.getVariables()); + + synchronized (sessionVarsCache) { + sessionVarsCache.clear(); + sessionVarsCache.putAll(updated); + } + return Collections.unmodifiableMap(sessionVarsCache); + } else { + throw new ServiceException(response.getMessage()); + } } }.withRetries(); } + public String getCachedSessionVariable(final String varname) { + if (sessionVarsCache.containsKey(varname)) { + return sessionVarsCache.get(varname); + } else { + throw new RuntimeException("No such session variable" + varname); + } + } + public String getSessionVariable(final String varname) throws ServiceException { return new ServerCallable(connPool, getTajoMasterAddr(), TajoMasterClientProtocol.class, false, true) { public String call(NettyClientBase client) throws ServiceException { // If a desired variable is client side one and exists in the cache, immediately return the variable. - if (clientSideSessionVars.containsKey(varname)) { - return clientSideSessionVars.get(varname); + if (sessionVarsCache.containsKey(varname)) { + return sessionVarsCache.get(varname); } checkSessionAndGet(client); @@ -261,6 +296,10 @@ public Boolean call(NettyClientBase client) throws ServiceException { }.withRetries(); } + public Map getCachedAllSessionVariables() { + return Collections.unmodifiableMap(sessionVarsCache); + } + public Map getAllSessionVariables() throws ServiceException { return new ServerCallable>(connPool, getTajoMasterAddr(), TajoMasterClientProtocol.class, false, true) { @@ -269,8 +308,7 @@ public Map call(NettyClientBase client) throws ServiceException checkSessionAndGet(client); TajoMasterClientProtocolService.BlockingInterface tajoMasterService = client.getStub(); - KeyValueSet keyValueSet = new KeyValueSet(tajoMasterService.getAllSessionVariables(null, sessionId)); - return keyValueSet.getAllKeyValus(); + return ProtoUtil.convertToMap(tajoMasterService.getAllSessionVariables(null, sessionId)); } }.withRetries(); } @@ -336,9 +374,10 @@ protected void checkSessionAndGet(NettyClientBase client) throws ServiceExceptio CreateSessionResponse response = tajoMasterService.createSession(null, builder.build()); - if (response.getState() == CreateSessionResponse.ResultState.SUCCESS) { + if (response.getResultCode() == ResultCode.OK) { sessionId = response.getSessionId(); + sessionVarsCache.putAll(ProtoUtil.convertToMap(response.getVariables())); if (LOG.isDebugEnabled()) { LOG.debug(String.format("Got session %s as a user '%s'.", sessionId.getId(), userInfo.getUserName())); } diff --git a/tajo-client/src/main/proto/ClientProtos.proto b/tajo-client/src/main/proto/ClientProtos.proto index 4118458843..fa12ab20fc 100644 --- a/tajo-client/src/main/proto/ClientProtos.proto +++ b/tajo-client/src/main/proto/ClientProtos.proto @@ -37,13 +37,10 @@ message CreateSessionRequest { } message CreateSessionResponse { - enum ResultState { - SUCCESS = 0; - FAILED = 1; - } - required ResultState state = 1; + required ResultCode resultCode = 1; optional SessionIdProto sessionId = 2; - optional string message = 3; + optional KeyValueSetProto variables = 3; + optional string message = 4; } message UpdateSessionVariableRequest { @@ -52,6 +49,12 @@ message UpdateSessionVariableRequest { repeated string unsetVariables = 3; } +message SessionUpdateResponse { + required ResultCode resultCode = 1; + optional KeyValueSetProto variables = 2; + optional string message = 3; +} + message SessionedStringProto { optional SessionIdProto sessionId = 1; required string value = 2; diff --git a/tajo-client/src/main/proto/TajoMasterClientProtocol.proto b/tajo-client/src/main/proto/TajoMasterClientProtocol.proto index bc59617b7d..10ca268540 100644 --- a/tajo-client/src/main/proto/TajoMasterClientProtocol.proto +++ b/tajo-client/src/main/proto/TajoMasterClientProtocol.proto @@ -33,7 +33,7 @@ service TajoMasterClientProtocolService { // Session APIs rpc createSession(CreateSessionRequest) returns (CreateSessionResponse); rpc removeSession(SessionIdProto) returns (BoolProto); - rpc updateSessionVariables(UpdateSessionVariableRequest) returns (BoolProto); + rpc updateSessionVariables(UpdateSessionVariableRequest) returns (SessionUpdateResponse); rpc existSessionVariable(SessionedStringProto) returns (BoolProto); rpc getSessionVariable(SessionedStringProto) returns (StringProto); rpc getAllSessionVariables(SessionIdProto) returns (KeyValueSetProto); diff --git a/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java b/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java index 8e3eb2add2..18d4f599c4 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java @@ -41,6 +41,11 @@ public class KeyValueSet implements ProtoObject, Cloneable, Gs public KeyValueSet() { keyVals = TUtil.newHashMap(); } + + public KeyValueSet(Map keyVals) { + this(); + putAll(keyVals); + } public KeyValueSet(KeyValueSetProto proto) { this.keyVals = TUtil.newHashMap(); diff --git a/tajo-common/src/main/java/org/apache/tajo/util/ProtoUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/ProtoUtil.java index 90ffb6422e..dbc987dd98 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/ProtoUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/ProtoUtil.java @@ -19,6 +19,7 @@ package org.apache.tajo.util; import java.util.Collection; +import java.util.Map; import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.*; @@ -39,4 +40,16 @@ public static StringListProto convertStrings(Collection strings) { public static Collection convertStrings(StringListProto strings) { return strings.getValuesList(); } + + public static Map convertToMap(KeyValueSetProto proto) { + Map keyVals = TUtil.newHashMap(); + for(KeyValueProto keyval : proto.getKeyvalList()) { + keyVals.put(keyval.getKey(), keyval.getValue()); + } + return keyVals; + } + + public static KeyValueSetProto convertFromMap(Map map) { + return new KeyValueSet(map).getProto(); + } } diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 index 62dffd15ee..7e18ee0c23 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 @@ -281,12 +281,13 @@ QUARTER : Q U A R T E R; RANGE : R A N G E; RANK : R A N K; REGEXP : R E G E X P; +RENAME : R E N A M E; +RESET : R E S E T; RLIKE : R L I K E; ROLLUP : R O L L U P; ROW : R O W; ROWS : R O W S; ROW_NUMBER : R O W UNDERLINE N U M B E R; -RENAME : R E N A M E; SECOND : S E C O N D; SESSION : S E S S I O N; diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 index f36ffafa20..6eccca0e6f 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 @@ -52,9 +52,10 @@ statement session_statement : SET CATALOG dbname = identifier - | SET TIME ZONE (Character_String_Literal | signed_numerical_literal | DEFAULT) + | SET TIME ZONE (TO | EQUAL)? (Character_String_Literal | signed_numerical_literal | DEFAULT) | SET (SESSION)? name=identifier (TO | EQUAL)? (Character_String_Literal | signed_numerical_literal | boolean_literal | DEFAULT) + | RESET name=identifier ; data_statement @@ -293,6 +294,7 @@ nonreserved_keywords | RANK | REGEXP | RENAME + | RESET | RLIKE | ROLLUP | ROW diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index 821d440032..f23a8d3cd3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -71,6 +71,7 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.TimeZone; import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME; import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto; @@ -212,7 +213,39 @@ private SubmitQueryResponse executeQueryInternal(QueryContext queryContext, responseBuilder.setIsForwarded(false); responseBuilder.setUserName(queryContext.get(SessionVars.USERNAME)); - if (PlannerUtil.checkIfDDLPlan(rootNode)) { + if (PlannerUtil.checkIfSetSession(rootNode)) { + + SetSessionNode setSessionNode = rootNode.getChild(); + + final String varName = setSessionNode.getName(); + + // SET CATALOG 'XXX' + if (varName.equals(SessionVars.CURRENT_DATABASE.name())) { + String databaseName = setSessionNode.getValue(); + + if (catalog.existDatabase(databaseName)) { + session.selectDatabase(setSessionNode.getValue()); + } else { + responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); + responseBuilder.setResultCode(ClientProtos.ResultCode.ERROR); + responseBuilder.setErrorMessage("database \"" + databaseName + "\" does not exists."); + return responseBuilder.build(); + } + + // others + } else { + if (setSessionNode.isDefaultValue()) { + session.removeVariable(varName); + } else { + session.setVariable(varName, setSessionNode.getValue()); + } + } + + context.getSystemMetrics().counter("Query", "numDDLQuery").inc(); + responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); + responseBuilder.setResultCode(ClientProtos.ResultCode.OK); + + } else if (PlannerUtil.checkIfDDLPlan(rootNode)) { context.getSystemMetrics().counter("Query", "numDDLQuery").inc(); updateQuery(queryContext, rootNode.getChild()); responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); @@ -461,6 +494,8 @@ public QueryId updateQuery(QueryContext queryContext, String sql, boolean isJson private boolean updateQuery(QueryContext queryContext, LogicalNode root) throws IOException { switch (root.getType()) { + case SET_SESSION: + case CREATE_DATABASE: CreateDatabaseNode createDatabase = (CreateDatabaseNode) root; createDatabase(queryContext, createDatabase.getDatabaseName(), null, createDatabase.isIfNotExists()); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java index 540bd7113f..a4f0259e7e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java @@ -143,17 +143,18 @@ public CreateSessionResponse createSession(RpcController controller, CreateSessi String sessionId = context.getSessionManager().createSession(request.getUsername(), databaseName); CreateSessionResponse.Builder builder = CreateSessionResponse.newBuilder(); - builder.setState(CreateSessionResponse.ResultState.SUCCESS); + builder.setResultCode(ResultCode.OK); builder.setSessionId(TajoIdProtos.SessionIdProto.newBuilder().setId(sessionId).build()); + builder.setVariables(ProtoUtil.convertFromMap(context.getSessionManager().getAllVariables(sessionId))); return builder.build(); } catch (NoSuchDatabaseException nsde) { CreateSessionResponse.Builder builder = CreateSessionResponse.newBuilder(); - builder.setState(CreateSessionResponse.ResultState.FAILED); + builder.setResultCode(ResultCode.ERROR); builder.setMessage(nsde.getMessage()); return builder.build(); } catch (InvalidSessionException e) { CreateSessionResponse.Builder builder = CreateSessionResponse.newBuilder(); - builder.setState(CreateSessionResponse.ResultState.FAILED); + builder.setResultCode(ResultCode.ERROR); builder.setMessage(e.getMessage()); return builder.build(); } @@ -162,14 +163,30 @@ public CreateSessionResponse createSession(RpcController controller, CreateSessi @Override public BoolProto removeSession(RpcController controller, TajoIdProtos.SessionIdProto request) throws ServiceException { + if (request != null) { context.getSessionManager().removeSession(request.getId()); } - return ProtoUtil.TRUE; + + return BOOL_TRUE; + } + + public SessionUpdateResponse buildSessionUpdateOnSuccess(Map variables) { + SessionUpdateResponse.Builder builder = SessionUpdateResponse.newBuilder(); + builder.setResultCode(ResultCode.OK); + builder.setVariables(new KeyValueSet(variables).getProto()); + return builder.build(); + } + + public SessionUpdateResponse buildSessionUpdateOnError(String message) { + SessionUpdateResponse.Builder builder = SessionUpdateResponse.newBuilder(); + builder.setResultCode(ResultCode.ERROR); + builder.setMessage(message); + return builder.build(); } @Override - public BoolProto updateSessionVariables(RpcController controller, UpdateSessionVariableRequest request) + public SessionUpdateResponse updateSessionVariables(RpcController controller, UpdateSessionVariableRequest request) throws ServiceException { try { String sessionId = request.getSessionId().getId(); @@ -179,9 +196,9 @@ public BoolProto updateSessionVariables(RpcController controller, UpdateSessionV for (String unsetVariable : request.getUnsetVariablesList()) { context.getSessionManager().removeVariable(sessionId, unsetVariable); } - return ProtoUtil.TRUE; + return buildSessionUpdateOnSuccess(context.getSessionManager().getAllVariables(sessionId)); } catch (Throwable t) { - throw new ServiceException(t); + return buildSessionUpdateOnError("Invalid Session Id" + request.getSessionId()); } } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSetSessionQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSetSessionQuery.java new file mode 100644 index 0000000000..5d463c4fe3 --- /dev/null +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSetSessionQuery.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.query; + +import org.apache.tajo.IntegrationTest; +import org.apache.tajo.QueryTestCaseBase; +import org.apache.tajo.TajoConstants; +import org.apache.tajo.catalog.CatalogUtil; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +@Category(IntegrationTest.class) +public class TestSetSessionQuery extends QueryTestCaseBase { + + public TestSetSessionQuery() { + super(TajoConstants.DEFAULT_DATABASE_NAME); + } + + @Test + public final void testSetCatalog1() throws Exception { + executeString("CREATE DATABASE testsetcatalog;").close(); + assertEquals(TajoConstants.DEFAULT_DATABASE_NAME, getClient().getCurrentDatabase()); + executeString("SET CATALOG testsetcatalog").close(); + assertEquals("testsetcatalog", getClient().getCurrentDatabase()); + executeString("SET CATALOG \"default\"").close(); + executeString("DROP DATABASE testsetcatalog;").close(); + } + + @Test + public final void testSetCatalog2() throws Exception { + executeString("CREATE DATABASE \"testSetCatalog\";").close(); + assertEquals(TajoConstants.DEFAULT_DATABASE_NAME, getClient().getCurrentDatabase()); + executeString("SET CATALOG \"testSetCatalog\"").close(); + assertEquals("testSetCatalog", getClient().getCurrentDatabase()); + executeString("SET CATALOG \"default\"").close(); + executeString("DROP DATABASE \"testSetCatalog\";").close(); + } + + @Test + public final void testSetTimezone() throws Exception { + assertFalse(getClient().existSessionVariable("TZ")); + executeString("SET TIME ZONE 'GMT+9'").close(); + assertTrue(getClient().existSessionVariable("TZ")); + executeString("SET TIME ZONE to DEFAULT").close(); + } + + @Test + public final void testSetSession1() throws Exception { + assertFalse(getClient().existSessionVariable("key1")); + executeString("SET SESSION key1 to true").close(); + assertTrue(getClient().existSessionVariable("key1")); + + executeString("SET SESSION key1 to true").close(); + executeString("SET SESSION key2 to 'val1'").close(); + assertTrue(getClient().existSessionVariable("key1")); + assertTrue(getClient().existSessionVariable("key2")); + executeString("RESET key1").close(); + executeString("SET SESSION key2 to DEFAULT").close(); + assertFalse(getClient().existSessionVariable("key2")); + } +} \ No newline at end of file diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java index 68f21869f7..7c2909930b 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java @@ -176,6 +176,13 @@ private static boolean hasAsterisk(NamedExpr [] namedExprs) { return rewrittenTargets.toArray(new NamedExpr[rewrittenTargets.size()]); } + @Override + public LogicalNode visitSetSession(LogicalPlanner.PlanContext ctx, Stack stack, SetSession expr) + throws PlanningException { + SetSessionNode setSession = ctx.plan.createNode(SetSessionNode.class); + return setSession; + } + @Override public LogicalNode visitProjection(LogicalPlanner.PlanContext ctx, Stack stack, Projection expr) throws PlanningException { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index f21bbb5108..d1539713d5 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -22,6 +22,7 @@ import com.google.common.base.Joiner; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import org.apache.commons.lang.math.NumberUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -187,6 +188,16 @@ public LogicalNode postHook(PlanContext context, Stack stack, Expr expr, L return current; } + @Override + public LogicalNode visitSetSession(PlanContext context, Stack stack, SetSession expr) throws PlanningException { + QueryBlock block = context.queryBlock; + + SetSessionNode setSessionNode = block.getNodeFromExpr(expr); + setSessionNode.init(expr.getName(), expr.getValue()); + + return setSessionNode; + } + public LogicalNode visitExplain(PlanContext ctx, Stack stack, Explain expr) throws PlanningException { ctx.plan.setExplain(); return visit(ctx, stack, expr.getChild()); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java index df1e341baa..1ac12c240c 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java @@ -24,6 +24,9 @@ import java.util.Stack; public interface AlgebraVisitor { + // Relational Operators + RESULT visitSetSession(CONTEXT ctx, Stack stack, SetSession expr) throws PlanningException; + // Relational Operators RESULT visitProjection(CONTEXT ctx, Stack stack, Projection expr) throws PlanningException; RESULT visitLimit(CONTEXT ctx, Stack stack, Limit expr) throws PlanningException; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java index 57ddfbd6d0..bd105144de 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java @@ -51,6 +51,9 @@ public RESULT visit(CONTEXT ctx, Stack stack, Expr expr) throws PlanningEx RESULT current; switch (expr.getType()) { + case SetSession: + current = visitSetSession(ctx, stack, (SetSession) expr); + break; case Projection: current = visitProjection(ctx, stack, (Projection) expr); @@ -294,6 +297,11 @@ private RESULT visitDefaultBinaryExpr(CONTEXT ctx, Stack stack, BinaryOper return child; } + @Override + public RESULT visitSetSession(CONTEXT ctx, Stack stack, SetSession expr) throws PlanningException { + return null; + } + @Override public RESULT visitProjection(CONTEXT ctx, Stack stack, Projection expr) throws PlanningException { stack.push(expr); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/NodeType.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/NodeType.java index 5d1874f711..9f01de92b2 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/NodeType.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/NodeType.java @@ -26,6 +26,8 @@ * This indicates a logical node type. */ public enum NodeType { + SET_SESSION(SetSessionNode.class), + ROOT(LogicalRootNode.class), EXPRS(EvalExprNode.class), PROJECTION(ProjectionNode.class), diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SetSessionNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SetSessionNode.java new file mode 100644 index 0000000000..ba5f83eb5c --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SetSessionNode.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.plan.logical; + +import com.google.gson.annotations.Expose; +import org.apache.tajo.plan.PlanString; + +public class SetSessionNode extends LogicalNode { + @Expose private String name; + @Expose private String value; + + public SetSessionNode(int pid) { + super(pid, NodeType.SET_SESSION); + } + + public void init(String name, String value) { + this.name = name; + this.value = value; + } + + public String getName() { + return name; + } + + public boolean isDefaultValue() { + return value == null; + } + + public String getValue() { + return value; + } + + @Override + public void preOrder(LogicalNodeVisitor visitor) { + visitor.visit(this); + } + + @Override + public void postOrder(LogicalNodeVisitor visitor) { + visitor.visit(this); + } + + @Override + public PlanString getPlanString() { + PlanString planString = new PlanString("SET SESSION "); + planString.appendTitle(name).appendTitle("="); + if (value != null) { + planString.appendTitle(String.valueOf(value)); + } + return planString; + } +} diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java index 8a9e1cac7d..e6fa8de710 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java @@ -48,6 +48,16 @@ public class PlannerUtil { + public static boolean checkIfSetSession(LogicalNode node) { + LogicalNode baseNode = node; + if (node instanceof LogicalRootNode) { + baseNode = ((LogicalRootNode) node).getChild(); + } + + return baseNode.getType() == NodeType.SET_SESSION; + + } + public static boolean checkIfDDLPlan(LogicalNode node) { LogicalNode baseNode = node; if (node instanceof LogicalRootNode) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java index f6d04ba2a0..e6ff0d8956 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java @@ -30,7 +30,9 @@ import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.algebra.BaseAlgebraVisitor; import org.apache.tajo.util.TUtil; +import org.apache.tajo.validation.ConstraintViolation; +import java.util.Collection; import java.util.Set; import java.util.Stack; @@ -58,6 +60,24 @@ public VerificationState verify(OverridableConf queryContext, VerificationState return context.state; } + @Override + public Expr visitSetSession(Context ctx, Stack stack, SetSession expr) throws PlanningException { + + // we should allow undefined session variables which can be used in query statements in the future. + if (SessionVars.exists(expr.getName())) { + SessionVars var = SessionVars.get(expr.getName()); + if (var.validator() != null) { + Collection violations = var.validator().validate(expr.getValue()); + + for (ConstraintViolation violation : violations) { + ctx.state.addVerification(violation.getMessage()); + } + } + } + + return expr; + } + public Expr visitProjection(Context context, Stack stack, Projection expr) throws PlanningException { super.visitProjection(context, stack, expr); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java index 89eb4a8475..d09710ee32 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java @@ -59,6 +59,9 @@ public RESULT visit(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock bl case ROOT: current = visitRoot(context, plan, block, (LogicalRootNode) node, stack); break; + case SET_SESSION: + current = visitSetSession(context, plan, block, (SetSessionNode) node, stack); + break; case EXPRS: return null; case PROJECTION: @@ -149,6 +152,12 @@ public RESULT visitRoot(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBloc return result; } + @Override + public RESULT visitSetSession(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, SetSessionNode node, + Stack stack) throws PlanningException { + return null; + } + @Override public RESULT visitProjection(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, ProjectionNode node, Stack stack) diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/LogicalPlanVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/LogicalPlanVisitor.java index 29807f179e..6a0c338e31 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/LogicalPlanVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/LogicalPlanVisitor.java @@ -25,9 +25,13 @@ import java.util.Stack; public interface LogicalPlanVisitor { + RESULT visitRoot(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, LogicalRootNode node, Stack stack) throws PlanningException; + RESULT visitSetSession(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, SetSessionNode node, + Stack stack) throws PlanningException; + RESULT visitProjection(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, ProjectionNode node, Stack stack) throws PlanningException; From 67b4bdedd3dae75db624e85783661c56ca3977cd Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 9 Dec 2014 18:47:27 +0900 Subject: [PATCH 3/8] Fix initlal bugs. --- .../apache/tajo/jdbc/TajoResultSetBase.java | 7 +- .../java/org/apache/tajo/SessionVars.java | 2 +- .../java/org/apache/tajo/conf/TajoConf.java | 71 ++++++------------- .../engine/function/datetime/CurrentDate.java | 2 +- .../function/datetime/DatePartFromTime.java | 2 +- .../datetime/DatePartFromTimestamp.java | 2 +- .../function/datetime/ToCharTimestamp.java | 2 +- .../function/datetime/ToTimestampText.java | 2 +- .../org/apache/tajo/TajoTestingCluster.java | 4 +- .../org/apache/tajo/cli/tsql/TestTajoCli.java | 29 ++++++++ .../apache/tajo/engine/eval/ExprTestBase.java | 2 +- .../tajo/engine/eval/TestSQLExpression.java | 5 +- .../function/TestDateTimeFunctions.java | 5 +- .../tajo/engine/query/TestSelectQuery.java | 36 +++++++++- .../TestTajoCli/testSetTimezone1.result | 4 ++ .../org/apache/tajo/plan/ExprAnnotator.java | 12 ++-- .../org/apache/tajo/plan/LogicalPlanner.java | 4 +- .../org/apache/tajo/plan/expr/CastEval.java | 5 +- .../apache/tajo/plan/util/PlannerUtil.java | 4 +- .../storage/TextSerializerDeserializer.java | 6 +- 20 files changed, 122 insertions(+), 84 deletions(-) create mode 100644 tajo-core/src/test/resources/results/TestTajoCli/testSetTimezone1.result diff --git a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java index 051ea75483..77cbbf278a 100644 --- a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java +++ b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java @@ -19,7 +19,6 @@ package org.apache.tajo.jdbc; import org.apache.tajo.SessionVars; -import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.Schema; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.*; @@ -53,11 +52,11 @@ public TajoResultSetBase(@Nullable Map clientSideSessionVars) { if (clientSideSessionVars != null) { - if (clientSideSessionVars.containsKey(SessionVars.TZ.name())) { - String timezoneId = clientSideSessionVars.get(SessionVars.TZ.name()); + if (clientSideSessionVars.containsKey(SessionVars.TIMEZONE.name())) { + String timezoneId = clientSideSessionVars.get(SessionVars.TIMEZONE.name()); this.timezone = TimeZone.getTimeZone(timezoneId); } else { - this.timezone = TimeZone.getTimeZone(TajoConstants.DEFAULT_SYSTEM_TIMEZONE); + this.timezone = TimeZone.getDefault(); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java index fa717d1230..d87bbef0b3 100644 --- a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java +++ b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java @@ -71,7 +71,7 @@ public enum SessionVars implements ConfigKey { ON_ERROR_STOP(ConfVars.$CLI_ERROR_STOP, "tsql will exist if an error occurs.", CLI_SIDE_VAR), // Timezone & Date ---------------------------------------------------------- - TZ(ConfVars.$TIMEZONE, "Sets timezone", CLI_SIDE_VAR), + TIMEZONE(ConfVars.$TIMEZONE, "Sets timezone", CLI_SIDE_VAR), DATE_ORDER(ConfVars.$DATE_ORDER, "date order (default is YMD)", CLI_SIDE_VAR), // Locales and Character set ------------------------------------------------ diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index b00b322deb..aee3dadb89 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -18,8 +18,8 @@ package org.apache.tajo.conf; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -39,16 +39,10 @@ import java.net.InetSocketAddress; import java.util.Map; import java.util.TimeZone; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; public class TajoConf extends Configuration { - private static TimeZone SYSTEM_TIMEZONE; private static int DATE_ORDER = -1; - private static final ReentrantReadWriteLock confLock = new ReentrantReadWriteLock(); - private static final Lock writeLock = confLock.writeLock(); - private static final Lock readLock = confLock.readLock(); private static final Map vars = TUtil.newHashMap(); @@ -59,8 +53,6 @@ public class TajoConf extends Configuration { Configuration.addDefaultResource("storage-site.xml"); Configuration.addDefaultResource("tajo-default.xml"); Configuration.addDefaultResource("tajo-site.xml"); - - confStaticInit(); for (ConfVars confVars: ConfVars.values()) { vars.put(confVars.keyname(), confVars); @@ -82,55 +74,36 @@ public TajoConf(Path path) { addResource(path); } - private static void confStaticInit() { - TimeZone.setDefault(getSystemTimezone()); - getDateOrder(); + public TimeZone getSystemTimezone() { + return TimeZone.getTimeZone(getVar(ConfVars.$TIMEZONE)); } - public static TimeZone getSystemTimezone() { - writeLock.lock(); - try { - if (SYSTEM_TIMEZONE == null) { - TajoConf tajoConf = new TajoConf(); - SYSTEM_TIMEZONE = TimeZone.getTimeZone(tajoConf.getVar(ConfVars.$TIMEZONE)); - } - return SYSTEM_TIMEZONE; - } finally { - writeLock.unlock(); - } + public void setSystemTimezone(TimeZone timezone) { + setVar(ConfVars.$TIMEZONE, timezone.getID()); } public static int getDateOrder() { - writeLock.lock(); - try { - if (DATE_ORDER < 0) { - TajoConf tajoConf = new TajoConf(); - String dateOrder = tajoConf.getVar(ConfVars.$DATE_ORDER); - if ("YMD".equals(dateOrder)) { - DATE_ORDER = DateTimeConstants.DATEORDER_YMD; - } else if ("DMY".equals(dateOrder)) { - DATE_ORDER = DateTimeConstants.DATEORDER_DMY; - } else if ("MDY".equals(dateOrder)) { - DATE_ORDER = DateTimeConstants.DATEORDER_MDY; - } else { - DATE_ORDER = DateTimeConstants.DATEORDER_YMD; - } + if (DATE_ORDER < 0) { + TajoConf tajoConf = new TajoConf(); + String dateOrder = tajoConf.getVar(ConfVars.$DATE_ORDER); + if ("YMD".equals(dateOrder)) { + DATE_ORDER = DateTimeConstants.DATEORDER_YMD; + } else if ("DMY".equals(dateOrder)) { + DATE_ORDER = DateTimeConstants.DATEORDER_DMY; + } else if ("MDY".equals(dateOrder)) { + DATE_ORDER = DateTimeConstants.DATEORDER_MDY; + } else { + DATE_ORDER = DateTimeConstants.DATEORDER_YMD; } - return DATE_ORDER; - } finally { - writeLock.unlock(); } + return DATE_ORDER; } + @VisibleForTesting public static int setDateOrder(int dateOrder) { - writeLock.lock(); - try { - int oldDateOrder = DATE_ORDER; - DATE_ORDER = dateOrder; - return oldDateOrder; - } finally { - writeLock.unlock(); - } + int oldDateOrder = DATE_ORDER; + DATE_ORDER = dateOrder; + return oldDateOrder; } public static enum ConfVars implements ConfigKey { @@ -349,7 +322,7 @@ public static enum ConfVars implements ConfigKey { $CLI_ERROR_STOP("tajo.cli.error.stop", false), // Timezone & Date ---------------------------------------------------------- - $TIMEZONE("tajo.timezone", TajoConstants.DEFAULT_SYSTEM_TIMEZONE), + $TIMEZONE("tajo.timezone", TimeZone.getDefault().getID()), $DATE_ORDER("tajo.date.order", "YMD"), // FILE FORMAT diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentDate.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentDate.java index ccfbd843f6..d57ac02c4b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentDate.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentDate.java @@ -53,7 +53,7 @@ public CurrentDate() { @Override public void init(OverridableConf context, FunctionEval.ParamType[] types) { - String timezoneId = context.get(SessionVars.TZ, TajoConstants.DEFAULT_SYSTEM_TIMEZONE); + String timezoneId = context.get(SessionVars.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE); timezone = TimeZone.getTimeZone(timezoneId); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTime.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTime.java index ae09197bae..85c76a2b8b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTime.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTime.java @@ -63,7 +63,7 @@ public DatePartFromTime() { @Override public void init(OverridableConf context, FunctionEval.ParamType [] types) { - String timezoneId = context.get(SessionVars.TZ, TajoConstants.DEFAULT_SYSTEM_TIMEZONE); + String timezoneId = context.get(SessionVars.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE); timezone = TimeZone.getTimeZone(timezoneId); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTimestamp.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTimestamp.java index cd195fb16a..57b3900711 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTimestamp.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTimestamp.java @@ -59,7 +59,7 @@ public DatePartFromTimestamp() { @Override public void init(OverridableConf context, FunctionEval.ParamType [] types) { - String timezoneId = context.get(SessionVars.TZ, TajoConstants.DEFAULT_SYSTEM_TIMEZONE); + String timezoneId = context.get(SessionVars.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE); timezone = TimeZone.getTimeZone(timezoneId); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java index 9e76677800..1053ecb55c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java @@ -62,7 +62,7 @@ public ToCharTimestamp() { @Override public void init(OverridableConf context, FunctionEval.ParamType[] paramTypes) { - String timezoneId = context.get(SessionVars.TZ, TajoConstants.DEFAULT_SYSTEM_TIMEZONE); + String timezoneId = context.get(SessionVars.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE); timezone = TimeZone.getTimeZone(timezoneId); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestampText.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestampText.java index e9beefbdd9..6b199f6cdd 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestampText.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestampText.java @@ -54,7 +54,7 @@ public ToTimestampText() { } public void init(OverridableConf queryContext, FunctionEval.ParamType [] paramTypes) { - String timezoneId = queryContext.get(SessionVars.TZ, TajoConstants.DEFAULT_SYSTEM_TIMEZONE); + String timezoneId = queryContext.get(SessionVars.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE); timezone = TimeZone.getTimeZone(timezoneId); } diff --git a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java index e5dfb7a103..757ba0fae8 100644 --- a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -118,7 +118,9 @@ void setTestingFlagProperties() { } void initPropertiesAndConfigs() { - TimeZone.setDefault(TimeZone.getTimeZone(TajoConstants.DEFAULT_SYSTEM_TIMEZONE)); + TimeZone testDefaultTZ = TimeZone.getTimeZone(TajoConstants.DEFAULT_SYSTEM_TIMEZONE); + conf.setSystemTimezone(testDefaultTZ); + TimeZone.setDefault(testDefaultTZ); if (System.getProperty(ConfVars.RESOURCE_MANAGER_CLASS.varname) != null) { String testResourceManager = System.getProperty(ConfVars.RESOURCE_MANAGER_CLASS.varname); diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java index b14bfa9519..d81b0eae27 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java @@ -350,6 +350,35 @@ public void testHelpSessionVars() throws Exception { assertOutputResult(new String(out.toByteArray())); } + @Test + public void testSetTimezone1() throws Exception { + tajoCli.executeMetaCommand("\\set TIMEZONE GMT"); + assertSessionVar(tajoCli, "TIMEZONE", "GMT"); + + setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); + Thread t = new Thread() { + public void run() { + try { + tajoCli.executeScript("select timestamp '2014-10-01 08:09:01'::text;"); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + }; + t.start(); + String consoleResult; + while (true) { + Thread.sleep(3 * 1000); + consoleResult = new String(out.toByteArray()); + if (consoleResult.indexOf("row") >= 0) { + t.interrupt(); + break; + } + } + assertOutputResult(consoleResult); + } + @Test public void testNonForwardQueryPause() throws Exception { final String sql = "select * from default.lineitem"; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index 8cd87a8943..e286b92858 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -218,7 +218,7 @@ public void testEval(OverridableConf context, Schema schema, String tableName, S queryContext.putAll(context); } - String timezoneId = queryContext.get(SessionVars.TZ); + String timezoneId = queryContext.get(SessionVars.TIMEZONE); TimeZone timeZone = TimeZone.getTimeZone(timezoneId); LazyTuple lazyTuple; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestSQLExpression.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestSQLExpression.java index 4a9e42ceef..52982c0091 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestSQLExpression.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestSQLExpression.java @@ -23,7 +23,6 @@ import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.exception.NoSuchFunctionException; import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.TimestampDatum; import org.apache.tajo.engine.query.QueryContext; @@ -856,7 +855,7 @@ public void testSigned() throws IOException { @Test public void testCastWithNestedFunction() throws IOException { QueryContext context = new QueryContext(getConf()); - context.put(SessionVars.TZ, "GMT-6"); + context.put(SessionVars.TIMEZONE, "GMT-6"); TimeZone tz = TimeZone.getTimeZone("GMT-6"); int unixtime = 1389071574; // (int) (System.currentTimeMillis() / 1000); @@ -868,7 +867,7 @@ public void testCastWithNestedFunction() throws IOException { @Test public void testCastFromTable() throws IOException { QueryContext queryContext = new QueryContext(getConf()); - queryContext.put(SessionVars.TZ, "GMT-6"); + queryContext.put(SessionVars.TIMEZONE, "GMT-6"); TimeZone tz = TimeZone.getTimeZone("GMT-6"); Schema schema = new Schema(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java index cb7856bac9..6c708fb109 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java @@ -21,7 +21,6 @@ import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.TimestampDatum; import org.apache.tajo.engine.eval.ExprTestBase; @@ -431,7 +430,7 @@ public void testDateTimeNow() throws IOException { TimeZone.setDefault(TimeZone.getTimeZone("GMT-6")); QueryContext context = new QueryContext(getConf()); - context.put(SessionVars.TZ, "GMT-6"); + context.put(SessionVars.TIMEZONE, "GMT-6"); try { Date expectedDate = new Date(System.currentTimeMillis()); @@ -455,7 +454,7 @@ public void testTimeValueKeyword() throws IOException { TimeZone.setDefault(TimeZone.getTimeZone("GMT-6")); QueryContext context = new QueryContext(getConf()); - context.put(SessionVars.TZ, "GMT-6"); + context.put(SessionVars.TIMEZONE, "GMT-6"); try { Date expectedDate = new Date(System.currentTimeMillis()); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java index 5b7641a5d8..12798d65be 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java @@ -566,7 +566,7 @@ public void testTimezonedTable2() throws Exception { @Test public void testTimezonedTable3() throws Exception { Map sessionVars = new HashMap(); - sessionVars.put(SessionVars.TZ.name(), "GMT+9"); + sessionVars.put(SessionVars.TIMEZONE.name(), "GMT+9"); getClient().updateSessionVariables(sessionVars); try { @@ -578,6 +578,38 @@ public void testTimezonedTable3() throws Exception { executeString("DROP TABLE IF EXISTS timezoned3"); } - getClient().unsetSessionVariables(Lists.newArrayList("TZ")); + getClient().unsetSessionVariables(Lists.newArrayList("TIMEZONE")); + } + + @Test + public void testCurrentTimestamp1() throws Exception { + + try { + Map sessionVars = new HashMap(); + sessionVars.put(SessionVars.TIMEZONE.name(), "GMT+9"); + getClient().updateSessionVariables(sessionVars); + + ResultSet res = executeString("SELECT current_timestamp;"); + System.out.println(resultSetToString(res)); + cleanupQuery(res); + } finally { + getClient().unsetSessionVariables(Lists.newArrayList("TIMEZONE")); + } + } + + @Test + public void testCurrentTimestamp2() throws Exception { + + try { + Map sessionVars = new HashMap(); + sessionVars.put(SessionVars.TIMEZONE.name(), "GMT+9"); + getClient().updateSessionVariables(sessionVars); + + ResultSet res = executeString("SELECT current_timestamp;"); + System.out.println(resultSetToString(res)); + cleanupQuery(res); + } finally { + getClient().unsetSessionVariables(Lists.newArrayList("TIMEZONE")); + } } } \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testSetTimezone1.result b/tajo-core/src/test/resources/results/TestTajoCli/testSetTimezone1.result new file mode 100644 index 0000000000..fdb2a2d546 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestTajoCli/testSetTimezone1.result @@ -0,0 +1,4 @@ +?timestamp +------------------------------- +2014-10-01 08:09:01 +(1 rows, , 0 B selected) \ No newline at end of file diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java index 62ba33c361..fcbbb212ca 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java @@ -783,8 +783,8 @@ public EvalNode visitCastExpr(Context ctx, Stack stack, CastExpr expr) thr // some cast operation may require earlier evaluation with timezone. TimeZone tz = null; - if (ctx.queryContext.containsKey(SessionVars.TZ)) { - String tzId = ctx.queryContext.get(SessionVars.TZ); + if (ctx.queryContext.containsKey(SessionVars.TIMEZONE)) { + String tzId = ctx.queryContext.get(SessionVars.TIMEZONE); tz = TimeZone.getTimeZone(tzId); } @@ -859,8 +859,8 @@ public EvalNode visitTimestampLiteral(Context ctx, Stack stack, TimestampL TimeMeta tm = new TimeMeta(); DateTimeUtil.toJulianTimeMeta(timestamp, tm); - if (ctx.queryContext.containsKey(SessionVars.TZ)) { - TimeZone tz = TimeZone.getTimeZone(ctx.queryContext.get(SessionVars.TZ)); + if (ctx.queryContext.containsKey(SessionVars.TIMEZONE)) { + TimeZone tz = TimeZone.getTimeZone(ctx.queryContext.get(SessionVars.TIMEZONE)); DateTimeUtil.toUTCTimezone(tm, tz); } @@ -889,8 +889,8 @@ public EvalNode visitTimeLiteral(Context ctx, Stack stack, TimeLiteral exp TimeDatum timeDatum = new TimeDatum(time); TimeMeta tm = timeDatum.toTimeMeta(); - if (ctx.queryContext.containsKey(SessionVars.TZ)) { - TimeZone tz = TimeZone.getTimeZone(ctx.queryContext.get(SessionVars.TZ)); + if (ctx.queryContext.containsKey(SessionVars.TIMEZONE)) { + TimeZone tz = TimeZone.getTimeZone(ctx.queryContext.get(SessionVars.TIMEZONE)); DateTimeUtil.toUTCTimezone(tm, tz); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index f21bbb5108..f93bacdcab 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -94,8 +94,8 @@ public PlanContext(OverridableConf context, LogicalPlan plan, QueryBlock block, this.evalOptimizer = evalOptimizer; // session's time zone - if (context.containsKey(SessionVars.TZ)) { - String timezoneId = context.get(SessionVars.TZ); + if (context.containsKey(SessionVars.TIMEZONE)) { + String timezoneId = context.get(SessionVars.TIMEZONE); timeZone = TimeZone.getTimeZone(timezoneId); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CastEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CastEval.java index 700913e9ff..3cad842558 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CastEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CastEval.java @@ -21,7 +21,6 @@ import com.google.gson.annotations.Expose; import org.apache.tajo.OverridableConf; import org.apache.tajo.SessionVars; -import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.Schema; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; @@ -40,8 +39,8 @@ public CastEval(OverridableConf context, EvalNode operand, DataType target) { super(EvalType.CAST, operand); this.target = target; - if (context.containsKey(SessionVars.TZ)) { - String timezoneId = context.get(SessionVars.TZ); + if (context.containsKey(SessionVars.TIMEZONE)) { + String timezoneId = context.get(SessionVars.TIMEZONE); timezone = TimeZone.getTimeZone(timezoneId); } } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java index 8a9e1cac7d..b14fc3aa35 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java @@ -792,8 +792,8 @@ public static void applySessionToTableProperties(OverridableConf sessionVars, tableProperties.set(StorageConstants.TEXT_NULL, sessionVars.get(SessionVars.NULL_CHAR)); } - if (sessionVars.containsKey(SessionVars.TZ)) { - tableProperties.set(StorageConstants.TIMEZONE, sessionVars.get(SessionVars.TZ)); + if (sessionVars.containsKey(SessionVars.TIMEZONE)) { + tableProperties.set(StorageConstants.TIMEZONE, sessionVars.get(SessionVars.TIMEZONE)); } } } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java index ce7b11d2ac..ab8816bc28 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java @@ -20,6 +20,7 @@ import com.google.protobuf.Message; import org.apache.commons.codec.binary.Base64; +import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; @@ -30,6 +31,7 @@ import java.io.IOException; import java.io.OutputStream; +import java.util.TimeZone; // Compatibility with Apache Hive @Deprecated @@ -85,12 +87,12 @@ public int serialize(Column col, Datum datum, OutputStream out, byte[] nullChara out.write(bytes); break; case TIME: - bytes = ((TimeDatum)datum).asChars(TajoConf.getSystemTimezone(), true).getBytes(); + bytes = ((TimeDatum)datum).asChars(TimeZone.getDefault(), true).getBytes(); length = bytes.length; out.write(bytes); break; case TIMESTAMP: - bytes = ((TimestampDatum)datum).asChars(TajoConf.getSystemTimezone(), true).getBytes(); + bytes = ((TimestampDatum)datum).asChars(TimeZone.getDefault(), true).getBytes(); length = bytes.length; out.write(bytes); break; From 73db3e1f0ca8ca7c062f49fc7cb747c8ee4ff120 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Wed, 10 Dec 2014 09:56:30 +0900 Subject: [PATCH 4/8] Fixed all bugs. --- .../apache/tajo/client/QueryClientImpl.java | 8 ++- .../apache/tajo/client/SessionConnection.java | 65 ++++++------------- .../apache/tajo/client/TajoClientUtil.java | 4 +- tajo-client/src/main/proto/ClientProtos.proto | 11 ++-- .../tajo/engine/parser/SQLAnalyzer.java | 2 +- .../org/apache/tajo/master/GlobalEngine.java | 4 +- .../tajo/master/TajoMasterClientService.java | 6 +- .../org/apache/tajo/QueryTestCaseBase.java | 8 ++- .../tajo/engine/query/TestSelectQuery.java | 35 +++------- .../TestSelectQuery/testTimezonedTable2.sql | 1 + .../TestSelectQuery/testTimezonedTable4.sql | 2 + 11 files changed, 63 insertions(+), 83 deletions(-) create mode 100644 tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable4.sql diff --git a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java index 6809fda2dc..dc35968789 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java @@ -37,6 +37,7 @@ import org.apache.tajo.rpc.NettyClientBase; import org.apache.tajo.rpc.ServerCallable; import org.apache.tajo.util.NetUtils; +import org.apache.tajo.util.ProtoUtil; import java.io.IOException; import java.net.InetSocketAddress; @@ -181,7 +182,11 @@ public ClientProtos.SubmitQueryResponse call(NettyClientBase client) throws Serv TajoMasterClientProtocolService.BlockingInterface tajoMasterService = client.getStub(); - return tajoMasterService.submitQuery(null, builder.build()); + SubmitQueryResponse response = tajoMasterService.submitQuery(null, builder.build()); + if (response.getResultCode() == ResultCode.OK) { + connection.updateSessionVarsCache(ProtoUtil.convertToMap(response.getSessionVars())); + } + return response; } }.withRetries(); } @@ -489,6 +494,7 @@ public Boolean call(NettyClientBase client) throws ServiceException { ClientProtos.UpdateQueryResponse response = tajoMasterService.updateQuery(null, builder.build()); if (response.getResultCode() == ClientProtos.ResultCode.OK) { + connection.updateSessionVarsCache(ProtoUtil.convertToMap(response.getSessionVars())); return true; } else { if (response.hasErrorMessage()) { diff --git a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java index dcec1a3e98..44b772b0b3 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java @@ -23,12 +23,12 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.security.UserGroupInformation; import org.apache.tajo.QueryId; -import org.apache.tajo.SessionVars; import org.apache.tajo.TajoIdProtos; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.ipc.ClientProtos; import org.apache.tajo.ipc.ClientProtos.ResultCode; +import org.apache.tajo.ipc.ClientProtos.SessionUpdateResponse; import org.apache.tajo.ipc.TajoMasterClientProtocol; import org.apache.tajo.rpc.NettyClientBase; import org.apache.tajo.rpc.RpcConnectionPool; @@ -190,32 +190,17 @@ public Map updateSessionVariables(final Map vari public Map call(NettyClientBase client) throws ServiceException { checkSessionAndGet(client); - // keep client-side session variables - for (Map.Entry entry : variables.entrySet()) { - String key = entry.getKey(); - if (SessionVars.exists(entry.getKey())) { - SessionVars configKey = SessionVars.get(key); - if (configKey.getMode() == SessionVars.VariableMode.CLI_SIDE_VAR) { - sessionVarsCache.put(key, entry.getValue()); - } - } - } - TajoMasterClientProtocolService.BlockingInterface tajoMasterService = client.getStub(); KeyValueSet keyValueSet = new KeyValueSet(); keyValueSet.putAll(variables); ClientProtos.UpdateSessionVariableRequest request = ClientProtos.UpdateSessionVariableRequest.newBuilder() .setSessionId(sessionId) - .setSetVariables(keyValueSet.getProto()).build(); + .setSessionVars(keyValueSet.getProto()).build(); - ClientProtos.SessionUpdateResponse response = tajoMasterService.updateSessionVariables(null, request); - if (response.getResultCode() == ResultCode.OK) { - Map updated = ProtoUtil.convertToMap(response.getVariables()); + SessionUpdateResponse response = tajoMasterService.updateSessionVariables(null, request); - synchronized (sessionVarsCache) { - sessionVarsCache.clear(); - sessionVarsCache.putAll(updated); - } + if (response.getResultCode() == ResultCode.OK) { + updateSessionVarsCache(ProtoUtil.convertToMap(response.getSessionVars())); return Collections.unmodifiableMap(sessionVarsCache); } else { throw new ServiceException(response.getMessage()); @@ -230,26 +215,15 @@ public Map unsetSessionVariables(final List variables) public Map call(NettyClientBase client) throws ServiceException { checkSessionAndGet(client); - // Remove matched session vars - for (String key : variables) { - if (sessionVarsCache.containsKey(key)) { - sessionVarsCache.remove(key); - } - } - TajoMasterClientProtocolService.BlockingInterface tajoMasterService = client.getStub(); ClientProtos.UpdateSessionVariableRequest request = ClientProtos.UpdateSessionVariableRequest.newBuilder() .setSessionId(sessionId) .addAllUnsetVariables(variables).build(); - ClientProtos.SessionUpdateResponse response = tajoMasterService.updateSessionVariables(null, request); - if (response.getResultCode() == ResultCode.OK) { - Map updated = ProtoUtil.convertToMap(response.getVariables()); + SessionUpdateResponse response = tajoMasterService.updateSessionVariables(null, request); - synchronized (sessionVarsCache) { - sessionVarsCache.clear(); - sessionVarsCache.putAll(updated); - } + if (response.getResultCode() == ResultCode.OK) { + updateSessionVarsCache(ProtoUtil.convertToMap(response.getSessionVars())); return Collections.unmodifiableMap(sessionVarsCache); } else { throw new ServiceException(response.getMessage()); @@ -258,11 +232,10 @@ public Map call(NettyClientBase client) throws ServiceException }.withRetries(); } - public String getCachedSessionVariable(final String varname) { - if (sessionVarsCache.containsKey(varname)) { - return sessionVarsCache.get(varname); - } else { - throw new RuntimeException("No such session variable" + varname); + void updateSessionVarsCache(Map variables) { + synchronized (sessionVarsCache) { + this.sessionVarsCache.clear(); + this.sessionVarsCache.putAll(variables); } } @@ -271,9 +244,11 @@ public String getSessionVariable(final String varname) throws ServiceException { public String call(NettyClientBase client) throws ServiceException { - // If a desired variable is client side one and exists in the cache, immediately return the variable. - if (sessionVarsCache.containsKey(varname)) { - return sessionVarsCache.get(varname); + synchronized (sessionVarsCache) { + // If a desired variable is client side one and exists in the cache, immediately return the variable. + if (sessionVarsCache.containsKey(varname)) { + return sessionVarsCache.get(varname); + } } checkSessionAndGet(client); @@ -297,7 +272,9 @@ public Boolean call(NettyClientBase client) throws ServiceException { } public Map getCachedAllSessionVariables() { - return Collections.unmodifiableMap(sessionVarsCache); + synchronized (sessionVarsCache) { + return Collections.unmodifiableMap(sessionVarsCache); + } } public Map getAllSessionVariables() throws ServiceException { @@ -377,7 +354,7 @@ protected void checkSessionAndGet(NettyClientBase client) throws ServiceExceptio if (response.getResultCode() == ResultCode.OK) { sessionId = response.getSessionId(); - sessionVarsCache.putAll(ProtoUtil.convertToMap(response.getVariables())); + updateSessionVarsCache(ProtoUtil.convertToMap(response.getSessionVars())); if (LOG.isDebugEnabled()) { LOG.debug(String.format("Got session %s as a user '%s'.", sessionId.getId(), userInfo.getUserName())); } diff --git a/tajo-client/src/main/java/org/apache/tajo/client/TajoClientUtil.java b/tajo-client/src/main/java/org/apache/tajo/client/TajoClientUtil.java index bf9d111051..744ba1d6ef 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/TajoClientUtil.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/TajoClientUtil.java @@ -72,8 +72,8 @@ public static ResultSet createResultSet(TajoConf conf, QueryClient client, Clien // non-forward query // select * from table1 [limit 10] int fetchRowNum = conf.getIntVar(TajoConf.ConfVars.$RESULT_SET_FETCH_ROWNUM); - if (response.hasSessionVariables()) { - for (PrimitiveProtos.KeyValueProto eachKeyValue: response.getSessionVariables().getKeyvalList()) { + if (response.hasSessionVars()) { + for (PrimitiveProtos.KeyValueProto eachKeyValue: response.getSessionVars().getKeyvalList()) { if (eachKeyValue.getKey().equals(SessionVars.FETCH_ROWNUM.keyname())) { fetchRowNum = Integer.parseInt(eachKeyValue.getValue()); } diff --git a/tajo-client/src/main/proto/ClientProtos.proto b/tajo-client/src/main/proto/ClientProtos.proto index fa12ab20fc..51db7630b4 100644 --- a/tajo-client/src/main/proto/ClientProtos.proto +++ b/tajo-client/src/main/proto/ClientProtos.proto @@ -39,19 +39,19 @@ message CreateSessionRequest { message CreateSessionResponse { required ResultCode resultCode = 1; optional SessionIdProto sessionId = 2; - optional KeyValueSetProto variables = 3; + optional KeyValueSetProto sessionVars = 3; optional string message = 4; } message UpdateSessionVariableRequest { optional SessionIdProto sessionId = 1; - optional KeyValueSetProto setVariables = 2; + optional KeyValueSetProto sessionVars = 2; repeated string unsetVariables = 3; } message SessionUpdateResponse { required ResultCode resultCode = 1; - optional KeyValueSetProto variables = 2; + optional KeyValueSetProto sessionVars = 2; optional string message = 3; } @@ -69,13 +69,14 @@ message ExplainQueryResponse { message QueryRequest { optional SessionIdProto sessionId = 1; required string query = 2; - repeated KeyValueProto setVariables = 3; + repeated KeyValueProto sessionVars = 3; required bool isJson = 4; } message UpdateQueryResponse { required ResultCode resultCode = 1; optional string errorMessage = 2; + optional KeyValueSetProto sessionVars = 3; } message GetQueryResultRequest { @@ -140,7 +141,7 @@ message SubmitQueryResponse { optional string errorMessage = 10; optional string errorTrace = 11; - optional KeyValueSetProto sessionVariables = 12; + optional KeyValueSetProto sessionVars = 12; } message GetQueryStatusResponse { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index ca04301562..7d6c951964 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@ -113,7 +113,7 @@ public Expr visitSession_statement(@NotNull SQLParser.Session_statementContext c } else { value = null; } - return new SetSession(SessionVars.TZ.name(), value); + return new SetSession(SessionVars.TIMEZONE.name(), value); } else { throw new SQLSyntaxError("Unsupported session statement"); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index f23a8d3cd3..399644c9de 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -65,6 +65,7 @@ import org.apache.tajo.plan.verifier.VerifyException; import org.apache.tajo.storage.*; import org.apache.tajo.util.CommonTestingUtil; +import org.apache.tajo.util.ProtoUtil; import org.apache.tajo.worker.TaskAttemptContext; import java.io.IOException; @@ -303,7 +304,6 @@ private SubmitQueryResponse executeQueryInternal(QueryContext queryContext, responseBuilder.setQueryId(queryId.getProto()); responseBuilder.setMaxRowNum(maxRow); responseBuilder.setTableDesc(desc.getProto()); - responseBuilder.setSessionVariables(session.getProto().getVariables()); responseBuilder.setResultCode(ClientProtos.ResultCode.OK); // NonFromQuery indicates a form of 'select a, x+y;' @@ -359,6 +359,8 @@ private SubmitQueryResponse executeQueryInternal(QueryContext queryContext, LOG.info("Query is forwarded to " + queryInfo.getQueryMasterHost() + ":" + queryInfo.getQueryMasterPort()); } } + + responseBuilder.setSessionVars(ProtoUtil.convertFromMap(session.getAllVariables())); SubmitQueryResponse response = responseBuilder.build(); return response; } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java index a4f0259e7e..b420a65f39 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java @@ -145,7 +145,7 @@ public CreateSessionResponse createSession(RpcController controller, CreateSessi CreateSessionResponse.Builder builder = CreateSessionResponse.newBuilder(); builder.setResultCode(ResultCode.OK); builder.setSessionId(TajoIdProtos.SessionIdProto.newBuilder().setId(sessionId).build()); - builder.setVariables(ProtoUtil.convertFromMap(context.getSessionManager().getAllVariables(sessionId))); + builder.setSessionVars(ProtoUtil.convertFromMap(context.getSessionManager().getAllVariables(sessionId))); return builder.build(); } catch (NoSuchDatabaseException nsde) { CreateSessionResponse.Builder builder = CreateSessionResponse.newBuilder(); @@ -174,7 +174,7 @@ public BoolProto removeSession(RpcController controller, TajoIdProtos.SessionIdP public SessionUpdateResponse buildSessionUpdateOnSuccess(Map variables) { SessionUpdateResponse.Builder builder = SessionUpdateResponse.newBuilder(); builder.setResultCode(ResultCode.OK); - builder.setVariables(new KeyValueSet(variables).getProto()); + builder.setSessionVars(new KeyValueSet(variables).getProto()); return builder.build(); } @@ -190,7 +190,7 @@ public SessionUpdateResponse updateSessionVariables(RpcController controller, Up throws ServiceException { try { String sessionId = request.getSessionId().getId(); - for (KeyValueProto kv : request.getSetVariables().getKeyvalList()) { + for (KeyValueProto kv : request.getSessionVars().getKeyvalList()) { context.getSessionManager().setVariable(sessionId, kv.getKey(), kv.getValue()); } for (String unsetVariable : request.getUnsetVariablesList()) { diff --git a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java index efcc69163f..ecfb9f5606 100644 --- a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -330,7 +330,13 @@ public ResultSet executeFile(String queryFileName) throws Exception { if (parsedResults.size() > 1) { assertNotNull("This script \"" + queryFileName + "\" includes two or more queries"); } - ResultSet result = client.executeQueryAndGetResult(parsedResults.get(0).getHistoryStatement()); + + int idx = 0; + for (; idx < parsedResults.size() - 1; idx++) { + client.executeQueryAndGetResult(parsedResults.get(idx).getHistoryStatement()).close(); + } + + ResultSet result = client.executeQueryAndGetResult(parsedResults.get(idx).getHistoryStatement()); assertNotNull("Query succeeded test", result); return result; } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java index 12798d65be..9154134882 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java @@ -541,6 +541,7 @@ public void testColumnEqualityButNotJoinCondition2() throws Exception { @Test public void testTimezonedTable1() throws Exception { + // default client time zone test without any time zone try { executeDDL("datetime_table_ddl.sql", "timezoned", new String[]{"timezoned1"}); ResultSet res = executeQuery(); @@ -553,6 +554,7 @@ public void testTimezonedTable1() throws Exception { @Test public void testTimezonedTable2() throws Exception { + // manually setting timezone to GMT try { executeDDL("datetime_table_timezoned_ddl.sql", "timezoned", new String[]{"timezoned2"}); ResultSet res = executeQuery(); @@ -565,6 +567,8 @@ public void testTimezonedTable2() throws Exception { @Test public void testTimezonedTable3() throws Exception { + // set time zone GMT+9 through TajoClient + Map sessionVars = new HashMap(); sessionVars.put(SessionVars.TIMEZONE.name(), "GMT+9"); getClient().updateSessionVariables(sessionVars); @@ -582,34 +586,15 @@ public void testTimezonedTable3() throws Exception { } @Test - public void testCurrentTimestamp1() throws Exception { - + public void testTimezonedTable4() throws Exception { + // set time zone GMT+9 in query statement try { - Map sessionVars = new HashMap(); - sessionVars.put(SessionVars.TIMEZONE.name(), "GMT+9"); - getClient().updateSessionVariables(sessionVars); - - ResultSet res = executeString("SELECT current_timestamp;"); - System.out.println(resultSetToString(res)); - cleanupQuery(res); - } finally { - getClient().unsetSessionVariables(Lists.newArrayList("TIMEZONE")); - } - } - - @Test - public void testCurrentTimestamp2() throws Exception { - - try { - Map sessionVars = new HashMap(); - sessionVars.put(SessionVars.TIMEZONE.name(), "GMT+9"); - getClient().updateSessionVariables(sessionVars); - - ResultSet res = executeString("SELECT current_timestamp;"); - System.out.println(resultSetToString(res)); + executeDDL("datetime_table_timezoned_ddl.sql", "timezoned", new String[]{"timezoned3"}); + ResultSet res = executeQuery(); + assertResultSet(res, "testTimezonedTable3.result"); cleanupQuery(res); } finally { - getClient().unsetSessionVariables(Lists.newArrayList("TIMEZONE")); + executeString("DROP TABLE IF EXISTS timezoned3"); } } } \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable2.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable2.sql index 722fc658b3..1fd9e36855 100644 --- a/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable2.sql +++ b/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable2.sql @@ -1 +1,2 @@ +SET TIME ZONE 'GMT'; SELECT * FROM timezoned2; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable4.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable4.sql new file mode 100644 index 0000000000..f2d9691ef8 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable4.sql @@ -0,0 +1,2 @@ +SET TIME ZONE 'GMT+9'; +SELECT * FROM timezoned3; \ No newline at end of file From e6239f36a4f70a2ad616f46381479d1a3275da31 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Wed, 10 Dec 2014 14:05:56 +0900 Subject: [PATCH 5/8] Fixed some compilation bug and removed the reserved word 'default' from antlr. --- .../src/main/java/org/apache/tajo/catalog/CatalogUtil.java | 7 +------ .../main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 | 2 +- .../main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 | 1 + 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 274033d034..8e8314c106 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -52,11 +52,6 @@ public class CatalogUtil { - public static boolean isValidIdentifier(String identifier) { - Preconditions.checkNotNull(identifier); - return identifier.length() > 0 && Character.isAlphabetic(identifier.charAt(0)); - } - /** * Normalize an identifier. Normalization means a translation from a identifier to be a refined identifier name. * @@ -705,7 +700,7 @@ public static void closeQuietly(Statement stmt, ResultSet res) { "AS", "ALL", "AND", "ANY", "ASYMMETRIC", "ASC", "BOTH", "CASE", "CAST", "CREATE", "CROSS", "CURRENT_DATE", "CURRENT_TIME", "CURRENT_TIMESTAMP", - "DEFAULT", "DESC", "DISTINCT", + "DESC", "DISTINCT", "END", "ELSE", "EXCEPT", "FALSE", "FULL", "FROM", "GROUP", diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 index 7e18ee0c23..3ba008fcf8 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 @@ -132,7 +132,6 @@ CURRENT_DATE: C U R R E N T UNDERLINE D A T E; CURRENT_TIME: C U R R E N T UNDERLINE T I M E; CURRENT_TIMESTAMP: C U R R E N T UNDERLINE T I M E S T A M P; -DEFAULT : D E F A U L T; DESC : D E S C; DISTINCT : D I S T I N C T; @@ -214,6 +213,7 @@ CUME_DIST : C U M E UNDERLINE D I S T; CURRENT : C U R R E N T; DAY : D A Y; +DEFAULT : D E F A U L T; DATABASE : D A T A B A S E; DEC : D E C; DECADE : D E C A D E; diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 index 6eccca0e6f..76afb6bb4d 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 @@ -243,6 +243,7 @@ nonreserved_keywords | DAY | DEC | DECADE + | DEFAULT | DENSE_RANK | DOW | DOY From fb981a1ba64d0d233646edf716ceba678620c7bc Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Wed, 10 Dec 2014 17:17:03 +0900 Subject: [PATCH 6/8] Fixed test bugs. --- .../org/apache/tajo/catalog/CatalogUtil.java | 5 ++++ .../tajo/cli/tsql/commands/SetCommand.java | 7 ++++- .../org/apache/tajo/cli/tsql/TestTajoCli.java | 29 ------------------- .../engine/query/TestSetSessionQuery.java | 4 +-- .../TestSQLAnalyzer/settimezone1.result | 2 +- .../TestSQLAnalyzer/settimezone2.result | 2 +- .../TestSQLAnalyzer/settimezone3.result | 2 +- .../TestTajoCli/testHelpSessionVars.result | 2 +- 8 files changed, 17 insertions(+), 36 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 8e8314c106..9057ecbb44 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -823,6 +823,11 @@ public static AlterTableDesc addNewColumn(String tableName, Column column, Alter public static final String ENABLE_DICTIONARY = "parquet.enable.dictionary"; public static final String VALIDATION = "parquet.validation"; + /** + * + * @param type + * @return + */ public static KeyValueSet newPhysicalProperties(StoreType type) { KeyValueSet options = new KeyValueSet(); if (StoreType.CSV == type || StoreType.TEXTFILE == type) { diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/tsql/commands/SetCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/tsql/commands/SetCommand.java index e7d2fe530f..21c4be51b5 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/tsql/commands/SetCommand.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/tsql/commands/SetCommand.java @@ -55,9 +55,14 @@ public void set(String key, String val) throws ServiceException { SessionVars sessionVar = null; if (SessionVars.exists(key)) { // if the variable is one of the session variables + sessionVar = SessionVars.get(key); // is it cli-side variable? - updateSessionVariable(key, val); + if (sessionVar.getMode() == VariableMode.CLI_SIDE_VAR) { + context.setCliSideVar(key, val); + } else { + updateSessionVariable(key, val); + } if (SessionVars.isDeprecated(key)) { context.getOutput().println("Warning: deprecated to directly use config key in TajoConf.ConfVars. " + diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java index d81b0eae27..b14bfa9519 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java @@ -350,35 +350,6 @@ public void testHelpSessionVars() throws Exception { assertOutputResult(new String(out.toByteArray())); } - @Test - public void testSetTimezone1() throws Exception { - tajoCli.executeMetaCommand("\\set TIMEZONE GMT"); - assertSessionVar(tajoCli, "TIMEZONE", "GMT"); - - setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); - Thread t = new Thread() { - public void run() { - try { - tajoCli.executeScript("select timestamp '2014-10-01 08:09:01'::text;"); - } catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - } - }; - t.start(); - String consoleResult; - while (true) { - Thread.sleep(3 * 1000); - consoleResult = new String(out.toByteArray()); - if (consoleResult.indexOf("row") >= 0) { - t.interrupt(); - break; - } - } - assertOutputResult(consoleResult); - } - @Test public void testNonForwardQueryPause() throws Exception { final String sql = "select * from default.lineitem"; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSetSessionQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSetSessionQuery.java index 5d463c4fe3..c9c73391b9 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSetSessionQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSetSessionQuery.java @@ -58,9 +58,9 @@ public final void testSetCatalog2() throws Exception { @Test public final void testSetTimezone() throws Exception { - assertFalse(getClient().existSessionVariable("TZ")); + assertFalse(getClient().existSessionVariable("TIMEZONE")); executeString("SET TIME ZONE 'GMT+9'").close(); - assertTrue(getClient().existSessionVariable("TZ")); + assertTrue(getClient().existSessionVariable("TIMEZONE")); executeString("SET TIME ZONE to DEFAULT").close(); } diff --git a/tajo-core/src/test/resources/results/TestSQLAnalyzer/settimezone1.result b/tajo-core/src/test/resources/results/TestSQLAnalyzer/settimezone1.result index 366e447d12..b7dda46292 100644 --- a/tajo-core/src/test/resources/results/TestSQLAnalyzer/settimezone1.result +++ b/tajo-core/src/test/resources/results/TestSQLAnalyzer/settimezone1.result @@ -1,5 +1,5 @@ { - "name": "TZ", + "name": "TIMEZONE", "value": "PDT", "OpType": "SetSession" } \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestSQLAnalyzer/settimezone2.result b/tajo-core/src/test/resources/results/TestSQLAnalyzer/settimezone2.result index 7653373e5a..10e4bfa551 100644 --- a/tajo-core/src/test/resources/results/TestSQLAnalyzer/settimezone2.result +++ b/tajo-core/src/test/resources/results/TestSQLAnalyzer/settimezone2.result @@ -1,5 +1,5 @@ { - "name": "TZ", + "name": "TIMEZONE", "value": "-7", "OpType": "SetSession" } \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestSQLAnalyzer/settimezone3.result b/tajo-core/src/test/resources/results/TestSQLAnalyzer/settimezone3.result index 8bb188ec15..31f78e19ce 100644 --- a/tajo-core/src/test/resources/results/TestSQLAnalyzer/settimezone3.result +++ b/tajo-core/src/test/resources/results/TestSQLAnalyzer/settimezone3.result @@ -1,4 +1,4 @@ { - "name": "TZ", + "name": "TIMEZONE", "OpType": "SetSession" } \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result b/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result index 25f1ae786c..b5b7c229c9 100644 --- a/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result +++ b/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result @@ -8,7 +8,7 @@ Available Session Variables: \set CLI_PAGING_ENABLED [true or false] - Enable paging of result display \set CLI_DISPLAY_ERROR_TRACE [true or false] - Enable display of error trace \set ON_ERROR_STOP [true or false] - tsql will exist if an error occurs. -\set TZ [text value] - Sets timezone +\set TIMEZONE [text value] - Sets timezone \set DATE_ORDER [text value] - date order (default is YMD) \set LANG [text value] - Language \set LC_ALL [text value] - String sort order From e4a30e4a274ef74094957b391305260ddacbacc7 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Thu, 11 Dec 2014 10:49:15 +0900 Subject: [PATCH 7/8] Add documentation. --- .../tajo/engine/query/TestSelectQuery.java | 4 +- tajo-docs/src/main/sphinx/index.rst | 1 + .../table_management/table_overview.rst | 67 +++------ tajo-docs/src/main/sphinx/time_zone.rst | 129 ++++++++++++++++++ 4 files changed, 148 insertions(+), 53 deletions(-) create mode 100644 tajo-docs/src/main/sphinx/time_zone.rst diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java index 0c459f13de..2d9b1f8302 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java @@ -542,7 +542,7 @@ public void testColumnEqualityButNotJoinCondition2() throws Exception { @Test public void testTimezonedTable1() throws Exception { - // Table - GMT (No table property) + // Table - GMT (No table property or no system timezone) // Client - GMT (default client time zone is used if no TIME ZONE session variable is given.) try { executeDDL("datetime_table_ddl.sql", "timezoned", new String[]{"timezoned1"}); @@ -606,7 +606,7 @@ public void testTimezonedTable4() throws Exception { @Test public void testTimezonedTable5() throws Exception { - // Table - timezone = GMT+9 (by system default timezone) + // Table - timezone = GMT+9 (by a specified system timezone) // TajoClient uses JVM default timezone (GMT+9) try { diff --git a/tajo-docs/src/main/sphinx/index.rst b/tajo-docs/src/main/sphinx/index.rst index bca40538ce..80cd8428e9 100644 --- a/tajo-docs/src/main/sphinx/index.rst +++ b/tajo-docs/src/main/sphinx/index.rst @@ -33,6 +33,7 @@ Table of Contents: configuration tsql sql_language + time_zone functions table_management table_partitioning diff --git a/tajo-docs/src/main/sphinx/table_management/table_overview.rst b/tajo-docs/src/main/sphinx/table_management/table_overview.rst index bb4b82741e..704d6aa992 100644 --- a/tajo-docs/src/main/sphinx/table_management/table_overview.rst +++ b/tajo-docs/src/main/sphinx/table_management/table_overview.rst @@ -5,6 +5,14 @@ Overview of Tajo Tables Overview ======== +Managed Table +================ + +.. todo:: + +External Table +================ + .. todo:: Table Properties @@ -39,60 +47,17 @@ Compression Time zone --------- -In Tajo, a table property ``timezone`` allows users to specify a time zone that the table uses for reading or writing. -When each table row are read or written, ```timestamp``` and ```time``` column values are adjusted by a given time zone if it is set. Time zone can be an abbreviation form like 'PST' or 'DST'. Also, it accepts an offset-based form like 'GMT+9' or UTC+9' or a location-based form like 'Asia/Seoul'. - -Each table has one time zone, and many tables can have different time zones. Internally, Tajo translates all tables data to offset-based values. So, complex queries like join with multiple time zones work well. - -.. note:: - - In many cases, offset-based forms or locaion-based forms are recommanded. In order to know the list of time zones, please refer to `List of tz database time zones `_ - -.. note:: - - Java 6 does not recognize many location-based time zones and an offset-based timezone using the prefix 'UTC'. We highly recommanded using the offset-based time zone using the prefix 'GMT'. In other words, you should use 'GMT-7' instead of 'UTC-7' in Java 6. - -How time zone works in Tajo -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -For example, consider that there is a list of delimited text lines where each rows are written with ``Asia/Seoul`` time zone (i.e., GMT + 9). - -.. code-block:: text - - 1980-4-1 01:50:30.010|1980-04-01 - 80/4/1 1:50:30 AM|80/4/1 - 1980 April 1 1:50:30|1980-04-01 - - -In order to register the table, we should put a table property ``'timezone'='Asia/Seoul'`` in ``CREATE TABLE`` statement as follows: - -.. code-block:: sql - - CREATE EXTERNAL TABLE table1 ( - t_timestamp TIMESTAMP, - t_date DATE - ) USING TEXTFILE WITH('text.delimiter'='|', 'timezone'='ASIA/Seoul') LOCATION '/path-to-table/' +In Tajo, a table property ``timezone`` allows users to specify a time zone that the table uses for reading or writing. -By default, ``tsql`` and ``TajoClient`` API use UTC time zone. So, timestamp values in the result are adjusted by the time zone offset. But, date is not adjusted because date type does not consider time zone. +You can specify a table time zone as follows: .. code-block:: sql - default> SELECT * FROM table1 - t_timestamp, t_date - ---------------------------------- - 1980-03-31 16:50:30.01, 1980-04-01 - 1980-03-31 16:50:30 , 1980-04-01 - 1980-03-31 16:50:30 , 1980-04-01 - -In addition, users can set client-side time zone by setting a session variable 'TZ'. It enables a client to translate timestamp or time values to user's time zoned ones. - -.. code-block:: sql + CREATE EXTERNAL TABLE table1 ( + t_timestamp TIMESTAMP, + t_date DATE + ) USING TEXTFILE WITH('timezone'='ASIA/Seoul') LOCATION '/path-to-table/' + - default> \set TZ 'Asia/Seoul' - default> SELECT * FROM table1 - t_timestamp, t_date - ---------------------------------- - 1980-04-01 01:50:30.01, 1980-04-01 - 1980-04-01 01:50:30 , 1980-04-01 - 1980-04-01 01:50:30 , 1980-04-01 \ No newline at end of file +In order to learn time zone, please refer to :doc:`/time_zone`. \ No newline at end of file diff --git a/tajo-docs/src/main/sphinx/time_zone.rst b/tajo-docs/src/main/sphinx/time_zone.rst new file mode 100644 index 0000000000..8225deedc2 --- /dev/null +++ b/tajo-docs/src/main/sphinx/time_zone.rst @@ -0,0 +1,129 @@ +****************** +Time Zone +****************** + +Time zone affects some data types (e.g., ``Timestamp`` and ``Time``) and operations (e.g., ``to_char``). Tables can have different time zones. Internally, Tajo translates all table rows to UTC values and processes them. It becomes easier for Tajo to handle multiple different time zones. + +In Tajo, there are some time zong settings. + +======================== +Server Cluster Time Zone +======================== + +One Tajo cluster has a system time zone which globally affects all tables in which the table property 'time zone' are not specified. + +You can set the system time zone in *conf/tajo-site.xml* file as follows: + +**tajo-site.xml** + +.. code-block:: xml + + tajo.timezone + GMT+9 + + +================== +Table Time Zone +================== + +In Tajo, a table property ``timezone`` allows users to specify a time zone that the table uses for reading or writing. +When each table row are read or written, ```timestamp``` and ```time``` column values are adjusted by a given time zone if it is set. + +You can specify a table time zone as follows: + +.. code-block:: sql + + CREATE EXTERNAL TABLE table1 ( + t_timestamp TIMESTAMP, + t_date DATE + ) USING TEXTFILE WITH('timezone'='ASIA/Seoul') LOCATION '/path-to-table/' + + +In order to learn table properties, please refer to :doc:`/table_management/table_overview`. + +================== +Client Time Zone +================== + +Each client has its own time zone setting. It translates retrieved timestamp and time values by time zone. In order to set client time zone, you should set the session variable ``TIMEZONE``. There are some ways to set this session variable. + +In ``tsql``, you can use ``\set timezone`` meta command as follows: + +**tsql** + +.. code-block:: sh + + default> \set timezone GMT+9 + + +The following ways use SQL statements. So, this way is available in ``tsql``, JDBC, and Tajo Java API. + +**SQL** + +.. code-block:: sql + + SET TIME ZONE 'GMT+9'; + + or + + SET SESSION TIMEZONE TO 'GMT+9'; + +============ +Time Zone ID +============ + +Time zone can be an abbreviation form like 'PST' or 'DST'. Also, it accepts an offset-based form like 'GMT+9' or UTC+9' or a location-based form like 'Asia/Seoul'. + +.. note:: + + In many cases, offset-based forms or locaion-based forms are recommanded. In order to know the list of time zones, please refer to `List of tz database time zones `_ + +.. note:: + + Java 6 does not recognize many location-based time zones and an offset-based timezone using the prefix 'UTC'. We highly recommanded using the offset-based time zone using the prefix 'GMT'. In other words, you should use 'GMT-7' instead of 'UTC-7' in Java 6. + +===================== +Examples of Time Zone +===================== + +For example, consider that there is a list of delimited text lines where each rows are written with ``Asia/Seoul`` time zone (i.e., GMT + 9). + +.. code-block:: text + + 1980-4-1 01:50:30.010|1980-04-01 + 80/4/1 1:50:30 AM|80/4/1 + 1980 April 1 1:50:30|1980-04-01 + + +In order to register the table, we should put a table property ``'timezone'='Asia/Seoul'`` in ``CREATE TABLE`` statement as follows: + +.. code-block:: sql + + CREATE EXTERNAL TABLE table1 ( + t_timestamp TIMESTAMP, + t_date DATE + ) USING TEXTFILE WITH('text.delimiter'='|', 'timezone'='ASIA/Seoul') LOCATION '/path-to-table/' + + +By default, ``tsql`` and ``TajoClient`` API use UTC time zone. So, timestamp values in the result are adjusted by the time zone offset. But, date is not adjusted because date type does not consider time zone. + +.. code-block:: sql + + default> SELECT * FROM table1 + t_timestamp, t_date + ---------------------------------- + 1980-03-31 16:50:30.01, 1980-04-01 + 1980-03-31 16:50:30 , 1980-04-01 + 1980-03-31 16:50:30 , 1980-04-01 + +In addition, users can set client-side time zone by setting a session variable 'TZ'. It enables a client to translate timestamp or time values to user's time zoned ones. + +.. code-block:: sql + + default> SET TIME ZONE 'Asia/Seoul' + default> SELECT * FROM table1 + t_timestamp, t_date + ---------------------------------- + 1980-04-01 01:50:30.01, 1980-04-01 + 1980-04-01 01:50:30 , 1980-04-01 + 1980-04-01 01:50:30 , 1980-04-01 \ No newline at end of file From 43cc50db2efb39690611fa03b89422388060a13e Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Thu, 11 Dec 2014 15:18:47 +0900 Subject: [PATCH 8/8] Add tajo-site.xml config documentation. --- .../java/org/apache/tajo/conf/TajoConf.java | 2 +- tajo-docs/src/main/sphinx/configuration.rst | 4 +++- .../sphinx/configuration/catalog-site-xml.rst | 3 +++ ...faults.rst => service_config_defaults.rst} | 6 ++--- .../sphinx/configuration/tajo-site-xml.rst | 24 +++++++++++++++++++ 5 files changed, 34 insertions(+), 5 deletions(-) create mode 100644 tajo-docs/src/main/sphinx/configuration/catalog-site-xml.rst rename tajo-docs/src/main/sphinx/configuration/{configuration_defaults.rst => service_config_defaults.rst} (94%) create mode 100644 tajo-docs/src/main/sphinx/configuration/tajo-site-xml.rst diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 674a9e1081..b474c0425d 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -323,7 +323,7 @@ public static enum ConfVars implements ConfigKey { // Timezone & Date ---------------------------------------------------------- $TIMEZONE("tajo.timezone", TimeZone.getDefault().getID()), - $DATE_ORDER("tajo.date.order", "YMD"), + $DATE_ORDER("tajo.datetime.date-order", "YMD"), // FILE FORMAT $TEXT_NULL("tajo.text.null", "\\\\N"), diff --git a/tajo-docs/src/main/sphinx/configuration.rst b/tajo-docs/src/main/sphinx/configuration.rst index f898969168..5214ea8cb6 100644 --- a/tajo-docs/src/main/sphinx/configuration.rst +++ b/tajo-docs/src/main/sphinx/configuration.rst @@ -10,5 +10,7 @@ Configuration configuration/tajo_master_configuration configuration/worker_configuration configuration/catalog_configuration - configuration/configuration_defaults configuration/ha_configuration + configuration/service_config_defaults + configuration/tajo-site-xml + configuration/catalog-site-xml \ No newline at end of file diff --git a/tajo-docs/src/main/sphinx/configuration/catalog-site-xml.rst b/tajo-docs/src/main/sphinx/configuration/catalog-site-xml.rst new file mode 100644 index 0000000000..96273b235d --- /dev/null +++ b/tajo-docs/src/main/sphinx/configuration/catalog-site-xml.rst @@ -0,0 +1,3 @@ +************************* +The catalog-site.xml File +************************* \ No newline at end of file diff --git a/tajo-docs/src/main/sphinx/configuration/configuration_defaults.rst b/tajo-docs/src/main/sphinx/configuration/service_config_defaults.rst similarity index 94% rename from tajo-docs/src/main/sphinx/configuration/configuration_defaults.rst rename to tajo-docs/src/main/sphinx/configuration/service_config_defaults.rst index e3967b1f0b..aaaf34e019 100644 --- a/tajo-docs/src/main/sphinx/configuration/configuration_defaults.rst +++ b/tajo-docs/src/main/sphinx/configuration/service_config_defaults.rst @@ -1,6 +1,6 @@ -********************** -Configuration Defaults -********************** +************************************** +Cluster Service Configuration Defaults +************************************** Tajo Master Configuration Defaults ==================================== diff --git a/tajo-docs/src/main/sphinx/configuration/tajo-site-xml.rst b/tajo-docs/src/main/sphinx/configuration/tajo-site-xml.rst new file mode 100644 index 0000000000..d0c359715c --- /dev/null +++ b/tajo-docs/src/main/sphinx/configuration/tajo-site-xml.rst @@ -0,0 +1,24 @@ +********************** +The tajo-site.xml File +********************** + +To the ``core-site.xml`` file on every host in your cluster, you must add the following information: + +====================== +System Config +====================== + + + +====================== +Date/Time Settings +====================== + ++--------------------------+----------------+--------------------------------------------------------+ +| Property Name | Property Value | Descriptions | ++==========================+================+========================================================+ +| tajo.timezone | Time zone id | Refer to :doc:`/time_zone` | ++--------------------------+----------------+--------------------------------------------------------+ +| tajo.datetime.date-order | Date order | Determine date order. It should be one of YMD, DMY, MDY| ++--------------------------+----------------+--------------------------------------------------------+ +