From 9e24edbfbaa75bcb7aae1b83e097450048b788bd Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Sat, 6 Dec 2014 23:30:28 +0900 Subject: [PATCH 1/6] TAJO-1234: Rearrange timezone in date/time types. --- .../java/org/apache/tajo/OverridableConf.java | 7 +- .../java/org/apache/tajo/TajoConstants.java | 1 + .../java/org/apache/tajo/conf/TajoConf.java | 16 ++-- .../java/org/apache/tajo/datum/Datum.java | 2 +- .../org/apache/tajo/datum/DatumFactory.java | 57 +++++++---- .../tajo/json/TimeZoneGsonSerdeAdapter.java | 40 ++++++++ .../apache/tajo/storage/StorageConstants.java | 21 +++- .../tajo/util/datetime/DateTimeUtil.java | 24 +++-- .../engine/function/builtin/AvgDouble.java | 3 - .../engine/function/datetime/CurrentDate.java | 20 +++- .../function/datetime/DatePartFromTime.java | 21 +++- .../datetime/DatePartFromTimestamp.java | 18 +++- .../function/datetime/ToCharTimestamp.java | 19 +++- .../tajo/engine/function/string/BTrim.java | 4 +- .../tajo/engine/function/string/LTrim.java | 4 +- .../tajo/engine/function/string/Lpad.java | 4 +- .../tajo/engine/function/string/RTrim.java | 4 +- .../engine/function/string/RegexpReplace.java | 4 +- .../tajo/engine/function/string/Rpad.java | 4 +- .../tajo/engine/json/CoreGsonHelper.java | 2 + .../tajo/engine/query/QueryContext.java | 4 +- .../apache/tajo/engine/eval/ExprTestBase.java | 59 ++++++++---- .../tajo/engine/eval/TestSQLExpression.java | 67 +++++++------ .../function/TestDateTimeFunctions.java | 45 +++++---- .../tajo/engine/query/TestSortQuery.java | 6 +- .../org/apache/tajo/jdbc/TestResultSet.java | 9 +- .../org/apache/tajo/jdbc/TestTajoJdbc.java | 6 +- .../org/apache/tajo/plan/ExprAnnotator.java | 96 ++++++++++++------- .../org/apache/tajo/plan/LogicalPlanner.java | 8 ++ .../org/apache/tajo/plan/expr/CastEval.java | 28 +++++- .../tajo/plan/expr/GeneralFunctionEval.java | 8 +- .../tajo/plan/function/GeneralFunction.java | 4 +- .../serder/EvalTreeProtoDeserializer.java | 7 +- .../plan/serder/EvalTreeProtoSerializer.java | 3 + .../tajo/plan/serder/PlanGsonHelper.java | 2 + .../apache/tajo/plan/util/PlannerUtil.java | 4 + tajo-plan/src/main/proto/Plan.proto | 1 + .../storage/text/CSVLineDeserializer.java | 2 +- .../tajo/storage/text/CSVLineSerializer.java | 2 +- .../text/TextFieldSerializerDeserializer.java | 45 +++++++-- 40 files changed, 484 insertions(+), 197 deletions(-) create mode 100644 tajo-common/src/main/java/org/apache/tajo/json/TimeZoneGsonSerdeAdapter.java diff --git a/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java b/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java index 84be00ebae..b7a5da71b3 100644 --- a/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java @@ -181,10 +181,13 @@ public void put(ConfigKey key, String val) { private void assertRegisteredEnum(ConfigKey key) { boolean registered = false; - for (ConfigType c : configTypes) { - registered = key.type() == c; + if (configTypes != null) { + for (ConfigType c : configTypes) { + registered = key.type() == c; + } } + // default permitted keys registered |= key.type() == ConfigType.SESSION || key.type() != ConfigType.SYSTEM; Preconditions.checkArgument(registered, key.keyname() + " (" + key.type() + ") is not allowed in " + diff --git a/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java b/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java index 1cc28af662..31e18fcb4b 100644 --- a/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java +++ b/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java @@ -31,6 +31,7 @@ public class TajoConstants { public static final String DEFAULT_DATABASE_NAME = "default"; public static final String DEFAULT_SCHEMA_NAME = "public"; + public static final String SYSTEM_DEFAULT_TIMEZONE = "UTC"; public static final String EMPTY_STRING = ""; diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 312abfb089..db2d6f0e80 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -18,6 +18,7 @@ package org.apache.tajo.conf; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; @@ -44,7 +45,7 @@ public class TajoConf extends Configuration { - private static TimeZone CURRENT_TIMEZONE; + private static TimeZone SYSTEM_TIMEZONE; private static int DATE_ORDER = -1; private static final ReentrantReadWriteLock confLock = new ReentrantReadWriteLock(); private static final Lock writeLock = confLock.writeLock(); @@ -90,21 +91,22 @@ private static void confStaticInit() { public static TimeZone getCurrentTimeZone() { writeLock.lock(); try { - if (CURRENT_TIMEZONE == null) { + if (SYSTEM_TIMEZONE == null) { TajoConf tajoConf = new TajoConf(); - CURRENT_TIMEZONE = TimeZone.getTimeZone(tajoConf.getVar(ConfVars.$TIMEZONE)); + SYSTEM_TIMEZONE = TimeZone.getTimeZone(tajoConf.getVar(ConfVars.$TIMEZONE)); } - return CURRENT_TIMEZONE; + return SYSTEM_TIMEZONE; } finally { writeLock.unlock(); } } + @VisibleForTesting @SuppressWarnings("unused") public static TimeZone setCurrentTimeZone(TimeZone timeZone) { writeLock.lock(); try { - TimeZone oldTimeZone = CURRENT_TIMEZONE; - CURRENT_TIMEZONE = timeZone; + TimeZone oldTimeZone = SYSTEM_TIMEZONE; + SYSTEM_TIMEZONE = timeZone; return oldTimeZone; } finally { writeLock.unlock(); @@ -360,7 +362,7 @@ public static enum ConfVars implements ConfigKey { $CLI_ERROR_STOP("tajo.cli.error.stop", false), // Timezone & Date ---------------------------------------------------------- - $TIMEZONE("tajo.timezone", TimeZone.getDefault().getID()), + $TIMEZONE("tajo.timezone", TajoConstants.SYSTEM_DEFAULT_TIMEZONE), $DATE_ORDER("tajo.date.order", "YMD"), // FILE FORMAT diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java b/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java index 442db71e91..cfff369ee7 100644 --- a/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java +++ b/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java @@ -100,7 +100,7 @@ public String asChars() { } public byte[] asTextBytes() { - return toString().getBytes(); + return asChars().getBytes(); } public boolean isNumeric() { diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java b/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java index 17cfc7a9b8..11ba791625 100644 --- a/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java +++ b/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java @@ -27,7 +27,9 @@ import org.apache.tajo.util.datetime.DateTimeUtil; import org.apache.tajo.util.datetime.TimeMeta; +import javax.annotation.Nullable; import java.io.IOException; +import java.util.TimeZone; public class DatumFactory { @@ -282,6 +284,12 @@ public static TimeDatum createTime(String timeStr) { return new TimeDatum(DateTimeUtil.toJulianTime(timeStr)); } + public static TimeDatum createTime(String timeStr, TimeZone tz) { + TimeMeta tm = DateTimeUtil.decodeDateTime(timeStr); + DateTimeUtil.toUTCTimezone(tm, tz); + return new TimeDatum(DateTimeUtil.toTime(tm)); + } + public static TimestampDatum createTimestmpDatumWithJavaMillis(long millis) { return new TimestampDatum(DateTimeUtil.javaTimeToJulianTime(millis)); } @@ -294,6 +302,12 @@ public static TimestampDatum createTimestamp(String datetimeStr) { return new TimestampDatum(DateTimeUtil.toJulianTimestamp(datetimeStr)); } + public static TimestampDatum createTimestamp(String datetimeStr, TimeZone tz) { + TimeMeta tm = DateTimeUtil.decodeDateTime(datetimeStr); + DateTimeUtil.toUTCTimezone(tm, tz); + return new TimestampDatum(DateTimeUtil.toJulianTimestamp(tm)); + } + public static IntervalDatum createInterval(String intervalStr) { return new IntervalDatum(intervalStr); } @@ -318,13 +332,17 @@ public static DateDatum createDate(Datum datum) { } } - public static TimeDatum createTime(Datum datum) { + public static TimeDatum createTime(Datum datum, @Nullable TimeZone tz) { switch (datum.type()) { case INT8: return new TimeDatum(datum.asInt8()); + case CHAR: + case VARCHAR: case TEXT: TimeMeta tm = DateTimeFormat.parseDateTime(datum.asChars(), "HH24:MI:SS.MS"); - DateTimeUtil.toUTCTimezone(tm); + if (tz != null) { + DateTimeUtil.toUTCTimezone(tm, tz); + } return new TimeDatum(DateTimeUtil.toTime(tm)); case TIME: return (TimeDatum) datum; @@ -333,10 +351,12 @@ public static TimeDatum createTime(Datum datum) { } } - public static TimestampDatum createTimestamp(Datum datum) { + public static TimestampDatum createTimestamp(Datum datum, @Nullable TimeZone tz) { switch (datum.type()) { + case CHAR: + case VARCHAR: case TEXT: - return parseTimestamp(datum.asChars()); + return parseTimestamp(datum.asChars(), tz); case TIMESTAMP: return (TimestampDatum) datum; default: @@ -349,8 +369,8 @@ public static TimestampDatum createTimestamp(long julianTimestamp) { return new TimestampDatum(julianTimestamp); } - public static TimestampDatum parseTimestamp(String str) { - return new TimestampDatum(DateTimeUtil.toJulianTimestampWithTZ(str)); + public static TimestampDatum parseTimestamp(String str, @Nullable TimeZone tz) { + return new TimestampDatum(DateTimeUtil.toJulianTimestampWithTZ(str, tz)); } public static BlobDatum createBlob(byte[] val) { @@ -381,7 +401,7 @@ public static Inet4Datum createInet4(String val) { return new Inet4Datum(val); } - public static Datum cast(Datum operandDatum, DataType target) { + public static Datum cast(Datum operandDatum, DataType target, @Nullable TimeZone tz) { switch (target.getType()) { case BOOLEAN: return DatumFactory.createBool(operandDatum.asBool()); @@ -398,21 +418,24 @@ public static Datum cast(Datum operandDatum, DataType target) { return DatumFactory.createFloat4(operandDatum.asFloat4()); case FLOAT8: return DatumFactory.createFloat8(operandDatum.asFloat8()); + case VARCHAR: case TEXT: switch (operandDatum.type()) { case TIMESTAMP: { TimestampDatum timestampDatum = (TimestampDatum)operandDatum; - TimeMeta tm = timestampDatum.toTimeMeta(); - DateTimeUtil.toUserTimezone(tm); - TimestampDatum convertedTimestampDatum = new TimestampDatum(DateTimeUtil.toJulianTimestamp(tm)); - return DatumFactory.createText(convertedTimestampDatum.asTextBytes()); + if (tz != null) { + return DatumFactory.createText(timestampDatum.asChars(tz, false)); + } else { + return DatumFactory.createText(timestampDatum.asChars()); + } } case TIME: { TimeDatum timeDatum = (TimeDatum)operandDatum; - TimeMeta tm = timeDatum.toTimeMeta(); - DateTimeUtil.toUserTimezone(tm); - TimeDatum convertedTimeDatum = new TimeDatum(DateTimeUtil.toTime(tm)); - return DatumFactory.createText(convertedTimeDatum.asTextBytes()); + if (tz != null) { + return DatumFactory.createText(timeDatum.asChars(tz, false)); + } else { + return DatumFactory.createText(timeDatum.asChars()); + } } default: return DatumFactory.createText(operandDatum.asTextBytes()); @@ -420,9 +443,9 @@ public static Datum cast(Datum operandDatum, DataType target) { case DATE: return DatumFactory.createDate(operandDatum); case TIME: - return DatumFactory.createTime(operandDatum); + return DatumFactory.createTime(operandDatum, tz); case TIMESTAMP: - return DatumFactory.createTimestamp(operandDatum); + return DatumFactory.createTimestamp(operandDatum, tz); case BLOB: return DatumFactory.createBlob(operandDatum.asByteArray()); case INET4: diff --git a/tajo-common/src/main/java/org/apache/tajo/json/TimeZoneGsonSerdeAdapter.java b/tajo-common/src/main/java/org/apache/tajo/json/TimeZoneGsonSerdeAdapter.java new file mode 100644 index 0000000000..38441d35a1 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/json/TimeZoneGsonSerdeAdapter.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + */ +package org.apache.tajo.json; + +import com.google.gson.*; + +import java.lang.reflect.Type; +import java.util.TimeZone; + +public class TimeZoneGsonSerdeAdapter implements GsonSerDerAdapter { + + @Override + public JsonElement serialize(TimeZone object, Type arg1, JsonSerializationContext arg2) { + return new JsonPrimitive(object.getID()); + } + + @Override + public TimeZone deserialize(JsonElement arg0, Type arg1, JsonDeserializationContext context) throws JsonParseException { + return TimeZone.getTimeZone(arg0.getAsJsonPrimitive().getAsString()); + } +} diff --git a/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java b/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java index 459c9c96ac..917da17a68 100644 --- a/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java +++ b/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java @@ -19,10 +19,18 @@ package org.apache.tajo.storage; public class StorageConstants { - // table properties + + // Common table properties ------------------------------------------------- + + // time zone + public static final String TIMEZONE = "timezone"; + public static final String DEFAULT_TIMEZONE = "UTC"; + + // compression public static final String COMPRESSION_CODEC = "compression.codec"; public static final String COMPRESSION_TYPE = "compression.type"; + // Text file properties ------------------------------------------------- @Deprecated public static final String CSVFILE_DELIMITER = "csvfile.delimiter"; @Deprecated @@ -47,12 +55,14 @@ public class StorageConstants { public static final String TEXT_ERROR_TOLERANCE_MAXNUM = "text.error-tolerance.max-num"; public static final String DEFAULT_TEXT_ERROR_TOLERANCE_MAXNUM = "0"; + // Sequence file properties ------------------------------------------------- @Deprecated public static final String SEQUENCEFILE_DELIMITER = "sequencefile.delimiter"; @Deprecated public static final String SEQUENCEFILE_NULL = "sequencefile.null"; public static final String SEQUENCEFILE_SERDE = "sequencefile.serde"; + // RC file properties ------------------------------------------------- @Deprecated public static final String RCFILE_NULL = "rcfile.null"; public static final String RCFILE_SERDE = "rcfile.serde"; @@ -61,17 +71,22 @@ public class StorageConstants { public static final String DEFAULT_BINARY_SERDE = "org.apache.tajo.storage.BinarySerializerDeserializer"; public static final String DEFAULT_TEXT_SERDE = "org.apache.tajo.storage.TextSerializerDeserializer"; + + // Parquet file properties ------------------------------------------------- public static final String PARQUET_DEFAULT_BLOCK_SIZE; public static final String PARQUET_DEFAULT_PAGE_SIZE; public static final String PARQUET_DEFAULT_COMPRESSION_CODEC_NAME; public static final String PARQUET_DEFAULT_IS_DICTIONARY_ENABLED; public static final String PARQUET_DEFAULT_IS_VALIDATION_ENABLED; + public static final int DEFAULT_BLOCK_SIZE = 128 * 1024 * 1024; + public static final int DEFAULT_PAGE_SIZE = 1 * 1024 * 1024; + + + // Avro file properties ------------------------------------------------- public static final String AVRO_SCHEMA_LITERAL = "avro.schema.literal"; public static final String AVRO_SCHEMA_URL = "avro.schema.url"; - public static final int DEFAULT_BLOCK_SIZE = 128 * 1024 * 1024; - public static final int DEFAULT_PAGE_SIZE = 1 * 1024 * 1024; static { PARQUET_DEFAULT_BLOCK_SIZE = Integer.toString(DEFAULT_BLOCK_SIZE); PARQUET_DEFAULT_PAGE_SIZE = Integer.toString(DEFAULT_PAGE_SIZE); diff --git a/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeUtil.java index a9dc4e82e8..5226142a9c 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeUtil.java @@ -25,6 +25,7 @@ import org.apache.tajo.util.datetime.DateTimeConstants.DateToken; import org.apache.tajo.util.datetime.DateTimeConstants.TokenField; +import javax.annotation.Nullable; import java.util.TimeZone; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -34,7 +35,6 @@ * This Class is originated from j2date in datetime.c of PostgreSQL. */ public class DateTimeUtil { - private static int MAX_FRACTION_LENGTH = 6; /** maximum possible number of fields in a date * string */ @@ -663,16 +663,20 @@ public static long toJulianTimestamp(String str) { /** - * Parse datetime string to julian time. - * The result is the local time basis. + * Parse datetime string to UTC-based julian time. + * The result julian time is adjusted by local timezone. + * * @param timestampStr - * @return + * @param tz Local timezone. If it is NULL, UTC will be used by default. + * @return UTC-based julian time */ - public static long toJulianTimestampWithTZ(String timestampStr) { + public static long toJulianTimestampWithTZ(String timestampStr, @Nullable TimeZone tz) { long timestamp = DateTimeUtil.toJulianTimestamp(timestampStr); TimeMeta tm = new TimeMeta(); DateTimeUtil.toJulianTimeMeta(timestamp, tm); - DateTimeUtil.toUTCTimezone(tm); + if (tz != null) { + DateTimeUtil.toUTCTimezone(tm, tz); + } return DateTimeUtil.toJulianTimestamp(tm); } @@ -2060,10 +2064,6 @@ public static int date2isoyearday(int year, int mon, int mday) { return date2j(year, mon, mday) - isoweek2j(date2isoyear(year, mon, mday), 1) + 1; } - public static void toUserTimezone(TimeMeta tm) { - toUserTimezone(tm, TajoConf.getCurrentTimeZone()); - } - public static void toUserTimezone(TimeMeta tm, TimeZone timeZone) { tm.plusMillis(timeZone.getRawOffset()); } @@ -2073,6 +2073,10 @@ public static void toUTCTimezone(TimeMeta tm) { tm.plusMillis(0 - timeZone.getRawOffset()); } + public static void toUTCTimezone(TimeMeta tm, TimeZone timeZone) { + tm.plusMillis(0 - timeZone.getRawOffset()); + } + public static String getTimeZoneDisplayTime(TimeZone timeZone) { return getTimeZoneDisplayTime(timeZone.getRawOffset() / 1000); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgDouble.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgDouble.java index 5961076a8b..f337c368c0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgDouble.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgDouble.java @@ -52,9 +52,6 @@ public AvgContext newContext() { return new AvgContext(); } - public void init() { - } - @Override public void eval(FunctionContext ctx, Tuple params) { AvgContext avgCtx = (AvgContext) ctx; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentDate.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentDate.java index f43a41f430..49295f5b54 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentDate.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentDate.java @@ -18,17 +18,24 @@ package org.apache.tajo.engine.function.datetime; +import com.google.gson.annotations.Expose; +import org.apache.tajo.OverridableConf; +import org.apache.tajo.SessionVars; +import org.apache.tajo.TajoConstants; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.DateDatum; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; +import org.apache.tajo.plan.expr.FunctionEval; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.storage.Tuple; import org.apache.tajo.util.datetime.DateTimeUtil; import org.apache.tajo.util.datetime.TimeMeta; +import java.util.TimeZone; + @Description( functionName = "current_date", description = "Get current date. Result is DATE type.", @@ -37,19 +44,26 @@ paramTypes = {@ParamTypes(paramTypes = {})} ) public class CurrentDate extends GeneralFunction { - DateDatum datum; + @Expose private TimeZone timezone; + private DateDatum datum; public CurrentDate() { super(NoArgs); } + @Override + public void init(OverridableConf context, FunctionEval.ParamType[] types) { + String timezoneId = context.get(SessionVars.TZ, TajoConstants.SYSTEM_DEFAULT_TIMEZONE); + timezone = TimeZone.getTimeZone(timezoneId); + } + @Override public Datum eval(Tuple params) { if (datum == null) { long julianTimestamp = DateTimeUtil.javaTimeToJulianTime(System.currentTimeMillis()); TimeMeta tm = new TimeMeta(); DateTimeUtil.toJulianTimeMeta(julianTimestamp, tm); - DateTimeUtil.toUserTimezone(tm); + DateTimeUtil.toUserTimezone(tm, timezone); datum = DatumFactory.createDate(tm.years, tm.monthOfYear, tm.dayOfMonth); } return datum; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTime.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTime.java index 66a54bf043..462c38d1ce 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTime.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTime.java @@ -18,20 +18,28 @@ package org.apache.tajo.engine.function.datetime; +import com.google.gson.annotations.Expose; +import org.apache.tajo.OverridableConf; +import org.apache.tajo.SessionVars; +import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.datum.TimeDatum; -import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; +import org.apache.tajo.plan.expr.FunctionEval; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.storage.Tuple; import org.apache.tajo.util.datetime.DateTimeConstants; import org.apache.tajo.util.datetime.DateTimeUtil; import org.apache.tajo.util.datetime.TimeMeta; +import java.util.TimeZone; + import static org.apache.tajo.common.TajoDataTypes.Type.FLOAT8; import static org.apache.tajo.common.TajoDataTypes.Type.TEXT; @@ -44,6 +52,9 @@ paramTypes = {@ParamTypes(paramTypes = {TajoDataTypes.Type.TEXT, TajoDataTypes.Type.TIME})} ) public class DatePartFromTime extends GeneralFunction { + @Expose private TimeZone timezone; + private DatePartExtractorFromTime extractor = null; + public DatePartFromTime() { super(new Column[] { new Column("target", FLOAT8), @@ -51,7 +62,11 @@ public DatePartFromTime() { }); } - private DatePartExtractorFromTime extractor = null; + @Override + public void init(OverridableConf context, FunctionEval.ParamType [] types) { + String timezoneId = context.get(SessionVars.TZ, TajoConstants.SYSTEM_DEFAULT_TIMEZONE); + timezone = TimeZone.getTimeZone(timezoneId); + } @Override public Datum eval(Tuple params) { @@ -93,7 +108,7 @@ public Datum eval(Tuple params) { } TimeMeta tm = time.toTimeMeta(); - DateTimeUtil.toUserTimezone(tm); + DateTimeUtil.toUserTimezone(tm, timezone); return extractor.extract(tm); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTimestamp.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTimestamp.java index 843c593b87..89526983d7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTimestamp.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTimestamp.java @@ -18,9 +18,14 @@ package org.apache.tajo.engine.function.datetime; +import com.google.gson.annotations.Expose; +import org.apache.tajo.OverridableConf; +import org.apache.tajo.SessionVars; +import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.*; +import org.apache.tajo.plan.expr.FunctionEval; import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; @@ -29,6 +34,8 @@ import org.apache.tajo.util.datetime.DateTimeUtil; import org.apache.tajo.util.datetime.TimeMeta; +import java.util.TimeZone; + import static org.apache.tajo.common.TajoDataTypes.Type.*; @Description( @@ -40,6 +47,9 @@ paramTypes = {@ParamTypes(paramTypes = {TajoDataTypes.Type.TEXT, TajoDataTypes.Type.TIMESTAMP})} ) public class DatePartFromTimestamp extends GeneralFunction { + @Expose private TimeZone timezone; + private DatePartExtractorFromTimestamp extractor = null; + public DatePartFromTimestamp() { super(new Column[] { new Column("target", FLOAT8), @@ -47,7 +57,11 @@ public DatePartFromTimestamp() { }); } - private DatePartExtractorFromTimestamp extractor = null; + @Override + public void init(OverridableConf context, FunctionEval.ParamType [] types) { + String timezoneId = context.get(SessionVars.TZ, TajoConstants.SYSTEM_DEFAULT_TIMEZONE); + timezone = TimeZone.getTimeZone(timezoneId); + } @Override public Datum eval(Tuple params) { @@ -115,7 +129,7 @@ public Datum eval(Tuple params) { } TimeMeta tm = timestamp.toTimeMeta(); - DateTimeUtil.toUserTimezone(tm); + DateTimeUtil.toUserTimezone(tm, timezone); return extractor.extract(tm); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java index 02b52e37c4..8dbddb9bc8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java @@ -18,6 +18,10 @@ package org.apache.tajo.engine.function.datetime; +import com.google.gson.annotations.Expose; +import org.apache.tajo.OverridableConf; +import org.apache.tajo.SessionVars; +import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; @@ -33,6 +37,9 @@ import org.apache.tajo.util.datetime.DateTimeUtil; import org.apache.tajo.util.datetime.TimeMeta; +import java.util.Objects; +import java.util.TimeZone; + import static org.apache.tajo.common.TajoDataTypes.Type.TEXT; import static org.apache.tajo.common.TajoDataTypes.Type.TIMESTAMP; @@ -45,6 +52,8 @@ paramTypes = {@ParamTypes(paramTypes = {TajoDataTypes.Type.TIMESTAMP, TajoDataTypes.Type.TEXT})} ) public class ToCharTimestamp extends GeneralFunction { + @Expose private TimeZone timezone; + public ToCharTimestamp() { super(new Column[] { new Column("timestamp", TIMESTAMP), @@ -53,7 +62,9 @@ public ToCharTimestamp() { } @Override - public void init(FunctionEval.ParamType[] paramTypes) { + public void init(OverridableConf context, FunctionEval.ParamType[] paramTypes) { + String timezoneId = context.get(SessionVars.TZ, TajoConstants.SYSTEM_DEFAULT_TIMEZONE); + timezone = TimeZone.getTimeZone(timezoneId); } @Override @@ -63,11 +74,11 @@ public Datum eval(Tuple params) { } TimestampDatum valueDatum = (TimestampDatum) params.get(0); - TimeMeta tm = valueDatum.toTimeMeta(); - DateTimeUtil.toUserTimezone(tm); - Datum pattern = params.get(1); + TimeMeta tm = valueDatum.toTimeMeta(); + DateTimeUtil.toUserTimezone(tm, timezone); + return DatumFactory.createText(DateTimeFormat.to_char(tm, pattern.asChars())); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/BTrim.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/BTrim.java index 3214a5eef0..9864b0de21 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/BTrim.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/BTrim.java @@ -20,6 +20,7 @@ import com.google.gson.annotations.Expose; import org.apache.commons.lang.StringUtils; +import org.apache.tajo.OverridableConf; import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; @@ -57,7 +58,8 @@ public BTrim() { }); } - public void init(FunctionEval.ParamType[] paramTypes) { + @Override + public void init(OverridableConf context, FunctionEval.ParamType[] paramTypes) { if (paramTypes.length == 2) { hasTrimCharacters = true; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/LTrim.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/LTrim.java index f3452939fc..5d95c38ec1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/LTrim.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/LTrim.java @@ -20,6 +20,7 @@ import com.google.gson.annotations.Expose; import org.apache.commons.lang.StringUtils; +import org.apache.tajo.OverridableConf; import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; @@ -57,7 +58,8 @@ public LTrim() { }); } - public void init(FunctionEval.ParamType[] paramTypes) { + @Override + public void init(OverridableConf context, FunctionEval.ParamType[] paramTypes) { if (paramTypes.length == 2) { hasTrimCharacters = true; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Lpad.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Lpad.java index 58c0951867..da11f194c0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Lpad.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Lpad.java @@ -20,6 +20,7 @@ import com.google.gson.annotations.Expose; import org.apache.commons.lang.StringUtils; +import org.apache.tajo.OverridableConf; import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; @@ -57,7 +58,8 @@ public Lpad() { }); } - public void init(FunctionEval.ParamType[] paramTypes) { + @Override + public void init(OverridableConf context, FunctionEval.ParamType[] paramTypes) { if (paramTypes.length == 3) { hasFillCharacters = true; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/RTrim.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/RTrim.java index 8e295fa7fa..03b9c25893 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/RTrim.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/RTrim.java @@ -20,6 +20,7 @@ import com.google.gson.annotations.Expose; import org.apache.commons.lang.StringUtils; +import org.apache.tajo.OverridableConf; import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; @@ -56,7 +57,8 @@ public RTrim() { }); } - public void init(FunctionEval.ParamType[] paramTypes) { + @Override + public void init(OverridableConf context, FunctionEval.ParamType[] paramTypes) { if (paramTypes.length == 2) { hasTrimCharacters = true; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/RegexpReplace.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/RegexpReplace.java index da06f732a3..6b888ec776 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/RegexpReplace.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/RegexpReplace.java @@ -19,6 +19,7 @@ package org.apache.tajo.engine.function.string; import com.google.gson.annotations.Expose; +import org.apache.tajo.OverridableConf; import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.BooleanDatum; @@ -70,7 +71,8 @@ public RegexpReplace() { }); } - public void init(ParamType[] paramTypes) { + @Override + public void init(OverridableConf context, ParamType[] paramTypes) { if (paramTypes[0] == ParamType.NULL || paramTypes[1] == ParamType.NULL || paramTypes[2] == ParamType.NULL) { isAlwaysNull = true; } else if (paramTypes[1] == ParamType.CONSTANT) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Rpad.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Rpad.java index 9d42848fa9..15091182e6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Rpad.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Rpad.java @@ -20,6 +20,7 @@ import com.google.gson.annotations.Expose; import org.apache.commons.lang.StringUtils; +import org.apache.tajo.OverridableConf; import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; @@ -58,7 +59,8 @@ public Rpad() { }); } - public void init(FunctionEval.ParamType[] paramTypes) { + @Override + public void init(OverridableConf context, FunctionEval.ParamType[] paramTypes) { if (paramTypes.length == 3) { hasFillCharacters = true; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/json/CoreGsonHelper.java b/tajo-core/src/main/java/org/apache/tajo/engine/json/CoreGsonHelper.java index 57f2536058..5a03bfd253 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/json/CoreGsonHelper.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/json/CoreGsonHelper.java @@ -38,6 +38,7 @@ import java.lang.reflect.Type; import java.util.Map; +import java.util.TimeZone; public class CoreGsonHelper { private static Gson gson; @@ -58,6 +59,7 @@ private static Map registerAdapters() { adapters.put(AggFunction.class, new FunctionAdapter()); adapters.put(Datum.class, new DatumAdapter()); adapters.put(DataType.class, new DataTypeAdapter()); + adapters.put(TimeZone.class, new TimeZoneGsonSerdeAdapter()); return adapters; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java index d8f7f080b8..47ead40fdd 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java @@ -74,12 +74,12 @@ public QueryContext(TajoConf conf) { } public QueryContext(TajoConf conf, Session session) { - super(conf); + super(conf, ConfigKey.ConfigType.QUERY, ConfigKey.ConfigType.SESSION); putAll(session.getAllVariables()); } public QueryContext(TajoConf conf, KeyValueSetProto proto) { - super(conf, proto, ConfigKey.ConfigType.QUERY); + super(conf, proto, ConfigKey.ConfigType.QUERY, ConfigKey.ConfigType.SESSION); } //----------------------------------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index df9ef65e68..c89c146790 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -58,6 +58,7 @@ import java.io.IOException; import java.util.List; +import java.util.TimeZone; import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME; @@ -77,6 +78,10 @@ public static String getUserTimeZoneDisplay() { return DateTimeUtil.getTimeZoneDisplayTime(TajoConf.getCurrentTimeZone()); } + public static String getUserTimeZoneDisplay(TimeZone tz) { + return DateTimeUtil.getTimeZoneDisplayTime(tz); + } + public ExprTestBase() { } @@ -164,7 +169,7 @@ private static Target[] getRawTargets(QueryContext context, String query, boolea assertJsonSerDer(t.getEvalTree()); } for (Target t : targets) { - assertEvalTreeProtoSerDer(t.getEvalTree()); + assertEvalTreeProtoSerDer(context, t.getEvalTree()); } return targets; } @@ -173,8 +178,19 @@ public void testSimpleEval(String query, String [] expected) throws IOException testEval(null, null, null, query, expected); } - public void testSimpleEval(String query, String [] expected, boolean condition) throws IOException { - testEval(null, null, null, null, query, expected, ',', condition); + public void testSimpleEval(OverridableConf context, String query, String [] expected) throws IOException { + testEval(context, null, null, null, query, expected); + } + + public void testSimpleEval(String query, String [] expected, boolean successOrFail) + throws IOException { + + testEval(null, null, null, null, query, expected, ',', successOrFail); + } + + public void testSimpleEval(OverridableConf context, String query, String [] expected, boolean successOrFail) + throws IOException { + testEval(context, null, null, null, query, expected, ',', successOrFail); } public void testEval(Schema schema, String tableName, String csvTuple, String query, String [] expected) @@ -183,10 +199,10 @@ public void testEval(Schema schema, String tableName, String csvTuple, String qu expected, ',', true); } - public void testEval(OverridableConf overideConf, Schema schema, String tableName, String csvTuple, String query, + public void testEval(OverridableConf context, Schema schema, String tableName, String csvTuple, String query, String [] expected) throws IOException { - testEval(overideConf, schema, tableName != null ? CatalogUtil.normalizeIdentifier(tableName) : null, csvTuple, + testEval(context, schema, tableName != null ? CatalogUtil.normalizeIdentifier(tableName) : null, csvTuple, query, expected, ',', true); } @@ -196,16 +212,19 @@ public void testEval(Schema schema, String tableName, String csvTuple, String qu query, expected, delimiter, condition); } - public void testEval(OverridableConf overideConf, Schema schema, String tableName, String csvTuple, String query, + public void testEval(OverridableConf context, Schema schema, String tableName, String csvTuple, String query, String [] expected, char delimiter, boolean condition) throws IOException { - QueryContext context; - if (overideConf == null) { - context = LocalTajoTestingUtility.createDummyContext(conf); + QueryContext queryContext; + if (context == null) { + queryContext = LocalTajoTestingUtility.createDummyContext(conf); } else { - context = LocalTajoTestingUtility.createDummyContext(conf); - context.putAll(overideConf); + queryContext = LocalTajoTestingUtility.createDummyContext(conf); + queryContext.putAll(context); } + String timezoneId = queryContext.get(SessionVars.TZ); + TimeZone timeZone = TimeZone.getTimeZone(timezoneId); + LazyTuple lazyTuple; VTuple vtuple = null; String qualifiedTableName = @@ -230,8 +249,8 @@ public void testEval(OverridableConf overideConf, Schema schema, String tableNam boolean nullDatum; Datum datum = lazyTuple.get(i); nullDatum = (datum instanceof TextDatum || datum instanceof CharDatum); - nullDatum = nullDatum && datum.asChars().equals("") || - datum.asChars().equals(context.get(SessionVars.NULL_CHAR)); + nullDatum = nullDatum && + datum.asChars().equals("") || datum.asChars().equals(queryContext.get(SessionVars.NULL_CHAR)); nullDatum |= datum.isNull(); if (nullDatum) { @@ -249,10 +268,10 @@ public void testEval(OverridableConf overideConf, Schema schema, String tableNam TajoClassLoader classLoader = new TajoClassLoader(); try { - targets = getRawTargets(context, query, condition); + targets = getRawTargets(queryContext, query, condition); EvalCodeGenerator codegen = null; - if (context.getBool(SessionVars.CODEGEN)) { + if (queryContext.getBool(SessionVars.CODEGEN)) { codegen = new EvalCodeGenerator(classLoader); } @@ -260,7 +279,7 @@ public void testEval(OverridableConf overideConf, Schema schema, String tableNam for (int i = 0; i < targets.length; i++) { EvalNode eval = targets[i].getEvalTree(); - if (context.getBool(SessionVars.CODEGEN)) { + if (queryContext.getBool(SessionVars.CODEGEN)) { eval = codegen.compile(inputSchema, eval); } @@ -277,9 +296,9 @@ public void testEval(OverridableConf overideConf, Schema schema, String tableNam Datum datum = outTuple.get(i); String outTupleAsChars; if (datum.type() == Type.TIMESTAMP) { - outTupleAsChars = ((TimestampDatum) datum).asChars(TajoConf.getCurrentTimeZone(), true); + outTupleAsChars = ((TimestampDatum) datum).asChars(timeZone, false); } else if (datum.type() == Type.TIME) { - outTupleAsChars = ((TimeDatum) datum).asChars(TajoConf.getCurrentTimeZone(), true); + outTupleAsChars = ((TimeDatum) datum).asChars(timeZone, false); } else { outTupleAsChars = datum.asChars(); } @@ -302,8 +321,8 @@ public void testEval(OverridableConf overideConf, Schema schema, String tableNam } } - public static void assertEvalTreeProtoSerDer(EvalNode evalNode) { + public static void assertEvalTreeProtoSerDer(OverridableConf context, EvalNode evalNode) { PlanProto.EvalTree converted = EvalTreeProtoSerializer.serialize(evalNode); - assertEquals(evalNode, EvalTreeProtoDeserializer.deserialize(converted)); + assertEquals(evalNode, EvalTreeProtoDeserializer.deserialize(context, converted)); } } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestSQLExpression.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestSQLExpression.java index 6c9892a704..4a9e42ceef 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestSQLExpression.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestSQLExpression.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.eval; +import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.exception.NoSuchFunctionException; @@ -25,6 +26,7 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.TimestampDatum; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.util.datetime.DateTimeUtil; import org.junit.Test; @@ -853,41 +855,46 @@ public void testSigned() throws IOException { @Test public void testCastWithNestedFunction() throws IOException { - int unixtime = 1389071574;//(int) (System.currentTimeMillis() / 1000); + QueryContext context = new QueryContext(getConf()); + context.put(SessionVars.TZ, "GMT-6"); + TimeZone tz = TimeZone.getTimeZone("GMT-6"); + + int unixtime = 1389071574; // (int) (System.currentTimeMillis() / 1000); TimestampDatum expected = DatumFactory.createTimestmpDatumWithUnixTime(unixtime); - testSimpleEval(String.format("select to_timestamp(CAST(split_part('%d.999', '.', 1) as INT8));", unixtime), - new String[] {expected.asChars(TajoConf.getCurrentTimeZone(), true)}); + testSimpleEval(context, String.format("select to_timestamp(CAST(split_part('%d.999', '.', 1) as INT8));", unixtime), + new String[] {expected.asChars(tz, false)}); } @Test public void testCastFromTable() throws IOException { - TimeZone originTimeZone = TajoConf.setCurrentTimeZone(TimeZone.getTimeZone("GMT-6")); - try { - Schema schema = new Schema(); - schema.addColumn("col1", TEXT); - schema.addColumn("col2", TEXT); - - testEval(schema, "table1", "123,234", "select cast(col1 as float) as b, cast(col2 as float) as a from table1", - new String[]{"123.0", "234.0"}); - testEval(schema, "table1", "123,234", "select col1::float, col2::float from table1", - new String[]{"123.0", "234.0"}); - - TimestampDatum timestamp = DatumFactory.createTimestamp("1980-04-01 01:50:01" + - DateTimeUtil.getTimeZoneDisplayTime(TajoConf.getCurrentTimeZone())); - - testEval(schema, "table1", "1980-04-01 01:50:01,234", "select col1::timestamp as t1, col2::float from table1 " + - "where t1 = '1980-04-01 01:50:01'::timestamp", - new String[]{timestamp.asChars(TajoConf.getCurrentTimeZone(), true), "234.0"} - ); - - testSimpleEval("select '1980-04-01 01:50:01'::timestamp;", - new String[]{timestamp.asChars(TajoConf.getCurrentTimeZone(), true)}); - testSimpleEval("select '1980-04-01 01:50:01'::timestamp::text", new String[]{"1980-04-01 01:50:01"}); - - testSimpleEval("select (cast ('99999'::int8 as text))::int4 + 1", new String[]{"100000"}); - } finally { - TajoConf.setCurrentTimeZone(originTimeZone); - } + QueryContext queryContext = new QueryContext(getConf()); + queryContext.put(SessionVars.TZ, "GMT-6"); + TimeZone tz = TimeZone.getTimeZone("GMT-6"); + + Schema schema = new Schema(); + schema.addColumn("col1", TEXT); + schema.addColumn("col2", TEXT); + + testEval(queryContext, schema, + "table1", + "123,234", + "select cast(col1 as float) as b, cast(col2 as float) as a from table1", + new String[]{"123.0", "234.0"}); + testEval(queryContext, schema, "table1", "123,234", "select col1::float, col2::float from table1", + new String[]{"123.0", "234.0"}); + + TimestampDatum timestamp = DatumFactory.createTimestamp("1980-04-01 01:50:01" + + DateTimeUtil.getTimeZoneDisplayTime(tz)); + + testEval(queryContext, schema, "table1", "1980-04-01 01:50:01,234", + "select col1::timestamp as t1, col2::float from table1 where t1 = '1980-04-01 01:50:01'::timestamp", + new String[]{timestamp.asChars(tz, false), "234.0"} + ); + + testSimpleEval("select '1980-04-01 01:50:01'::timestamp;", new String[]{timestamp.asChars(tz, false)}); + testSimpleEval("select '1980-04-01 01:50:01'::timestamp::text", new String[]{"1980-04-01 01:50:01"}); + + testSimpleEval("select (cast ('99999'::int8 as text))::int4 + 1", new String[]{"100000"}); } @Test diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java index 7cca13d2a4..54704cf869 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java @@ -19,11 +19,13 @@ package org.apache.tajo.engine.function; +import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.Schema; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.TimestampDatum; import org.apache.tajo.engine.eval.ExprTestBase; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.util.datetime.DateTimeUtil; import org.apache.tajo.util.datetime.TimeMeta; import org.junit.Test; @@ -34,6 +36,7 @@ import java.util.TimeZone; import static org.apache.tajo.common.TajoDataTypes.Type.*; +import static org.junit.Assert.assertEquals; public class TestDateTimeFunctions extends ExprTestBase { @Test @@ -43,7 +46,7 @@ public void testToTimestamp() throws IOException { // (expectedTimestamp / 1000) means the translation from millis seconds to unix timestamp String q1 = String.format("select to_timestamp(%d);", (expectedTimestamp / 1000)); - testSimpleEval(q1, new String[]{expected.toString(TajoConf.getCurrentTimeZone(), true)}); + testSimpleEval(q1, new String[]{expected.toString()}); testSimpleEval("select to_timestamp('1997-12-30 11:40:50.345', 'YYYY-MM-DD HH24:MI:SS.MS');", new String[]{"1997-12-30 11:40:50.345" + getUserTimeZoneDisplay()}); @@ -212,6 +215,10 @@ public void testExtract() throws IOException { @Test public void testDatePart() throws IOException { + TimeZone timeZone = TimeZone.getDefault(); + assertEquals("UTC", timeZone.getID()); + assertEquals("UTC", TajoConf.getCurrentTimeZone().getID()); + Schema schema2 = new Schema(); schema2.addColumn("col1", TIMESTAMP); testEval(schema2, "table1", @@ -387,45 +394,49 @@ public void testAddDays() throws IOException { @Test public void testDateTimeNow() throws IOException { - TimeZone originTimeZone = TajoConf.setCurrentTimeZone(TimeZone.getTimeZone("GMT-6")); - TimeZone systemOriginTimeZone = TimeZone.getDefault(); + TimeZone originalTimezone = TimeZone.getDefault(); TimeZone.setDefault(TimeZone.getTimeZone("GMT-6")); + + QueryContext context = new QueryContext(getConf()); + context.put(SessionVars.TZ, "GMT-6"); + try { Date expectedDate = new Date(System.currentTimeMillis()); - testSimpleEval("select to_char(now(), 'yyyy-MM-dd');", + testSimpleEval(context, "select to_char(now(), 'yyyy-MM-dd');", new String[]{dateFormat(expectedDate, "yyyy-MM-dd")}); - testSimpleEval("select cast(extract(year from now()) as INT4);", + testSimpleEval(context, "select cast(extract(year from now()) as INT4);", new String[]{dateFormat(expectedDate, "yyyy")}); - testSimpleEval("select current_date();", + testSimpleEval(context, "select current_date();", new String[]{dateFormat(expectedDate, "yyyy-MM-dd")}); - testSimpleEval("select cast(extract(hour from current_time()) as INT4);", + testSimpleEval(context, "select cast(extract(hour from current_time()) as INT4);", new String[]{String.valueOf(Integer.parseInt(dateFormat(expectedDate, "HH")))}); } finally { - TajoConf.setCurrentTimeZone(originTimeZone); - TimeZone.setDefault(systemOriginTimeZone); + TimeZone.setDefault(originalTimezone); } } @Test public void testTimeValueKeyword() throws IOException { - TimeZone originTimeZone = TajoConf.setCurrentTimeZone(TimeZone.getTimeZone("GMT-6")); - TimeZone systemOriginTimeZone = TimeZone.getDefault(); + TimeZone originTimeZone = TimeZone.getDefault(); TimeZone.setDefault(TimeZone.getTimeZone("GMT-6")); + + QueryContext context = new QueryContext(getConf()); + context.put(SessionVars.TZ, "GMT-6"); + try { Date expectedDate = new Date(System.currentTimeMillis()); - testSimpleEval("select to_char(current_timestamp, 'yyyy-MM-dd');", + testSimpleEval(context, "select to_char(current_timestamp, 'yyyy-MM-dd');", new String[]{dateFormat(expectedDate, "yyyy-MM-dd")}); - testSimpleEval("select cast(extract(year from current_timestamp) as INT4);", + testSimpleEval(context, "select cast(extract(year from current_timestamp) as INT4);", new String[]{dateFormat(expectedDate, "yyyy")}); - testSimpleEval("select current_date;", + testSimpleEval(context, "select current_date;", new String[]{dateFormat(expectedDate, "yyyy-MM-dd")}); - testSimpleEval("select cast(extract(hour from current_time) as INT4);", + testSimpleEval(context, "select cast(extract(hour from current_time) as INT4);", new String[]{String.valueOf(Integer.parseInt(dateFormat(expectedDate, "HH")))}); } finally { - TajoConf.setCurrentTimeZone(originTimeZone); - TimeZone.setDefault(systemOriginTimeZone); + TimeZone.setDefault(originTimeZone); } } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java index 37454e633c..dd738e7177 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java @@ -125,8 +125,7 @@ public final void testSortAfterGroupbyWithAlias() throws Exception { public final void testSortWithDate() throws Exception { // skip this test if catalog uses HCatalogStore. // It is because HCatalogStore does not support Time data type. - TimeZone oldTimeZone = TajoConf.setCurrentTimeZone(TimeZone.getTimeZone("UTC")); - TimeZone systemOldTimeZone = TimeZone.getDefault(); + TimeZone originalTimezone = TimeZone.getDefault(); TimeZone.setDefault(TimeZone.getTimeZone("UTC")); try { if (!testingCluster.isHCatalogStoreRunning()) { @@ -138,8 +137,7 @@ public final void testSortWithDate() throws Exception { cleanupQuery(res); } } finally { - TajoConf.setCurrentTimeZone(oldTimeZone); - TimeZone.setDefault(systemOldTimeZone); + TimeZone.setDefault(originalTimezone); } } diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java index f491c63d51..bcff78e63f 100644 --- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java +++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java @@ -133,10 +133,7 @@ public void testDateTimeType() throws Exception { // Hcatalog does not support date type, time type in hive-0.12.0 if(util.isHCatalogStoreRunning()) return; - TimeZone tajoCurrentTimeZone = TajoConf.getCurrentTimeZone(); - TajoConf.setCurrentTimeZone(TimeZone.getTimeZone("UTC")); - - TimeZone systemCurrentTimeZone = TimeZone.getDefault(); + TimeZone originalTimezone = TimeZone.getDefault(); TimeZone.setDefault(TimeZone.getTimeZone("UTC")); ResultSet res = null; @@ -212,8 +209,8 @@ public void testDateTimeType() throws Exception { assertNotNull(timestamp); assertEquals("2014-01-01 10:00:00.0", timestamp.toString()); } finally { - TajoConf.setCurrentTimeZone(tajoCurrentTimeZone); - TimeZone.setDefault(systemCurrentTimeZone); + TimeZone.setDefault(originalTimezone); + if (res != null) { res.close(); } diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java index a004baa829..d9c1354d60 100644 --- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java +++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java @@ -543,8 +543,7 @@ public void testSortWithDateTime() throws Exception { // skip this test if catalog uses HCatalogStore. // It is because HCatalogStore does not support Time data type. - TimeZone oldTimeZone = TajoConf.setCurrentTimeZone(TimeZone.getTimeZone("UTC")); - TimeZone systemOldTimeZone = TimeZone.getDefault(); + TimeZone originalTimezone = TimeZone.getDefault(); TimeZone.setDefault(TimeZone.getTimeZone("UTC")); try { @@ -575,8 +574,7 @@ public void testSortWithDateTime() throws Exception { } } finally { - TajoConf.setCurrentTimeZone(oldTimeZone); - TimeZone.setDefault(systemOldTimeZone); + TimeZone.setDefault(originalTimezone); cleanupQuery(res); if (stmt != null) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java index 406cdfc535..62ba33c361 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java @@ -19,6 +19,8 @@ package org.apache.tajo.plan; import com.google.common.collect.Sets; +import org.apache.tajo.OverridableConf; +import org.apache.tajo.SessionVars; import org.apache.tajo.algebra.*; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.CatalogUtil; @@ -43,6 +45,7 @@ import java.util.Set; import java.util.Stack; +import java.util.TimeZone; import static org.apache.tajo.algebra.WindowSpec.WindowFrameEndBoundType; import static org.apache.tajo.algebra.WindowSpec.WindowFrameStartBoundType; @@ -64,11 +67,16 @@ public ExprAnnotator(CatalogService catalog) { } static class Context { + OverridableConf queryContext; + TimeZone timeZone; LogicalPlan plan; LogicalPlan.QueryBlock currentBlock; NameResolvingMode columnRsvLevel; public Context(LogicalPlanner.PlanContext planContext, NameResolvingMode colRsvLevel) { + this.queryContext = planContext.queryContext; + this.timeZone = planContext.timeZone; + this.plan = planContext.plan; this.currentBlock = planContext.queryBlock; this.columnRsvLevel = colRsvLevel; @@ -95,7 +103,7 @@ public static void assertEval(boolean condition, String message) throws Planning * @param rhs right hand side term * @return a pair including left/right hand side terms */ - public static Pair convertTypesIfNecessary(EvalNode lhs, EvalNode rhs) { + private static Pair convertTypesIfNecessary(Context ctx, EvalNode lhs, EvalNode rhs) { Type lhsType = lhs.getValueType().getType(); Type rhsType = rhs.getValueType().getType(); @@ -108,10 +116,10 @@ public static Pair convertTypesIfNecessary(EvalNode lhs, Eva if (toBeCasted != null) { // if not null, one of either should be converted to another type. // Overwrite lhs, rhs, or both with cast expression. if (lhsType != toBeCasted) { - lhs = convertType(lhs, CatalogUtil.newSimpleDataType(toBeCasted)); + lhs = convertType(ctx, lhs, CatalogUtil.newSimpleDataType(toBeCasted)); } if (rhsType != toBeCasted) { - rhs = convertType(rhs, CatalogUtil.newSimpleDataType(toBeCasted)); + rhs = convertType(ctx, rhs, CatalogUtil.newSimpleDataType(toBeCasted)); } } @@ -126,7 +134,7 @@ public static Pair convertTypesIfNecessary(EvalNode lhs, Eva * @param toType target type * @return type converted expression. */ - private static EvalNode convertType(EvalNode evalNode, DataType toType) { + private static EvalNode convertType(Context ctx, EvalNode evalNode, DataType toType) { // if original and toType is the same, we don't need type conversion. if (evalNode.getValueType().equals(toType)) { @@ -140,9 +148,9 @@ private static EvalNode convertType(EvalNode evalNode, DataType toType) { if (evalNode.getType() == EvalType.BETWEEN) { BetweenPredicateEval between = (BetweenPredicateEval) evalNode; - between.setPredicand(convertType(between.getPredicand(), toType)); - between.setBegin(convertType(between.getBegin(), toType)); - between.setEnd(convertType(between.getEnd(), toType)); + between.setPredicand(convertType(ctx, between.getPredicand(), toType)); + between.setBegin(convertType(ctx, between.getBegin(), toType)); + between.setEnd(convertType(ctx, between.getEnd(), toType)); return between; @@ -150,11 +158,11 @@ private static EvalNode convertType(EvalNode evalNode, DataType toType) { CaseWhenEval caseWhenEval = (CaseWhenEval) evalNode; for (CaseWhenEval.IfThenEval ifThen : caseWhenEval.getIfThenEvals()) { - ifThen.setResult(convertType(ifThen.getResult(), toType)); + ifThen.setResult(convertType(ctx, ifThen.getResult(), toType)); } if (caseWhenEval.hasElse()) { - caseWhenEval.setElseResult(convertType(caseWhenEval.getElse(), toType)); + caseWhenEval.setElseResult(convertType(ctx, caseWhenEval.getElse(), toType)); } return caseWhenEval; @@ -166,7 +174,7 @@ private static EvalNode convertType(EvalNode evalNode, DataType toType) { Datum[] convertedDatum = new Datum[datums.length]; for (int i = 0; i < datums.length; i++) { - convertedDatum[i] = DatumFactory.cast(datums[i], toType); + convertedDatum[i] = DatumFactory.cast(datums[i], toType, ctx.timeZone); } RowConstantEval convertedRowConstant = new RowConstantEval(convertedDatum); @@ -175,11 +183,11 @@ private static EvalNode convertType(EvalNode evalNode, DataType toType) { } else if (evalNode.getType() == EvalType.CONST) { ConstEval original = (ConstEval) evalNode; - ConstEval newConst = new ConstEval(DatumFactory.cast(original.getValue(), toType)); + ConstEval newConst = new ConstEval(DatumFactory.cast(original.getValue(), toType, ctx.timeZone)); return newConst; } else { - return new CastEval(evalNode, toType); + return new CastEval(ctx.queryContext, evalNode, toType); } } @@ -279,7 +287,7 @@ public EvalNode visitCommonComparison(Context ctx, Stack stack, BinaryOper throw new IllegalStateException("Wrong Expr Type: " + expr.getType()); } - return createBinaryNode(evalType, left, right); + return createBinaryNode(ctx, evalType, left, right); } /////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -308,7 +316,7 @@ public EvalNode visitBetween(Context ctx, Stack stack, BetweenPredicate be between.isSymmetric(), predicand, begin, end); - betweenEval = (BetweenPredicateEval) convertType(betweenEval, widestType); + betweenEval = (BetweenPredicateEval) convertType(ctx, betweenEval, widestType); return betweenEval; } @@ -342,7 +350,7 @@ public EvalNode visitCaseWhen(Context ctx, Stack stack, CaseWhenPredicate assertEval(widestType != null, "Invalid Type Conversion for CaseWhen"); // implicit type conversion - caseWhenEval = (CaseWhenEval) convertType(caseWhenEval, widestType); + caseWhenEval = (CaseWhenEval) convertType(ctx, caseWhenEval, widestType); return caseWhenEval; } @@ -362,7 +370,7 @@ public EvalNode visitInPredicate(Context ctx, Stack stack, InPredicate exp RowConstantEval rowConstantEval = (RowConstantEval) visit(ctx, stack, expr.getInValue()); stack.pop(); - Pair pair = convertTypesIfNecessary(lhs, rowConstantEval); + Pair pair = convertTypesIfNecessary(ctx, lhs, rowConstantEval); return new InEval(pair.getFirst(), (RowConstantEval) pair.getSecond(), expr.isNot()); } @@ -415,10 +423,10 @@ public EvalNode visitConcatenate(Context ctx, Stack stack, BinaryOperator stack.pop(); if (lhs.getValueType().getType() != Type.TEXT) { - lhs = convertType(lhs, CatalogUtil.newSimpleDataType(Type.TEXT)); + lhs = convertType(ctx, lhs, CatalogUtil.newSimpleDataType(Type.TEXT)); } if (rhs.getValueType().getType() != Type.TEXT) { - rhs = convertType(rhs, CatalogUtil.newSimpleDataType(Type.TEXT)); + rhs = convertType(ctx, rhs, CatalogUtil.newSimpleDataType(Type.TEXT)); } return new BinaryEval(EvalType.CONCATENATE, lhs, rhs); @@ -448,8 +456,8 @@ private EvalNode visitPatternMatchPredicate(Context ctx, Stack stack, Patt // Arithmetic Operators /////////////////////////////////////////////////////////////////////////////////////////////////////////// - private static BinaryEval createBinaryNode(EvalType type, EvalNode lhs, EvalNode rhs) { - Pair pair = convertTypesIfNecessary(lhs, rhs); // implicit type conversion if necessary + private static BinaryEval createBinaryNode(Context ctx, EvalType type, EvalNode lhs, EvalNode rhs) { + Pair pair = convertTypesIfNecessary(ctx, lhs, rhs); // implicit type conversion if necessary return new BinaryEval(type, pair.getFirst(), pair.getSecond()); } @@ -460,7 +468,7 @@ public EvalNode visitPlus(Context ctx, Stack stack, BinaryOperator expr) t EvalNode right = visit(ctx, stack, expr.getRight()); stack.pop(); - return createBinaryNode(EvalType.PLUS, left, right); + return createBinaryNode(ctx, EvalType.PLUS, left, right); } @Override @@ -470,7 +478,7 @@ public EvalNode visitMinus(Context ctx, Stack stack, BinaryOperator expr) EvalNode right = visit(ctx, stack, expr.getRight()); stack.pop(); - return createBinaryNode(EvalType.MINUS, left, right); + return createBinaryNode(ctx, EvalType.MINUS, left, right); } @Override @@ -480,7 +488,7 @@ public EvalNode visitMultiply(Context ctx, Stack stack, BinaryOperator exp EvalNode right = visit(ctx, stack, expr.getRight()); stack.pop(); - return createBinaryNode(EvalType.MULTIPLY, left, right); + return createBinaryNode(ctx, EvalType.MULTIPLY, left, right); } @Override @@ -490,7 +498,7 @@ public EvalNode visitDivide(Context ctx, Stack stack, BinaryOperator expr) EvalNode right = visit(ctx, stack, expr.getRight()); stack.pop(); - return createBinaryNode(EvalType.DIVIDE, left, right); + return createBinaryNode(ctx, EvalType.DIVIDE, left, right); } @Override @@ -500,7 +508,7 @@ public EvalNode visitModular(Context ctx, Stack stack, BinaryOperator expr EvalNode right = visit(ctx, stack, expr.getRight()); stack.pop(); - return createBinaryNode(EvalType.MODULAR, left, right); + return createBinaryNode(ctx, EvalType.MODULAR, left, right); } /////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -584,14 +592,14 @@ public EvalNode visitFunction(Context ctx, Stack stack, FunctionExpr expr) } else { lastDataType = CatalogUtil.newSimpleDataType(CatalogUtil.getPrimitiveTypeOf(lastDataType.getType())); } - givenArgs[i] = convertType(givenArgs[i], lastDataType); + givenArgs[i] = convertType(ctx, givenArgs[i], lastDataType); } } else { assertEval(funcDesc.getParamTypes().length == givenArgs.length, "The number of parameters is mismatched to the function definition: " + funcDesc.toString()); // According to our function matching method, each given argument can be casted to the definition parameter. for (int i = 0; i < givenArgs.length; i++) { - givenArgs[i] = convertType(givenArgs[i], funcDesc.getParamTypes()[i]); + givenArgs[i] = convertType(ctx, givenArgs[i], funcDesc.getParamTypes()[i]); } } @@ -600,7 +608,7 @@ public EvalNode visitFunction(Context ctx, Stack stack, FunctionExpr expr) FunctionType functionType = funcDesc.getFuncType(); if (functionType == FunctionType.GENERAL || functionType == FunctionType.UDF) { - return new GeneralFunctionEval(funcDesc, (GeneralFunction) funcDesc.newInstance(), givenArgs); + return new GeneralFunctionEval(ctx.queryContext, funcDesc, (GeneralFunction) funcDesc.newInstance(), givenArgs); } else if (functionType == FunctionType.AGGREGATION || functionType == FunctionType.UDA) { if (!ctx.currentBlock.hasNode(NodeType.GROUP_BY)) { @@ -768,11 +776,23 @@ public EvalNode visitDataType(Context ctx, Stack stack, DataTypeExpr expr) public EvalNode visitCastExpr(Context ctx, Stack stack, CastExpr expr) throws PlanningException { EvalNode child = super.visitCastExpr(ctx, stack, expr); - if (child.getType() == EvalType.CONST) { // if it is a casting operation for a constant value - ConstEval constEval = (ConstEval) child; // it will be pre-computed and casted to a constant value - return new ConstEval(DatumFactory.cast(constEval.getValue(), LogicalPlanner.convertDataType(expr.getTarget()))); + // if it is a casting operation for a constant value, it will be pre-computed and casted to a constant value. + + if (child.getType() == EvalType.CONST) { + ConstEval constEval = (ConstEval) child; + + // some cast operation may require earlier evaluation with timezone. + TimeZone tz = null; + if (ctx.queryContext.containsKey(SessionVars.TZ)) { + String tzId = ctx.queryContext.get(SessionVars.TZ); + tz = TimeZone.getTimeZone(tzId); + } + + return new ConstEval( + DatumFactory.cast(constEval.getValue(), LogicalPlanner.convertDataType(expr.getTarget()), tz)); + } else { - return new CastEval(child, LogicalPlanner.convertDataType(expr.getTarget())); + return new CastEval(ctx.queryContext, child, LogicalPlanner.convertDataType(expr.getTarget())); } } @@ -838,7 +858,11 @@ public EvalNode visitTimestampLiteral(Context ctx, Stack stack, TimestampL TimeMeta tm = new TimeMeta(); DateTimeUtil.toJulianTimeMeta(timestamp, tm); - DateTimeUtil.toUTCTimezone(tm); + + if (ctx.queryContext.containsKey(SessionVars.TZ)) { + TimeZone tz = TimeZone.getTimeZone(ctx.queryContext.get(SessionVars.TZ)); + DateTimeUtil.toUTCTimezone(tm, tz); + } return new ConstEval(new TimestampDatum(DateTimeUtil.toJulianTimestamp(tm))); } @@ -864,7 +888,11 @@ public EvalNode visitTimeLiteral(Context ctx, Stack stack, TimeLiteral exp } TimeDatum timeDatum = new TimeDatum(time); TimeMeta tm = timeDatum.toTimeMeta(); - DateTimeUtil.toUTCTimezone(tm); + + if (ctx.queryContext.containsKey(SessionVars.TZ)) { + TimeZone tz = TimeZone.getTimeZone(ctx.queryContext.get(SessionVars.TZ)); + DateTimeUtil.toUTCTimezone(tm, tz); + } return new ConstEval(new TimeDatum(DateTimeUtil.toTime(tm))); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 69c0e4b5a0..f21bbb5108 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -83,6 +83,7 @@ public static class PlanContext { LogicalPlan plan; QueryBlock queryBlock; EvalTreeOptimizer evalOptimizer; + TimeZone timeZone; boolean debugOrUnitTests; public PlanContext(OverridableConf context, LogicalPlan plan, QueryBlock block, EvalTreeOptimizer evalOptimizer, @@ -91,6 +92,13 @@ public PlanContext(OverridableConf context, LogicalPlan plan, QueryBlock block, this.plan = plan; this.queryBlock = block; this.evalOptimizer = evalOptimizer; + + // session's time zone + if (context.containsKey(SessionVars.TZ)) { + String timezoneId = context.get(SessionVars.TZ); + timeZone = TimeZone.getTimeZone(timezoneId); + } + this.debugOrUnitTests = debugOrUnitTests; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CastEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CastEval.java index d625a11ba9..700913e9ff 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CastEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CastEval.java @@ -19,19 +19,31 @@ package org.apache.tajo.plan.expr; import com.google.gson.annotations.Expose; +import org.apache.tajo.OverridableConf; +import org.apache.tajo.SessionVars; +import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.Schema; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.storage.Tuple; +import org.apache.tajo.util.TUtil; + +import java.util.TimeZone; import static org.apache.tajo.common.TajoDataTypes.DataType; public class CastEval extends UnaryEval { @Expose private DataType target; + @Expose private TimeZone timezone; - public CastEval(EvalNode operand, DataType target) { + public CastEval(OverridableConf context, EvalNode operand, DataType target) { super(EvalType.CAST, operand); this.target = target; + + if (context.containsKey(SessionVars.TZ)) { + String timezoneId = context.get(SessionVars.TZ); + timezone = TimeZone.getTimeZone(timezoneId); + } } public EvalNode getOperand() { @@ -43,6 +55,14 @@ public DataType getValueType() { return target; } + public boolean hasTimeZone() { + return this.timezone != null; + } + + public TimeZone getTimezone() { + return this.timezone; + } + @Override public String getName() { return target.getType().name(); @@ -54,7 +74,7 @@ public Datum eval(Schema schema, Tuple tuple) { return operandDatum; } - return DatumFactory.cast(operandDatum, target); + return DatumFactory.cast(operandDatum, target, timezone); } public String toString() { @@ -66,7 +86,9 @@ public boolean equals(Object obj) { boolean valid = obj != null && obj instanceof CastEval; if (valid) { CastEval another = (CastEval) obj; - return child.equals(another.child) && target.equals(another.target); + return child.equals(another.child) && + target.equals(another.target) && + TUtil.checkEquals(timezone, another.timezone); } else { return false; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/GeneralFunctionEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/GeneralFunctionEval.java index e28e5f3f12..0c6f1e2bc4 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/GeneralFunctionEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/GeneralFunctionEval.java @@ -20,6 +20,7 @@ import com.google.common.base.Objects; import com.google.gson.annotations.Expose; +import org.apache.tajo.OverridableConf; import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.catalog.Schema; import org.apache.tajo.datum.Datum; @@ -28,14 +29,17 @@ import org.apache.tajo.storage.VTuple; import org.apache.tajo.util.TUtil; +import javax.annotation.Nullable; + public class GeneralFunctionEval extends FunctionEval { @Expose protected GeneralFunction instance; private Tuple params = null; - public GeneralFunctionEval(FunctionDesc desc, GeneralFunction instance, EvalNode[] givenArgs) { + public GeneralFunctionEval(@Nullable OverridableConf queryContext, FunctionDesc desc, GeneralFunction instance, + EvalNode[] givenArgs) { super(EvalType.FUNCTION, desc, givenArgs); this.instance = instance; - this.instance.init(getParamType()); + this.instance.init(queryContext, getParamType()); } /* (non-Javadoc) diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java index 006449ffee..39db5c6a33 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java @@ -18,6 +18,7 @@ package org.apache.tajo.plan.function; +import org.apache.tajo.OverridableConf; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos; @@ -36,7 +37,8 @@ public GeneralFunction(Column[] definedArgs) { /** * This method gives hints to an actual function instance. */ - public void init(FunctionEval.ParamType [] paramTypes) {} + @SuppressWarnings("unused") + public void init(OverridableConf queryContext, FunctionEval.ParamType [] paramTypes) {} public abstract Datum eval(Tuple params); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoDeserializer.java index e6d54b1fbc..89b4fc0653 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoDeserializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoDeserializer.java @@ -20,6 +20,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.tajo.OverridableConf; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.catalog.exception.NoSuchFunctionException; @@ -43,7 +44,7 @@ */ public class EvalTreeProtoDeserializer { - public static EvalNode deserialize(PlanProto.EvalTree tree) { + public static EvalNode deserialize(OverridableConf context, PlanProto.EvalTree tree) { Map evalNodeMap = Maps.newHashMap(); // sort serialized eval nodes in an ascending order of their IDs. @@ -79,7 +80,7 @@ public int compare(PlanProto.EvalNode o1, PlanProto.EvalNode o2) { current = new IsNullEval(unaryProto.getNegative(), child); break; case CAST: - current = new CastEval(child, unaryProto.getCastingType()); + current = new CastEval(context, child, unaryProto.getCastingType()); break; case SIGNED: current = new SignedEval(unaryProto.getNegative(), child); @@ -153,7 +154,7 @@ public int compare(PlanProto.EvalNode o1, PlanProto.EvalNode o2) { funcDesc = new FunctionDesc(funcProto.getFuncion()); if (type == EvalType.FUNCTION) { GeneralFunction instance = (GeneralFunction) funcDesc.newInstance(); - current = new GeneralFunctionEval(new FunctionDesc(funcProto.getFuncion()), instance, params); + current = new GeneralFunctionEval(context, new FunctionDesc(funcProto.getFuncion()), instance, params); } else if (type == EvalType.AGG_FUNCTION || type == EvalType.WINDOW_FUNCTION) { AggFunction instance = (AggFunction) funcDesc.newInstance(); if (type == EvalType.AGG_FUNCTION) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java index 9f22c20bc8..92a245f98e 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java @@ -111,6 +111,9 @@ public EvalNode visitUnaryEval(EvalTreeProtoBuilderContext context, Stack registerAdapters() { adapters.put(AggFunction.class, new FunctionAdapter()); adapters.put(Datum.class, new DatumAdapter()); adapters.put(DataType.class, new DataTypeAdapter()); + adapters.put(TimeZone.class, new TimeZoneGsonSerdeAdapter()); return adapters; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java index c55c20315e..8a9e1cac7d 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java @@ -791,6 +791,10 @@ public static void applySessionToTableProperties(OverridableConf sessionVars, if (sessionVars.containsKey(SessionVars.NULL_CHAR)) { tableProperties.set(StorageConstants.TEXT_NULL, sessionVars.get(SessionVars.NULL_CHAR)); } + + if (sessionVars.containsKey(SessionVars.TZ)) { + tableProperties.set(StorageConstants.TIMEZONE, sessionVars.get(SessionVars.TZ)); + } } } } diff --git a/tajo-plan/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto index 8639117c81..0f82d870c2 100644 --- a/tajo-plan/src/main/proto/Plan.proto +++ b/tajo-plan/src/main/proto/Plan.proto @@ -150,6 +150,7 @@ message UnaryEval { required int32 child_id = 1; optional DataType castingType = 2; optional bool negative = 3; + optional string timezone = 4; } message BinaryEval { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java index f2eebc6e23..1599f62ad4 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java @@ -45,7 +45,7 @@ public void init() { } nullChars = TextLineSerDe.getNullChars(meta); - fieldSerDer = new TextFieldSerializerDeserializer(); + fieldSerDer = new TextFieldSerializerDeserializer(meta); } public void deserialize(final ByteBuf lineBuf, Tuple output) throws IOException, TextLineParsingError { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerializer.java index 73970009fa..c0fc18fa17 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerializer.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerializer.java @@ -45,7 +45,7 @@ public void init() { delimiter = CSVLineSerDe.getFieldDelimiter(meta); columnNum = schema.size(); - serde = new TextFieldSerializerDeserializer(); + serde = new TextFieldSerializerDeserializer(meta); } @Override diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextFieldSerializerDeserializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextFieldSerializerDeserializer.java index 95d0407673..3becb8e14b 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextFieldSerializerDeserializer.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextFieldSerializerDeserializer.java @@ -23,24 +23,34 @@ import io.netty.util.CharsetUtil; import org.apache.commons.codec.binary.Base64; import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.*; import org.apache.tajo.datum.protobuf.ProtobufJsonFormat; import org.apache.tajo.storage.FieldSerializerDeserializer; +import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.NumberUtil; import java.io.IOException; import java.io.OutputStream; import java.nio.charset.CharsetDecoder; +import java.util.TimeZone; -//Compatibility with Apache Hive public class TextFieldSerializerDeserializer implements FieldSerializerDeserializer { public static final byte[] trueBytes = "true".getBytes(); public static final byte[] falseBytes = "false".getBytes(); private static ProtobufJsonFormat protobufJsonFormat = ProtobufJsonFormat.getInstance(); private final CharsetDecoder decoder = CharsetUtil.getDecoder(CharsetUtil.UTF_8); + private final boolean hasTimezone; + private final TimeZone timezone; + + public TextFieldSerializerDeserializer(TableMeta meta) { + hasTimezone = meta.containsOption(StorageConstants.TIMEZONE); + timezone = TimeZone.getTimeZone(meta.getOption(StorageConstants.TIMEZONE, StorageConstants.DEFAULT_TIMEZONE)); + } + private static boolean isNull(ByteBuf val, ByteBuf nullBytes) { return !val.isReadable() || nullBytes.equals(val); } @@ -50,7 +60,8 @@ private static boolean isNullText(ByteBuf val, ByteBuf nullBytes) { } @Override - public int serialize(OutputStream out, Datum datum, Column col, int columnIndex, byte[] nullChars) throws IOException { + public int serialize(OutputStream out, Datum datum, Column col, int columnIndex, byte[] nullChars) + throws IOException { byte[] bytes; int length = 0; TajoDataTypes.DataType dataType = col.getDataType(); @@ -95,12 +106,20 @@ public int serialize(OutputStream out, Datum datum, Column col, int columnIndex, out.write(bytes); break; case TIME: - bytes = ((TimeDatum) datum).asChars(TajoConf.getCurrentTimeZone(), true).getBytes(); + if (hasTimezone) { + bytes = ((TimeDatum) datum).asChars(timezone, true).getBytes(); + } else { + bytes = datum.asTextBytes(); + } length = bytes.length; out.write(bytes); break; case TIMESTAMP: - bytes = ((TimestampDatum) datum).asChars(TajoConf.getCurrentTimeZone(), true).getBytes(); + if (hasTimezone) { + bytes = ((TimestampDatum) datum).asChars(timezone, true).getBytes(); + } else { + bytes = datum.asTextBytes(); + } length = bytes.length; out.write(bytes); break; @@ -178,12 +197,22 @@ public Datum deserialize(ByteBuf buf, Column col, int columnIndex, ByteBuf nullC decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString()); break; case TIME: - datum = DatumFactory.createTime( - decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString()); + if (hasTimezone) { + datum = DatumFactory.createTime( + decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString(), timezone); + } else { + datum = DatumFactory.createTime( + decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString()); + } break; case TIMESTAMP: - datum = DatumFactory.createTimestamp( - decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString()); + if (hasTimezone) { + datum = DatumFactory.createTimestamp( + decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString(), timezone); + } else { + datum = DatumFactory.createTimestamp( + decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString()); + } break; case INTERVAL: datum = DatumFactory.createInterval( From bf8fa157d0b2fc22b172d125c8dd33b4c0d2f614 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Sun, 7 Dec 2014 13:45:27 +0900 Subject: [PATCH 2/6] Added more unit tests and changed some variables. --- .../org/apache/tajo/client/QueryClient.java | 2 + .../apache/tajo/client/QueryClientImpl.java | 8 +- .../apache/tajo/client/SessionConnection.java | 33 ++++ .../apache/tajo/client/TajoClientUtil.java | 3 +- .../org/apache/tajo/jdbc/FetchResultSet.java | 1 + .../apache/tajo/jdbc/TajoMemoryResultSet.java | 5 +- .../org/apache/tajo/jdbc/TajoResultSet.java | 2 + .../apache/tajo/jdbc/TajoResultSetBase.java | 37 ++++- .../java/org/apache/tajo/SessionVars.java | 4 +- .../java/org/apache/tajo/TajoConstants.java | 2 +- .../java/org/apache/tajo/conf/TajoConf.java | 18 +-- .../tajo/util/datetime/DateTimeUtil.java | 5 - .../apache/tajo/datum/TestTimestampDatum.java | 5 +- .../engine/function/datetime/CurrentDate.java | 2 +- .../function/datetime/DatePartFromTime.java | 3 +- .../datetime/DatePartFromTimestamp.java | 2 +- .../function/datetime/ToCharTimestamp.java | 3 +- .../function/datetime/ToTimestampText.java | 15 +- .../org/apache/tajo/QueryTestCaseBase.java | 8 + .../apache/tajo/engine/eval/ExprTestBase.java | 4 - .../tajo/engine/eval/TestIntervalType.java | 46 +++--- .../engine/eval/TestSQLDateTimeTypes.java | 18 +-- .../function/TestConditionalExpressions.java | 16 +- .../function/TestDateTimeFunctions.java | 147 +++++++++++------- .../tajo/engine/query/TestSelectQuery.java | 47 +++++- .../TestSelectQuery/timezoned/table1.tbl | 3 + .../TestSelectQuery/datetime_table_ddl.sql | 4 + .../datetime_table_timezoned_ddl.sql | 4 + .../TestSelectQuery/testTimezonedTable1.sql | 1 + .../TestSelectQuery/testTimezonedTable2.sql | 1 + .../TestSelectQuery/testTimezonedTable3.sql | 1 + .../testTimezonedTable1.result | 5 + .../testTimezonedTable2.result | 5 + .../testTimezonedTable3.result | 5 + .../tajo/jdbc/TajoMetaDataResultSet.java | 2 + .../storage/TextSerializerDeserializer.java | 7 +- 36 files changed, 318 insertions(+), 156 deletions(-) create mode 100644 tajo-core/src/test/resources/dataset/TestSelectQuery/timezoned/table1.tbl create mode 100644 tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_ddl.sql create mode 100644 tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql create mode 100644 tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable1.sql create mode 100644 tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable2.sql create mode 100644 tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable3.sql create mode 100644 tajo-core/src/test/resources/results/TestSelectQuery/testTimezonedTable1.result create mode 100644 tajo-core/src/test/resources/results/TestSelectQuery/testTimezonedTable2.result create mode 100644 tajo-core/src/test/resources/results/TestSelectQuery/testTimezonedTable3.result diff --git a/tajo-client/src/main/java/org/apache/tajo/client/QueryClient.java b/tajo-client/src/main/java/org/apache/tajo/client/QueryClient.java index 9b246639ea..32ef97d9d9 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/QueryClient.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/QueryClient.java @@ -42,6 +42,8 @@ public interface QueryClient extends Closeable { public SessionIdProto getSessionId(); + public Map getClientSideSessionVars(); + public String getBaseDatabase(); @Override diff --git a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java index 5b789596a9..1cee51572b 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java @@ -72,6 +72,11 @@ public TajoIdProtos.SessionIdProto getSessionId() { return connection.getSessionId(); } + @Override + public Map getClientSideSessionVars() { + return connection.getClientSideSessionVars(); + } + @Override public String getBaseDatabase() { return connection.getBaseDatabase(); @@ -455,7 +460,8 @@ public ClientProtos.SerializedResultSet call(NettyClientBase client) throws Serv return new TajoMemoryResultSet( new Schema(serializedResultSet.getSchema()), serializedResultSet.getSerializedTuplesList(), - serializedResultSet.getSerializedTuplesCount()); + serializedResultSet.getSerializedTuplesCount(), + getClientSideSessionVars()); } catch (Exception e) { throw new ServiceException(e.getMessage(), e); } diff --git a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java index 42085a229d..922984f4e5 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java @@ -23,6 +23,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.security.UserGroupInformation; import org.apache.tajo.QueryId; +import org.apache.tajo.SessionVars; import org.apache.tajo.TajoIdProtos; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.conf.TajoConf; @@ -39,6 +40,7 @@ import java.io.Closeable; import java.io.IOException; import java.net.InetSocketAddress; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -68,6 +70,9 @@ public class SessionConnection implements Closeable { private AtomicBoolean closed = new AtomicBoolean(false); + /** session variable cache */ + Map clientSideSessionVars = new ConcurrentHashMap(); + public SessionConnection(TajoConf conf) throws IOException { this(conf, NetUtils.createSocketAddr(conf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS)), null); @@ -105,6 +110,10 @@ public SessionConnection(TajoConf conf, InetSocketAddress addr, @Nullable String this.baseDatabase = baseDatabase != null ? baseDatabase : null; } + public Map getClientSideSessionVars() { + return Collections.unmodifiableMap(clientSideSessionVars); + } + public T getStub(QueryId queryId, Class protocolClass, boolean asyncMode) throws NoSuchMethodException, ConnectTimeoutException, ClassNotFoundException { InetSocketAddress addr = queryMasterMap.get(queryId); @@ -177,6 +186,17 @@ public Boolean updateSessionVariables(final Map variables) throw public Boolean call(NettyClientBase client) throws ServiceException { checkSessionAndGet(client); + // keep client-side session variables + for (Map.Entry entry : variables.entrySet()) { + String key = entry.getKey(); + if (SessionVars.exists(entry.getKey())) { + SessionVars configKey = SessionVars.get(key); + if (configKey.getMode() == SessionVars.VariableMode.CLI_SIDE_VAR) { + clientSideSessionVars.put(key, entry.getValue()); + } + } + } + TajoMasterClientProtocolService.BlockingInterface tajoMasterService = client.getStub(); KeyValueSet keyValueSet = new KeyValueSet(); keyValueSet.putAll(variables); @@ -195,6 +215,13 @@ public Boolean unsetSessionVariables(final List variables) throws Servi public Boolean call(NettyClientBase client) throws ServiceException { checkSessionAndGet(client); + // Remove matched session vars + for (String key : variables) { + if (clientSideSessionVars.containsKey(key)) { + clientSideSessionVars.remove(key); + } + } + TajoMasterClientProtocolService.BlockingInterface tajoMasterService = client.getStub(); ClientProtos.UpdateSessionVariableRequest request = ClientProtos.UpdateSessionVariableRequest.newBuilder() .setSessionId(sessionId) @@ -208,6 +235,12 @@ public String getSessionVariable(final String varname) throws ServiceException { return new ServerCallable(connPool, getTajoMasterAddr(), TajoMasterClientProtocol.class, false, true) { public String call(NettyClientBase client) throws ServiceException { + + // If a desired variable is client side one and exists in the cache, immediately return the variable. + if (clientSideSessionVars.containsKey(varname)) { + return clientSideSessionVars.get(varname); + } + checkSessionAndGet(client); TajoMasterClientProtocolService.BlockingInterface tajoMasterService = client.getStub(); diff --git a/tajo-client/src/main/java/org/apache/tajo/client/TajoClientUtil.java b/tajo-client/src/main/java/org/apache/tajo/client/TajoClientUtil.java index 7aed335f10..bf9d111051 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/TajoClientUtil.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/TajoClientUtil.java @@ -88,7 +88,8 @@ public static ResultSet createResultSet(TajoConf conf, QueryClient client, Clien return new TajoMemoryResultSet( new Schema(serializedResultSet.getSchema()), serializedResultSet.getSerializedTuplesList(), - response.getMaxRowNum()); + response.getMaxRowNum(), + client.getClientSideSessionVars()); } } } diff --git a/tajo-client/src/main/java/org/apache/tajo/jdbc/FetchResultSet.java b/tajo-client/src/main/java/org/apache/tajo/jdbc/FetchResultSet.java index 78674b15b3..18b7c1a3e2 100644 --- a/tajo-client/src/main/java/org/apache/tajo/jdbc/FetchResultSet.java +++ b/tajo-client/src/main/java/org/apache/tajo/jdbc/FetchResultSet.java @@ -35,6 +35,7 @@ public class FetchResultSet extends TajoResultSetBase { private boolean finished = false; public FetchResultSet(QueryClient tajoClient, Schema schema, QueryId queryId, int fetchRowNum) { + super(tajoClient.getClientSideSessionVars()); this.tajoClient = tajoClient; this.queryId = queryId; this.fetchRowNum = fetchRowNum; diff --git a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoMemoryResultSet.java b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoMemoryResultSet.java index 84fafda20f..d0898f5787 100644 --- a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoMemoryResultSet.java +++ b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoMemoryResultSet.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.sql.SQLException; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; public class TajoMemoryResultSet extends TajoResultSetBase { @@ -33,7 +34,9 @@ public class TajoMemoryResultSet extends TajoResultSetBase { private AtomicBoolean closed = new AtomicBoolean(false); private RowStoreUtil.RowStoreDecoder decoder; - public TajoMemoryResultSet(Schema schema, List serializedTuples, int maxRowNum) { + public TajoMemoryResultSet(Schema schema, List serializedTuples, int maxRowNum, + Map clientSideSessionVars) { + super(clientSideSessionVars); this.schema = schema; this.totalRow = maxRowNum; this.serializedTuples = serializedTuples; diff --git a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java index e99c0ff762..e2ac3b1913 100644 --- a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java +++ b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java @@ -56,12 +56,14 @@ public class TajoResultSet extends TajoResultSetBase { private AtomicBoolean closed = new AtomicBoolean(false); public TajoResultSet(QueryClient tajoClient, QueryId queryId) { + super(tajoClient.getClientSideSessionVars()); this.tajoClient = tajoClient; this.queryId = queryId; init(); } public TajoResultSet(QueryClient tajoClient, QueryId queryId, TajoConf conf, TableDesc table) throws IOException { + super(tajoClient.getClientSideSessionVars()); this.tajoClient = tajoClient; this.queryId = queryId; this.conf = conf; diff --git a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java index 78d8bde4aa..2f9b6e419f 100644 --- a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java +++ b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java @@ -18,6 +18,8 @@ package org.apache.tajo.jdbc; +import org.apache.tajo.SessionVars; +import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.Schema; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; @@ -26,6 +28,7 @@ import org.apache.tajo.util.datetime.DateTimeUtil; import org.apache.tajo.util.datetime.TimeMeta; +import javax.annotation.Nullable; import java.io.IOException; import java.io.InputStream; import java.io.Reader; @@ -37,12 +40,30 @@ import java.util.TimeZone; public abstract class TajoResultSetBase implements ResultSet { + protected final Map clientSideSessionVars; + protected TimeZone timezone; + protected int curRow; protected long totalRow; protected boolean wasNull; protected Schema schema; protected Tuple cur; + public TajoResultSetBase(@Nullable Map clientSideSessionVars) { + this.clientSideSessionVars = clientSideSessionVars; + + if (clientSideSessionVars != null) { + + if (clientSideSessionVars.containsKey(SessionVars.TZ.name())) { + String timezoneId = clientSideSessionVars.get(SessionVars.TZ.name()); + this.timezone = TimeZone.getTimeZone(timezoneId); + } else { + this.timezone = TimeZone.getTimeZone(TajoConstants.UTC_TIMEZONE); + } + + } + } + protected void init() { cur = null; curRow = 0; @@ -226,13 +247,13 @@ public Object getObject(int fieldId) throws SQLException { case FLOAT8: return d.asFloat8(); case NUMERIC: return d.asFloat8(); case DATE: { - return getDate((DateDatum)d, TajoConf.getCurrentTimeZone()); + return getDate((DateDatum)d, timezone); } case TIME: { - return getTime((TimeDatum)d, TajoConf.getCurrentTimeZone()); + return getTime((TimeDatum)d, timezone); } case TIMESTAMP: { - return getTimestamp((TimestampDatum) d, TajoConf.getCurrentTimeZone()); + return getTimestamp((TimestampDatum) d, timezone); } default: return d.asChars(); } @@ -289,10 +310,10 @@ private String getString(Datum datum, int fieldId) throws SQLException { case BOOLEAN: return String.valueOf(datum.asBool()); case TIME: { - return ((TimeDatum)datum).asChars(TajoConf.getCurrentTimeZone(), false); + return ((TimeDatum)datum).asChars(timezone, false); } case TIMESTAMP: { - return ((TimestampDatum)datum).asChars(TajoConf.getCurrentTimeZone(), false); + return ((TimestampDatum)datum).asChars(timezone, false); } default : return datum.asChars(); @@ -307,7 +328,7 @@ public Date getDate(int fieldId) throws SQLException { return null; } - return getDate((DateDatum)datum, TajoConf.getCurrentTimeZone()); + return getDate((DateDatum)datum, null); } @Override @@ -347,7 +368,7 @@ public Time getTime(int fieldId) throws SQLException { return null; } - return getTime((TimeDatum)datum, TajoConf.getCurrentTimeZone()); + return getTime((TimeDatum)datum, timezone); } @@ -388,7 +409,7 @@ public Timestamp getTimestamp(int fieldId) throws SQLException { return null; } - return getTimestamp((TimestampDatum)datum, TajoConf.getCurrentTimeZone()); + return getTimestamp((TimestampDatum)datum, timezone); } @Override diff --git a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java index dd33e127e2..fa717d1230 100644 --- a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java +++ b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java @@ -71,8 +71,8 @@ public enum SessionVars implements ConfigKey { ON_ERROR_STOP(ConfVars.$CLI_ERROR_STOP, "tsql will exist if an error occurs.", CLI_SIDE_VAR), // Timezone & Date ---------------------------------------------------------- - TZ(ConfVars.$TIMEZONE, "Sets timezone", FROM_SHELL_ENV), - DATE_ORDER(ConfVars.$DATE_ORDER, "date order (default is YMD)", FROM_SHELL_ENV), + TZ(ConfVars.$TIMEZONE, "Sets timezone", CLI_SIDE_VAR), + DATE_ORDER(ConfVars.$DATE_ORDER, "date order (default is YMD)", CLI_SIDE_VAR), // Locales and Character set ------------------------------------------------ // TODO - they are reserved variables, and we should support them. diff --git a/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java b/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java index 31e18fcb4b..4d1e4604f5 100644 --- a/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java +++ b/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java @@ -31,7 +31,7 @@ public class TajoConstants { public static final String DEFAULT_DATABASE_NAME = "default"; public static final String DEFAULT_SCHEMA_NAME = "public"; - public static final String SYSTEM_DEFAULT_TIMEZONE = "UTC"; + public static final String UTC_TIMEZONE = "UTC"; public static final String EMPTY_STRING = ""; diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index db2d6f0e80..1213775d2a 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -84,11 +84,11 @@ public TajoConf(Path path) { } private static void confStaticInit() { - TimeZone.setDefault(getCurrentTimeZone()); + TimeZone.setDefault(getSystemTimezone()); getDateOrder(); } - public static TimeZone getCurrentTimeZone() { + public static TimeZone getSystemTimezone() { writeLock.lock(); try { if (SYSTEM_TIMEZONE == null) { @@ -101,18 +101,6 @@ public static TimeZone getCurrentTimeZone() { } } - @VisibleForTesting @SuppressWarnings("unused") - public static TimeZone setCurrentTimeZone(TimeZone timeZone) { - writeLock.lock(); - try { - TimeZone oldTimeZone = SYSTEM_TIMEZONE; - SYSTEM_TIMEZONE = timeZone; - return oldTimeZone; - } finally { - writeLock.unlock(); - } - } - public static int getDateOrder() { writeLock.lock(); try { @@ -362,7 +350,7 @@ public static enum ConfVars implements ConfigKey { $CLI_ERROR_STOP("tajo.cli.error.stop", false), // Timezone & Date ---------------------------------------------------------- - $TIMEZONE("tajo.timezone", TajoConstants.SYSTEM_DEFAULT_TIMEZONE), + $TIMEZONE("tajo.timezone", TajoConstants.UTC_TIMEZONE), $DATE_ORDER("tajo.date.order", "YMD"), // FILE FORMAT diff --git a/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeUtil.java index 5226142a9c..7acb544cba 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeUtil.java @@ -2068,11 +2068,6 @@ public static void toUserTimezone(TimeMeta tm, TimeZone timeZone) { tm.plusMillis(timeZone.getRawOffset()); } - public static void toUTCTimezone(TimeMeta tm) { - TimeZone timeZone = TajoConf.getCurrentTimeZone(); - tm.plusMillis(0 - timeZone.getRawOffset()); - } - public static void toUTCTimezone(TimeMeta tm, TimeZone timeZone) { tm.plusMillis(0 - timeZone.getRawOffset()); } diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java index 58860832c3..a925954488 100644 --- a/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java +++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java @@ -19,7 +19,6 @@ package org.apache.tajo.datum; import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.conf.TajoConf; import org.apache.tajo.exception.InvalidCastException; import org.apache.tajo.json.CommonGsonHelper; import org.apache.tajo.util.datetime.DateTimeUtil; @@ -29,8 +28,8 @@ import java.util.Calendar; import java.util.TimeZone; +import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.*; -import static org.hamcrest.CoreMatchers.*; public class TestTimestampDatum { private static long javatime; @@ -40,7 +39,7 @@ public class TestTimestampDatum { @BeforeClass public static void setUp() { javatime = System.currentTimeMillis(); - calendar = Calendar.getInstance(TajoConf.getCurrentTimeZone()); + calendar = Calendar.getInstance(TimeZone.getTimeZone("PST")); calendar.setTimeInMillis(javatime); unixtime = (int) (javatime / 1000); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentDate.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentDate.java index 49295f5b54..41f66948e5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentDate.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentDate.java @@ -53,7 +53,7 @@ public CurrentDate() { @Override public void init(OverridableConf context, FunctionEval.ParamType[] types) { - String timezoneId = context.get(SessionVars.TZ, TajoConstants.SYSTEM_DEFAULT_TIMEZONE); + String timezoneId = context.get(SessionVars.TZ, TajoConstants.UTC_TIMEZONE); timezone = TimeZone.getTimeZone(timezoneId); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTime.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTime.java index 462c38d1ce..a8f9d129a8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTime.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTime.java @@ -23,7 +23,6 @@ import org.apache.tajo.SessionVars; import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; @@ -64,7 +63,7 @@ public DatePartFromTime() { @Override public void init(OverridableConf context, FunctionEval.ParamType [] types) { - String timezoneId = context.get(SessionVars.TZ, TajoConstants.SYSTEM_DEFAULT_TIMEZONE); + String timezoneId = context.get(SessionVars.TZ, TajoConstants.UTC_TIMEZONE); timezone = TimeZone.getTimeZone(timezoneId); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTimestamp.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTimestamp.java index 89526983d7..c0cb383358 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTimestamp.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTimestamp.java @@ -59,7 +59,7 @@ public DatePartFromTimestamp() { @Override public void init(OverridableConf context, FunctionEval.ParamType [] types) { - String timezoneId = context.get(SessionVars.TZ, TajoConstants.SYSTEM_DEFAULT_TIMEZONE); + String timezoneId = context.get(SessionVars.TZ, TajoConstants.UTC_TIMEZONE); timezone = TimeZone.getTimeZone(timezoneId); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java index 8dbddb9bc8..b071d60840 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java @@ -37,7 +37,6 @@ import org.apache.tajo.util.datetime.DateTimeUtil; import org.apache.tajo.util.datetime.TimeMeta; -import java.util.Objects; import java.util.TimeZone; import static org.apache.tajo.common.TajoDataTypes.Type.TEXT; @@ -63,7 +62,7 @@ public ToCharTimestamp() { @Override public void init(OverridableConf context, FunctionEval.ParamType[] paramTypes) { - String timezoneId = context.get(SessionVars.TZ, TajoConstants.SYSTEM_DEFAULT_TIMEZONE); + String timezoneId = context.get(SessionVars.TZ, TajoConstants.UTC_TIMEZONE); timezone = TimeZone.getTimeZone(timezoneId); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestampText.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestampText.java index e50bacc85a..2108df4127 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestampText.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestampText.java @@ -18,9 +18,13 @@ package org.apache.tajo.engine.function.datetime; +import org.apache.tajo.OverridableConf; +import org.apache.tajo.SessionVars; +import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.*; +import org.apache.tajo.plan.expr.FunctionEval; import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; @@ -29,6 +33,8 @@ import org.apache.tajo.util.datetime.DateTimeUtil; import org.apache.tajo.util.datetime.TimeMeta; +import java.util.TimeZone; + import static org.apache.tajo.common.TajoDataTypes.Type.TEXT; @Description( @@ -41,10 +47,17 @@ paramTypes = {@ParamTypes(paramTypes = {TajoDataTypes.Type.TEXT, TajoDataTypes.Type.TEXT})} ) public class ToTimestampText extends GeneralFunction { + private TimeZone timezone; + public ToTimestampText() { super(new Column[]{new Column("DateTimeText", TEXT), new Column("Pattern", TEXT)}); } + public void init(OverridableConf queryContext, FunctionEval.ParamType [] paramTypes) { + String timezoneId = queryContext.get(SessionVars.TZ, TajoConstants.UTC_TIMEZONE); + timezone = TimeZone.getTimeZone(timezoneId); + } + @Override public Datum eval(Tuple params) { if(params.isNull(0) || params.isNull(1)) { @@ -55,7 +68,7 @@ public Datum eval(Tuple params) { TextDatum patternDatum = (TextDatum) params.get(1); TimeMeta tm = DateTimeFormat.parseDateTime(dateTimeTextDatum.asChars(), patternDatum.asChars()); - DateTimeUtil.toUTCTimezone(tm); + DateTimeUtil.toUTCTimezone(tm, timezone); return new TimestampDatum(DateTimeUtil.toJulianTimestamp(tm)); } diff --git a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java index 4e4964e42e..efcc69163f 100644 --- a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -532,6 +532,14 @@ public List executeDDL(String ddlFileName, @Nullable String [] args) thr * replaced by the first and second elements of args respectively. It uses zero-based index. * * + * Example ddl + *
+   *   CREATE EXTERNAL TABLE ${0} (
+   *     t_timestamp  TIMESTAMP,
+   *     t_date    DATE
+   *   ) USING CSV LOCATION ${table.path}
+   * 
+ * * @param ddlFileName A file name, containing a data definition statement. * @param dataFileName A file name, containing data rows, which columns have to be separated by vertical bar '|'. * This file name is used for replacing some format string indicating an external table location. diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index c89c146790..8cd87a8943 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -74,10 +74,6 @@ public class ExprTestBase { private static LogicalOptimizer optimizer; private static LogicalPlanVerifier annotatedPlanVerifier; - public static String getUserTimeZoneDisplay() { - return DateTimeUtil.getTimeZoneDisplayTime(TajoConf.getCurrentTimeZone()); - } - public static String getUserTimeZoneDisplay(TimeZone tz) { return DateTimeUtil.getTimeZoneDisplayTime(tz); } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestIntervalType.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestIntervalType.java index c054fd1824..cde370d349 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestIntervalType.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestIntervalType.java @@ -32,28 +32,28 @@ public void testIntervalPostgresqlCase() throws IOException { // http://www.postgresql.org/docs/8.2/static/functions-datetime.html testSimpleEval("select date '2001-09-28' + 7", new String[]{"2001-10-05"}); testSimpleEval("select date '2001-09-28' + interval '1 hour'", - new String[]{"2001-09-28 01:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2001-09-28 01:00:00"}); testSimpleEval("select date '2001-09-28' + time '03:00'", - new String[]{"2001-09-28 03:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2001-09-28 03:00:00"}); testSimpleEval("select time '03:00' + date '2001-09-28'", - new String[]{"2001-09-28 03:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2001-09-28 03:00:00"}); testSimpleEval("select interval '1 day' + interval '1 hour'", new String[]{"1 day 01:00:00"}); testSimpleEval("select timestamp '2001-09-28 01:00' + interval '23 hours'", - new String[]{"2001-09-29 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2001-09-29 00:00:00"}); - testSimpleEval("select time '01:00' + interval '3 hours'", new String[]{"04:00:00" + getUserTimeZoneDisplay()}); + testSimpleEval("select time '01:00' + interval '3 hours'", new String[]{"04:00:00"}); testSimpleEval("select date '2001-10-01' - date '2001-09-28'", new String[]{"3"}); testSimpleEval("select date '2001-10-01' - 7", new String[]{"2001-09-24"}); testSimpleEval("select date '2001-09-28' - interval '1 hour'", - new String[]{"2001-09-27 23:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2001-09-27 23:00:00"}); testSimpleEval("select time '05:00' - time '03:00'", new String[]{"02:00:00"}); - testSimpleEval("select time '05:00' - interval '2 hours'", new String[]{"03:00:00" + getUserTimeZoneDisplay()}); + testSimpleEval("select time '05:00' - interval '2 hours'", new String[]{"03:00:00"}); testSimpleEval("select timestamp '2001-09-28 23:00' - interval '23 hours'", - new String[]{"2001-09-28 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2001-09-28 00:00:00"}); testSimpleEval("select interval '1 day' - interval '1 hour'", new String[]{"23:00:00"}); @@ -67,32 +67,32 @@ public void testIntervalPostgresqlCase() throws IOException { @Test public void testCaseByCase() throws Exception { testSimpleEval("select date '2001-08-28' + interval '10 day 1 hour'", - new String[]{"2001-09-07 01:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2001-09-07 01:00:00"}); testSimpleEval("select interval '10 day 01:00:00' + date '2001-08-28'", - new String[]{"2001-09-07 01:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2001-09-07 01:00:00"}); testSimpleEval("select time '10:20:30' + interval '1 day 01:00:00'", - new String[]{"11:20:30" + getUserTimeZoneDisplay()}); + new String[]{"11:20:30"}); testSimpleEval("select interval '1 day 01:00:00' + time '10:20:30'", - new String[]{"11:20:30" + getUserTimeZoneDisplay()}); + new String[]{"11:20:30"}); testSimpleEval("select time '10:20:30' - interval '1 day 01:00:00'", - new String[]{"09:20:30" + getUserTimeZoneDisplay()}); + new String[]{"09:20:30"}); testSimpleEval("select (interval '1 month 20 day' + interval '50 day')", new String[]{"1 month 70 days"}); testSimpleEval("select date '2013-01-01' + interval '1 month 70 day'", - new String[]{"2013-04-12 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2013-04-12 00:00:00"}); testSimpleEval("select date '2013-01-01' + (interval '1 month 20 day' + interval '50 day')", - new String[]{"2013-04-12 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2013-04-12 00:00:00"}); testSimpleEval("select interval '1 month 70 day' + date '2013-01-01'", - new String[]{"2013-04-12 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2013-04-12 00:00:00"}); testSimpleEval("select date '2013-01-01' - interval '1 month 70 day'", - new String[]{"2012-09-22 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2012-09-22 00:00:00"}); testSimpleEval("select timestamp '2001-09-28 23:00' - interval '1 month 2 day 10:20:30'", - new String[]{"2001-08-26 12:39:30" + getUserTimeZoneDisplay()}); + new String[]{"2001-08-26 12:39:30"}); testSimpleEval("select timestamp '2001-09-28 23:00' + interval '1 month 2 day 10:20:30'", - new String[]{"2001-10-31 09:20:30" + getUserTimeZoneDisplay()}); + new String[]{"2001-10-31 09:20:30"}); testSimpleEval("select interval '1 month 2 day 10:20:30' + timestamp '2001-09-28 23:00'", - new String[]{"2001-10-31 09:20:30" + getUserTimeZoneDisplay()}); + new String[]{"2001-10-31 09:20:30"}); testSimpleEval("select interval '5 month' / 3", new String[]{"1 month 20 days"}); @@ -104,13 +104,13 @@ public void testCaseByCase() throws Exception { testSimpleEval("select interval '3 year 5 month 1 hour' / 1.5", new String[]{"2 years 3 months 10 days 00:40:00"}); testSimpleEval("select date '2001-09-28' - time '03:00'", - new String[]{"2001-09-27 21:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2001-09-27 21:00:00"}); testSimpleEval("select date '2014-03-20' + interval '1 day'", - new String[]{"2014-03-21 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2014-03-21 00:00:00"}); testSimpleEval("select date '2014-03-20' - interval '1 day'", - new String[]{"2014-03-19 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2014-03-19 00:00:00"}); } @Test diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestSQLDateTimeTypes.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestSQLDateTimeTypes.java index 60f7dcd12f..fc74339356 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestSQLDateTimeTypes.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestSQLDateTimeTypes.java @@ -26,14 +26,10 @@ public class TestSQLDateTimeTypes extends ExprTestBase { @Test public void testTimestamp() throws IOException { - testSimpleEval("select TIMESTAMP '1970-01-17 10:09:37';", - new String[]{"1970-01-17 10:09:37" + getUserTimeZoneDisplay()}); - testSimpleEval("select TIMESTAMP '1970-01-17 10:09:37.5';", - new String[]{"1970-01-17 10:09:37.5" + getUserTimeZoneDisplay()}); - testSimpleEval("select TIMESTAMP '1970-01-17 10:09:37.01';", - new String[]{"1970-01-17 10:09:37.01" + getUserTimeZoneDisplay()}); - testSimpleEval("select TIMESTAMP '1970-01-17 10:09:37.003';", - new String[]{"1970-01-17 10:09:37.003" + getUserTimeZoneDisplay()}); + testSimpleEval("select TIMESTAMP '1970-01-17 10:09:37';", new String[]{"1970-01-17 10:09:37"}); + testSimpleEval("select TIMESTAMP '1970-01-17 10:09:37.5';", new String[]{"1970-01-17 10:09:37.5"}); + testSimpleEval("select TIMESTAMP '1970-01-17 10:09:37.01';", new String[]{"1970-01-17 10:09:37.01"}); + testSimpleEval("select TIMESTAMP '1970-01-17 10:09:37.003';",new String[]{"1970-01-17 10:09:37.003"}); } @Test @@ -44,14 +40,12 @@ public void testToTimestamp() throws IOException { @Test public void testTimeLiteral() throws IOException { - testSimpleEval("select TIME '10:09:37';", - new String[]{"10:09:37" + getUserTimeZoneDisplay()}); + testSimpleEval("select TIME '10:09:37';", new String[]{"10:09:37"}); } @Test public void testDateLiteral() throws IOException { - testSimpleEval("select DATE '1970-01-17';", - new String[]{"1970-01-17"}); + testSimpleEval("select DATE '1970-01-17';", new String[]{"1970-01-17"}); } } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestConditionalExpressions.java b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestConditionalExpressions.java index 54c8722493..bec8cd37e9 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestConditionalExpressions.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestConditionalExpressions.java @@ -212,25 +212,25 @@ public void testCoalesceBoolean() throws Exception { @Test public void testCoalesceTimestamp() throws Exception { testSimpleEval("select coalesce(null, timestamp '2014-01-01 00:00:00');", - new String[]{"2014-01-01 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2014-01-01 00:00:00"}); testSimpleEval("select coalesce(null, null, timestamp '2014-01-01 00:00:00');", - new String[]{"2014-01-01 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2014-01-01 00:00:00"}); testSimpleEval("select coalesce(timestamp '2014-01-01 00:00:00', null, timestamp '2014-01-02 00:00:00');", - new String[]{"2014-01-01 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2014-01-01 00:00:00"}); testSimpleEval("select coalesce(null, timestamp '2014-01-01 00:00:00', timestamp '2014-02-01 00:00:00');", - new String[]{"2014-01-01 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2014-01-01 00:00:00"}); } @Test public void testCoalesceTime() throws Exception { testSimpleEval("select coalesce(null, time '12:00:00');", - new String[]{"12:00:00" + getUserTimeZoneDisplay()}); + new String[]{"12:00:00"}); testSimpleEval("select coalesce(null, null, time '12:00:00');", - new String[]{"12:00:00" + getUserTimeZoneDisplay()}); + new String[]{"12:00:00"}); testSimpleEval("select coalesce(time '12:00:00', null, time '13:00:00');", - new String[]{"12:00:00" + getUserTimeZoneDisplay()}); + new String[]{"12:00:00"}); testSimpleEval("select coalesce(null, time '12:00:00', time '13:00:00');", - new String[]{"12:00:00" + getUserTimeZoneDisplay()}); + new String[]{"12:00:00"}); } @Test diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java index 54704cf869..59dc8fd71c 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java @@ -49,55 +49,55 @@ public void testToTimestamp() throws IOException { testSimpleEval(q1, new String[]{expected.toString()}); testSimpleEval("select to_timestamp('1997-12-30 11:40:50.345', 'YYYY-MM-DD HH24:MI:SS.MS');", - new String[]{"1997-12-30 11:40:50.345" + getUserTimeZoneDisplay()}); + new String[]{"1997-12-30 11:40:50.345"}); testSimpleEval("select to_timestamp('1997-12-30 11:40:50.345 PM', 'YYYY-MM-DD HH24:MI:SS.MS PM');", - new String[]{"1997-12-30 23:40:50.345" + getUserTimeZoneDisplay()}); + new String[]{"1997-12-30 23:40:50.345"}); testSimpleEval("select to_timestamp('0097/Feb/16 --> 08:14:30', 'YYYY/Mon/DD --> HH:MI:SS');", - new String[]{"0097-02-16 08:14:30" + getUserTimeZoneDisplay()}); + new String[]{"0097-02-16 08:14:30"}); testSimpleEval("select to_timestamp('97/2/16 8:14:30', 'FMYYYY/FMMM/FMDD FMHH:FMMI:FMSS');", - new String[]{"0097-02-16 08:14:30" + getUserTimeZoneDisplay()}); + new String[]{"0097-02-16 08:14:30"}); testSimpleEval("select to_timestamp('1985 September 12', 'YYYY FMMonth DD');", - new String[]{"1985-09-12 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"1985-09-12 00:00:00"}); testSimpleEval("select to_timestamp('1,582nd VIII 21', 'Y,YYYth FMRM DD');", - new String[]{"1582-08-21 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"1582-08-21 00:00:00"}); testSimpleEval("select to_timestamp('05121445482000', 'MMDDHH24MISSYYYY');", - new String[]{"2000-05-12 14:45:48" + getUserTimeZoneDisplay()}); + new String[]{"2000-05-12 14:45:48"}); testSimpleEval("select to_timestamp('2000January09Sunday', 'YYYYFMMonthDDFMDay');", - new String[]{"2000-01-09 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2000-01-09 00:00:00"}); testSimpleEval("select to_timestamp('97/Feb/16', 'YY/Mon/DD');", - new String[]{"1997-02-16 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"1997-02-16 00:00:00"}); testSimpleEval("select to_timestamp('19971116', 'YYYYMMDD');", - new String[]{"1997-11-16 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"1997-11-16 00:00:00"}); testSimpleEval("select to_timestamp('20000-1116', 'YYYY-MMDD');", - new String[]{"20000-11-16 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"20000-11-16 00:00:00"}); testSimpleEval("select to_timestamp('9-1116', 'Y-MMDD');", - new String[]{"2009-11-16 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2009-11-16 00:00:00"}); testSimpleEval("select to_timestamp('95-1116', 'YY-MMDD');", - new String[]{"1995-11-16 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"1995-11-16 00:00:00"}); testSimpleEval("select to_timestamp('995-1116', 'YYY-MMDD');", - new String[]{"1995-11-16 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"1995-11-16 00:00:00"}); testSimpleEval("select to_timestamp('2005426', 'YYYYWWD');", - new String[]{"2005-10-15 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2005-10-15 00:00:00"}); testSimpleEval("select to_timestamp('2005300', 'YYYYDDD');", - new String[]{"2005-10-27 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2005-10-27 00:00:00"}); testSimpleEval("select to_timestamp('2005527', 'IYYYIWID');", - new String[]{"2006-01-01 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2006-01-01 00:00:00"}); testSimpleEval("select to_timestamp('005527', 'IYYIWID');", - new String[]{"2006-01-01 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2006-01-01 00:00:00"}); testSimpleEval("select to_timestamp('05527', 'IYIWID');", - new String[]{"2006-01-01 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2006-01-01 00:00:00"}); testSimpleEval("select to_timestamp('5527', 'IIWID');", - new String[]{"2006-01-01 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2006-01-01 00:00:00"}); testSimpleEval("select to_timestamp('2005364', 'IYYYIDDD');", - new String[]{"2006-01-01 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2006-01-01 00:00:00"}); testSimpleEval("select to_timestamp('20050302', 'YYYYMMDD');", - new String[]{"2005-03-02 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2005-03-02 00:00:00"}); testSimpleEval("select to_timestamp('2005 03 02', 'YYYYMMDD');", - new String[]{"2005-03-02 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2005-03-02 00:00:00"}); testSimpleEval("select to_timestamp(' 2005 03 02', 'YYYYMMDD');", - new String[]{"2005-03-02 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2005-03-02 00:00:00"}); testSimpleEval("select to_timestamp(' 20050302', 'YYYYMMDD');", - new String[]{"2005-03-02 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2005-03-02 00:00:00"}); } @Test @@ -118,10 +118,21 @@ public void testToChar() throws IOException { @Test public void testExtract() throws IOException { + TimeZone UTC = TimeZone.getTimeZone("UTC"); + TimeZone PST = TimeZone.getTimeZone("PST"); + Schema schema2 = new Schema(); schema2.addColumn("col1", TIMESTAMP); testEval(schema2, "table1", - "1970-01-17 10:09:37" + getUserTimeZoneDisplay(), + "1970-01-17 10:09:37", + "select extract(year from col1), extract(month from col1), extract(day from col1) from table1;", + new String[]{"1970.0", "1.0", "17.0"}); + testEval(schema2, "table1", + "1970-01-17 10:09:37" + getUserTimeZoneDisplay(UTC), + "select extract(year from col1), extract(month from col1), extract(day from col1) from table1;", + new String[]{"1970.0", "1.0", "17.0"}); + testEval(schema2, "table1", + "1970-01-17 10:09:37" + getUserTimeZoneDisplay(PST), "select extract(year from col1), extract(month from col1), extract(day from col1) from table1;", new String[]{"1970.0", "1.0", "17.0"}); @@ -129,9 +140,17 @@ public void testExtract() throws IOException { Schema schema3 = new Schema(); schema3.addColumn("col1", TIME); testEval(schema3, "table1", - "10:09:37.5" + getUserTimeZoneDisplay(), + "10:09:37.5", "select extract(hour from col1), extract(minute from col1), extract(second from col1) from table1;", new String[]{"10.0", "9.0", "37.5"}); + testEval(schema3, "table1", + "10:09:37.5" + getUserTimeZoneDisplay(UTC), + "select extract(hour from col1), extract(minute from col1), extract(second from col1) from table1;", + new String[]{"10.0", "9.0", "37.5"}); + testEval(schema3, "table1", + "10:09:37.5" + getUserTimeZoneDisplay(PST), + "select extract(hour from col1), extract(minute from col1), extract(second from col1) from table1;", + new String[]{"18.0", "9.0", "37.5"}); Schema schema4 = new Schema(); schema4.addColumn("col1", DATE); @@ -215,22 +234,36 @@ public void testExtract() throws IOException { @Test public void testDatePart() throws IOException { - TimeZone timeZone = TimeZone.getDefault(); - assertEquals("UTC", timeZone.getID()); - assertEquals("UTC", TajoConf.getCurrentTimeZone().getID()); + TimeZone UTC = TimeZone.getTimeZone("UTC"); + TimeZone PST = TimeZone.getTimeZone("PST"); Schema schema2 = new Schema(); schema2.addColumn("col1", TIMESTAMP); + + testEval(schema2, "table1", + "1970-01-17 22:09:37", + "select date_part('year', col1), date_part('month', col1), date_part('day', col1) from table1;", + new String[]{"1970.0", "1.0", "17.0"}); testEval(schema2, "table1", - "1970-01-17 22:09:37" + getUserTimeZoneDisplay(), + "1970-01-17 22:09:37" + getUserTimeZoneDisplay(UTC), "select date_part('year', col1), date_part('month', col1), date_part('day', col1) from table1;", new String[]{"1970.0", "1.0", "17.0"}); + testEval(schema2, "table1", + "1970-01-17 22:09:37" + getUserTimeZoneDisplay(PST), + "select date_part('year', col1), date_part('month', col1), date_part('day', col1) from table1;", + new String[]{"1970.0", "1.0", "18.0"}); Schema schema3 = new Schema(); schema3.addColumn("col1", TIME); - testEval(schema3, "table1", "10:09:37.5" + getUserTimeZoneDisplay(), + testEval(schema3, "table1", "10:09:37.5", "select date_part('hour', col1), date_part('minute', col1), date_part('second', col1) from table1;", new String[]{"10.0", "9.0", "37.5"}); + testEval(schema3, "table1", "10:09:37.5" + getUserTimeZoneDisplay(UTC), + "select date_part('hour', col1), date_part('minute', col1), date_part('second', col1) from table1;", + new String[]{"10.0", "9.0", "37.5"}); + testEval(schema3, "table1", "10:09:37.5" + getUserTimeZoneDisplay(PST), + "select date_part('hour', col1), date_part('minute', col1), date_part('second', col1) from table1;", + new String[]{"18.0", "9.0", "37.5"}); Schema schema4 = new Schema(); schema4.addColumn("col1", DATE); @@ -325,7 +358,7 @@ public void testUtcUsecTo() throws IOException { public void testToDate() throws IOException { testSimpleEval("select to_date('2014-01-04', 'YYYY-MM-DD')", new String[]{"2014-01-04"}); testSimpleEval("select to_date('2014-01-04', 'YYYY-MM-DD') + interval '1 day'", - new String[]{"2014-01-05 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2014-01-05 00:00:00"}); testSimpleEval("SELECT to_date('201404', 'yyyymm');", new String[]{"2014-04-01"}); } @@ -333,63 +366,63 @@ public void testToDate() throws IOException { @Test public void testAddMonths() throws Exception { testSimpleEval("SELECT add_months(date '2013-12-17', 2::INT2);", - new String[]{"2014-02-17 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2014-02-17 00:00:00"}); testSimpleEval("SELECT add_months(date '2013-12-17', 2::INT4);", - new String[]{"2014-02-17 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2014-02-17 00:00:00"}); testSimpleEval("SELECT add_months(date '2013-12-17', 2::INT8);", - new String[]{"2014-02-17 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2014-02-17 00:00:00"}); testSimpleEval("SELECT add_months(timestamp '2013-12-17 12:10:20', 2::INT2);", - new String[]{"2014-02-17 12:10:20" + getUserTimeZoneDisplay()}); + new String[]{"2014-02-17 12:10:20"}); testSimpleEval("SELECT add_months(timestamp '2013-12-17 12:10:20', 2::INT4);", - new String[]{"2014-02-17 12:10:20" + getUserTimeZoneDisplay()}); + new String[]{"2014-02-17 12:10:20"}); testSimpleEval("SELECT add_months(timestamp '2013-12-17 12:10:20', 2::INT8);", - new String[]{"2014-02-17 12:10:20" + getUserTimeZoneDisplay()}); + new String[]{"2014-02-17 12:10:20"}); testSimpleEval("SELECT add_months(date '2014-02-05', -3::INT2);", - new String[]{"2013-11-05 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2013-11-05 00:00:00"}); testSimpleEval("SELECT add_months(date '2014-02-05', -3::INT4);", - new String[]{"2013-11-05 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2013-11-05 00:00:00"}); testSimpleEval("SELECT add_months(date '2014-02-05', -3::INT8);", - new String[]{"2013-11-05 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2013-11-05 00:00:00"}); testSimpleEval("SELECT add_months(timestamp '2014-02-05 12:10:20', -3::INT2);", - new String[]{"2013-11-05 12:10:20" + getUserTimeZoneDisplay()}); + new String[]{"2013-11-05 12:10:20"}); testSimpleEval("SELECT add_months(timestamp '2014-02-05 12:10:20', -3::INT4);", - new String[]{"2013-11-05 12:10:20" + getUserTimeZoneDisplay()}); + new String[]{"2013-11-05 12:10:20"}); testSimpleEval("SELECT add_months(timestamp '2014-02-05 12:10:20', -3::INT8);", - new String[]{"2013-11-05 12:10:20" + getUserTimeZoneDisplay()}); + new String[]{"2013-11-05 12:10:20"}); } @Test public void testAddDays() throws IOException { testSimpleEval("SELECT add_days(date '2013-12-30', 5::INT2);", - new String[]{"2014-01-04 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2014-01-04 00:00:00"}); testSimpleEval("SELECT add_days(date '2013-12-30', 5::INT4);", - new String[]{"2014-01-04 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2014-01-04 00:00:00"}); testSimpleEval("SELECT add_days(date '2013-12-30', 5::INT8);", - new String[]{"2014-01-04 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2014-01-04 00:00:00"}); testSimpleEval("SELECT add_days(timestamp '2013-12-30 12:10:20', 5::INT2);", - new String[]{"2014-01-04 12:10:20" + getUserTimeZoneDisplay()}); + new String[]{"2014-01-04 12:10:20"}); testSimpleEval("SELECT add_days(timestamp '2013-12-30 12:10:20', 5::INT4);", - new String[]{"2014-01-04 12:10:20" + getUserTimeZoneDisplay()}); + new String[]{"2014-01-04 12:10:20"}); testSimpleEval("SELECT add_days(timestamp '2013-12-30 12:10:20', 5::INT8);", - new String[]{"2014-01-04 12:10:20" + getUserTimeZoneDisplay()}); + new String[]{"2014-01-04 12:10:20"}); testSimpleEval("SELECT add_days(date '2013-12-05', -7::INT2);", - new String[]{"2013-11-28 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2013-11-28 00:00:00"}); testSimpleEval("SELECT add_days(date '2013-12-05', -7::INT4);", - new String[]{"2013-11-28 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2013-11-28 00:00:00"}); testSimpleEval("SELECT add_days(date '2013-12-05', -7::INT8);", - new String[]{"2013-11-28 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2013-11-28 00:00:00"}); testSimpleEval("SELECT add_days(timestamp '2013-12-05 12:10:20', -7::INT2);", - new String[]{"2013-11-28 12:10:20" + getUserTimeZoneDisplay()}); + new String[]{"2013-11-28 12:10:20"}); testSimpleEval("SELECT add_days(timestamp '2013-12-05 12:10:20', -7::INT4);", - new String[]{"2013-11-28 12:10:20" + getUserTimeZoneDisplay()}); + new String[]{"2013-11-28 12:10:20"}); testSimpleEval("SELECT add_days(timestamp '2013-12-05 12:10:20', -7::INT8);", - new String[]{"2013-11-28 12:10:20" + getUserTimeZoneDisplay()}); + new String[]{"2013-11-28 12:10:20"}); } @Test diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java index 5e6d05b019..1399f2c459 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java @@ -18,11 +18,8 @@ package org.apache.tajo.engine.query; -import org.apache.tajo.IntegrationTest; -import org.apache.tajo.QueryTestCaseBase; -import org.apache.tajo.TajoConstants; +import org.apache.tajo.*; import org.apache.tajo.TajoProtos.QueryState; -import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; @@ -37,6 +34,8 @@ import org.junit.experimental.categories.Category; import java.sql.ResultSet; +import java.util.HashMap; +import java.util.Map; import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; import static org.junit.Assert.*; @@ -538,4 +537,44 @@ public void testColumnEqualityButNotJoinCondition2() throws Exception { assertResultSet(res); cleanupQuery(res); } + + @Test + public void testTimezonedTable1() throws Exception { + try { + executeDDL("datetime_table_ddl.sql", "timezoned", new String[]{"timezoned1"}); + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } finally { + executeString("DROP TABLE IF EXISTS timezoned1"); + } + } + + @Test + public void testTimezonedTable2() throws Exception { + try { + executeDDL("datetime_table_timezoned_ddl.sql", "timezoned", new String[]{"timezoned2"}); + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } finally { + executeString("DROP TABLE IF EXISTS timezoned2"); + } + } + + @Test + public void testTimezonedTable3() throws Exception { + Map sessionVars = new HashMap(); + sessionVars.put(SessionVars.TZ.name(), "Asia/Seoul"); + getClient().updateSessionVariables(sessionVars); + + try { + executeDDL("datetime_table_timezoned_ddl.sql", "timezoned", new String[]{"timezoned3"}); + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } finally { + executeString("DROP TABLE IF EXISTS timezoned3"); + } + } } \ No newline at end of file diff --git a/tajo-core/src/test/resources/dataset/TestSelectQuery/timezoned/table1.tbl b/tajo-core/src/test/resources/dataset/TestSelectQuery/timezoned/table1.tbl new file mode 100644 index 0000000000..38e8bd9dca --- /dev/null +++ b/tajo-core/src/test/resources/dataset/TestSelectQuery/timezoned/table1.tbl @@ -0,0 +1,3 @@ +1980-4-1 01:50:30.010|1980-04-01 +80/4/1 1:50:30 AM|80/4/1 +1980 April 1 1:50:30|1980-04-01 \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_ddl.sql new file mode 100644 index 0000000000..beb19b7f7d --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_ddl.sql @@ -0,0 +1,4 @@ +CREATE EXTERNAL TABLE ${0} ( + t_timestamp TIMESTAMP, + t_date DATE +) USING TEXTFILE LOCATION ${table.path} \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql new file mode 100644 index 0000000000..d8d59c189a --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql @@ -0,0 +1,4 @@ +CREATE EXTERNAL TABLE ${0} ( + t_timestamp TIMESTAMP, + t_date DATE +) USING TEXTFILE WITH ('timezone' = 'ASIA/Seoul') LOCATION ${table.path} \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable1.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable1.sql new file mode 100644 index 0000000000..38c9e90ff1 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable1.sql @@ -0,0 +1 @@ +SELECT * FROM timezoned1; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable2.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable2.sql new file mode 100644 index 0000000000..722fc658b3 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable2.sql @@ -0,0 +1 @@ +SELECT * FROM timezoned2; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable3.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable3.sql new file mode 100644 index 0000000000..32b9c3a4ad --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable3.sql @@ -0,0 +1 @@ +SELECT * FROM timezoned3; \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testTimezonedTable1.result b/tajo-core/src/test/resources/results/TestSelectQuery/testTimezonedTable1.result new file mode 100644 index 0000000000..39f593b0c8 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSelectQuery/testTimezonedTable1.result @@ -0,0 +1,5 @@ +t_timestamp,t_date +------------------------------- +1980-04-01 01:50:30.01,1980-04-01 +1980-04-01 01:50:30,1980-04-01 +1980-04-01 01:50:30,1980-04-01 \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testTimezonedTable2.result b/tajo-core/src/test/resources/results/TestSelectQuery/testTimezonedTable2.result new file mode 100644 index 0000000000..916f4be8dd --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSelectQuery/testTimezonedTable2.result @@ -0,0 +1,5 @@ +t_timestamp,t_date +------------------------------- +1980-03-31 16:50:30.01,1980-04-01 +1980-03-31 16:50:30,1980-04-01 +1980-03-31 16:50:30,1980-04-01 \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testTimezonedTable3.result b/tajo-core/src/test/resources/results/TestSelectQuery/testTimezonedTable3.result new file mode 100644 index 0000000000..39f593b0c8 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSelectQuery/testTimezonedTable3.result @@ -0,0 +1,5 @@ +t_timestamp,t_date +------------------------------- +1980-04-01 01:50:30.01,1980-04-01 +1980-04-01 01:50:30,1980-04-01 +1980-04-01 01:50:30,1980-04-01 \ No newline at end of file diff --git a/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoMetaDataResultSet.java b/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoMetaDataResultSet.java index faa058d774..8f5bed682a 100644 --- a/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoMetaDataResultSet.java +++ b/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoMetaDataResultSet.java @@ -31,12 +31,14 @@ public class TajoMetaDataResultSet extends TajoResultSetBase { private List values; public TajoMetaDataResultSet(Schema schema, List values) { + super(null); init(); this.schema = schema; setDataTuples(values); } public TajoMetaDataResultSet(List columns, List types, List values) { + super(null); init(); schema = new Schema(); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java index 094d285a26..ce7b11d2ac 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java @@ -31,14 +31,13 @@ import java.io.IOException; import java.io.OutputStream; -//Compatibility with Apache Hive +// Compatibility with Apache Hive @Deprecated public class TextSerializerDeserializer implements SerializerDeserializer { public static final byte[] trueBytes = "true".getBytes(); public static final byte[] falseBytes = "false".getBytes(); private ProtobufJsonFormat protobufJsonFormat = ProtobufJsonFormat.getInstance(); - @Override public int serialize(Column col, Datum datum, OutputStream out, byte[] nullCharacters) throws IOException { @@ -86,12 +85,12 @@ public int serialize(Column col, Datum datum, OutputStream out, byte[] nullChara out.write(bytes); break; case TIME: - bytes = ((TimeDatum)datum).asChars(TajoConf.getCurrentTimeZone(), true).getBytes(); + bytes = ((TimeDatum)datum).asChars(TajoConf.getSystemTimezone(), true).getBytes(); length = bytes.length; out.write(bytes); break; case TIMESTAMP: - bytes = ((TimestampDatum)datum).asChars(TajoConf.getCurrentTimeZone(), true).getBytes(); + bytes = ((TimestampDatum)datum).asChars(TajoConf.getSystemTimezone(), true).getBytes(); length = bytes.length; out.write(bytes); break; From bbc0f97bba1ea6f9355bf17074fbf05af69abbd1 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Sun, 7 Dec 2014 15:03:19 +0900 Subject: [PATCH 3/6] Add documentation for time zone. --- .../src/main/sphinx/table_management.rst | 5 +- .../src/main/sphinx/table_management/csv.rst | 4 +- .../table_management/table_overview.rst | 94 +++++++++++++++++++ 3 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 tajo-docs/src/main/sphinx/table_management/table_overview.rst diff --git a/tajo-docs/src/main/sphinx/table_management.rst b/tajo-docs/src/main/sphinx/table_management.rst index 62a8e45198..2b21ddc305 100644 --- a/tajo-docs/src/main/sphinx/table_management.rst +++ b/tajo-docs/src/main/sphinx/table_management.rst @@ -2,8 +2,11 @@ Table Management ****************** +In Tajo, a table is a logical view of one data sources. Logically, one table consists of a logical schema, partitions, URL, and various properties. Physically, A table can be a directory in HDFS, a single file, one HBase table, or a RDBMS table. In order to make good use of Tajo, users need to understand features and physical characteristics of their physical layout. This section explains all about table management. + .. toctree:: :maxdepth: 1 + table_management/table_overview table_management/file_formats - table_management/compression + table_management/compression \ No newline at end of file diff --git a/tajo-docs/src/main/sphinx/table_management/csv.rst b/tajo-docs/src/main/sphinx/table_management/csv.rst index 71313d6c1d..37dc2baffe 100644 --- a/tajo-docs/src/main/sphinx/table_management/csv.rst +++ b/tajo-docs/src/main/sphinx/table_management/csv.rst @@ -39,9 +39,9 @@ Now, the CSV storage format provides the following physical properties. * ``text.delimiter``: delimiter character. ``|`` or ``\u0001`` is usually used, and the default field delimiter is ``|``. * ``text.null``: NULL character. The default NULL character is an empty string ``''``. Hive's default NULL character is ``'\\N'``. * ``compression.codec``: Compression codec. You can enable compression feature and set specified compression algorithm. The compression algorithm used to compress files. The compression codec name should be the fully qualified class name inherited from `org.apache.hadoop.io.compress.CompressionCodec `_. By default, compression is disabled. -* ``csvfile.serde``: custom (De)serializer class. ``org.apache.tajo.storage.TextSerializerDeserializer`` is the default (De)serializer class. +* ``csvfile.serde`` (deprecated): custom (De)serializer class. ``org.apache.tajo.storage.TextSerializerDeserializer`` is the default (De)serializer class. +* ``timezone``: the time zone that the table uses for writting. When table rows are read or written, ```timestamp``` and ```time``` column values are adjusted by this timezone if it is set. Time zone can be an abbreviation form like 'PST' or 'DST'. Also, it accepts an offset-based form like 'UTC+9' or a location-based form like 'Asia/Seoul'. * ``text.error-tolerance.max-num``: the maximum number of permissible parsing errors. This value should be an integer value. By default, ``text.error-tolerance.max-num`` is ``0``. According to the value, parsing errors will be handled in different ways. - * If ``text.error-tolerance.max-num < 0``, all parsing errors are ignored. * If ``text.error-tolerance.max-num == 0``, any parsing error is not allowed. If any error occurs, the query will be failed. (default) * If ``text.error-tolerance.max-num > 0``, the given number of parsing errors in each task will be pemissible. diff --git a/tajo-docs/src/main/sphinx/table_management/table_overview.rst b/tajo-docs/src/main/sphinx/table_management/table_overview.rst new file mode 100644 index 0000000000..5361e014ae --- /dev/null +++ b/tajo-docs/src/main/sphinx/table_management/table_overview.rst @@ -0,0 +1,94 @@ +************************************* +Overview of Tajo Tables +************************************* + +Overview +======== + +.. todo:: + +Table Properties +================ +All table formats provide parameters for enabling or disabling features and adjusting physical parameters. +The ``WITH`` clause in the CREATE TABLE statement allows users to set those properties. + +The following example is to set a custom field delimiter, NULL character, and compression codec: + +.. code-block:: sql + + CREATE TABLE table1 ( + id int, + name text, + score float, + type text + ) USING CSV WITH('text.delimiter'='\u0001', + 'text.null'='\\N', + 'compression.codec'='org.apache.hadoop.io.compress.SnappyCodec'); + +Each physical table layout has its own specialized properties. They will be addressed in :doc:`/table_management/file_formats`. + + +Common Table Properties +======================= + +There are some common table properties which are used in most tables. + +Compression +----------- +.. todo:: + +Time zone +--------- +In Tajo, a table property ``timezone`` allows users to specify a time zone that the table uses for reading or writing. +When each table row are read or written, ```timestamp``` and ```time``` column values are adjusted by a given time zone if it is set. Time zone can be an abbreviation form like 'PST' or 'DST'. Also, it accepts an offset-based form like 'UTC+9' or a location-based form like 'Asia/Seoul'. + +Each table has one time zone, and many tables can have different time zones. Internally, Tajo translates all tables data to UTC-based values. So, complex queries like join with multiple time zones work well. + +.. note:: + + In many cases, offset-based forms or locaion-based forms are recommanded. In order to know the list of time zones, please refer to `List of tz database time zones `_ + +How time zone works in Tajo +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For example, consider that there is a list of delimited text lines where each rows are written with ``Asia/Seoul`` time zone (i.e., GMT + 9). + +.. code-block:: text + + 1980-4-1 01:50:30.010|1980-04-01 + 80/4/1 1:50:30 AM|80/4/1 + 1980 April 1 1:50:30|1980-04-01 + + +In order to register the table, we should put a table property ``'timezone'='Asia/Seoul'`` in ``CREATE TABLE`` statement as follows: + +.. code-block:: sql + + CREATE EXTERNAL TABLE table1 ( + t_timestamp TIMESTAMP, + t_date DATE + ) USING TEXTFILE WITH('text.delimiter'='|', 'timezone'='ASIA/Seoul') LOCATION '/path-to-table/' + + +By default, ``tsql`` and ``TajoClient`` API use UTC time zone. So, timestamp values in the result are adjusted by the time zone offset. But, date is not adjusted because date type does not consider time zone. + +.. code-block:: sql + + default> SELECT * FROM table1 + t_timestamp, t_date + ---------------------------------- + 1980-03-31 16:50:30.01, 1980-04-01 + 1980-03-31 16:50:30 , 1980-04-01 + 1980-03-31 16:50:30 , 1980-04-01 + +In addition, users can set client-side time zone by setting a session variable 'TZ'. It enables a client to translate timestamp or time values to user's time zoned ones. + +.. code-block:: sql + + default> \set TZ 'Asia/Seoul' + default> SELECT * FROM table1 + t_timestamp, t_date + ---------------------------------- + 1980-04-01 01:50:30.01, 1980-04-01 + 1980-04-01 01:50:30 , 1980-04-01 + 1980-04-01 01:50:30 , 1980-04-01 \ No newline at end of file From e58ae797fdbff1b2b1176524ad48a8d8f04ba74c Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Sun, 7 Dec 2014 23:21:18 +0900 Subject: [PATCH 4/6] Fixed test failures. --- .../test/java/org/apache/tajo/TajoTestingCluster.java | 3 +++ .../org/apache/tajo/engine/query/TestSelectQuery.java | 5 ++++- .../test/java/org/apache/tajo/jdbc/TestResultSet.java | 10 +++------- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java index 603da0c8e2..fe4c432bd2 100644 --- a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -59,6 +59,7 @@ import java.sql.ResultSet; import java.util.ArrayList; import java.util.List; +import java.util.TimeZone; import java.util.UUID; public class TajoTestingCluster { @@ -117,6 +118,8 @@ void setTestingFlagProperties() { } void initPropertiesAndConfigs() { + TimeZone.setDefault(TimeZone.getTimeZone(TajoConstants.UTC_TIMEZONE)); + if (System.getProperty(ConfVars.RESOURCE_MANAGER_CLASS.varname) != null) { String testResourceManager = System.getProperty(ConfVars.RESOURCE_MANAGER_CLASS.varname); Preconditions.checkState(testResourceManager.equals(TajoWorkerResourceManager.class.getCanonicalName())); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java index 1399f2c459..382d2d06ae 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.query; +import com.google.common.collect.Lists; import org.apache.tajo.*; import org.apache.tajo.TajoProtos.QueryState; import org.apache.tajo.catalog.CatalogService; @@ -565,7 +566,7 @@ public void testTimezonedTable2() throws Exception { @Test public void testTimezonedTable3() throws Exception { Map sessionVars = new HashMap(); - sessionVars.put(SessionVars.TZ.name(), "Asia/Seoul"); + sessionVars.put(SessionVars.TZ.name(), "UTC+9"); getClient().updateSessionVariables(sessionVars); try { @@ -576,5 +577,7 @@ public void testTimezonedTable3() throws Exception { } finally { executeString("DROP TABLE IF EXISTS timezoned3"); } + + getClient().unsetSessionVariables(Lists.newArrayList("TZ")); } } \ No newline at end of file diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java index bcff78e63f..543c17ac07 100644 --- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java +++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java @@ -32,6 +32,7 @@ import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.client.QueryClientImpl; import org.apache.tajo.client.TajoClient; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; @@ -105,8 +106,8 @@ public static void terminate() throws IOException { } @Test - public void test() throws IOException, SQLException { - TajoResultSet rs = new TajoResultSet(null, null, conf, desc); + public void test() throws Exception { + TajoResultSet rs = new TajoResultSet(TajoTestingCluster.newTajoClient(), null, conf, desc); ResultSetMetaData meta = rs.getMetaData(); assertNotNull(meta); Schema schema = scoreSchema; @@ -133,9 +134,6 @@ public void testDateTimeType() throws Exception { // Hcatalog does not support date type, time type in hive-0.12.0 if(util.isHCatalogStoreRunning()) return; - TimeZone originalTimezone = TimeZone.getDefault(); - TimeZone.setDefault(TimeZone.getTimeZone("UTC")); - ResultSet res = null; TajoClient client = TajoTestingCluster.newTajoClient(); try { @@ -209,8 +207,6 @@ public void testDateTimeType() throws Exception { assertNotNull(timestamp); assertEquals("2014-01-01 10:00:00.0", timestamp.toString()); } finally { - TimeZone.setDefault(originalTimezone); - if (res != null) { res.close(); } From 332eacffdbbacc3a7850e6400734213448cf4ac0 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 8 Dec 2014 09:47:17 +0900 Subject: [PATCH 5/6] Fixed the test failure in Java 6. --- .../org/apache/tajo/engine/query/TestSelectQuery.java | 2 +- .../TestSelectQuery/datetime_table_timezoned_ddl.sql | 2 +- .../src/main/sphinx/table_management/table_overview.rst | 8 ++++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java index 382d2d06ae..5b7641a5d8 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java @@ -566,7 +566,7 @@ public void testTimezonedTable2() throws Exception { @Test public void testTimezonedTable3() throws Exception { Map sessionVars = new HashMap(); - sessionVars.put(SessionVars.TZ.name(), "UTC+9"); + sessionVars.put(SessionVars.TZ.name(), "GMT+9"); getClient().updateSessionVariables(sessionVars); try { diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql index d8d59c189a..11014d86dd 100644 --- a/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql @@ -1,4 +1,4 @@ CREATE EXTERNAL TABLE ${0} ( t_timestamp TIMESTAMP, t_date DATE -) USING TEXTFILE WITH ('timezone' = 'ASIA/Seoul') LOCATION ${table.path} \ No newline at end of file +) USING TEXTFILE WITH ('timezone' = 'GMT+9') LOCATION ${table.path} \ No newline at end of file diff --git a/tajo-docs/src/main/sphinx/table_management/table_overview.rst b/tajo-docs/src/main/sphinx/table_management/table_overview.rst index 5361e014ae..bb4b82741e 100644 --- a/tajo-docs/src/main/sphinx/table_management/table_overview.rst +++ b/tajo-docs/src/main/sphinx/table_management/table_overview.rst @@ -40,14 +40,18 @@ Compression Time zone --------- In Tajo, a table property ``timezone`` allows users to specify a time zone that the table uses for reading or writing. -When each table row are read or written, ```timestamp``` and ```time``` column values are adjusted by a given time zone if it is set. Time zone can be an abbreviation form like 'PST' or 'DST'. Also, it accepts an offset-based form like 'UTC+9' or a location-based form like 'Asia/Seoul'. +When each table row are read or written, ```timestamp``` and ```time``` column values are adjusted by a given time zone if it is set. Time zone can be an abbreviation form like 'PST' or 'DST'. Also, it accepts an offset-based form like 'GMT+9' or UTC+9' or a location-based form like 'Asia/Seoul'. -Each table has one time zone, and many tables can have different time zones. Internally, Tajo translates all tables data to UTC-based values. So, complex queries like join with multiple time zones work well. +Each table has one time zone, and many tables can have different time zones. Internally, Tajo translates all tables data to offset-based values. So, complex queries like join with multiple time zones work well. .. note:: In many cases, offset-based forms or locaion-based forms are recommanded. In order to know the list of time zones, please refer to `List of tz database time zones `_ +.. note:: + + Java 6 does not recognize many location-based time zones and an offset-based timezone using the prefix 'UTC'. We highly recommanded using the offset-based time zone using the prefix 'GMT'. In other words, you should use 'GMT-7' instead of 'UTC-7' in Java 6. + How time zone works in Tajo ^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 071316eb055778fcb1892ae47a75c04910e5ef14 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 8 Dec 2014 10:10:24 +0900 Subject: [PATCH 6/6] Fixed remain UTC usages. --- .../java/org/apache/tajo/TajoConstants.java | 3 ++- .../apache/tajo/storage/StorageConstants.java | 4 +++- .../apache/tajo/datum/TestTimestampDatum.java | 2 +- .../function/TestDateTimeFunctions.java | 12 +++++------ .../tajo/engine/query/TestSortQuery.java | 21 +++++++------------ .../org/apache/tajo/jdbc/TestTajoJdbc.java | 4 ---- 6 files changed, 20 insertions(+), 26 deletions(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java b/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java index 4d1e4604f5..77572a608b 100644 --- a/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java +++ b/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java @@ -31,7 +31,8 @@ public class TajoConstants { public static final String DEFAULT_DATABASE_NAME = "default"; public static final String DEFAULT_SCHEMA_NAME = "public"; - public static final String UTC_TIMEZONE = "UTC"; + /** Java 6 only recognize GMT instead of UTC. So, we should keep using GMT. */ + public static final String UTC_TIMEZONE = "GMT"; public static final String EMPTY_STRING = ""; diff --git a/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java b/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java index 917da17a68..b146686414 100644 --- a/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java +++ b/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java @@ -18,13 +18,15 @@ package org.apache.tajo.storage; +import org.apache.tajo.TajoConstants; + public class StorageConstants { // Common table properties ------------------------------------------------- // time zone public static final String TIMEZONE = "timezone"; - public static final String DEFAULT_TIMEZONE = "UTC"; + public static final String DEFAULT_TIMEZONE = TajoConstants.UTC_TIMEZONE; // compression public static final String COMPRESSION_CODEC = "compression.codec"; diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java index a925954488..277eefcfa6 100644 --- a/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java +++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java @@ -150,7 +150,7 @@ public final void testTimestampConstructor() { assertEquals(datum2, datum); for (int i = 0; i < 100; i++) { - TimeZone timeZone = TimeZone.getTimeZone("UTC"); + TimeZone timeZone = TimeZone.getTimeZone("GMT"); Calendar cal = Calendar.getInstance(timeZone); long jTime = System.currentTimeMillis(); int uTime = (int)(jTime / 1000); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java index 59dc8fd71c..cb7856bac9 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java @@ -118,7 +118,7 @@ public void testToChar() throws IOException { @Test public void testExtract() throws IOException { - TimeZone UTC = TimeZone.getTimeZone("UTC"); + TimeZone GMT = TimeZone.getTimeZone("GMT"); TimeZone PST = TimeZone.getTimeZone("PST"); Schema schema2 = new Schema(); @@ -128,7 +128,7 @@ public void testExtract() throws IOException { "select extract(year from col1), extract(month from col1), extract(day from col1) from table1;", new String[]{"1970.0", "1.0", "17.0"}); testEval(schema2, "table1", - "1970-01-17 10:09:37" + getUserTimeZoneDisplay(UTC), + "1970-01-17 10:09:37" + getUserTimeZoneDisplay(GMT), "select extract(year from col1), extract(month from col1), extract(day from col1) from table1;", new String[]{"1970.0", "1.0", "17.0"}); testEval(schema2, "table1", @@ -144,7 +144,7 @@ public void testExtract() throws IOException { "select extract(hour from col1), extract(minute from col1), extract(second from col1) from table1;", new String[]{"10.0", "9.0", "37.5"}); testEval(schema3, "table1", - "10:09:37.5" + getUserTimeZoneDisplay(UTC), + "10:09:37.5" + getUserTimeZoneDisplay(GMT), "select extract(hour from col1), extract(minute from col1), extract(second from col1) from table1;", new String[]{"10.0", "9.0", "37.5"}); testEval(schema3, "table1", @@ -234,7 +234,7 @@ public void testExtract() throws IOException { @Test public void testDatePart() throws IOException { - TimeZone UTC = TimeZone.getTimeZone("UTC"); + TimeZone GMT = TimeZone.getTimeZone("GMT"); TimeZone PST = TimeZone.getTimeZone("PST"); Schema schema2 = new Schema(); @@ -245,7 +245,7 @@ public void testDatePart() throws IOException { "select date_part('year', col1), date_part('month', col1), date_part('day', col1) from table1;", new String[]{"1970.0", "1.0", "17.0"}); testEval(schema2, "table1", - "1970-01-17 22:09:37" + getUserTimeZoneDisplay(UTC), + "1970-01-17 22:09:37" + getUserTimeZoneDisplay(GMT), "select date_part('year', col1), date_part('month', col1), date_part('day', col1) from table1;", new String[]{"1970.0", "1.0", "17.0"}); testEval(schema2, "table1", @@ -258,7 +258,7 @@ public void testDatePart() throws IOException { testEval(schema3, "table1", "10:09:37.5", "select date_part('hour', col1), date_part('minute', col1), date_part('second', col1) from table1;", new String[]{"10.0", "9.0", "37.5"}); - testEval(schema3, "table1", "10:09:37.5" + getUserTimeZoneDisplay(UTC), + testEval(schema3, "table1", "10:09:37.5" + getUserTimeZoneDisplay(GMT), "select date_part('hour', col1), date_part('minute', col1), date_part('second', col1) from table1;", new String[]{"10.0", "9.0", "37.5"}); testEval(schema3, "table1", "10:09:37.5" + getUserTimeZoneDisplay(PST), diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java index dd738e7177..48a1464d22 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java @@ -125,19 +125,14 @@ public final void testSortAfterGroupbyWithAlias() throws Exception { public final void testSortWithDate() throws Exception { // skip this test if catalog uses HCatalogStore. // It is because HCatalogStore does not support Time data type. - TimeZone originalTimezone = TimeZone.getDefault(); - TimeZone.setDefault(TimeZone.getTimeZone("UTC")); - try { - if (!testingCluster.isHCatalogStoreRunning()) { - // create external table table1 (col1 timestamp, col2 date, col3 time) ... - executeDDL("create_table_with_date_ddl.sql", "table1"); - - ResultSet res = executeQuery(); - assertResultSet(res); - cleanupQuery(res); - } - } finally { - TimeZone.setDefault(originalTimezone); + + if (!testingCluster.isHCatalogStoreRunning()) { + // create external table table1 (col1 timestamp, col2 date, col3 time) ... + executeDDL("create_table_with_date_ddl.sql", "table1"); + + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); } } diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java index d9c1354d60..99baeba9c5 100644 --- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java +++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java @@ -543,8 +543,6 @@ public void testSortWithDateTime() throws Exception { // skip this test if catalog uses HCatalogStore. // It is because HCatalogStore does not support Time data type. - TimeZone originalTimezone = TimeZone.getDefault(); - TimeZone.setDefault(TimeZone.getTimeZone("UTC")); try { if (!testingCluster.isHCatalogStoreRunning()) { @@ -574,8 +572,6 @@ public void testSortWithDateTime() throws Exception { } } finally { - TimeZone.setDefault(originalTimezone); - cleanupQuery(res); if (stmt != null) { stmt.close();