From 4545eb76368136023d6d8d590db78b2160ed6808 Mon Sep 17 00:00:00 2001 From: Mehant Baid Date: Thu, 17 Apr 2014 19:02:25 -0700 Subject: [PATCH] DRILL-549: Implement functions for date and interval data types Jira, DRILL-549 has the complete list of functions implemented. --- exec/java-exec/src/main/codegen/config.fmpp | 5 +- .../main/codegen/data/DateIntervalFunc.tdd | 19 ++ .../src/main/codegen/data/ExtractTypes.tdd | 2 +- .../codegen/data/IntervalNumericTypes.tdd | 19 ++ .../codegen/templates/CastVarCharDate.java | 2 +- .../DateDateArithmeticFunctions.java | 75 ++++++ .../DateIntervalArithmeticFunctions.java | 242 ++++++++++++++++++ .../DateToCharFunctions.java | 79 ++++++ .../Extract.java | 7 +- .../IntervalIntervalArithmetic.java | 117 +++++++++ .../IntervalNumericArithmetic.java | 146 +++++++++++ .../ToDateTypeFunctions.java | 77 ++++++ .../codegen/templates/FixedValueVectors.java | 2 +- .../exec/expr/fn/impl/CastBigIntDate.java | 2 +- .../expr/fn/impl/CastBigIntTimeStamp.java | 2 +- .../expr/fn/impl/CastBigIntTimeStampTZ.java | 55 ++++ .../drill/exec/expr/fn/impl/CastIntTime.java | 46 ++++ .../exec/expr/fn/impl/DateTypeFunctions.java | 201 +++++++++------ .../drill/exec/expr/fn/impl/DateUtility.java | 2 + .../drill/exec/ops/FragmentContext.java | 8 +- .../planner/fragment/SimpleParallelizer.java | 5 + .../exec/planner/logical/DrillOptiq.java | 19 ++ .../drill/exec/fn/impl/TestDateFunctions.java | 143 +++++++++++ .../date/date_difference_arithmetic.json | 36 +++ .../date/date_interval_arithmetic.json | 46 ++++ .../functions/date/interval_arithmetic.json | 44 ++++ .../resources/functions/date/to_char.json | 36 +++ .../functions/date/to_date_type.json | 36 +++ .../apache/drill/exec/proto/BitControl.java | 107 +++++++- protocol/src/main/protobuf/BitControl.proto | 3 +- 30 files changed, 1479 insertions(+), 104 deletions(-) create mode 100644 exec/java-exec/src/main/codegen/data/DateIntervalFunc.tdd create mode 100644 exec/java-exec/src/main/codegen/data/IntervalNumericTypes.tdd create mode 100644 exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateDateArithmeticFunctions.java create mode 100644 exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateIntervalArithmeticFunctions.java create mode 100644 exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateToCharFunctions.java rename exec/java-exec/src/main/codegen/templates/{ => DateIntervalFunctionTemplates}/Extract.java (95%) create mode 100644 exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/IntervalIntervalArithmetic.java create mode 100644 exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/IntervalNumericArithmetic.java create mode 100644 exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/ToDateTypeFunctions.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastBigIntTimeStampTZ.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastIntTime.java create mode 100644 exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateFunctions.java create mode 100644 exec/java-exec/src/test/resources/functions/date/date_difference_arithmetic.json create mode 100644 exec/java-exec/src/test/resources/functions/date/date_interval_arithmetic.json create mode 100644 exec/java-exec/src/test/resources/functions/date/interval_arithmetic.json create mode 100644 exec/java-exec/src/test/resources/functions/date/to_char.json create mode 100644 exec/java-exec/src/test/resources/functions/date/to_date_type.json diff --git a/exec/java-exec/src/main/codegen/config.fmpp b/exec/java-exec/src/main/codegen/config.fmpp index 49e0614a16f..bdc415777e8 100644 --- a/exec/java-exec/src/main/codegen/config.fmpp +++ b/exec/java-exec/src/main/codegen/config.fmpp @@ -26,7 +26,10 @@ data: { date: tdd(../data/DateTypes.tdd), extract: tdd(../data/ExtractTypes.tdd), parser: tdd(../data/Parser.tdd), - decimal: tdd(../data/DecimalTypes.tdd) + decimal: tdd(../data/DecimalTypes.tdd), + dateIntervalFunc: tdd(../data/DateIntervalFunc.tdd), + intervalNumericTypes: tdd(../data/IntervalNumericTypes.tdd), + extract: tdd(../data/ExtractTypes.tdd) } freemarkerLinks: { includes: includes/ diff --git a/exec/java-exec/src/main/codegen/data/DateIntervalFunc.tdd b/exec/java-exec/src/main/codegen/data/DateIntervalFunc.tdd new file mode 100644 index 00000000000..ea295b38c23 --- /dev/null +++ b/exec/java-exec/src/main/codegen/data/DateIntervalFunc.tdd @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:# www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +{ + {dates: ["Date", "TimeStamp", "Time", "TimeStampTZ"] }, + {intervals: ["Interval", "IntervalDay", "IntervalYear", "Int", "BigInt"] } +} diff --git a/exec/java-exec/src/main/codegen/data/ExtractTypes.tdd b/exec/java-exec/src/main/codegen/data/ExtractTypes.tdd index 70d78617777..594f78fd22f 100644 --- a/exec/java-exec/src/main/codegen/data/ExtractTypes.tdd +++ b/exec/java-exec/src/main/codegen/data/ExtractTypes.tdd @@ -16,5 +16,5 @@ { toTypes: [Second, Minute, Hour, Day, Month, Year], - fromTypes: [Date, Time, TimeStamp, Interval, IntervalDay, IntervalYear] + fromTypes: [Date, Time, TimeStamp, TimeStampTZ, Interval, IntervalDay, IntervalYear] } diff --git a/exec/java-exec/src/main/codegen/data/IntervalNumericTypes.tdd b/exec/java-exec/src/main/codegen/data/IntervalNumericTypes.tdd new file mode 100644 index 00000000000..1646820bf85 --- /dev/null +++ b/exec/java-exec/src/main/codegen/data/IntervalNumericTypes.tdd @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:# www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +{ + {interval: ["Interval", "IntervalDay", "IntervalYear"] }, + {numeric: ["Int", "BigInt", "Float4", "Float8"] } +} \ No newline at end of file diff --git a/exec/java-exec/src/main/codegen/templates/CastVarCharDate.java b/exec/java-exec/src/main/codegen/templates/CastVarCharDate.java index ab846c0dcc6..39b8b41ce9e 100644 --- a/exec/java-exec/src/main/codegen/templates/CastVarCharDate.java +++ b/exec/java-exec/src/main/codegen/templates/CastVarCharDate.java @@ -72,7 +72,7 @@ public void eval() { <#elseif type.to == "Time"> org.joda.time.format.DateTimeFormatter f = org.apache.drill.exec.expr.fn.impl.DateUtility.getTimeFormatter(); - out.value = (int) (org.joda.time.DateMidnight.parse(input, f).withZoneRetainFields(org.joda.time.DateTimeZone.UTC)).getMillis(); + out.value = (int) ((f.parseDateTime(input)).withZoneRetainFields(org.joda.time.DateTimeZone.UTC).getMillis()); } diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateDateArithmeticFunctions.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateDateArithmeticFunctions.java new file mode 100644 index 00000000000..181acf0e527 --- /dev/null +++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateDateArithmeticFunctions.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.drill.exec.expr.annotations.Workspace; + +<@pp.dropOutputFile /> + +<#list dateIntervalFunc.dates as type> + +<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/G${type}Difference.java" /> + +<#include "/@includes/license.ftl" /> + +package org.apache.drill.exec.expr.fn.impl; + +import org.apache.drill.exec.expr.DrillSimpleFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.annotations.Workspace; +import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.holders.*; +import org.apache.drill.exec.record.RecordBatch; + +import io.netty.buffer.ByteBuf; + +@SuppressWarnings("unused") +@FunctionTemplate(names = {"date_diff", "subtract"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) +public class G${type}Difference implements DrillSimpleFunc { + + @Param ${type}Holder left; + @Param ${type}Holder right; + @Output IntervalDayHolder out; + + public void setup(RecordBatch b) { + } + + public void eval() { + <#if type == "Time"> + out.milliSeconds = left.value - right.value; + <#elseif type == "Date"> + out.days = (int) ((left.value - right.value) / org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis); + <#elseif type == "TimeStamp"> + long difference = (left.value - right.value); + out.milliSeconds = (int) (difference % org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis); + out.days = (int) (difference / org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis); + <#elseif type == "TimeStampTZ"> + if (left.index != right.index) { + // The two inputs are in different time zones convert one of them + org.joda.time.DateTime leftInput = new org.joda.time.DateTime(left.value, org.joda.time.DateTimeZone.forID(org.apache.drill.exec.expr.fn.impl.DateUtility.timezoneList[left.index])); + // convert left timestamp to the time zone of the right timestamp + left.value = (leftInput.withZone(org.joda.time.DateTimeZone.forID(org.apache.drill.exec.expr.fn.impl.DateUtility.timezoneList[right.index]))).getMillis(); + } + long difference = (left.value - right.value); + out.milliSeconds = (int) (difference % org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis); + out.days = (int) (difference / org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis); + + } +} + \ No newline at end of file diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateIntervalArithmeticFunctions.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateIntervalArithmeticFunctions.java new file mode 100644 index 00000000000..9103a864044 --- /dev/null +++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateIntervalArithmeticFunctions.java @@ -0,0 +1,242 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +<@pp.dropOutputFile /> + + +<#list dateIntervalFunc.dates as datetype> +<#list dateIntervalFunc.intervals as intervaltype> + +<#if datetype == "Date" || datetype == "TimeStamp" || datetype == "TimeStampTZ"> +<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/${datetype}${intervaltype}Functions.java" /> + +<#include "/@includes/license.ftl" /> + +package org.apache.drill.exec.expr.fn.impl; + +import io.netty.buffer.ByteBuf; + +import org.apache.drill.exec.expr.DrillSimpleFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.annotations.Workspace; +import org.apache.drill.exec.expr.holders.*; +import org.apache.drill.exec.record.RecordBatch; +import org.joda.time.MutableDateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.DateMidnight; +import org.apache.drill.exec.expr.fn.impl.DateUtility; + + +public class ${datetype}${intervaltype}Functions { + +<#macro dateIntervalArithmeticBlock left right temp op output intervaltype datetype> + + <#-- Throw exception if we are adding integer to a TIMESTAMP --> + <#if (datetype == "TimeStamp" || datetype == "TimeStampTZ") && (intervaltype == "Int" || intervaltype == "BigInt")> + if (1 == 1) { + /* Since this will be included in the run time generated code, there might be other logic that follows this + * if the exception is raised without a condition, we will hit compilation issues while compiling run time code + * with the error: unreachable code. + */ + throw new UnsupportedOperationException("Cannot add integer to TIMESTAMP, cast it to specific interval"); + } + <#else> + ${temp}.setMillis(${left}.value); + + <#if intervaltype == "Interval"> + ${temp}.addMonths(${right}.months <#if op == '-'> * -1 ); + ${temp}.addDays(${right}.days <#if op == '-'> * -1 ); + ${temp}.add(${right}.milliSeconds <#if op == '-'> * -1 ); + <#elseif intervaltype == "IntervalYear"> + ${temp}.addMonths(${right}.value <#if op == '-'> * -1 ); + <#elseif intervaltype == "IntervalDay"> + ${temp}.addDays(${right}.days <#if op == '-'> * -1 ); + ${temp}.add(${right}.milliSeconds <#if op == '-'> * -1 ); + <#elseif intervaltype == "Int" || intervaltype == "BigInt"> + ${temp}.addDays((int) ${right}.value <#if op == '-'> * -1 ); + + + <#-- copy the time zone index if its a timestamp --> + <#if datetype == "TimeStampTZ"> + ${output}.index = ${left}.index; + + + ${output}.value = ${temp}.getMillis(); + + + + @SuppressWarnings("unused") + @FunctionTemplate(names = {"date_add", "add"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL) + public static class ${datetype}${intervaltype}AddFunction implements DrillSimpleFunc { + @Param ${datetype}Holder left; + @Param ${intervaltype}Holder right; + @Workspace org.joda.time.MutableDateTime temp; + @Output ${datetype}Holder out; + + public void setup(RecordBatch incoming) { + <#if datetype == "TimeStampTZ"> + temp = new org.joda.time.MutableDateTime(org.joda.time.DateTimeZone.forID(org.apache.drill.exec.expr.fn.impl.DateUtility.timezoneList[left.index])); + <#else> + temp = new org.joda.time.MutableDateTime(org.joda.time.DateTimeZone.UTC); + + } + + public void eval() { + <@dateIntervalArithmeticBlock left="left" right="right" temp = "temp" op = "+" output="out" intervaltype=intervaltype datetype = datetype/> + } + } + + <#-- Below function is the same as above except the arguments are in reverse order. We use macros to avoid having the logic in multiple places --> + @SuppressWarnings("unused") + @FunctionTemplate(names = {"date_add", "add"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL) + public static class ${intervaltype}${datetype}AddFunction implements DrillSimpleFunc { + + @Param ${intervaltype}Holder right; + @Param ${datetype}Holder left; + @Workspace org.joda.time.MutableDateTime temp; + @Output ${datetype}Holder out; + + public void setup(RecordBatch incoming) { + <#if datetype == "TimeStampTZ"> + temp = new org.joda.time.MutableDateTime(org.joda.time.DateTimeZone.forID(org.apache.drill.exec.expr.fn.impl.DateUtility.timezoneList[left.index])); + <#else> + temp = new org.joda.time.MutableDateTime(org.joda.time.DateTimeZone.UTC); + + } + + public void eval() { + + <@dateIntervalArithmeticBlock left="left" right="right" temp = "temp" op = "+" output="out" intervaltype=intervaltype datetype = datetype/> + } + } + + @SuppressWarnings("unused") + @FunctionTemplate(names = {"date_sub", "subtract"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL) + public static class ${datetype}${intervaltype}SubtractFunction implements DrillSimpleFunc { + @Param ${datetype}Holder left; + @Param ${intervaltype}Holder right; + @Workspace org.joda.time.MutableDateTime temp; + @Output ${datetype}Holder out; + + public void setup(RecordBatch incoming) { + <#if datetype == "TimeStampTZ"> + temp = new org.joda.time.MutableDateTime(org.joda.time.DateTimeZone.forID(org.apache.drill.exec.expr.fn.impl.DateUtility.timezoneList[left.index])); + <#else> + temp = new org.joda.time.MutableDateTime(org.joda.time.DateTimeZone.UTC); + + } + + public void eval() { + <@dateIntervalArithmeticBlock left="left" right="right" temp = "temp" op = "-" output="out" intervaltype=intervaltype datetype = datetype/> + } + } +} +<#elseif datetype == "Time"> +<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/${datetype}${intervaltype}Functions.java" /> + +<#include "/@includes/license.ftl" /> + +package org.apache.drill.exec.expr.fn.impl; + +import io.netty.buffer.ByteBuf; + +import org.apache.drill.exec.expr.DrillSimpleFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.annotations.Workspace; +import org.apache.drill.exec.expr.holders.*; +import org.apache.drill.exec.record.RecordBatch; +import org.joda.time.MutableDateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.DateMidnight; +import org.apache.drill.exec.expr.fn.impl.DateUtility; + +public class ${datetype}${intervaltype}Functions { +<#macro timeIntervalArithmeticBlock left right temp op output intervaltype> + <#if intervaltype == "Interval"> + if (${right}.months != 0 || ${right}.days != 0) { + throw new UnsupportedOperationException("Cannot add interval type with months or days to TIME"); + } + ${output} = ${left}.value ${op} ${right}.milliSeconds; + <#elseif intervaltype == "IntervalYear"> + if (1 == 1) { + throw new UnsupportedOperationException("Cannot add IntervalYear to TIME"); + } + <#elseif intervaltype == "IntervalDay"> + if (${right}.days != 0) { + throw new UnsupportedOperationException("Cannot add days to TIME"); + } + ${output} = ${left}.value ${op} ${right}.milliSeconds; + <#elseif intervaltype == "Int" || intervaltype == "BigInt"> + if (1 == 1) { + throw new UnsupportedOperationException("Cannot add integer to TIME, cast it to specific interval"); + } + + + + @SuppressWarnings("unused") + @FunctionTemplate(names = {"date_add", "add"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL) + public static class ${datetype}${intervaltype}AddFunction implements DrillSimpleFunc { + @Param ${datetype}Holder left; + @Param ${intervaltype}Holder right; + @Output ${datetype}Holder out; + + public void setup(RecordBatch incoming) { + } + + public void eval() { + <@timeIntervalArithmeticBlock left="left" right="right" temp = "temp" op = "+" output="out.value" intervaltype=intervaltype /> + } + } + + @SuppressWarnings("unused") + @FunctionTemplate(names = {"date_add", "add"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL) + public static class ${intervaltype}${datetype}AddFunction implements DrillSimpleFunc { + @Param ${intervaltype}Holder right; + @Param ${datetype}Holder left; + @Output ${datetype}Holder out; + + public void setup(RecordBatch incoming) { + } + public void eval() { + <@timeIntervalArithmeticBlock left="left" right="right" temp = "temp" op = "+" output="out.value" intervaltype=intervaltype /> + } + } + + @SuppressWarnings("unused") + @FunctionTemplate(names = {"date_sub", "subtract"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL) + public static class ${datetype}${intervaltype}SubtractFunction implements DrillSimpleFunc { + @Param ${datetype}Holder left; + @Param ${intervaltype}Holder right; + @Output ${datetype}Holder out; + + public void setup(RecordBatch incoming) { + } + + public void eval() { + <@timeIntervalArithmeticBlock left="left" right="right" temp = "temp" op = "-" output="out.value" intervaltype=intervaltype /> + } + } +} + + + \ No newline at end of file diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateToCharFunctions.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateToCharFunctions.java new file mode 100644 index 00000000000..cd5466c8efd --- /dev/null +++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateToCharFunctions.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.drill.exec.expr.annotations.Workspace; + +<@pp.dropOutputFile /> + +<#list dateIntervalFunc.dates as type> + +<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/G${type}ToChar.java" /> + +<#include "/@includes/license.ftl" /> + +package org.apache.drill.exec.expr.fn.impl; + +import org.apache.drill.exec.expr.DrillSimpleFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.annotations.Workspace; +import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.holders.*; +import org.apache.drill.exec.record.RecordBatch; + +import io.netty.buffer.ByteBuf; + +@SuppressWarnings("unused") +@FunctionTemplate(name = "to_char", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) +public class G${type}ToChar implements DrillSimpleFunc { + + @Param ${type}Holder left; + @Param VarCharHolder right; + @Workspace ByteBuf buffer; + @Workspace org.joda.time.MutableDateTime temp; + @Workspace org.joda.time.format.DateTimeFormatter format; + @Output VarCharHolder out; + + public void setup(RecordBatch b) { + temp = new org.joda.time.MutableDateTime(0, org.joda.time.DateTimeZone.UTC); + buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[100]); + + // Get the desired output format and create a DateTimeFormatter + byte[] buf = new byte[right.end - right.start]; + right.buffer.getBytes(right.start, buf, 0, right.end - right.start); + String input = new String(buf); + format = org.joda.time.format.DateTimeFormat.forPattern(input); + } + + public void eval() { + temp.setMillis(left.value); + <#if type == "TimeStampTZ"> + temp.setZoneRetainFields(org.joda.time.DateTimeZone.forID(org.apache.drill.exec.expr.fn.impl.DateUtility.timezoneList[left.index])); + + + // print current value in the desired format + String str = format.print(temp); + + out.buffer = buffer; + out.start = 0; + out.end = Math.min(100, str.length()); // truncate if target type has length smaller than that of input's string + out.buffer.setBytes(0, str.substring(0,out.end).getBytes()); + } +} + \ No newline at end of file diff --git a/exec/java-exec/src/main/codegen/templates/Extract.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/Extract.java similarity index 95% rename from exec/java-exec/src/main/codegen/templates/Extract.java rename to exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/Extract.java index bef0614214e..b8ff73b142a 100644 --- a/exec/java-exec/src/main/codegen/templates/Extract.java +++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/Extract.java @@ -33,7 +33,7 @@ public class ${className} { <#list extract.fromTypes as fromUnit> <#list extract.toTypes as toUnit> -<#if fromUnit == "Date" || fromUnit == "Time" || fromUnit == "TimeStamp"> +<#if fromUnit == "Date" || fromUnit == "Time" || fromUnit == "TimeStamp" || fromUnit == "TimeStampTZ"> @FunctionTemplate(name = "extract${toUnit}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL) public static class ${toUnit}From${fromUnit} implements DrillSimpleFunc { @@ -48,6 +48,11 @@ public void setup(RecordBatch incoming) { public void eval() { dateTime.setMillis(in.value); + + <#if fromUnit == "TimeStampTZ"> + dateTime.setZoneRetainFields(org.joda.time.DateTimeZone.forID(org.apache.drill.exec.expr.fn.impl.DateUtility.timezoneList[in.index])); + + <#if toUnit == "Second"> out.value = dateTime.getSecondOfMinute(); <#elseif toUnit = "Minute"> diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/IntervalIntervalArithmetic.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/IntervalIntervalArithmetic.java new file mode 100644 index 00000000000..ad769e14997 --- /dev/null +++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/IntervalIntervalArithmetic.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +<@pp.dropOutputFile /> + + +<#list intervalNumericTypes.interval as intervaltype> + +<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/${intervaltype}Functions.java" /> + +<#include "/@includes/license.ftl" /> + +package org.apache.drill.exec.expr.fn.impl; + +import io.netty.buffer.ByteBuf; + +import org.apache.drill.exec.expr.DrillSimpleFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.annotations.Workspace; +import org.apache.drill.exec.expr.holders.*; +import org.apache.drill.exec.record.RecordBatch; +import org.joda.time.MutableDateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.DateMidnight; +import org.apache.drill.exec.expr.fn.impl.DateUtility; + + +public class ${intervaltype}Functions { + + @SuppressWarnings("unused") + @FunctionTemplate(name = "add", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL) + public static class ${intervaltype}AddFunction implements DrillSimpleFunc { + @Param ${intervaltype}Holder left; + @Param ${intervaltype}Holder right; + @Output ${intervaltype}Holder out; + + public void setup(RecordBatch incoming) { + } + + public void eval() { + <#if intervaltype == "Interval"> + out.months = left.months + right.months; + out.days = left.days + right.days; + out.milliSeconds = left.milliSeconds + right.milliSeconds; + <#elseif intervaltype == "IntervalYear"> + out.value = left.value + right.value; + <#elseif intervaltype == "IntervalDay"> + out.days = left.days + right.days; + out.milliSeconds = left.milliSeconds + right.milliSeconds; + + } + } + + @SuppressWarnings("unused") + @FunctionTemplate(name = "subtract", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL) + public static class ${intervaltype}SubtractFunction implements DrillSimpleFunc { + @Param ${intervaltype}Holder left; + @Param ${intervaltype}Holder right; + @Output ${intervaltype}Holder out; + + public void setup(RecordBatch incoming) { + } + + public void eval() { + <#if intervaltype == "Interval"> + out.months = left.months - right.months; + out.days = left.days - right.days; + out.milliSeconds = left.milliSeconds - right.milliSeconds; + <#elseif intervaltype == "IntervalYear"> + out.value = left.value - right.value; + <#elseif intervaltype == "IntervalDay"> + out.days = left.days - right.days; + out.milliSeconds = left.milliSeconds - right.milliSeconds; + + } + } + @SuppressWarnings("unused") + @FunctionTemplate(names = {"negative", "u-", "-"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL) + public static class ${intervaltype}NegateFunction implements DrillSimpleFunc { + @Param ${intervaltype}Holder left; + @Output ${intervaltype}Holder out; + + public void setup(RecordBatch incoming) { + } + + public void eval() { + <#if intervaltype == "Interval"> + out.months = -left.months; + out.days = -left.days; + out.milliSeconds = -left.milliSeconds; + <#elseif intervaltype == "IntervalYear"> + out.value = -left.value; + <#elseif intervaltype == "IntervalDay"> + out.days = -left.days; + out.milliSeconds = -left.milliSeconds; + + } + } +} + \ No newline at end of file diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/IntervalNumericArithmetic.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/IntervalNumericArithmetic.java new file mode 100644 index 00000000000..9a56c1b116a --- /dev/null +++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/IntervalNumericArithmetic.java @@ -0,0 +1,146 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +<@pp.dropOutputFile /> + +<#list intervalNumericTypes.interval as intervaltype> +<#list intervalNumericTypes.numeric as numerictype> + +<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/${intervaltype}${numerictype}Functions.java" /> + +<#include "/@includes/license.ftl" /> + +package org.apache.drill.exec.expr.fn.impl; + +import io.netty.buffer.ByteBuf; + +import org.apache.drill.exec.expr.DrillSimpleFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.annotations.Workspace; +import org.apache.drill.exec.expr.holders.*; +import org.apache.drill.exec.record.RecordBatch; +import org.joda.time.MutableDateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.DateMidnight; +import org.apache.drill.exec.expr.fn.impl.DateUtility; + + +public class ${intervaltype}${numerictype}Functions { + +<#-- Macro block to multiply interval data type with integers --> +<#macro intervalIntegerMultiplyBlock left right temp out intervaltype> + <#if intervaltype == "Interval"> + ${out}.months = (int) (${left}.months * ${right}.value); + ${out}.days = (int) (${left}.days * ${right}.value); + ${out}.milliSeconds = (int) (${left}.milliSeconds * ${right}.value); + <#elseif intervaltype == "IntervalYear"> + ${out}.months = (int) (${left}.value * ${right}.value); + <#elseif intervaltype == "IntervalDay"> + ${out}.days = (int) (${left}.days * ${right}.value); + ${out}.milliSeconds = (int) (${left}.milliSeconds * ${right}.value); + + + +<#-- Macro block to multiply and divide interval data type with integers, floating point values --> +<#macro intervalNumericArithmeticBlock left right temp op out intervaltype> + double fractionalMonths = 0; + double fractionalDays = 0; + double fractionalMillis = 0; + + <#if intervaltype == "Interval"> + fractionalMonths = (${left}.months ${op} (double) ${right}.value); + fractionalDays = (${left}.days ${op} (double) ${right}.value); + fractionalMillis = (${left}.milliSeconds ${op} (double) ${right}.value); + <#elseif intervaltype == "IntervalYear"> + fractionalMonths = (${left}.value ${op} (double) ${right}.value); + <#elseif intervaltype == "IntervalDay"> + fractionalDays = (${left}.days ${op} ${right}.value); + fractionalMillis = (${left}.milliSeconds ${op} (double) ${right}.value); + + + ${out}.months = (int) fractionalMonths; + + // Transfer fractional part to days + fractionalMonths = fractionalMonths - (long) fractionalMonths; + fractionalDays += fractionalMonths * org.apache.drill.exec.expr.fn.impl.DateUtility.monthToStandardDays; + ${out}.days = (int) fractionalDays; + + // Transfer fractional part to millis + fractionalDays = fractionalDays - (long) fractionalDays; + fractionalMillis += fractionalDays * org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis; + + ${out}.milliSeconds = (int) fractionalMillis; + + + @SuppressWarnings("unused") + @FunctionTemplate(name = "multiply", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL) + public static class ${intervaltype}${numerictype}MultiplyFunction implements DrillSimpleFunc { + @Param ${intervaltype}Holder left; + @Param ${numerictype}Holder right; + @Output IntervalHolder out; + + public void setup(RecordBatch incoming) { + } + + public void eval() { + <#if numerictype == "Int" || numerictype == "BigInt"> + <@intervalIntegerMultiplyBlock left="left" right="right" temp = "temp" out = "out" intervaltype=intervaltype /> + <#elseif numerictype == "Float4" || numerictype == "Float8"> + <@intervalNumericArithmeticBlock left="left" right="right" temp = "temp" op = "*" out = "out" intervaltype=intervaltype /> + + } + } + + @SuppressWarnings("unused") + @FunctionTemplate(name = "multiply", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL) + public static class ${numerictype}${intervaltype}MultiplyFunction implements DrillSimpleFunc { + @Param ${numerictype}Holder right; + @Param ${intervaltype}Holder left; + @Output IntervalHolder out; + + public void setup(RecordBatch incoming) { + } + + public void eval() { + <#if numerictype == "Int" || numerictype == "BigInt"> + <@intervalIntegerMultiplyBlock left="left" right="right" temp = "temp" out = "out" intervaltype=intervaltype /> + <#elseif numerictype == "Float4" || numerictype == "Float8"> + <@intervalNumericArithmeticBlock left="left" right="right" temp = "temp" op = "*" out = "out" intervaltype=intervaltype /> + + } + } + + @SuppressWarnings("unused") + @FunctionTemplate(names = {"divide", "div"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL) + public static class ${intervaltype}${numerictype}DivideFunction implements DrillSimpleFunc { + @Param ${intervaltype}Holder left; + @Param ${numerictype}Holder right; + @Output IntervalHolder out; + + public void setup(RecordBatch incoming) { + } + + public void eval() { + <@intervalNumericArithmeticBlock left="left" right="right" temp = "temp" op = "/" out = "out" intervaltype=intervaltype /> + } + } +} + + \ No newline at end of file diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/ToDateTypeFunctions.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/ToDateTypeFunctions.java new file mode 100644 index 00000000000..30dfc89cabf --- /dev/null +++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/ToDateTypeFunctions.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.drill.exec.expr.annotations.Workspace; + +<@pp.dropOutputFile /> + +<#list dateIntervalFunc.dates as type> + +<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/GTo${type}.java" /> + +<#include "/@includes/license.ftl" /> + +package org.apache.drill.exec.expr.fn.impl; + +import org.apache.drill.exec.expr.DrillSimpleFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.annotations.Workspace; +import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.holders.*; +import org.apache.drill.exec.record.RecordBatch; + +@FunctionTemplate(name = "to_${type?lower_case}" , scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) +public class GTo${type} implements DrillSimpleFunc { + + + @Param VarCharHolder left; + @Param VarCharHolder right; + @Workspace org.joda.time.format.DateTimeFormatter format; + @Output ${type}Holder out; + + public void setup(RecordBatch b) { + // Get the desired output format + byte[] buf = new byte[right.end - right.start]; + right.buffer.getBytes(right.start, buf, 0, right.end - right.start); + String formatString = new String(buf); + format = org.joda.time.format.DateTimeFormat.forPattern(formatString); + } + + public void eval() { + + // Get the input + byte[] buf1 = new byte[left.end - left.start]; + left.buffer.getBytes(left.start, buf1, 0, left.end - left.start); + String input = new String(buf1); + + <#if type == "Date"> + out.value = (org.joda.time.DateMidnight.parse(input, format).withZoneRetainFields(org.joda.time.DateTimeZone.UTC)).getMillis(); + <#elseif type == "TimeStamp"> + out.value = org.joda.time.DateTime.parse(input, format).withZoneRetainFields(org.joda.time.DateTimeZone.UTC).getMillis(); + <#elseif type == "TimeStampTZ"> + org.joda.time.DateTime dt = org.joda.time.DateTime.parse(input, format); + out.value = dt.getMillis(); + out.index = org.apache.drill.exec.expr.fn.impl.DateUtility.getIndex(dt.getZone().toString()); + <#elseif type == "Time"> + out.value = (int) ((format.parseDateTime(input)).withZoneRetainFields(org.joda.time.DateTimeZone.UTC).getMillis()); + + } +} + \ No newline at end of file diff --git a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java index cba9f97272b..d13fcdbb986 100644 --- a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java +++ b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java @@ -392,7 +392,7 @@ public Object getObject(int index) { org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC); date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); - return new Date(date.getMillis()); + return new Timestamp(date.getMillis()); } <#elseif minor.class == "IntervalYear"> public Object getObject(int index) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastBigIntDate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastBigIntDate.java index e2586d8a5ab..9c0703e429d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastBigIntDate.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastBigIntDate.java @@ -27,7 +27,7 @@ import org.apache.drill.exec.record.RecordBatch; @SuppressWarnings("unused") -@FunctionTemplate(name = "castDATE", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls= NullHandling.NULL_IF_NULL) +@FunctionTemplate(names = {"castDATE", "to_date"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls= NullHandling.NULL_IF_NULL) public class CastBigIntDate implements DrillSimpleFunc { @Param diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastBigIntTimeStamp.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastBigIntTimeStamp.java index ea92f3c336d..c53d7473c97 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastBigIntTimeStamp.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastBigIntTimeStamp.java @@ -27,7 +27,7 @@ import org.apache.drill.exec.record.RecordBatch; @SuppressWarnings("unused") -@FunctionTemplate(name = "castTIMESTAMP", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls= NullHandling.NULL_IF_NULL) +@FunctionTemplate(names = {"castTIMESTAMP", "to_timestamp"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls= NullHandling.NULL_IF_NULL) public class CastBigIntTimeStamp implements DrillSimpleFunc { @Param diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastBigIntTimeStampTZ.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastBigIntTimeStampTZ.java new file mode 100644 index 00000000000..4d355076fdb --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastBigIntTimeStampTZ.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.expr.fn.impl; + +import org.apache.drill.exec.expr.DrillSimpleFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.annotations.Workspace; +import org.apache.drill.exec.expr.holders.BigIntHolder; +import org.apache.drill.exec.expr.holders.TimeStampTZHolder; +import org.apache.drill.exec.record.RecordBatch; +import org.joda.time.DateTime; + +@SuppressWarnings("unused") +@FunctionTemplate(names = {"castTIMESTAMPTZ", "to_timestamptz"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls= NullHandling.NULL_IF_NULL) +public class CastBigIntTimeStampTZ implements DrillSimpleFunc { + + @Param + BigIntHolder in; + @Workspace + int timeZoneIndex; + @Output + TimeStampTZHolder out; + + @Override + public void setup(RecordBatch incoming) { + org.joda.time.DateTime temp = new org.joda.time.DateTime(); + + // Store the local time zone index + timeZoneIndex = org.apache.drill.exec.expr.fn.impl.DateUtility.getIndex(temp.getZone().toString()); + } + + @Override + public void eval() { + out.value = in.value; + out.index = timeZoneIndex; + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastIntTime.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastIntTime.java new file mode 100644 index 00000000000..f4a4ed0285a --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastIntTime.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.expr.fn.impl; + +import org.apache.drill.exec.expr.DrillSimpleFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.holders.IntHolder; +import org.apache.drill.exec.expr.holders.TimeHolder; +import org.apache.drill.exec.record.RecordBatch; + +@SuppressWarnings("unused") +@FunctionTemplate(names = {"castTIME", "to_time"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls= NullHandling.NULL_IF_NULL) +public class CastIntTime implements DrillSimpleFunc { + + @Param + IntHolder in; + @Output + TimeHolder out; + + @Override + public void setup(RecordBatch incoming) { + } + + @Override + public void eval() { + out.value = in.value; + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DateTypeFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DateTypeFunctions.java index 3d7504824e3..bbda0873245 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DateTypeFunctions.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DateTypeFunctions.java @@ -18,6 +18,7 @@ package org.apache.drill.exec.expr.fn.impl; +import io.netty.buffer.ByteBuf; import org.apache.drill.common.expression.*; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.common.types.Types; @@ -27,8 +28,17 @@ import org.apache.drill.exec.expr.annotations.Output; import org.apache.drill.exec.expr.annotations.Param; import org.apache.drill.exec.expr.annotations.Workspace; -import org.apache.drill.exec.expr.holders.*; +import org.apache.drill.exec.expr.holders.BigIntHolder; +import org.apache.drill.exec.expr.holders.TimeHolder; +import org.apache.drill.exec.expr.holders.VarCharHolder; +import org.apache.drill.exec.expr.holders.TimeStampHolder; +import org.apache.drill.exec.expr.holders.TimeStampTZHolder; +import org.apache.drill.exec.expr.holders.DateHolder; +import org.apache.drill.exec.expr.holders.IntervalHolder; +import org.apache.drill.exec.expr.holders.IntervalDayHolder; +import org.apache.drill.exec.expr.holders.IntervalYearHolder; import org.apache.drill.exec.record.RecordBatch; +import org.joda.time.DateTimeZone; public class DateTypeFunctions { @@ -201,132 +211,109 @@ public void eval() { } } - @FunctionTemplate(names = {"date_add", "add"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL) - public static class DateIntervalAdd implements DrillSimpleFunc{ - - @Param DateHolder dateType; - @Param IntervalHolder intervalType; + @FunctionTemplate(name = "current_date", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) + public static class CurrentDate implements DrillSimpleFunc { + @Workspace long queryStartDate; @Output DateHolder out; - public void setup(RecordBatch b){} - - public void eval(){ - - org.joda.time.MutableDateTime temp = new org.joda.time.MutableDateTime(dateType.value, org.joda.time.DateTimeZone.UTC); - temp.addMonths(intervalType.months); - temp.addDays(intervalType.days); - temp.add(intervalType.milliSeconds); + public void setup(RecordBatch incoming) { - out.value = temp.getMillis(); + int timeZoneIndex = incoming.getContext().getRootFragmentTimeZone(); + org.joda.time.DateTimeZone timeZone = org.joda.time.DateTimeZone.forID(org.apache.drill.exec.expr.fn.impl.DateUtility.getTimeZone(timeZoneIndex)); + org.joda.time.DateTime now = new org.joda.time.DateTime(incoming.getContext().getQueryStartTime(), timeZone); + queryStartDate = (new org.joda.time.DateMidnight(now.getYear(), now.getMonthOfYear(), now.getDayOfMonth(), timeZone)).getMillis(); } - } - - @FunctionTemplate(names = {"date_add", "add"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL) - public static class IntervalDateAdd implements DrillSimpleFunc{ - - @Param IntervalHolder intervalType; - @Param DateHolder dateType; - @Output DateHolder out; - - public void setup(RecordBatch b){} - - public void eval(){ - org.joda.time.MutableDateTime temp = new org.joda.time.MutableDateTime(dateType.value, org.joda.time.DateTimeZone.UTC); - temp.addMonths(intervalType.months); - temp.addDays(intervalType.days); - temp.add(intervalType.milliSeconds); - - out.value = temp.getMillis(); + public void eval() { + out.value = queryStartDate; } - } - - - @FunctionTemplate(names = {"date_sub", "subtract"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL) - public static class DateIntervalSub implements DrillSimpleFunc{ - - @Param DateHolder dateType; - @Param IntervalHolder intervalType; - @Output DateHolder out; - public void setup(RecordBatch b){} + } - public void eval(){ + @FunctionTemplate(names = {"current_timestamp", "now"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) + public static class CurrentTimeStamp implements DrillSimpleFunc { + @Workspace long queryStartDate; + @Workspace int timezoneIndex; + @Output TimeStampTZHolder out; - org.joda.time.MutableDateTime temp = new org.joda.time.MutableDateTime(dateType.value, org.joda.time.DateTimeZone.UTC); - temp.addMonths(intervalType.months * -1); - temp.addDays(intervalType.days * -1); - temp.add(intervalType.milliSeconds * -1); + public void setup(RecordBatch incoming) { - out.value = temp.getMillis(); + int timeZoneIndex = incoming.getContext().getRootFragmentTimeZone(); + org.joda.time.DateTimeZone timeZone = org.joda.time.DateTimeZone.forID(org.apache.drill.exec.expr.fn.impl.DateUtility.getTimeZone(timeZoneIndex)); + org.joda.time.DateTime now = new org.joda.time.DateTime(incoming.getContext().getQueryStartTime(), timeZone); + queryStartDate = now.getMillis(); + timezoneIndex = org.apache.drill.exec.expr.fn.impl.DateUtility.getIndex(now.getZone().toString()); } - } - - @FunctionTemplate(names = {"date_sub", "subtract"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL) - public static class IntervalDateSub implements DrillSimpleFunc{ - @Param IntervalHolder intervalType; - @Param DateHolder dateType; - @Output DateHolder out; + public void eval() { + out.value = queryStartDate; + out.index = timezoneIndex; + } - public void setup(RecordBatch b){} + } - public void eval(){ + @FunctionTemplate(name = "clock_timestamp", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL, isRandom = true) + public static class ClockTimeStamp implements DrillSimpleFunc { + @Workspace int timezoneIndex; + @Output TimeStampTZHolder out; - org.joda.time.MutableDateTime temp = new org.joda.time.MutableDateTime(dateType.value, org.joda.time.DateTimeZone.UTC); - temp.addMonths(intervalType.months * -1); - temp.addDays(intervalType.days * -1); - temp.add(intervalType.milliSeconds * -1); + public void setup(RecordBatch incoming) { + org.joda.time.DateTime now = new org.joda.time.DateTime(); + timezoneIndex = org.apache.drill.exec.expr.fn.impl.DateUtility.getIndex(now.getZone().toString()); + } - out.value = temp.getMillis(); + public void eval() { + org.joda.time.DateTime now = new org.joda.time.DateTime(); + out.value = now.getMillis(); + out.index = timezoneIndex; } } - @FunctionTemplate(name = "current_date", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) - public static class CurrentDate implements DrillSimpleFunc { - @Workspace long queryStartDate; - @Output DateHolder out; + @FunctionTemplate(name = "timeofday", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL, isRandom = true) + public static class TimeOfDay implements DrillSimpleFunc { + @Workspace ByteBuf buffer; + @Output VarCharHolder out; public void setup(RecordBatch incoming) { - - org.joda.time.DateTime now = new org.joda.time.DateTime(incoming.getContext().getQueryStartTime()); - queryStartDate = (new org.joda.time.DateMidnight(now.getYear(), now.getMonthOfYear(), now.getDayOfMonth(), org.joda.time.DateTimeZone.UTC)).getMillis(); + buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[100]); } public void eval() { - out.value = queryStartDate; + org.joda.time.DateTime temp = new org.joda.time.DateTime(); + String str = org.apache.drill.exec.expr.fn.impl.DateUtility.formatTimeStampTZ.print(temp); + out.buffer = buffer; + out.start = 0; + out.end = Math.min(100, str.length()); // truncate if target type has length smaller than that of input's string + out.buffer.setBytes(0, str.substring(0,out.end).getBytes()); } - } - @FunctionTemplate(name = "current_timestamptz", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) - public static class CurrentTimeStampTZ implements DrillSimpleFunc { + @FunctionTemplate(name = "localtimestamp", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) + public static class LocalTimeStamp implements DrillSimpleFunc { @Workspace long queryStartDate; - @Workspace int timezoneIndex; - @Output TimeStampTZHolder out; + @Output TimeStampHolder out; public void setup(RecordBatch incoming) { - org.joda.time.DateTime now = new org.joda.time.DateTime(incoming.getContext().getQueryStartTime()); + org.joda.time.DateTime now = (new org.joda.time.DateTime(incoming.getContext().getQueryStartTime())).withZoneRetainFields(org.joda.time.DateTimeZone.UTC); queryStartDate = now.getMillis(); - timezoneIndex = org.apache.drill.exec.expr.fn.impl.DateUtility.getIndex(now.getZone().toString()); } public void eval() { out.value = queryStartDate; - out.index = timezoneIndex; } - } - @FunctionTemplate(name = "current_time", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) + @FunctionTemplate(names = {"current_time", "localtime"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) public static class CurrentTime implements DrillSimpleFunc { @Workspace int queryStartTime; @Output TimeHolder out; public void setup(RecordBatch incoming) { - org.joda.time.DateTime now = new org.joda.time.DateTime(incoming.getContext().getQueryStartTime()); + int timeZoneIndex = incoming.getContext().getRootFragmentTimeZone(); + org.joda.time.DateTimeZone timeZone = org.joda.time.DateTimeZone.forID(org.apache.drill.exec.expr.fn.impl.DateUtility.getTimeZone(timeZoneIndex)); + org.joda.time.DateTime now = new org.joda.time.DateTime(incoming.getContext().getQueryStartTime(), timeZone); queryStartTime= (int) ((now.getHourOfDay() * org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis) + (now.getMinuteOfHour() * org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis) + (now.getSecondOfMinute() * org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis) + @@ -336,6 +323,56 @@ public void setup(RecordBatch incoming) { public void eval() { out.value = queryStartTime; } + } + + @SuppressWarnings("unused") + @FunctionTemplate(names = {"date_add", "add"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL) + public static class DateTimeAddFunction implements DrillSimpleFunc { + @Param DateHolder left; + @Param TimeHolder right; + @Output TimeStampHolder out; + + public void setup(RecordBatch incoming) { + } + + public void eval() { + out.value = left.value + right.value; + } + } + + @SuppressWarnings("unused") + @FunctionTemplate(names = {"date_add", "add"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL) + public static class TimeDateAddFunction implements DrillSimpleFunc { + @Param TimeHolder right; + @Param DateHolder left; + @Output TimeStampHolder out; + + public void setup(RecordBatch incoming) { + } + public void eval() { + out.value = left.value + right.value; + } + } + + /* Dummy function template to allow Optiq to validate this function call. + * At DrillOptiq time we rewrite all date_part() functions to extract functions, + * since they are essentially the same + */ + @SuppressWarnings("unused") + @FunctionTemplate(names = "date_part", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL) + public static class DatePartFunction implements DrillSimpleFunc { + @Param VarCharHolder left; + @Param DateHolder right; + @Output BigIntHolder out; + + public void setup(RecordBatch incoming) { + } + + public void eval() { + if (1 == 1) { + throw new UnsupportedOperationException("date_part function should be rewritten as extract() functions"); + } + } } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DateUtility.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DateUtility.java index c9e66a1a41e..2ac866de03f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DateUtility.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DateUtility.java @@ -1199,6 +1199,8 @@ public class DateUtility { public static final int hoursToMillis = 60 * 60 * 1000; public static final int minutesToMillis = 60 * 1000; public static final int secondsToMillis = 1000; + public static final int monthToStandardDays = 30; + public static final int daysToStandardMillis = 24 * 60 * 60 * 1000; public static int getIndex(String timezone) { return timezoneMap.get(timezone); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java index ddc5025a570..2035aa0c057 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java @@ -66,6 +66,7 @@ public class FragmentContext implements Closeable { private List daemonThreads = Lists.newLinkedList(); private IncomingBuffers buffers; private final long queryStartTime; + private final int rootFragmentTimeZone; private volatile Throwable failureCause; private volatile boolean failed = false; @@ -80,6 +81,7 @@ public FragmentContext(DrillbitContext dbContext, PlanFragment fragment, UserCli this.fragment = fragment; this.funcRegistry = funcRegistry; this.queryStartTime = fragment.getQueryStartTime(); + this.rootFragmentTimeZone = fragment.getTimeZone(); logger.debug("Getting initial memory allocation of {}", fragment.getMemInitial()); this.allocator = context.getAllocator().getChildAllocator(fragment.getHandle(), fragment.getMemInitial(), fragment.getMemMax()); } @@ -117,7 +119,11 @@ public FragmentStats getStats(){ } public long getQueryStartTime() { - return this.queryStartTime; + return this.queryStartTime; + } + + public int getRootFragmentTimeZone() { + return this.rootFragmentTimeZone; } /** diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java index 2560302de4b..581499d1ef2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java @@ -36,6 +36,9 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import org.apache.drill.exec.expr.fn.impl.DateUtility; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; /** * The simple parallelizer determines the level of parallelization of a plan based on the cost of the underlying @@ -79,6 +82,7 @@ private QueryWorkUnit generateWorkUnit(DrillbitEndpoint foremanNode, QueryId que FragmentRoot rootOperator = null; long queryStartTime = System.currentTimeMillis(); + int timeZone = DateUtility.getIndex(System.getProperty("user.timezone")); // now we generate all the individual plan fragments and associated assignments. Note, we need all endpoints // assigned before we can materialize, so we start a new loop here rather than utilizing the previous one. @@ -128,6 +132,7 @@ private QueryWorkUnit generateWorkUnit(DrillbitEndpoint foremanNode, QueryId que .setAssignment(wrapper.getAssignedEndpoint(minorFragmentId)) // .setLeafFragment(isLeafFragment) // .setQueryStartTime(queryStartTime) + .setTimeZone(timeZone) .build(); if (isRootNode) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java index f46f012ab37..2d0389fef47 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java @@ -29,6 +29,7 @@ import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.expression.ValueExpressions; +import org.apache.drill.common.expression.ValueExpressions.QuotedString; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.TypeProtos.MinorType; @@ -106,6 +107,7 @@ public LogicalExpression visitCall(RexCall call) { return lastArg; case FUNCTION: + case FUNCTION_ID: logger.debug("Function"); return getDrillFunctionFromOptiqCall(call); case POSTFIX: @@ -261,6 +263,23 @@ private LogicalExpression getDrillFunctionFromOptiqCall(RexCall call){ args.add(n.accept(this)); } + // Rewrite DATE_PART functions as extract functions + if (call.getOperator().getName().equalsIgnoreCase("DATE_PART")) { + + // assert that the function has exactly two arguments + assert args.size() == 2; + + /* Based on the first input to the date_part function we rewrite the function as the + * appropriate extract function. For example + * date_part('year', date '2008-2-23') ------> extractYear(date '2008-2-23') + */ + assert args.get(0) instanceof QuotedString; + + QuotedString extractString = (QuotedString) args.get(0); + String functionPostfix = extractString.value.substring(0, 1).toUpperCase() + extractString.value.substring(1).toLowerCase(); + return FunctionCallFactory.createExpression("extract" + functionPostfix, args.subList(1, 2)); + } + return FunctionCallFactory.createExpression(call.getOperator().getName().toLowerCase(), args); } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateFunctions.java new file mode 100644 index 00000000000..92c49b9eb3d --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateFunctions.java @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.fn.impl; + +import com.google.common.base.Charsets; +import com.google.common.io.Files; +import org.apache.drill.common.util.FileUtils; +import org.apache.drill.exec.client.DrillClient; +import org.apache.drill.exec.pop.PopUnitTestBase; +import org.apache.drill.exec.proto.UserProtos; +import org.apache.drill.exec.record.RecordBatchLoader; +import org.apache.drill.exec.record.VectorWrapper; +import org.apache.drill.exec.rpc.user.QueryResultBatch; +import org.apache.drill.exec.server.Drillbit; +import org.apache.drill.exec.server.RemoteServiceSet; +import org.apache.drill.exec.vector.ValueVector; +import org.apache.drill.exec.vector.BigIntVector; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertTrue; + +public class TestDateFunctions extends PopUnitTestBase { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestDateFunctions.class); + + public void testCommon(String[] expectedResults, String physicalPlan, String resourceFile) throws Exception { + try (RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet(); + Drillbit bit = new Drillbit(CONFIG, serviceSet); + DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) { + + // run query. + bit.run(); + client.connect(); + List results = client.runQuery(UserProtos.QueryType.PHYSICAL, + Files.toString(FileUtils.getResourceAsFile(physicalPlan), Charsets.UTF_8) + .replace("#{TEST_FILE}", resourceFile)); + + RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator()); + + QueryResultBatch batch = results.get(0); + assertTrue(batchLoader.load(batch.getHeader().getDef(), batch.getData())); + + + int i = 0; + for (VectorWrapper v : batchLoader) { + + ValueVector.Accessor accessor = v.getValueVector().getAccessor(); + System.out.println(accessor.getObject(0)); + assertTrue((accessor.getObject(0)).toString().equals(expectedResults[i++])); + } + + batchLoader.clear(); + for(QueryResultBatch b : results){ + b.release(); + } + } + } + + @Test + public void testDateIntervalArithmetic() throws Exception { + String expectedResults[] = {"2009-02-23", + "2008-02-24", + "13:20:33", + "2008-02-24 12:00:00.0", + "2009-04-23 12:00:00.0", + "2008-02-24 12:00:00.0", + "2009-04-23 12:00:00.0", + "2009-02-23", + "2008-02-24", + "13:20:33", + "2008-02-24 12:00:00.0", + "2009-04-23 12:00:00.0", + "2008-02-24 12:00:00.0", + "2009-04-23 12:00:00.0"}; + + testCommon(expectedResults, "/functions/date/date_interval_arithmetic.json", "/test_simple_date.json"); + } + + @Test + public void testDateDifferenceArithmetic() throws Exception { + + String[] expectedResults = {"365 days 0:0:0.0", + "-366 days 0:-1:0.0", + "0 days 3:0:0.0", + "0 days 11:0:0.0"}; + testCommon(expectedResults, "/functions/date/date_difference_arithmetic.json", "/test_simple_date.json"); + } + + @Test + public void testIntervalArithmetic() throws Exception { + + String[] expectedResults = {"2 years 2 months ", + "2 days 1:2:3.0", + "0 years 2 months ", + "0 days 1:2:3.0", + "2 years 4 months 0 days 0:0:0.0", + "0 years 0 months 0 days 2:0:6.0", + "0 years 7 months 0 days 0:0:0.0", + "0 years 0 months 0 days 0:30:1.500", + "2 years 9 months 18 days 0:0:0.0", + "0 years 0 months 0 days 2:24:7.200", + "0 years 6 months 19 days 23:59:59.999", + "0 years 0 months 0 days 0:28:35.714"}; + testCommon(expectedResults, "/functions/date/interval_arithmetic.json", "/test_simple_date.json"); + } + + @Test + public void testToChar() throws Exception { + + String expectedResults[] = {"2008-Feb-23", + "12 20 30", + "2008 Feb 23 12:00:00", + "2008-02-23 20:00:00 America/Los_Angeles"}; + testCommon(expectedResults, "/functions/date/to_char.json", "/test_simple_date.json"); + } + + @Test + public void testToDateType() throws Exception { + String expectedResults[] = {"2008-02-23", + "12:20:30", + "2008-02-23 12:00:00.0", + "2008-02-23 12:00:00.0"}; + + testCommon(expectedResults, "/functions/date/to_date_type.json", "/test_simple_date.json"); + } + +} diff --git a/exec/java-exec/src/test/resources/functions/date/date_difference_arithmetic.json b/exec/java-exec/src/test/resources/functions/date/date_difference_arithmetic.json new file mode 100644 index 00000000000..bd8386c23c2 --- /dev/null +++ b/exec/java-exec/src/test/resources/functions/date/date_difference_arithmetic.json @@ -0,0 +1,36 @@ +{ + "head" : { + "version" : 1, + "generator" : { + "type" : "org.apache.drill.exec.planner.logical.DrillImplementor", + "info" : "" + }, + "type" : "APACHE_DRILL_PHYSICAL", + "resultMode" : "EXEC" + }, + graph:[ + { + @id:1, + pop:"fs-scan", + format: {type: "json"}, + storage:{type: "file", connection: "classpath:///"}, + files:["#{TEST_FILE}"] + }, + { + pop:"project", + @id:2, + child: 1, + exprs: [ + { ref: "Date1", expr: "cast('2008-2-23' as date) - cast('2007-2-23' as date)"}, + { ref: "TimeStamp1", expr: "cast('2008-2-23 12:00:00' as timestamp) - cast('2009-2-23 12:01:00' as timestamp) "}, + { ref: "TimeStampTZ1", expr: "cast('2008-2-23 12:00:00 America/Los_Angeles' as timestamptz) - cast('2008-2-23 12:00:00 America/New_York' as timestamptz)"}, + { ref: "Time2", expr: "cast('12:20:30' as time) - cast('1:20:30' as time)"} + ] + }, + { + @id: 3, + child: 2, + pop: "screen" + } + ] +} diff --git a/exec/java-exec/src/test/resources/functions/date/date_interval_arithmetic.json b/exec/java-exec/src/test/resources/functions/date/date_interval_arithmetic.json new file mode 100644 index 00000000000..c1e10a3c9c6 --- /dev/null +++ b/exec/java-exec/src/test/resources/functions/date/date_interval_arithmetic.json @@ -0,0 +1,46 @@ +{ + "head" : { + "version" : 1, + "generator" : { + "type" : "org.apache.drill.exec.planner.logical.DrillImplementor", + "info" : "" + }, + "type" : "APACHE_DRILL_PHYSICAL", + "resultMode" : "EXEC" + }, + graph:[ + { + @id:1, + pop:"fs-scan", + format: {type: "json"}, + storage:{type: "file", connection: "classpath:///"}, + files:["#{TEST_FILE}"] + }, + { + pop:"project", + @id:2, + child: 1, + exprs: [ + { ref: "Date1", expr: "cast('2008-2-23' as date) + cast('P1Y' as intervalyear)"}, + { ref: "Date2", expr: "cast('2008-2-23' as date) + cast('P1D' as intervalday)"}, + { ref: "Time1", expr: "cast('12:20:30' as time) + cast('PT1H0M3S' as intervalday)"}, + { ref: "TimeStamp1", expr: "cast('2008-2-23 12:00:00' as timestamp) + cast('P1D' as intervalday)"}, + { ref: "TimeStamp2", expr: "cast('2008-2-23 12:00:00' as timestamp) + cast('P1Y2M' as intervalyear)"}, + { ref: "TimeStampTZ1", expr: "cast('2008-2-23 12:00:00 America/Los_Angeles' as timestamptz) + cast('P1D' as intervalday)"}, + { ref: "TimeStampTZ2", expr: "cast('2008-2-23 12:00:00 America/Los_Angeles' as timestamptz) + cast('P1Y2M' as intervalyear)"}, + { ref: "Date3", expr: "cast('P1Y' as intervalyear) + cast('2008-2-23' as date)"}, + { ref: "Date4", expr: "cast('P1D' as intervalday) + cast('2008-2-23' as date) "}, + { ref: "Time2", expr: "cast('PT1H0M3S' as intervalday) + cast('12:20:30' as time)"}, + { ref: "TimeStamp3", expr: "cast('P1D' as intervalday) + cast('2008-2-23 12:00:00' as timestamp)"}, + { ref: "TimeStamp4", expr: "cast('P1Y2M' as intervalyear) + cast('2008-2-23 12:00:00' as timestamp)"}, + { ref: "TimeStampTZ3", expr: "cast('P1D' as intervalday) + cast('2008-2-23 12:00:00 America/Los_Angeles' as timestamptz) "}, + { ref: "TimeStampTZ4", expr: "cast('P1Y2M' as intervalyear) + cast('2008-2-23 12:00:00 America/Los_Angeles' as timestamptz) "} + ] + }, + { + @id: 3, + child: 2, + pop: "screen" + } + ] +} diff --git a/exec/java-exec/src/test/resources/functions/date/interval_arithmetic.json b/exec/java-exec/src/test/resources/functions/date/interval_arithmetic.json new file mode 100644 index 00000000000..50ae92bcd26 --- /dev/null +++ b/exec/java-exec/src/test/resources/functions/date/interval_arithmetic.json @@ -0,0 +1,44 @@ +{ + "head" : { + "version" : 1, + "generator" : { + "type" : "org.apache.drill.exec.planner.logical.DrillImplementor", + "info" : "" + }, + "type" : "APACHE_DRILL_PHYSICAL", + "resultMode" : "EXEC" + }, + graph:[ + { + @id:1, + pop:"fs-scan", + format: {type: "json"}, + storage:{type: "file", connection: "classpath:///"}, + files:["#{TEST_FILE}"] + }, + { + pop:"project", + @id:2, + child: 1, + exprs: [ + { ref: "IntervalYear1", expr: "cast('P1Y2M' as intervalyear) + cast('P1Y' as intervalyear)"}, + { ref: "IntervalDay1", expr: "cast('P1DT1H2M3S' as intervalday) + cast('P1D' as intervalday)"}, + { ref: "IntervalYear2", expr: "cast('P1Y2M' as intervalyear) - cast('P1Y' as intervalyear)"}, + { ref: "IntervalDay2", expr: "cast('P1DT1H2M3S' as intervalday) - cast('P1D' as intervalday)"}, + { ref: "IntervalYear3", expr: "cast('P1Y2M' as intervalyear) * 2"}, + { ref: "IntervalDay3", expr: " 2 * cast('PT1H0M3S' as intervalday)"}, + { ref: "IntervalYear4", expr: "cast('P1Y2M' as intervalyear) / 2"}, + { ref: "IntervalDay4", expr: " cast('PT1H0M3S' as intervalday) / 2"}, + { ref: "IntervalYear5", expr: "cast('P1Y2M' as intervalyear) * 2.4"}, + { ref: "IntervalDay6", expr: " 2.4 * cast('PT1H0M3S' as intervalday)"}, + { ref: "IntervalYear6", expr: "cast('P1Y2M' as intervalyear) / 2.1"}, + { ref: "IntervalDay6", expr: " cast('PT1H0M3S' as intervalday) / 2.1"} + ] + }, + { + @id: 3, + child: 2, + pop: "screen" + } + ] +} diff --git a/exec/java-exec/src/test/resources/functions/date/to_char.json b/exec/java-exec/src/test/resources/functions/date/to_char.json new file mode 100644 index 00000000000..005940138bf --- /dev/null +++ b/exec/java-exec/src/test/resources/functions/date/to_char.json @@ -0,0 +1,36 @@ +{ + "head" : { + "version" : 1, + "generator" : { + "type" : "org.apache.drill.exec.planner.logical.DrillImplementor", + "info" : "" + }, + "type" : "APACHE_DRILL_PHYSICAL", + "resultMode" : "EXEC" + }, + graph:[ + { + @id:1, + pop:"fs-scan", + format: {type: "json"}, + storage:{type: "file", connection: "classpath:///"}, + files:["#{TEST_FILE}"] + }, + { + pop:"project", + @id:2, + child: 1, + exprs: [ + { ref: "Date1", expr: "to_char((cast('2008-2-23' as date)), 'yyyy-MMM-dd')"}, + { ref: "Time1", expr: "to_char(cast('12:20:30' as time), 'HH mm ss')"}, + { ref: "TimeStamp2", expr: "to_char(cast('2008-2-23 12:00:00' as timestamp), 'yyyy MMM dd HH:mm:ss')"}, + { ref: "TimeStampTZ1", expr: "to_char(cast('2008-2-23 12:00:00 America/Los_Angeles' as timestamptz), 'yyyy-MM-dd HH:mm:ss ZZZ')"} + ] + }, + { + @id: 3, + child: 2, + pop: "screen" + } + ] +} diff --git a/exec/java-exec/src/test/resources/functions/date/to_date_type.json b/exec/java-exec/src/test/resources/functions/date/to_date_type.json new file mode 100644 index 00000000000..ab429d15559 --- /dev/null +++ b/exec/java-exec/src/test/resources/functions/date/to_date_type.json @@ -0,0 +1,36 @@ +{ + "head" : { + "version" : 1, + "generator" : { + "type" : "org.apache.drill.exec.planner.logical.DrillImplementor", + "info" : "" + }, + "type" : "APACHE_DRILL_PHYSICAL", + "resultMode" : "EXEC" + }, + graph:[ + { + @id:1, + pop:"fs-scan", + format: {type: "json"}, + storage:{type: "file", connection: "classpath:///"}, + files:["#{TEST_FILE}"] + }, + { + pop:"project", + @id:2, + child: 1, + exprs: [ + { ref: "Date1", expr: "to_date('2008-FEB-23', 'yyyy-MMM-dd')"}, + { ref: "Time1", expr: "to_time('12:20:30', 'HH:mm:ss')"}, + { ref: "TimeStamp2", expr: "to_timestamp('2008-2-23 12:00:00', 'yyyy-MM-dd HH:mm:ss')"}, + { ref: "TimeStampTZ1", expr: "to_timestamptz('2008-2-23 12:00:00 America/Los_Angeles', 'yyyy-MM-dd HH:mm:ss ZZZ')"} + ] + }, + { + @id: 3, + child: 2, + pop: "screen" + } + ] +} diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/BitControl.java b/protocol/src/main/java/org/apache/drill/exec/proto/BitControl.java index f91a010b5d9..fe3752179d3 100644 --- a/protocol/src/main/java/org/apache/drill/exec/proto/BitControl.java +++ b/protocol/src/main/java/org/apache/drill/exec/proto/BitControl.java @@ -3145,6 +3145,16 @@ public interface PlanFragmentOrBuilder * optional .exec.shared.UserCredentials credentials = 15; */ org.apache.drill.exec.proto.UserBitShared.UserCredentialsOrBuilder getCredentialsOrBuilder(); + + // optional int32 time_zone = 16; + /** + * optional int32 time_zone = 16; + */ + boolean hasTimeZone(); + /** + * optional int32 time_zone = 16; + */ + int getTimeZone(); } /** * Protobuf type {@code exec.bit.control.PlanFragment} @@ -3294,6 +3304,11 @@ private PlanFragment( bitField0_ |= 0x00001000; break; } + case 128: { + bitField0_ |= 0x00002000; + timeZone_ = input.readInt32(); + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -3617,6 +3632,22 @@ public org.apache.drill.exec.proto.UserBitShared.UserCredentialsOrBuilder getCre return credentials_; } + // optional int32 time_zone = 16; + public static final int TIME_ZONE_FIELD_NUMBER = 16; + private int timeZone_; + /** + * optional int32 time_zone = 16; + */ + public boolean hasTimeZone() { + return ((bitField0_ & 0x00002000) == 0x00002000); + } + /** + * optional int32 time_zone = 16; + */ + public int getTimeZone() { + return timeZone_; + } + private void initFields() { handle_ = org.apache.drill.exec.proto.ExecProtos.FragmentHandle.getDefaultInstance(); networkCost_ = 0F; @@ -3631,6 +3662,7 @@ private void initFields() { memMax_ = 20000000000L; queryStartTime_ = 0L; credentials_ = org.apache.drill.exec.proto.UserBitShared.UserCredentials.getDefaultInstance(); + timeZone_ = 0; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -3683,6 +3715,9 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) if (((bitField0_ & 0x00001000) == 0x00001000)) { output.writeMessage(15, credentials_); } + if (((bitField0_ & 0x00002000) == 0x00002000)) { + output.writeInt32(16, timeZone_); + } getUnknownFields().writeTo(output); } @@ -3744,6 +3779,10 @@ public int getSerializedSize() { size += com.google.protobuf.CodedOutputStream .computeMessageSize(15, credentials_); } + if (((bitField0_ & 0x00002000) == 0x00002000)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(16, timeZone_); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -3906,6 +3945,8 @@ public Builder clear() { credentialsBuilder_.clear(); } bitField0_ = (bitField0_ & ~0x00001000); + timeZone_ = 0; + bitField0_ = (bitField0_ & ~0x00002000); return this; } @@ -4002,6 +4043,10 @@ public org.apache.drill.exec.proto.BitControl.PlanFragment buildPartial() { } else { result.credentials_ = credentialsBuilder_.build(); } + if (((from_bitField0_ & 0x00002000) == 0x00002000)) { + to_bitField0_ |= 0x00002000; + } + result.timeZone_ = timeZone_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -4059,6 +4104,9 @@ public Builder mergeFrom(org.apache.drill.exec.proto.BitControl.PlanFragment oth if (other.hasCredentials()) { mergeCredentials(other.getCredentials()); } + if (other.hasTimeZone()) { + setTimeZone(other.getTimeZone()); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -4940,6 +4988,39 @@ public org.apache.drill.exec.proto.UserBitShared.UserCredentialsOrBuilder getCre return credentialsBuilder_; } + // optional int32 time_zone = 16; + private int timeZone_ ; + /** + * optional int32 time_zone = 16; + */ + public boolean hasTimeZone() { + return ((bitField0_ & 0x00002000) == 0x00002000); + } + /** + * optional int32 time_zone = 16; + */ + public int getTimeZone() { + return timeZone_; + } + /** + * optional int32 time_zone = 16; + */ + public Builder setTimeZone(int value) { + bitField0_ |= 0x00002000; + timeZone_ = value; + onChanged(); + return this; + } + /** + * optional int32 time_zone = 16; + */ + public Builder clearTimeZone() { + bitField0_ = (bitField0_ & ~0x00002000); + timeZone_ = 0; + onChanged(); + return this; + } + // @@protoc_insertion_point(builder_scope:exec.bit.control.PlanFragment) } @@ -5672,7 +5753,7 @@ public Builder clearReportTime() { "shared.DrillPBError\022\024\n\014running_time\030\t \001(" + "\003\"k\n\rFragmentState\022\013\n\007SENDING\020\000\022\027\n\023AWAIT" + "ING_ALLOCATION\020\001\022\013\n\007RUNNING\020\002\022\014\n\010FINISHE" + - "D\020\003\022\r\n\tCANCELLED\020\004\022\n\n\006FAILED\020\005\"\225\003\n\014PlanF" + + "D\020\003\022\r\n\tCANCELLED\020\004\022\n\n\006FAILED\020\005\"\250\003\n\014PlanF" + "ragment\022(\n\006handle\030\001 \001(\0132\030.exec.bit.Fragm", "entHandle\022\024\n\014network_cost\030\004 \001(\002\022\020\n\010cpu_c" + "ost\030\005 \001(\002\022\021\n\tdisk_cost\030\006 \001(\002\022\023\n\013memory_c" + @@ -5682,17 +5763,17 @@ public Builder clearReportTime() { "xec.DrillbitEndpoint\022\035\n\013mem_initial\030\014 \001(" + "\003:\01020000000\022\034\n\007mem_max\030\r \001(\003:\0132000000000" + "0\022\030\n\020query_start_time\030\016 \001(\003\0221\n\013credentia" + - "ls\030\017 \001(\0132\034.exec.shared.UserCredentials\"f" + - "\n\017WorkQueueStatus\022(\n\010endpoint\030\001 \001(\0132\026.ex", - "ec.DrillbitEndpoint\022\024\n\014queue_length\030\002 \001(" + - "\005\022\023\n\013report_time\030\003 \001(\003*\332\001\n\007RpcType\022\r\n\tHA" + - "NDSHAKE\020\000\022\007\n\003ACK\020\001\022\013\n\007GOODBYE\020\002\022\033\n\027REQ_I" + - "NIATILIZE_FRAGMENT\020\003\022\027\n\023REQ_CANCEL_FRAGM" + - "ENT\020\006\022\027\n\023REQ_FRAGMENT_STATUS\020\007\022\022\n\016REQ_BI" + - "T_STATUS\020\010\022\030\n\024RESP_FRAGMENT_HANDLE\020\t\022\030\n\024" + - "RESP_FRAGMENT_STATUS\020\n\022\023\n\017RESP_BIT_STATU" + - "S\020\013B+\n\033org.apache.drill.exec.protoB\nBitC" + - "ontrolH\001" + "ls\030\017 \001(\0132\034.exec.shared.UserCredentials\022\021" + + "\n\ttime_zone\030\020 \001(\005\"f\n\017WorkQueueStatus\022(\n\010", + "endpoint\030\001 \001(\0132\026.exec.DrillbitEndpoint\022\024" + + "\n\014queue_length\030\002 \001(\005\022\023\n\013report_time\030\003 \001(" + + "\003*\332\001\n\007RpcType\022\r\n\tHANDSHAKE\020\000\022\007\n\003ACK\020\001\022\013\n" + + "\007GOODBYE\020\002\022\033\n\027REQ_INIATILIZE_FRAGMENT\020\003\022" + + "\027\n\023REQ_CANCEL_FRAGMENT\020\006\022\027\n\023REQ_FRAGMENT" + + "_STATUS\020\007\022\022\n\016REQ_BIT_STATUS\020\010\022\030\n\024RESP_FR" + + "AGMENT_HANDLE\020\t\022\030\n\024RESP_FRAGMENT_STATUS\020" + + "\n\022\023\n\017RESP_BIT_STATUS\020\013B+\n\033org.apache.dri" + + "ll.exec.protoB\nBitControlH\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -5722,7 +5803,7 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors( internal_static_exec_bit_control_PlanFragment_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_exec_bit_control_PlanFragment_descriptor, - new java.lang.String[] { "Handle", "NetworkCost", "CpuCost", "DiskCost", "MemoryCost", "FragmentJson", "Assignment", "LeafFragment", "Foreman", "MemInitial", "MemMax", "QueryStartTime", "Credentials", }); + new java.lang.String[] { "Handle", "NetworkCost", "CpuCost", "DiskCost", "MemoryCost", "FragmentJson", "Assignment", "LeafFragment", "Foreman", "MemInitial", "MemMax", "QueryStartTime", "Credentials", "TimeZone", }); internal_static_exec_bit_control_WorkQueueStatus_descriptor = getDescriptor().getMessageTypes().get(4); internal_static_exec_bit_control_WorkQueueStatus_fieldAccessorTable = new diff --git a/protocol/src/main/protobuf/BitControl.proto b/protocol/src/main/protobuf/BitControl.proto index 25cae116b96..a738646eba5 100644 --- a/protocol/src/main/protobuf/BitControl.proto +++ b/protocol/src/main/protobuf/BitControl.proto @@ -75,7 +75,8 @@ message PlanFragment { optional int64 mem_initial = 12 [default = 20000000]; // 20 megs optional int64 mem_max = 13 [default = 20000000000]; // 20 gigs optional int64 query_start_time = 14; // start time of query in milliseconds - optional exec.shared.UserCredentials credentials = 15; + optional exec.shared.UserCredentials credentials = 15; + optional int32 time_zone = 16; } message WorkQueueStatus{