Skip to content

Commit

Permalink
DRILL-4782 / DRILL-7139: Fix DATE_ADD and TO_TIME functions
Browse files Browse the repository at this point in the history
- cast function for the day interval changed to round milliseconds to complete days
- ToDateTypeFunctions#toTime now returning milliseconds of day
- updated the way how DayInterval subtracts and adds, to follow the cast function logic

UT core updates:

- added vectorValue function to the queryBuilder to simplify retrieving value of the vector
- refactored singleton query result functions at queryBuilder
  • Loading branch information
dgrinchenko authored and sachouche committed May 29, 2019
1 parent e8b921b commit 2615d68
Show file tree
Hide file tree
Showing 7 changed files with 304 additions and 79 deletions.
Expand Up @@ -86,24 +86,33 @@ public void eval() {
org.joda.time.Period period = org.joda.time.Period.parse(input);
<#if type.to == "Interval" || type.to == "NullableInterval">
out.months = (period.getYears() * org.apache.drill.exec.vector.DateUtilities.yearsToMonths) + period.getMonths();
out.months = period.getYears() * org.apache.drill.exec.vector.DateUtilities.yearsToMonths + period.getMonths();

out.days = period.getDays();

out.milliseconds = (period.getHours() * org.apache.drill.exec.vector.DateUtilities.hoursToMillis) +
(period.getMinutes() * org.apache.drill.exec.vector.DateUtilities.minutesToMillis) +
(period.getSeconds() * org.apache.drill.exec.vector.DateUtilities.secondsToMillis) +
(period.getMillis());
out.milliseconds = period.getHours() * org.apache.drill.exec.vector.DateUtilities.hoursToMillis +
period.getMinutes() * org.apache.drill.exec.vector.DateUtilities.minutesToMillis +
period.getSeconds() * org.apache.drill.exec.vector.DateUtilities.secondsToMillis +
period.getMillis();

<#elseif type.to == "IntervalDay" || type.to == "NullableIntervalDay">
out.days = period.getDays();

out.milliseconds = (period.getHours() * org.apache.drill.exec.vector.DateUtilities.hoursToMillis) +
(period.getMinutes() * org.apache.drill.exec.vector.DateUtilities.minutesToMillis) +
(period.getSeconds() * org.apache.drill.exec.vector.DateUtilities.secondsToMillis) +
(period.getMillis());
long millis = period.getHours() * (long) org.apache.drill.exec.vector.DateUtilities.hoursToMillis +
period.getMinutes() * (long) org.apache.drill.exec.vector.DateUtilities.minutesToMillis +
period.getSeconds() * (long) org.apache.drill.exec.vector.DateUtilities.secondsToMillis +
period.getMillis();

if (millis >= org.apache.drill.exec.vector.DateUtilities.daysToStandardMillis) {
int daysFromMillis = (int) (millis / org.apache.drill.exec.vector.DateUtilities.daysToStandardMillis);
millis -= daysFromMillis * org.apache.drill.exec.vector.DateUtilities.daysToStandardMillis;
out.days += daysFromMillis;
}

out.milliseconds = (int) millis;

<#elseif type.to == "IntervalYear" || type.to == "NullableIntervalYear">
out.value = (period.getYears() * org.apache.drill.exec.vector.DateUtilities.yearsToMonths) + period.getMonths();
out.value = period.getYears() * org.apache.drill.exec.vector.DateUtilities.yearsToMonths + period.getMonths();
</#if>
}
}
Expand Down
Expand Up @@ -65,7 +65,15 @@ public void eval() {
out.value = left.value + right.value;
<#elseif intervaltype == "IntervalDay">
out.days = left.days + right.days;
out.milliseconds = left.milliseconds + right.milliseconds;
long millis = (long) left.milliseconds + right.milliseconds;

if (millis >= org.apache.drill.exec.vector.DateUtilities.daysToStandardMillis) {
int daysFromMillis = (int) (millis / org.apache.drill.exec.vector.DateUtilities.daysToStandardMillis);

millis -= daysFromMillis * org.apache.drill.exec.vector.DateUtilities.daysToStandardMillis;
out.days += daysFromMillis;
}
out.milliseconds = (int) millis;
</#if>
}
}
Expand All @@ -90,6 +98,18 @@ public void eval() {
<#elseif intervaltype == "IntervalDay">
out.days = left.days - right.days;
out.milliseconds = left.milliseconds - right.milliseconds;

int daysFromMillis = out.milliseconds/org.apache.drill.exec.vector.DateUtilities.daysToStandardMillis;
if (daysFromMillis != 0) {
out.milliseconds -= org.apache.drill.exec.vector.DateUtilities.daysToStandardMillis*daysFromMillis;
out.days -= Math.abs(daysFromMillis);
}

// if milliseconds are bellow zero, substract them from the days
if (out.milliseconds < 0 && out.days > 0) {
out.days -= 1;
out.milliseconds = org.apache.drill.exec.vector.DateUtilities.daysToStandardMillis + out.milliseconds;
}
</#if>
}
}
Expand Down
Expand Up @@ -82,7 +82,7 @@ public void eval() {
<#elseif convert.to == "TimeStamp" || convert.to == "NullableTimeStamp">
out.value = org.joda.time.DateTime.parse(input, format).withZoneRetainFields(org.joda.time.DateTimeZone.UTC).getMillis();
<#elseif convert.to == "Time" || convert.to == "NullableTime">
out.value = (int) ((format.parseDateTime(input)).withZoneRetainFields(org.joda.time.DateTimeZone.UTC).getMillis());
out.value = format.parseDateTime(input).withZoneRetainFields(org.joda.time.DateTimeZone.UTC).getMillisOfDay();
</#if>
}
}
Expand Down
@@ -0,0 +1,95 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.drill.exec.expr.fn.impl;

import org.apache.drill.exec.vector.IntervalDayVector;
import org.apache.drill.test.ClusterFixture;
import org.apache.drill.test.ClusterTest;
import org.apache.drill.test.QueryBuilder;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;

public class TestIntervalDayFunctions extends ClusterTest {
@BeforeClass
public static void setUp() throws Exception {
startCluster(ClusterFixture.builder(dirTestWatcher));
}

@Test
public void testIntervalDaySubtractFunction() throws Exception {
QueryBuilder.VectorQueryReader<String,IntervalDayVector> vectorQueryReader =
(recordsCount, vector) -> vector.getAccessor().getAsStringBuilder(0).toString();

String result = queryBuilder()
.sql("select cast('P6D' as interval day) - cast('P5DT4M20S' as interval day) as i")
.vectorValue("i", IntervalDayVector.class, vectorQueryReader);
Assert.assertEquals("0 days 23:55:40", result);

result = queryBuilder()
.sql("select cast('P4D' as interval day) - cast('P4DT4M' as interval day) as i")
.vectorValue("i", IntervalDayVector.class, vectorQueryReader);
Assert.assertEquals("0 days 0:-4:00", result);

result = queryBuilder()
.sql("select cast('P4D' as interval day) - cast('PT4M' as interval day) as i")
.vectorValue("i", IntervalDayVector.class, vectorQueryReader);
Assert.assertEquals("3 days 23:56:00", result);

result = queryBuilder()
.sql("select cast('P4D' as interval day) - cast('P5D' as interval day) as i")
.vectorValue("i", IntervalDayVector.class, vectorQueryReader);
Assert.assertEquals("-1 day 0:00:00", result);

result = queryBuilder()
.sql("select cast('P4D' as interval day) - cast('P4DT23H59M59S' as interval day) as i")
.vectorValue("i", IntervalDayVector.class, vectorQueryReader);
Assert.assertEquals("0 days -23:-59:-59", result);

result = queryBuilder()
.sql("select cast('P4D' as interval day) - cast('P5DT23H59S' as interval day) as i")
.vectorValue("i", IntervalDayVector.class, vectorQueryReader);
Assert.assertEquals("-1 day -23:00:-59", result);

result = queryBuilder()
.sql("select cast('P2D' as interval day) - cast('P1DT4M' as interval day) as i")
.vectorValue("i", IntervalDayVector.class, vectorQueryReader);
Assert.assertEquals("0 days 23:56:00", result);
}

@Test
public void testIntervalDayPlusFunction() throws Exception {
QueryBuilder.VectorQueryReader<String,IntervalDayVector> vectorQueryReader =
(recordsCount, vector) -> vector.getAccessor().getAsStringBuilder(0).toString();

String result = queryBuilder()
.sql("select cast('P1D' as interval day) + cast('P2DT1H' as interval day) as i")
.vectorValue("i", IntervalDayVector.class, vectorQueryReader);
Assert.assertEquals("3 days 1:00:00", result);

result = queryBuilder()
.sql("select cast('PT12H' as interval day) + cast('PT12H' as interval day) as i")
.vectorValue("i", IntervalDayVector.class, vectorQueryReader);
Assert.assertEquals("1 day 0:00:00", result);

result = queryBuilder()
.sql("select cast('PT11H' as interval day) + cast('PT12H59M60S' as interval day) as i")
.vectorValue("i", IntervalDayVector.class, vectorQueryReader);
Assert.assertEquals("1 day 0:00:00", result);
}
}
Expand Up @@ -30,15 +30,13 @@
import org.apache.drill.categories.SqlFunctionTest;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.common.exceptions.UserRemoteException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.record.metadata.SchemaBuilder;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.vector.IntervalDayVector;
import org.apache.drill.exec.vector.IntervalYearVector;
import org.apache.drill.test.ClusterFixture;
import org.apache.drill.test.ClusterTest;
Expand Down Expand Up @@ -698,30 +696,53 @@ public void testCastVarCharDecimalOverflow() throws Exception {

@Test // DRILL-6783
public void testCastVarCharIntervalYear() throws Exception {
String query = "select cast('P31M' as interval month) as i from cp.`employee.json` limit 10";
List<QueryDataBatch> result = queryBuilder().sql(query).results();
RecordBatchLoader loader = new RecordBatchLoader(cluster.drillbit().getContext().getAllocator());

QueryDataBatch b = result.get(0);
loader.load(b.getHeader().getDef(), b.getData());

IntervalYearVector vector = (IntervalYearVector) loader.getValueAccessorById(
IntervalYearVector.class,
loader.getValueVectorId(SchemaPath.getCompoundPath("i")).getFieldIds())
.getValueVector();

Set<String> resultSet = new HashSet<>();
for (int i = 0; i < loader.getRecordCount(); i++) {
String displayValue = vector.getAccessor().getAsStringBuilder(i).toString();
resultSet.add(displayValue);
}
Set<String> results = queryBuilder()
.sql("select cast('P31M' as interval month) as i from cp.`employee.json` limit 10")
.vectorValue(
"i",
IntervalYearVector.class,
(recordCount, vector) -> {
Set<String> r = new HashSet<>();
for (int i = 0; i < recordCount; i++) {
r.add(vector.getAccessor().getAsStringBuilder(i).toString());
}
return r;
}
);

Assert.assertEquals(
"Casting literal string as INTERVAL should yield the same result for each row", 1, resultSet.size());
Assert.assertThat(resultSet, hasItem("2 years 7 months"));
"Casting literal string as INTERVAL should yield the same result for each row", 1, results.size());
Assert.assertThat(results, hasItem("2 years 7 months"));
}

b.release();
loader.clear();
@Test
public void testCastVarCharIntervalDay() throws Exception {
String result = queryBuilder()
.sql("select cast('PT1H' as interval minute) as i from (values(1))")
.vectorValue(
"i",
IntervalDayVector.class,
(recordsCount, vector) -> vector.getAccessor().getAsStringBuilder(0).toString()
);
Assert.assertEquals(result, "0 days 1:00:00");

result = queryBuilder()
.sql("select cast(concat('PT',107374,'M') as interval minute) as i from (values(1))")
.vectorValue(
"i",
IntervalDayVector.class,
(recordsCount, vector) -> vector.getAccessor().getAsStringBuilder(0).toString()
);
Assert.assertEquals(result, "74 days 13:34:00");

result = queryBuilder()
.sql("select cast(concat('PT',107375,'M') as interval minute) as i from (values(1))")
.vectorValue(
"i",
IntervalDayVector.class,
(recordsCount, vector) -> vector.getAccessor().getAsStringBuilder(0).toString()
);
Assert.assertEquals(result, "74 days 13:35:00");
}

@Test // DRILL-6959
Expand Down
Expand Up @@ -107,6 +107,18 @@ public void testJodaTime() throws Exception {
.go();
}

@Test
public void testToTimeWithDateTimePatternFormat() throws Exception {
mockUsDateFormatSymbols();

testBuilder()
.sqlQuery("select TO_TIME('2016-03-03 00:00', 'yyyy-MM-dd HH:mm') as `result` from (values(1))")
.unOrdered()
.baselineColumns("result")
.baselineValues(LocalTime.of(0,0,0))
.go();
}

@Test
public void testPostgresTime() throws Exception {
mockUsDateFormatSymbols();
Expand Down

0 comments on commit 2615d68

Please sign in to comment.