- * Drill dates are in the local time zone, so conversion is needed.
- * Drill dates are stores in ms, which is odd.
+ * Parse local time dates. Stored internally in a local epoch
+ * offset from the local epoch, in ms. Does no time zone conversions,
+ * simply asserts that the date is in in the same time zone as the
+ * Drillbit.
*/
public class DateValueListener extends ScalarListener {
@@ -46,6 +44,9 @@ public void onValue(JsonToken token, TokenIterator tokenizer) {
case VALUE_NULL:
setNull();
break;
+ case VALUE_NUMBER_INT:
+ writer.setLong(tokenizer.longValue());
+ break;
case VALUE_STRING:
try {
@@ -55,9 +56,9 @@ public void onValue(JsonToken token, TokenIterator tokenizer) {
// want to copy the offset since the epoch from UTC to our local
// time, so that we retain the date, even if the span of the date
// is different locally than UTC. A mess.
- LocalDate localDate = LocalDate.parse(tokenizer.stringValue(), DateUtility.isoFormatDate);
- ZonedDateTime utc = localDate.atStartOfDay(ZoneOffset.UTC);
- writer.setLong(utc.toEpochSecond() * 1000);
+ LocalDate localDate = LocalDate.parse(tokenizer.stringValue());
+ writer.setLong(Duration.between(TimestampValueListener.LOCAL_EPOCH,
+ localDate.atStartOfDay()).toMillis());
} catch (Exception e) {
throw loader.dataConversionError(schema(), "date", tokenizer.stringValue());
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/values/TimeValueListener.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/values/TimeValueListener.java
index bf90d721dbc..31d2fb755d6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/values/TimeValueListener.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/values/TimeValueListener.java
@@ -47,6 +47,9 @@ public void onValue(JsonToken token, TokenIterator tokenizer) {
case VALUE_NULL:
setNull();
break;
+ case VALUE_NUMBER_INT:
+ writer.setInt((int) tokenizer.longValue());
+ break;
case VALUE_STRING:
try {
LocalTime localTime = LocalTime.parse(tokenizer.stringValue(), TIME_FORMAT);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/values/TimestampValueListener.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/values/TimestampValueListener.java
index 7472ce92fe5..56a998eaef6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/values/TimestampValueListener.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/values/TimestampValueListener.java
@@ -17,8 +17,8 @@
*/
package org.apache.drill.exec.store.easy.json.values;
-import java.time.Instant;
-import java.time.ZoneId;
+import java.time.Duration;
+import java.time.LocalDateTime;
import org.apache.drill.exec.store.easy.json.loader.JsonLoaderImpl;
import org.apache.drill.exec.store.easy.json.parser.TokenIterator;
@@ -27,21 +27,13 @@
import com.fasterxml.jackson.core.JsonToken;
/**
- * Per the
- * V1 docs:
- *
- * In Shell mode, {@code
- * In Strict mode, {@code
- *
- * Drill dates are in the local time zone, so conversion is needed. + * Drill-flavored version of a timestamp parser. Assumes the date time is in + * a local (unspecified) time zone, interpreted to be the default time zone + * of the Drillbit machine. Does no time zone conversions. */ public class TimestampValueListener extends ScalarListener { - private final ZoneId localZoneId = ZoneId.systemDefault(); + public static final LocalDateTime LOCAL_EPOCH = LocalDateTime.of(1970, 1, 1, 0, 0, 0); public TimestampValueListener(JsonLoaderImpl loader, ScalarWriter writer) { super(loader, writer); @@ -49,17 +41,17 @@ public TimestampValueListener(JsonLoaderImpl loader, ScalarWriter writer) { @Override public void onValue(JsonToken token, TokenIterator tokenizer) { - Instant instant; switch (token) { case VALUE_NULL: setNull(); return; case VALUE_NUMBER_INT: - instant = Instant.ofEpochMilli(tokenizer.longValue()); + writer.setLong(tokenizer.longValue()); break; case VALUE_STRING: try { - instant = Instant.parse(tokenizer.stringValue()); + LocalDateTime localDT = LocalDateTime.parse(tokenizer.stringValue()); + writer.setLong(Duration.between(LOCAL_EPOCH, localDT).toMillis()); } catch (Exception e) { throw loader.dataConversionError(schema(), "date", tokenizer.stringValue()); } @@ -67,6 +59,5 @@ public void onValue(JsonToken token, TokenIterator tokenizer) { default: throw tokenizer.invalidValue(token); } - writer.setLong(instant.toEpochMilli() + localZoneId.getRules().getOffset(instant).getTotalSeconds() * 1000); } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/values/UtcDateValueListener.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/values/UtcDateValueListener.java new file mode 100644 index 00000000000..ab45ef32920 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/values/UtcDateValueListener.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.easy.json.values; + +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; + +import org.apache.drill.exec.expr.fn.impl.DateUtility; +import org.apache.drill.exec.store.easy.json.loader.JsonLoaderImpl; +import org.apache.drill.exec.store.easy.json.parser.TokenIterator; +import org.apache.drill.exec.vector.accessor.ScalarWriter; + +import com.fasterxml.jackson.core.JsonToken; + +/** + * Drill-specific extension to allow dates only, expressed in UTC + * to be consistent with Mongo timestamps. + *
+ * Drill dates are in the local time zone, so conversion is needed.
+ * Drill dates are stored in ms, which is odd.
+ */
+public class UtcDateValueListener extends ScalarListener {
+
+ public UtcDateValueListener(JsonLoaderImpl loader, ScalarWriter writer) {
+ super(loader, writer);
+ }
+
+ @Override
+ public void onValue(JsonToken token, TokenIterator tokenizer) {
+ switch (token) {
+ case VALUE_NULL:
+ setNull();
+ break;
+ case VALUE_NUMBER_INT:
+ writer.setLong(tokenizer.longValue());
+ break;
+ case VALUE_STRING:
+ try {
+
+ // A Drill date is ms since the epoch, local time. Our input
+ // is in UTC. We DO NOT want to convert from the date, midnight, UTC
+ // to local time since that will change the date. Instead, we just
+ // want to copy the offset since the epoch from UTC to our local
+ // time, so that we retain the date, even if the span of the date
+ // is different locally than UTC. A mess.
+ LocalDate localDate = LocalDate.parse(tokenizer.stringValue(), DateUtility.isoFormatDate);
+ ZonedDateTime utc = localDate.atStartOfDay(ZoneOffset.UTC);
+ writer.setLong(utc.toEpochSecond() * 1000);
+ } catch (Exception e) {
+ throw loader.dataConversionError(schema(), "date", tokenizer.stringValue());
+ }
+ break;
+ default:
+ throw tokenizer.invalidValue(token);
+ }
+ }
+}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/values/UtcTimestampValueListener.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/values/UtcTimestampValueListener.java
new file mode 100644
index 00000000000..53ed5439c98
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/values/UtcTimestampValueListener.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.easy.json.values;
+
+import java.time.Instant;
+import java.time.ZoneId;
+
+import org.apache.drill.exec.store.easy.json.loader.JsonLoaderImpl;
+import org.apache.drill.exec.store.easy.json.parser.TokenIterator;
+import org.apache.drill.exec.vector.accessor.ScalarWriter;
+
+import com.fasterxml.jackson.core.JsonToken;
+
+/**
+ * Per the
+ * V1 docs:
+ *
+ * In Shell mode, {@code
+ * In Strict mode, {@code
+ *
+ * Drill dates are in the local time zone, so conversion is needed. + */ +public class UtcTimestampValueListener extends ScalarListener { + + private final ZoneId localZoneId = ZoneId.systemDefault(); + + public UtcTimestampValueListener(JsonLoaderImpl loader, ScalarWriter writer) { + super(loader, writer); + } + + @Override + public void onValue(JsonToken token, TokenIterator tokenizer) { + Instant instant; + switch (token) { + case VALUE_NULL: + setNull(); + return; + case VALUE_NUMBER_INT: + instant = Instant.ofEpochMilli(tokenizer.longValue()); + break; + case VALUE_STRING: + try { + instant = Instant.parse(tokenizer.stringValue()); + } catch (Exception e) { + throw loader.dataConversionError(schema(), "date", tokenizer.stringValue()); + } + break; + default: + throw tokenizer.invalidValue(token); + } + writer.setLong(instant.toEpochMilli() + localZoneId.getRules().getOffset(instant).getTotalSeconds() * 1000); + } +} diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/loader/TestBasics.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/loader/TestBasics.java index 8d9cdc1faf2..4178fa34a43 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/loader/TestBasics.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/loader/TestBasics.java @@ -32,6 +32,7 @@ import org.apache.drill.exec.physical.rowSet.RowSetTestUtils; import org.apache.drill.exec.record.metadata.SchemaBuilder; import org.apache.drill.exec.record.metadata.TupleMetadata; +import org.apache.drill.exec.store.easy.json.loader.BaseJsonLoaderTest.JsonLoaderFixture; import org.apache.drill.test.rowSet.RowSetUtilities; import org.junit.Test; @@ -125,14 +126,16 @@ public void testCaseInsensitive() { @Test public void testProjection() { - final String json = - "{a: 1, b: [[{x: [[{y: []}]]}]]}\n" + - "{a: 2}\n" + - "{b: \"bar\"}"; + String json = + "{a: 10, b: true}\n" + + "{a: 20, b: [\"what?\"]}\n" + + "{a: 30, b: {c: \"oh, my!\"}}" + + "{a: 40}" + + "{a: 50, b: [[{x: [[{y: []}]]}]]}"; + JsonLoaderFixture loader = new JsonLoaderFixture(); - ProjectionFilter projectionFilter = ProjectionFilter.projectionFilter( - Projections.parse(RowSetTestUtils.projectList("a")), EmptyErrorContext.INSTANCE); - loader.rsLoaderOptions.projectionFilter(projectionFilter); + loader.rsLoaderOptions.projection( + Projections.parse(RowSetTestUtils.projectList("a"))); loader.open(json); RowSet results = loader.next(); assertNotNull(results); @@ -141,9 +144,11 @@ public void testProjection() { .addNullable("a", MinorType.BIGINT) .build(); RowSet expected = fixture.rowSetBuilder(expectedSchema) - .addRow(1) - .addRow(2) - .addSingleCol(null) + .addRow(10) + .addRow(20) + .addRow(30) + .addRow(40) + .addRow(50) .build(); RowSetUtilities.verify(expected, results); assertNull(loader.next()); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/loader/TestScalars.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/loader/TestScalars.java index de642486875..7f7a097519e 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/loader/TestScalars.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/loader/TestScalars.java @@ -17,17 +17,19 @@ */ package org.apache.drill.exec.store.easy.json.loader; +import static org.apache.drill.test.rowSet.RowSetUtilities.dec; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.time.Duration; +import java.time.LocalDateTime; + import org.apache.drill.categories.JsonTest; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.types.TypeProtos.MinorType; -import org.apache.drill.exec.physical.resultSet.project.Projections; import org.apache.drill.exec.physical.rowSet.RowSet; -import org.apache.drill.exec.physical.rowSet.RowSetTestUtils; import org.apache.drill.exec.record.metadata.SchemaBuilder; import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.test.rowSet.RowSetUtilities; @@ -527,26 +529,159 @@ public void testStringWithSchema() { } @Test - public void testProjection() { + public void testProvidedSchemaNumbers() { String json = - "{a: 10, b: true}\n" + - "{a: 20, b: [\"what?\"]}\n" + - "{a: 30, b: {c: \"oh, my!\"}}"; + + // null is ambiguous + "{s: null, i: null, bi: null, f4: null, f8: null, d: null}\n" + + // Strings are also + "{s: \"10\", i: \"10\", bi: \"10\", f4: \"10\", f8: \"10\", d: \"10\"}\n" + + "{ f4: \"10.5\", f8: \"10.5\", d: \"10.5\"}\n" + + "{ f4: \"-1e5\", f8: \"-1e5\", d: \"-1e5\"}\n" + + + // Float-only values + "{ f4: \"NaN\", f8: \"NaN\"}\n" + + "{ f4: \"Infinity\", f8: \"Infinity\"}\n" + + "{ f4: \"-Infinity\", f8: \"-Infinity\"}\n" + + + // Large decimal + "{d: \"123456789012345678901234.5678\" }\n" + + + // Ambiguous numbers + "{s: 10, i: 10, bi: 10, f4: 10, f8: 10, d: 10}\n" + + "{ f4: 10.5, f8: 10.5, d: 10.5}\n" + + "{ f4: -1e5, f8: -1e5, d: -1e5}\n" + + + // Float-only values + "{ f4: NaN, f8: NaN}\n" + + "{ f4: Infinity, f8: Infinity}\n" + + "{ f4: -Infinity, f8: -Infinity}\n" + + + // Large decimal + "{d: 123456789012345678901234.5678 }\n"; + TupleMetadata schema = new SchemaBuilder() + .addNullable("s", MinorType.SMALLINT) + .addNullable("i", MinorType.INT) + .addNullable("bi", MinorType.BIGINT) + .addNullable("f4", MinorType.FLOAT4) + .addNullable("f8", MinorType.FLOAT8) + .addNullable("d", MinorType.VARDECIMAL, 38, 4) + .build(); JsonLoaderFixture loader = new JsonLoaderFixture(); - loader.rsLoaderOptions.projection( - Projections.parse(RowSetTestUtils.projectList("a"))); + loader.jsonOptions.allowNanInf = true; + loader.builder.providedSchema(schema); loader.open(json); RowSet results = loader.next(); assertNotNull(results); - TupleMetadata expectedSchema = new SchemaBuilder() - .addNullable("a", MinorType.BIGINT) + RowSet expected = fixture.rowSetBuilder(schema) + // s i bi f4 f8 d + .addRow(null, null, null, null, null, null) + .addRow(10, 10, 10, 10, 10, dec("10")) + .addRow(null, null, null, 10.5, 10.5D, dec("10.5")) + .addRow(null, null, null, -1e5, -1e5D, dec("-1e5")) + .addRow(null, null, null, Float.NaN, Double.NaN, null) + .addRow(null, null, null, Float.POSITIVE_INFINITY, Double.POSITIVE_INFINITY, null) + .addRow(null, null, null, Float.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY, null) + .addRow(null, null, null, null, null, dec("123456789012345678901234.5678")) + .addRow(10, 10, 10, 10, 10, dec("10")) + .addRow(null, null, null, 10.5, 10.5D, dec("10.5")) + .addRow(null, null, null, -1e5, -1e5D, dec("-1e5")) + .addRow(null, null, null, Float.NaN, Double.NaN, null) + .addRow(null, null, null, Float.POSITIVE_INFINITY, Double.POSITIVE_INFINITY, null) + .addRow(null, null, null, Float.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY, null) + .addRow(null, null, null, null, null, dec("123456789012345678901234.5678")) .build(); - RowSet expected = fixture.rowSetBuilder(expectedSchema) - .addRow(10) - .addRow(20) - .addRow(30) + RowSetUtilities.verify(expected, results); + assertNull(loader.next()); + loader.close(); + } + + @Test + public void testProvidedSchemaWithDates() { + LocalDateTime local = LocalDateTime.of(2020, 4, 21, 11, 22, 33, 456_000_000); + LocalDateTime localEpoch = LocalDateTime.of(1970, 1, 1, 0, 0, 0); + long localTs = Duration.between(localEpoch, local).toMillis(); + LocalDateTime localDate = LocalDateTime.of(2020, 4, 21, 0, 0, 0); + long localDateTs = Duration.between(localEpoch, localDate).toMillis(); + int localTimeTs = (int) (localTs - localDateTs); + String json = + "{ts: null, d: null, t: null}\n" + + "{ts: \"2020-04-21T11:22:33.456\", d: \"2020-04-21\", t: \"11:22:33.456\"}\n" + + "{ts: " + localTs + ", d: " + localDateTs + ", t: " + localTimeTs + "}\n"; + TupleMetadata schema = new SchemaBuilder() + .addNullable("ts", MinorType.TIMESTAMP) + .addNullable("d", MinorType.DATE) + .addNullable("t", MinorType.TIME) + .build(); + + JsonLoaderFixture loader = new JsonLoaderFixture(); + loader.builder.providedSchema(schema); + loader.open(json); + RowSet results = loader.next(); + assertNotNull(results); + + RowSet expected = fixture.rowSetBuilder(schema) + .addRow(null, null, null) + .addRow(localTs, localDateTs, localTimeTs) + .addRow(localTs, localDateTs, localTimeTs) + .build(); + RowSetUtilities.verify(expected, results); + assertNull(loader.next()); + loader.close(); + } + + @Test + public void testProvidedSchemaWithIntervals() { + String json = + "{i: null, iy: null, id: null}\n" + + "{i: \"P1Y2M3DT4H5M6S\", iy: \"P1Y2M\", id: \"P3DT4H5M6S\"}"; + TupleMetadata schema = new SchemaBuilder() + .addNullable("i", MinorType.INTERVAL) + .addNullable("iy", MinorType.INTERVALYEAR) + .addNullable("id", MinorType.INTERVALDAY) + .build(); + + JsonLoaderFixture loader = new JsonLoaderFixture(); + loader.builder.providedSchema(schema); + loader.open(json); + RowSet results = loader.next(); + assertNotNull(results); + + org.joda.time.Period full = org.joda.time.Period.years(1).withMonths(2) + .withDays(3).withHours(4).withMinutes(5).withSeconds(6); + org.joda.time.Period ym = org.joda.time.Period.years(1).withMonths(2); + org.joda.time.Period dhms = org.joda.time.Period.days(3).withHours(4) + .withMinutes(5).withSeconds(6); + RowSet expected = fixture.rowSetBuilder(schema) + .addRow(null, null, null) + .addRow(full, ym, dhms) + .build(); + RowSetUtilities.verify(expected, results); + assertNull(loader.next()); + loader.close(); + } + + @Test + public void testProvidedSchemaWithBinary() { + String json = + "{b: null}\n" + + "{b: \"ZHJpbGw=\"}"; + TupleMetadata schema = new SchemaBuilder() + .addNullable("b", MinorType.VARBINARY) + .build(); + + JsonLoaderFixture loader = new JsonLoaderFixture(); + loader.builder.providedSchema(schema); + loader.open(json); + RowSet results = loader.next(); + assertNotNull(results); + + byte[] bytes = "Drill".getBytes(); + RowSet expected = fixture.rowSetBuilder(schema) + .addSingleCol(null) + .addSingleCol(bytes) .build(); RowSetUtilities.verify(expected, results); assertNull(loader.next());