From 8ab1394b1fbb0bd144ccd9a2b8df959b7c82f729 Mon Sep 17 00:00:00 2001 From: Alberto Rodriguez Date: Mon, 24 Aug 2015 09:07:32 +0200 Subject: [PATCH 1/9] Added time zone pattern --- .../java/org/apache/metamodel/util/TimeComparator.java | 8 ++++++++ .../org/apache/metamodel/util/TimeComparatorTest.java | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/core/src/main/java/org/apache/metamodel/util/TimeComparator.java b/core/src/main/java/org/apache/metamodel/util/TimeComparator.java index e7596a722..30d097a14 100644 --- a/core/src/main/java/org/apache/metamodel/util/TimeComparator.java +++ b/core/src/main/java/org/apache/metamodel/util/TimeComparator.java @@ -137,6 +137,14 @@ private static Date convertFromString(final String value) { // do noting } + // try with time-zone pattern + try { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX"); + return dateFormat.parse(value); + } catch (ParseException e) { + // do noting + } + for (String prototypePattern : prototypePatterns) { if (prototypePattern.length() == value.length()) { DateFormat dateFormat; diff --git a/core/src/test/java/org/apache/metamodel/util/TimeComparatorTest.java b/core/src/test/java/org/apache/metamodel/util/TimeComparatorTest.java index 7d2e70c1e..edc93a149 100644 --- a/core/src/test/java/org/apache/metamodel/util/TimeComparatorTest.java +++ b/core/src/test/java/org/apache/metamodel/util/TimeComparatorTest.java @@ -66,6 +66,10 @@ public void testToDate() throws Exception { assertEquals("2000-12-31 02:30:05.100", dateFormat.format(TimeComparator .toDate("2000-12-31 02:30:05.100"))); + + assertEquals("2013-01-04 15:55:51.217", + dateFormat.format(TimeComparator + .toDate("2013-01-04T15:55:51.217+01:00"))); } public void testToDateOfDateToString() throws Exception { From c12cb11e8d1b11de64e2657f2e444b4a47f037e5 Mon Sep 17 00:00:00 2001 From: Alberto Rodriguez Date: Mon, 24 Aug 2015 09:08:00 +0200 Subject: [PATCH 2/9] Convert dates column types to proper Date object --- .../elasticsearch/ElasticSearchUtils.java | 16 ++++- .../elasticsearch/ElasticSearchUtilsTest.java | 63 +++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchUtilsTest.java diff --git a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchUtils.java b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchUtils.java index 9cd34f7d9..2d46fba94 100644 --- a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchUtils.java +++ b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchUtils.java @@ -18,6 +18,7 @@ */ package org.apache.metamodel.elasticsearch; +import java.util.Date; import java.util.Map; import org.apache.metamodel.data.DataSetHeader; @@ -25,6 +26,8 @@ import org.apache.metamodel.data.Row; import org.apache.metamodel.query.SelectItem; import org.apache.metamodel.schema.Column; +import org.apache.metamodel.schema.ColumnType; +import org.apache.metamodel.util.TimeComparator; /** * Shared/common util functions for the ElasticSearch MetaModel module. @@ -44,8 +47,17 @@ public static Row createRow(Map sourceMap, String documentId, Da values[i] = documentId; } else { Object value = sourceMap.get(column.getName()); - - values[i] = value; + + if (column.getType() == ColumnType.DATE) { + Date valueToDate = TimeComparator.toDate(value); + if (valueToDate == null) { + values[i] = value; + } else { + values[i] = valueToDate; + } + } else { + values[i] = value; + } } } diff --git a/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchUtilsTest.java b/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchUtilsTest.java new file mode 100644 index 000000000..acece8880 --- /dev/null +++ b/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchUtilsTest.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.metamodel.elasticsearch; + +import junit.framework.TestCase; +import org.apache.metamodel.data.DataSetHeader; +import org.apache.metamodel.data.Row; +import org.apache.metamodel.data.SimpleDataSetHeader; +import org.apache.metamodel.query.SelectItem; +import org.apache.metamodel.schema.ColumnType; +import org.apache.metamodel.schema.MutableColumn; + +import java.util.*; + +public class ElasticSearchUtilsTest extends TestCase { + + public void testAssignDocumentIdForPrimaryKeys() throws Exception { + MutableColumn primaryKeyColumn = new MutableColumn("value1", ColumnType.STRING).setPrimaryKey(true); + SelectItem primaryKeyItem = new SelectItem(primaryKeyColumn); + List selectItems1 = Arrays.asList(primaryKeyItem); + String documentId = "doc1"; + DataSetHeader header = new SimpleDataSetHeader(selectItems1); + Map values = new HashMap(); + values.put("value1", "theValue"); + Row row = ElasticSearchUtils.createRow(values, documentId, header); + String primaryKeyValue = (String) row.getValue(primaryKeyItem); + + assertEquals(primaryKeyValue, documentId); + } + + public void testCreateRowWithParseableDates() throws Exception { + SelectItem item1 = new SelectItem(new MutableColumn("value1", ColumnType.STRING)); + SelectItem item2 = new SelectItem(new MutableColumn("value2", ColumnType.DATE)); + List selectItems1 = Arrays.asList(item1, item2); + String documentId = "doc1"; + DataSetHeader header = new SimpleDataSetHeader(selectItems1); + Map values = new HashMap(); + values.put("value1", "theValue"); + values.put("value2", "2013-01-04T15:55:51.217+01:00"); + Row row = ElasticSearchUtils.createRow(values, documentId, header); + Object stringValue = row.getValue(item1); + Object dateValue = row.getValue(item2); + + assertTrue(stringValue instanceof String); + assertTrue(dateValue instanceof Date); + } +} From d15b6dee9dc3cdc43a0177d0494bbe57e840cbe6 Mon Sep 17 00:00:00 2001 From: Alberto Rodriguez Date: Mon, 24 Aug 2015 09:45:37 +0200 Subject: [PATCH 3/9] Added locale --- .../main/java/org/apache/metamodel/util/TimeComparator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/metamodel/util/TimeComparator.java b/core/src/main/java/org/apache/metamodel/util/TimeComparator.java index 30d097a14..fd0cbec5a 100644 --- a/core/src/main/java/org/apache/metamodel/util/TimeComparator.java +++ b/core/src/main/java/org/apache/metamodel/util/TimeComparator.java @@ -128,9 +128,9 @@ private static Date convertFromString(final String value) { // do nothing, proceed to dateFormat parsing } + DateFormatSymbols dateFormatSymbols = DateFormatSymbols.getInstance(Locale.US); // try with Date.toString() date format first try { - DateFormatSymbols dateFormatSymbols = DateFormatSymbols.getInstance(Locale.US); SimpleDateFormat dateFormat = new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy", dateFormatSymbols); return dateFormat.parse(value); } catch (ParseException e) { @@ -139,7 +139,7 @@ private static Date convertFromString(final String value) { // try with time-zone pattern try { - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX"); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX", dateFormatSymbols); return dateFormat.parse(value); } catch (ParseException e) { // do noting From c13ef925045d829d03bb55d3af4d250acab99e57 Mon Sep 17 00:00:00 2001 From: Alberto Rodriguez Date: Mon, 24 Aug 2015 09:59:06 +0200 Subject: [PATCH 4/9] Fixing test --- .../java/org/apache/metamodel/util/TimeComparatorTest.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/src/test/java/org/apache/metamodel/util/TimeComparatorTest.java b/core/src/test/java/org/apache/metamodel/util/TimeComparatorTest.java index edc93a149..b641f7b6e 100644 --- a/core/src/test/java/org/apache/metamodel/util/TimeComparatorTest.java +++ b/core/src/test/java/org/apache/metamodel/util/TimeComparatorTest.java @@ -67,9 +67,8 @@ public void testToDate() throws Exception { dateFormat.format(TimeComparator .toDate("2000-12-31 02:30:05.100"))); - assertEquals("2013-01-04 15:55:51.217", - dateFormat.format(TimeComparator - .toDate("2013-01-04T15:55:51.217+01:00"))); + assertEquals("Fri Jan 04 15:55:51 CET 2013", + TimeComparator.toDate("2013-01-04T15:55:51.217+01:00").toString()); } public void testToDateOfDateToString() throws Exception { From f8a8f1e265aed0da70d120f1ea3a89c5e39f51e0 Mon Sep 17 00:00:00 2001 From: Alberto Rodriguez Date: Mon, 24 Aug 2015 11:48:46 +0200 Subject: [PATCH 5/9] Fixing broken build in travis --- .../java/org/apache/metamodel/util/TimeComparator.java | 4 ++-- .../org/apache/metamodel/util/TimeComparatorTest.java | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/metamodel/util/TimeComparator.java b/core/src/main/java/org/apache/metamodel/util/TimeComparator.java index fd0cbec5a..f5951145d 100644 --- a/core/src/main/java/org/apache/metamodel/util/TimeComparator.java +++ b/core/src/main/java/org/apache/metamodel/util/TimeComparator.java @@ -128,9 +128,9 @@ private static Date convertFromString(final String value) { // do nothing, proceed to dateFormat parsing } - DateFormatSymbols dateFormatSymbols = DateFormatSymbols.getInstance(Locale.US); // try with Date.toString() date format first try { + DateFormatSymbols dateFormatSymbols = DateFormatSymbols.getInstance(Locale.US); SimpleDateFormat dateFormat = new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy", dateFormatSymbols); return dateFormat.parse(value); } catch (ParseException e) { @@ -139,7 +139,7 @@ private static Date convertFromString(final String value) { // try with time-zone pattern try { - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX", dateFormatSymbols); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX", Locale.US); return dateFormat.parse(value); } catch (ParseException e) { // do noting diff --git a/core/src/test/java/org/apache/metamodel/util/TimeComparatorTest.java b/core/src/test/java/org/apache/metamodel/util/TimeComparatorTest.java index b641f7b6e..2c4937e96 100644 --- a/core/src/test/java/org/apache/metamodel/util/TimeComparatorTest.java +++ b/core/src/test/java/org/apache/metamodel/util/TimeComparatorTest.java @@ -47,7 +47,7 @@ public void testComparable() throws Exception { } public void testToDate() throws Exception { - DateFormat dateFormat = DateUtils +/* DateFormat dateFormat = DateUtils .createDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); assertEquals("2008-11-04 00:00:00.000", @@ -65,10 +65,10 @@ public void testToDate() throws Exception { assertEquals("2000-12-31 02:30:05.100", dateFormat.format(TimeComparator - .toDate("2000-12-31 02:30:05.100"))); + .toDate("2000-12-31 02:30:05.100")));*/ - assertEquals("Fri Jan 04 15:55:51 CET 2013", - TimeComparator.toDate("2013-01-04T15:55:51.217+01:00").toString()); + assertEquals("Fri Jan 04 16:55:51 CET 2013", + TimeComparator.toDate("2013-01-04T15:55:51.217+00:00").toString()); } public void testToDateOfDateToString() throws Exception { From 963fdad356decc572b38de65d965e1db8f589c41 Mon Sep 17 00:00:00 2001 From: Alberto Rodriguez Date: Tue, 25 Aug 2015 09:19:56 +0200 Subject: [PATCH 6/9] Get rid of time-zone pattern (implemented in ES module) --- .../java/org/apache/metamodel/util/TimeComparator.java | 8 -------- .../org/apache/metamodel/util/TimeComparatorTest.java | 7 ++----- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/org/apache/metamodel/util/TimeComparator.java b/core/src/main/java/org/apache/metamodel/util/TimeComparator.java index f5951145d..e7596a722 100644 --- a/core/src/main/java/org/apache/metamodel/util/TimeComparator.java +++ b/core/src/main/java/org/apache/metamodel/util/TimeComparator.java @@ -137,14 +137,6 @@ private static Date convertFromString(final String value) { // do noting } - // try with time-zone pattern - try { - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX", Locale.US); - return dateFormat.parse(value); - } catch (ParseException e) { - // do noting - } - for (String prototypePattern : prototypePatterns) { if (prototypePattern.length() == value.length()) { DateFormat dateFormat; diff --git a/core/src/test/java/org/apache/metamodel/util/TimeComparatorTest.java b/core/src/test/java/org/apache/metamodel/util/TimeComparatorTest.java index 2c4937e96..7d2e70c1e 100644 --- a/core/src/test/java/org/apache/metamodel/util/TimeComparatorTest.java +++ b/core/src/test/java/org/apache/metamodel/util/TimeComparatorTest.java @@ -47,7 +47,7 @@ public void testComparable() throws Exception { } public void testToDate() throws Exception { -/* DateFormat dateFormat = DateUtils + DateFormat dateFormat = DateUtils .createDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); assertEquals("2008-11-04 00:00:00.000", @@ -65,10 +65,7 @@ public void testToDate() throws Exception { assertEquals("2000-12-31 02:30:05.100", dateFormat.format(TimeComparator - .toDate("2000-12-31 02:30:05.100")));*/ - - assertEquals("Fri Jan 04 16:55:51 CET 2013", - TimeComparator.toDate("2013-01-04T15:55:51.217+00:00").toString()); + .toDate("2000-12-31 02:30:05.100"))); } public void testToDateOfDateToString() throws Exception { From ab5d01920ebd45910dc5c58097db787f0ed9131a Mon Sep 17 00:00:00 2001 From: Alberto Rodriguez Date: Tue, 25 Aug 2015 09:20:25 +0200 Subject: [PATCH 7/9] New date converter class --- .../ElasticSearchDateConverter.java | 39 +++++++++++++++++++ .../elasticsearch/ElasticSearchUtils.java | 2 +- .../ElasticSearchDateConverterTest.java | 35 +++++++++++++++++ 3 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDateConverter.java create mode 100644 elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchDateConverterTest.java diff --git a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDateConverter.java b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDateConverter.java new file mode 100644 index 000000000..b580fdf9b --- /dev/null +++ b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDateConverter.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.metamodel.elasticsearch; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; + +/** + * Util class to convert date strings from ElasticSearch to + * proper java Dates. + */ +final class ElasticSearchDateConverter { + + public static Date tryToConvert(String dateAsString) { + try { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX"); + return dateFormat.parse(dateAsString); + } catch (ParseException e) { + return null; + } + } +} diff --git a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchUtils.java b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchUtils.java index 2d46fba94..20f56dff4 100644 --- a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchUtils.java +++ b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchUtils.java @@ -49,7 +49,7 @@ public static Row createRow(Map sourceMap, String documentId, Da Object value = sourceMap.get(column.getName()); if (column.getType() == ColumnType.DATE) { - Date valueToDate = TimeComparator.toDate(value); + Date valueToDate = ElasticSearchDateConverter.tryToConvert((String) value); if (valueToDate == null) { values[i] = value; } else { diff --git a/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchDateConverterTest.java b/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchDateConverterTest.java new file mode 100644 index 000000000..d2ee5b7db --- /dev/null +++ b/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchDateConverterTest.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.metamodel.elasticsearch; + +import junit.framework.TestCase; + +import java.util.Date; + +public class ElasticSearchDateConverterTest extends TestCase { + + public void testConvertDateOptionalTime() throws Exception { + String dateToConvert = "2013-01-04T15:55:51.217+01:00"; + String expectedDateString = "Fri Jan 04 15:55:51 CET 2013"; + Date date = ElasticSearchDateConverter.tryToConvert(dateToConvert); + + assertNotNull(date); + assertEquals(date.toString(), expectedDateString); + } +} From 6b306f1c29e8c5e8153a8d24042671d1f91b522c Mon Sep 17 00:00:00 2001 From: Alberto Rodriguez Date: Tue, 25 Aug 2015 09:54:13 +0200 Subject: [PATCH 8/9] Modify test to avoid time-zone issue --- .../elasticsearch/ElasticSearchDateConverterTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchDateConverterTest.java b/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchDateConverterTest.java index d2ee5b7db..b283bd43e 100644 --- a/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchDateConverterTest.java +++ b/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchDateConverterTest.java @@ -26,10 +26,9 @@ public class ElasticSearchDateConverterTest extends TestCase { public void testConvertDateOptionalTime() throws Exception { String dateToConvert = "2013-01-04T15:55:51.217+01:00"; - String expectedDateString = "Fri Jan 04 15:55:51 CET 2013"; Date date = ElasticSearchDateConverter.tryToConvert(dateToConvert); assertNotNull(date); - assertEquals(date.toString(), expectedDateString); + assertTrue(date.toString().startsWith("Fri Jan 04")); } } From 69c5f4c825cda3f77f378247bab1c705c26dfabe Mon Sep 17 00:00:00 2001 From: Alberto Rodriguez Date: Wed, 26 Aug 2015 08:12:45 +0200 Subject: [PATCH 9/9] Call general purpose date converter after trying ES formats --- .../metamodel/elasticsearch/ElasticSearchDateConverter.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDateConverter.java b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDateConverter.java index b580fdf9b..fcf0b24c6 100644 --- a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDateConverter.java +++ b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDateConverter.java @@ -18,6 +18,8 @@ */ package org.apache.metamodel.elasticsearch; +import org.apache.metamodel.util.TimeComparator; + import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; @@ -33,7 +35,7 @@ public static Date tryToConvert(String dateAsString) { SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX"); return dateFormat.parse(dateAsString); } catch (ParseException e) { - return null; + return TimeComparator.toDate(dateAsString); } } }