#1193 fix ingestion error using current timestamp field and apply tim…

…ezon to select/groupby query
metatron-app · Jan 29, 2019 · 22a2a55 · 22a2a55
1 parent c7eb6d8
commit 22a2a55
Show file tree

Hide file tree

Showing 14 changed files with 161 additions and 73 deletions.
diff --git a/discovery-server/src/main/java/app/metatron/discovery/domain/datasource/DataSource.java b/discovery-server/src/main/java/app/metatron/discovery/domain/datasource/DataSource.java
@@ -440,13 +440,15 @@ public List<app.metatron.discovery.domain.workbook.configurations.field.Field> g
 
       switch (field.getRole()) {
         case DIMENSION:
-          resultFields.add(new DimensionField(fieldName, ref));
+          resultFields.add(new DimensionField(fieldName, ref, field.getFormatObject()));
           break;
         case MEASURE:
           resultFields.add(new MeasureField(fieldName, ref));
           break;
         case TIMESTAMP:
-          resultFields.add(new TimestampField(fieldName, ref));
+          if (BooleanUtils.isNotTrue(field.getDerived())) {
+            resultFields.add(new TimestampField(fieldName, ref, field.getFormatObject()));
+          }
           break;
       }
     }

diff --git a/discovery-server/src/main/java/app/metatron/discovery/domain/datasource/Field.java b/discovery-server/src/main/java/app/metatron/discovery/domain/datasource/Field.java
@@ -614,7 +614,7 @@ public void setIngestionRule(String ingestionRule) {
   public FieldFormat getFormatObject() {
     FieldFormat fieldFormat = GlobalObjectMapper.readValue(format, FieldFormat.class);
     if (getLogicalType() == LogicalType.TIMESTAMP && fieldFormat == null) {
-      fieldFormat = new CustomDateTimeFormat(getTimeFormat());
+      fieldFormat = new CustomDateTimeFormat(format);
     }
     return fieldFormat;
   }

diff --git a/...rc/main/java/app/metatron/discovery/domain/workbook/configurations/filter/TimeFilter.java b/...rc/main/java/app/metatron/discovery/domain/workbook/configurations/filter/TimeFilter.java
@@ -42,7 +42,7 @@ public abstract class TimeFilter extends Filter {
 
   public static final DateTime MAX_DATETIME = DateTime.parse("2051-01-01T00:00:00.000Z");
 
-  public static final List<String> DEFAULT_INTERVAL = Lists.newArrayList(MAX_DATETIME.toString() + "/" + MAX_DATETIME.toString());
+  public static final List<String> DEFAULT_INTERVAL = Lists.newArrayList(MIN_DATETIME.toString() + "/" + MAX_DATETIME.toString());
 
   public static final List<String> filterOptionTypes = Lists.newArrayList("RELATIVE", "RANGE");
 

diff --git a/...va/app/metatron/discovery/domain/workbook/configurations/format/CustomDateTimeFormat.java b/...va/app/metatron/discovery/domain/workbook/configurations/format/CustomDateTimeFormat.java
@@ -17,6 +17,8 @@
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
+import org.apache.commons.lang3.StringUtils;
+
 public class CustomDateTimeFormat extends TimeFieldFormat implements FieldFormat {
 
   String format;
@@ -28,7 +30,7 @@ public CustomDateTimeFormat(
       @JsonProperty("locale") String locale,
       @JsonProperty("filteringType") String filteringType) {
     super(timeZone, locale, filteringType);
-    this.format = format;
+    this.format = StringUtils.isEmpty(format) ? TimeFieldFormat.DEFAULT_DATETIME_FORMAT : format;
   }
 
   public CustomDateTimeFormat(String format) {

diff --git a/discovery-server/src/main/java/app/metatron/discovery/query/druid/AbstractQueryBuilder.java b/discovery-server/src/main/java/app/metatron/discovery/query/druid/AbstractQueryBuilder.java
@@ -88,6 +88,7 @@
 import app.metatron.discovery.domain.workbook.configurations.filter.TimeListFilter;
 import app.metatron.discovery.domain.workbook.configurations.filter.TimestampFilter;
 import app.metatron.discovery.domain.workbook.configurations.format.TimeFieldFormat;
+import app.metatron.discovery.domain.workbook.configurations.format.UnixTimeFormat;
 import app.metatron.discovery.query.druid.aggregations.AreaAggregation;
 import app.metatron.discovery.query.druid.aggregations.CountAggregation;
 import app.metatron.discovery.query.druid.aggregations.GenericMaxAggregation;
@@ -107,6 +108,7 @@
 import app.metatron.discovery.query.druid.filters.OrFilter;
 import app.metatron.discovery.query.druid.filters.RegExpFilter;
 import app.metatron.discovery.query.druid.filters.SelectorFilter;
+import app.metatron.discovery.query.druid.funtions.CastFunc;
 import app.metatron.discovery.query.druid.funtions.DateTimeMillisFunc;
 import app.metatron.discovery.query.druid.funtions.InFunc;
 import app.metatron.discovery.query.druid.funtions.TimeFormatFunc;
@@ -122,6 +124,7 @@
 import app.metatron.discovery.query.druid.virtualcolumns.VirtualColumn;
 import app.metatron.discovery.query.polaris.ComputationalField;
 import app.metatron.discovery.util.PolarisUtils;
+import app.metatron.discovery.util.TimeUnits;
 
 import static app.metatron.discovery.domain.datasource.DataSourceErrorCodes.CONFUSING_FIELD_CODE;
 import static app.metatron.discovery.domain.datasource.Field.FieldRole.DIMENSION;
@@ -348,6 +351,34 @@ protected void extractPartitions(List<app.metatron.discovery.domain.workbook.con
     }
   }
 
+  protected TimeFormatFunc createTimeFormatFunc(String fieldName, TimeFieldFormat originalTimeFormat, TimeFieldFormat timeFormat) {
+    TimeFormatFunc timeFormatFunc;
+    if (originalTimeFormat instanceof UnixTimeFormat) {
+      CastFunc func = new CastFunc(fieldName, CastFunc.CastType.LONG);
+      String expr = func.toExpression();
+      if (((UnixTimeFormat) originalTimeFormat).getUnit() == TimeUnits.SECOND) {
+        expr += expr + "* 1000";
+      }
+
+      timeFormatFunc = new TimeFormatFunc(expr,
+                                          timeFormat.enableSortField() ? timeFormat.getSortFormat() : timeFormat.getFormat(),
+                                          timeFormat.getTimeZone(),
+                                          timeFormat.getLocale());
+
+    } else {
+      timeFormatFunc = new TimeFormatFunc("\"" + fieldName + "\"",
+                                          originalTimeFormat.getFormat(),
+                                          originalTimeFormat.getTimeZone(),
+                                          originalTimeFormat.getLocale(),
+                                          timeFormat.enableSortField() ? timeFormat.getSortFormat() : timeFormat.getFormat(),
+                                          timeFormat.getTimeZone(),
+                                          timeFormat.getLocale());
+
+    }
+
+    return timeFormatFunc;
+  }
+
 
   protected List<String> getAllOutputFieldName() {
 

diff --git a/discovery-server/src/main/java/app/metatron/discovery/query/druid/Dimension.java b/discovery-server/src/main/java/app/metatron/discovery/query/druid/Dimension.java
@@ -20,13 +20,15 @@
 
 import app.metatron.discovery.common.datasource.LogicalType;
 import app.metatron.discovery.query.druid.dimensions.DefaultDimension;
+import app.metatron.discovery.query.druid.dimensions.ExpressionDimension;
 import app.metatron.discovery.query.druid.dimensions.ExtractionDimension;
 import app.metatron.discovery.query.druid.dimensions.LookupDimension;
 
 @JsonTypeInfo(use=JsonTypeInfo.Id.NAME, include= JsonTypeInfo.As.EXTERNAL_PROPERTY, property="type")
 @JsonSubTypes({
     @JsonSubTypes.Type(value = DefaultDimension.class, name = "default"),
     @JsonSubTypes.Type(value = LookupDimension.class, name = "lookup"),
+    @JsonSubTypes.Type(value = ExpressionDimension.class, name = "expression"),
     @JsonSubTypes.Type(value = ExtractionDimension.class, name = "extraction")
 })
 public interface Dimension {

diff --git a/...rver/src/main/java/app/metatron/discovery/query/druid/dimensions/ExpressionDimension.java b/...rver/src/main/java/app/metatron/discovery/query/druid/dimensions/ExpressionDimension.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specic language governing permissions and
+ * limitations under the License.
+ */
+
+package app.metatron.discovery.query.druid.dimensions;
+
+import com.fasterxml.jackson.annotation.JsonTypeName;
+
+import app.metatron.discovery.common.datasource.LogicalType;
+import app.metatron.discovery.query.druid.Dimension;
+
+@JsonTypeName("expression")
+public class ExpressionDimension implements Dimension {
+
+  String outputName;
+
+  String expression;
+
+  public ExpressionDimension() {
+  }
+
+  public ExpressionDimension(String outputName, String expression) {
+    this.outputName = outputName;
+    this.expression = expression;
+  }
+
+  @Override
+  public String getDimension() {
+    return null;
+  }
+
+  public String getExpression() {
+    return expression;
+  }
+
+  @Override
+  public String getOutputName() {
+    return outputName;
+  }
+
+  @Override
+  public LogicalType getLogicalType() {
+    return null;
+  }
+}
diff --git a/discovery-server/src/main/java/app/metatron/discovery/query/druid/funtions/CastFunc.java b/discovery-server/src/main/java/app/metatron/discovery/query/druid/funtions/CastFunc.java
@@ -32,8 +32,8 @@ public CastFunc(String fieldName, CastType type) {
 
   public String toExpression() {
     StringBuilder sb = new StringBuilder();
-    sb.append(FUNC_NAME).append("(");
-    sb.append(fieldName).append(", '");
+    sb.append(FUNC_NAME).append("(\"");
+    sb.append(fieldName).append("\", '");
     sb.append(type.name());
     sb.append("')");
 

diff --git a/...-server/src/main/java/app/metatron/discovery/query/druid/queries/GroupByQueryBuilder.java b/...-server/src/main/java/app/metatron/discovery/query/druid/queries/GroupByQueryBuilder.java
@@ -239,18 +239,12 @@ public GroupByQueryBuilder fields(List<Field> reqFields) {
             break;
 
           case TIMESTAMP:
-            TimeFieldFormat fieldFormat = (TimeFieldFormat) datasourceField.getFormatObject();
+            TimeFieldFormat originalTimeFormat = (TimeFieldFormat) datasourceField.getFormatObject();
             TimeFieldFormat timeFormat = (TimeFieldFormat) format;
 
             String innerFieldName = aliasName + Query.POSTFIX_INNER_FIELD;
 
-            TimeFormatFunc timeFormatFunc = new TimeFormatFunc("\"" + fieldName + "\"",
-                                                               fieldFormat.getFormat(),
-                                                               fieldFormat.getTimeZone(),
-                                                               fieldFormat.getLocale(),
-                                                               timeFormat.enableSortField() ? timeFormat.getSortFormat() : timeFormat.getFormat(),
-                                                               timeFormat.getTimeZone(),
-                                                               timeFormat.getLocale());
+            TimeFormatFunc timeFormatFunc = createTimeFormatFunc(fieldName, originalTimeFormat, timeFormat);
 
             ExprVirtualColumn exprVirtualColumn = new ExprVirtualColumn(timeFormatFunc.toExpression(), innerFieldName);
             virtualColumns.put(aliasName, exprVirtualColumn);
@@ -266,14 +260,14 @@ public GroupByQueryBuilder fields(List<Field> reqFields) {
               }
 
               if (postProcessor instanceof PostAggregationProcessor) {
-                // Sort 를 위한 Format 으로 모든 연산 수행 후, 최종 클라이언트가 지정한 Format 으로 변경
+                // Change date-time format that set from the query, after calculating date-time format for sort
                 TimeFormatFunc postFormatFunc = new TimeFormatFunc("\"" + aliasName + "\"",
-                                                                   fieldFormat.getFormat(),
-                                                                   fieldFormat.getTimeZone(),
-                                                                   fieldFormat.getLocale(),
+                                                                   timeFormat.enableSortField() ? timeFormat.getSortFormat() : timeFormat.getFormat(),
+                                                                   null,
+                                                                   null,
                                                                    timeFormat.getFormat(),
-                                                                   timeFormat.getTimeZone(),
-                                                                   timeFormat.getLocale());
+                                                                   null,
+                                                                   null);
 
                 ((PostAggregationProcessor) postProcessor)
                     .addPostAggregation(

diff --git a/...y-server/src/main/java/app/metatron/discovery/query/druid/queries/SelectQueryBuilder.java b/...y-server/src/main/java/app/metatron/discovery/query/druid/queries/SelectQueryBuilder.java
@@ -70,10 +70,10 @@
 import app.metatron.discovery.query.druid.AbstractQueryBuilder;
 import app.metatron.discovery.query.druid.Dimension;
 import app.metatron.discovery.query.druid.dimensions.DefaultDimension;
+import app.metatron.discovery.query.druid.dimensions.ExpressionDimension;
 import app.metatron.discovery.query.druid.dimensions.ExtractionDimension;
 import app.metatron.discovery.query.druid.dimensions.LookupDimension;
 import app.metatron.discovery.query.druid.extractionfns.ExpressionFunction;
-import app.metatron.discovery.query.druid.extractionfns.TimeParsingFunction;
 import app.metatron.discovery.query.druid.filters.AndFilter;
 import app.metatron.discovery.query.druid.funtions.TimeFormatFunc;
 import app.metatron.discovery.query.druid.granularities.SimpleGranularity;
@@ -120,10 +120,9 @@ public SelectQueryBuilder fields(List<Field> reqFields) {
     // 별도 forward context 추가시 Projection 항목 지정 위함
     projections = reqFields;
 
-    // 필드정보가 없을 경우 모든 필드를 대상으로 지정하는 것을 가정
+    // If fields are empty in the search query, it will get field information from datasource.
     if (CollectionUtils.isEmpty(reqFields)) {
-      reqFields = Lists.newArrayList();
-      reqFields.addAll(getAllFieldsByMapping());
+      reqFields = Lists.newArrayList(getAllFieldsByMapping());
     }
 
     for (Field reqField : reqFields) {
@@ -135,8 +134,8 @@ public SelectQueryBuilder fields(List<Field> reqFields) {
 
       String aliasName = reqField.getAlias();
 
-      if(UserDefinedField.REF_NAME.equals(reqField.getRef())) {
-        if(reqField instanceof DimensionField) {
+      if (UserDefinedField.REF_NAME.equals(reqField.getRef())) {
+        if (reqField instanceof DimensionField) {
           dimensions.add(new DefaultDimension(fieldName, aliasName));
         } else {
           virtualColumns.put(aliasName, new ExprVirtualColumn(fieldName, aliasName));
@@ -149,11 +148,16 @@ public SelectQueryBuilder fields(List<Field> reqFields) {
       if (reqField instanceof DimensionField) {
 
         DimensionField dimensionField = (DimensionField) reqField;
-        FieldFormat format = dimensionField.getFormat();
         app.metatron.discovery.domain.datasource.Field datasourceField = metaFieldMap.get(fieldName);
 
-        if(datasourceField == null) {
-          throw new QueryTimeExcetpion("'"+ fieldName +"' not found  in datasource ( " + dataSource.getName() + " )");
+        FieldFormat originalFormat = datasourceField.getFormatObject();
+        FieldFormat format = dimensionField.getFormat();
+        if (format == null) {
+          format = originalFormat;
+        }
+
+        if (datasourceField == null) {
+          throw new QueryTimeExcetpion("'" + fieldName + "' not found  in datasource ( " + dataSource.getName() + " )");
         }
 
         // In case of GEO Type, druid engine recognizes it as metric
@@ -178,22 +182,14 @@ public SelectQueryBuilder fields(List<Field> reqFields) {
               dimensions.add(new ExtractionDimension(fieldName, aliasName,
                                                      new ExpressionFunction(((DefaultFormat) format).getFormat(), fieldName)));
               break;
-            case TIMESTAMP: // TODO: 추후 별도의 Timestamp 처리 확인 해볼것
-              TimeFieldFormat fieldFormat = (TimeFieldFormat) datasourceField.getFormatObject();
+            case TIMESTAMP:
+              TimeFieldFormat originalTimeFormat = (TimeFieldFormat) originalFormat;
               TimeFieldFormat timeFormat = (TimeFieldFormat) format;
-              ExtractionDimension extractionDimension = new ExtractionDimension();
-              extractionDimension.setDimension(fieldName);
-              extractionDimension.setOutputName(aliasName);
-
-              extractionDimension.setExtractionFn(
-                  new TimeParsingFunction(fieldFormat.getFormat(),
-                                          fieldFormat.getTimeZone(),
-                                          fieldFormat.getLocale(),
-                                          timeFormat.getFormat(),
-                                          timeFormat.getLocale(),
-                                          timeFormat.getTimeZone())
-              );
-              dimensions.add(extractionDimension);
+
+              TimeFormatFunc timeFormatFunc = createTimeFormatFunc(fieldName, originalTimeFormat, timeFormat);
+
+              dimensions.add(new ExpressionDimension(aliasName, timeFormatFunc.toExpression()));
+
               break;
             default:
               dimensions.add(new DefaultDimension(fieldName, aliasName,
@@ -218,31 +214,26 @@ public SelectQueryBuilder fields(List<Field> reqFields) {
         //        dimensions.add(new DefaultDimension(vcName, aliasName));
 
       } else if (reqField instanceof TimestampField) {
+
         if (!this.metaFieldMap.containsKey(fieldName)) {
           continue;
         }
 
+        app.metatron.discovery.domain.datasource.Field datasourceField = metaFieldMap.get(fieldName);
+        TimeFieldFormat originalTimeFormat = (TimeFieldFormat) datasourceField.getFormatObject();
+
         TimestampField timestampField = (TimestampField) reqField;
         TimeFieldFormat timeFormat = (TimeFieldFormat) timestampField.getFormat();
-        TimeFormatFunc timeFormatFunc = null;
-        if (timeFormat != null) {
-          timeFormatFunc = new TimeFormatFunc(timestampField.getPredefinedColumn(dataSource instanceof MappingDataSource),
-                                              timeFormat.getFormat(),
-                                              timeFormat.getTimeZone(),
-                                              timeFormat.getLocale());
-        } else {
-          app.metatron.discovery.domain.datasource.Field datasourceField = metaFieldMap.get(fieldName);
-
-          timeFormatFunc = new TimeFormatFunc(timestampField.getPredefinedColumn(dataSource instanceof MappingDataSource),
-                                              datasourceField.getFormat() == null ?
-                                                  TimeFieldFormat.DEFAULT_DATETIME_FORMAT : datasourceField.getFormat(),
-                                              null,
-                                              null);
+        if (timeFormat == null) {
+          timeFormat = originalTimeFormat;
         }
 
-        ExprVirtualColumn exprVirtualColumn = new ExprVirtualColumn(timeFormatFunc.toExpression(), timestampField.getColunm());
-        virtualColumns.put(timestampField.getColunm(), exprVirtualColumn);
-        dimensions.add(new DefaultDimension(fieldName, aliasName));
+        TimeFormatFunc timeFormatFunc = new TimeFormatFunc(timestampField.getPredefinedColumn(dataSource instanceof MappingDataSource),
+                                                           timeFormat.getFormat(),
+                                                           timeFormat.getTimeZone(),
+                                                           timeFormat.getLocale());
+
+        dimensions.add(new ExpressionDimension(aliasName, timeFormatFunc.toExpression()));
       }
     }
 
@@ -264,9 +255,9 @@ public SelectQueryBuilder limit(Limit reqLimit) {
       pagingSpec.setThreshold(reqLimit.getLimit());
 
       for (Sort sort : reqLimit.getSort()) {
-        if(this.metaFieldMap.containsKey(sort.getField())) {
+        if (this.metaFieldMap.containsKey(sort.getField())) {
           app.metatron.discovery.domain.datasource.Field field = this.metaFieldMap.get(sort.getField());
-          if(field.getRole() == app.metatron.discovery.domain.datasource.Field.FieldRole.TIMESTAMP) {
+          if (field.getRole() == app.metatron.discovery.domain.datasource.Field.FieldRole.TIMESTAMP) {
             descending = sort.getDirection() == Sort.Direction.DESC ? true : false;
           } // Ignore any sorting on the rest of the field of timestamp role
         }