#2294 changed validation rule for timestamp column

Validation target must not include null or empty value.
metatron-app · Jul 17, 2019 · 4eeb249 · 4eeb249
1 parent f7b1104
commit 4eeb249
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 12 deletions.
diff --git a/...rc/app/data-storage/component/schema-configure/schema-configure-field-detail.component.ts b/...rc/app/data-storage/component/schema-configure/schema-configure-field-detail.component.ts
@@ -46,6 +46,7 @@ import {FieldConfigService} from "../../service/field-config.service";
 import {StringUtil} from "../../../common/util/string.util";
 import {DatasourceService} from "../../../datasource/service/datasource.service";
 import {TimeZoneObject, TimezoneService} from "../../service/timezone.service";
+import {GranularityService} from "../../service/granularity.service";
 
 declare const moment;
 
@@ -125,6 +126,7 @@ export class SchemaConfigureFieldDetailComponent extends AbstractComponent imple
               private fieldConfigService: FieldConfigService,
               private datasourceService: DatasourceService,
               private timezoneService: TimezoneService,
+              private granularityService: GranularityService,
               protected element: ElementRef,
               protected injector: Injector) {
     super(element, injector);
@@ -262,7 +264,7 @@ export class SchemaConfigureFieldDetailComponent extends AbstractComponent imple
   public getDefaultIngestionRuleValue(): string | number {
     switch (this.selectedField.logicalType) {
       case LogicalType.TIMESTAMP:
-        return (this.selectedField.format && this.selectedField.format.type === FieldFormatType.DATE_TIME) ? this.dataList[0] : this._currentMilliseconds;
+        return (this.selectedField.format && this.selectedField.format.type === FieldFormatType.DATE_TIME) ? this.granularityService.getAvailableStartData(this.dataList, 0) : this._currentMilliseconds;
       case LogicalType.BOOLEAN:
         return 'false';
       case LogicalType.TEXT:

diff --git a/discovery-frontend/src/app/data-storage/service/granularity.service.ts b/discovery-frontend/src/app/data-storage/service/granularity.service.ts
@@ -14,7 +14,7 @@
  */
 
 import {Injectable, Injector} from '@angular/core';
-import { TranslateService } from '@ngx-translate/core';
+import {TranslateService} from '@ngx-translate/core';
 import {FieldFormatType, FieldFormatUnit} from "../../domain/datasource/datasource";
 
 declare let moment: any;
@@ -103,7 +103,7 @@ export class GranularityService {
    * @return {GranularityIntervalInfo}
    */
   public getInitializedInterval(fieldDataList: any[], format: string, granularity: GranularityObject, type: FieldFormatType, unit: FieldFormatUnit): GranularityIntervalInfo {
-    const firstMoment = this._getConvertedMoment(fieldDataList[0], format, type, unit);
+    const firstMoment = this._getConvertedMoment(this.getAvailableStartData(fieldDataList, 0), format, type, unit);
     const endMoment = this._getConvertedMoment(this._getAvailableEndData(fieldDataList, fieldDataList.length-1), format, type, unit);
     // init
     const result: GranularityIntervalInfo = {
@@ -161,6 +161,16 @@ export class GranularityService {
     return result;
   }
 
+  /**
+   * Get available Start Data
+   * @param {any[]} dataList
+   * @param {number} startNumber
+   * @return {any}
+   */
+  public getAvailableStartData(dataList: any[], startNumber: number): any {
+    return dataList[startNumber] || (startNumber === dataList.length -1 ? moment() : this.getAvailableStartData(dataList, startNumber + 1));
+  }
+
   /**
    * Is valid interval
    * @param {string} dateTime

diff --git a/.../src/main/java/app/metatron/discovery/domain/datasource/format/DateTimeFormatChecker.java b/.../src/main/java/app/metatron/discovery/domain/datasource/format/DateTimeFormatChecker.java
@@ -107,14 +107,17 @@ public boolean checkTimeFormat(TimeFormatCheckRequest request) {
 
   public boolean checkUnixTimeFormat(List<String> samples) {
 
+    int totalCount = samples.size();
     int count = 0;
     for (String timeStr : samples) {
-      if(StringUtils.isNumeric(timeStr)) {
+      if (StringUtils.isBlank(timeStr)) {
+        totalCount--;
+      } else if(StringUtils.isNumeric(timeStr)) {
         count++;
       }
     }
 
-    if (count >= samples.size() * 0.5) {
+    if (count >= totalCount * 0.5) {
       return true;
     }
 
@@ -124,18 +127,23 @@ public boolean checkUnixTimeFormat(List<String> samples) {
 
   public boolean checkTimeFormat(DateTimeFormatter formatter, List<String> samples) {
 
+    int totalCount = samples.size();
     int count = 0;
     for (String timeStr : samples) {
-      try {
-        formatter.withLocale(Locale.ENGLISH).parseDateTime(timeStr);
-      } catch (Exception e) {
-        LOGGER.warn("Invalid date format - no match : " + e.getMessage());
-        continue;
+      if (StringUtils.isBlank(timeStr)) {
+        totalCount--;
+      } else {
+        try {
+          formatter.withLocale(Locale.ENGLISH).parseDateTime(timeStr);
+        } catch (Exception e) {
+          LOGGER.warn("Invalid date format - no match : " + e.getMessage());
+          continue;
+        }
+        count++;
       }
-      count++;
     }
 
-    if (count >= samples.size() * 0.5) {
+    if (count >= totalCount * 0.5) {
       return true;
     }