[CARBONDATA-3114]Remove Null Values in all types of columns for Range…

…Filters
apache · Nov 21, 2018 · fb0912f · fb0912f
1 parent c2ae987
commit fb0912f
Show file tree

Hide file tree

Showing 6 changed files with 47 additions and 12 deletions.
diff --git a/...rg/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/...rg/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
@@ -453,9 +453,8 @@ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
     } else if (!dimensionColumnPage.isAdaptiveEncoded()) {
       defaultValue = CarbonCommonConstants.EMPTY_BYTE_ARRAY;
     }
-    if (dimensionColumnPage.isNoDicitionaryColumn()) {
-      FilterUtil.removeNullValues(dimensionColumnPage, bitSet, defaultValue);
-    }
+    FilterUtil.removeNullValues(dimensionColumnPage, bitSet, defaultValue);
+
     return bitSet;
   }
 

diff --git a/...e/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/...e/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
@@ -457,9 +457,7 @@ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
     } else if (!dimensionColumnPage.isAdaptiveEncoded()) {
       defaultValue = CarbonCommonConstants.EMPTY_BYTE_ARRAY;
     }
-    if (dimensionColumnPage.isNoDicitionaryColumn()) {
-      FilterUtil.removeNullValues(dimensionColumnPage, bitSet, defaultValue);
-    }
+    FilterUtil.removeNullValues(dimensionColumnPage, bitSet, defaultValue);
     return bitSet;
   }
 

diff --git a/...he/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/...he/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -452,9 +452,8 @@ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
     if (dimColEvaluatorInfoList.get(0).getDimension().getDataType() == DataTypes.STRING) {
       defaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY;
     }
-    if (dimensionColumnPage.isNoDicitionaryColumn()) {
-      FilterUtil.removeNullValues(dimensionColumnPage, bitSet, defaultValue);
-    }
+    FilterUtil.removeNullValues(dimensionColumnPage, bitSet, defaultValue);
+
     return bitSet;
   }
 

diff --git a/.../apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java b/.../apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java
@@ -448,9 +448,8 @@ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
     if (dimColEvaluatorInfoList.get(0).getDimension().getDataType() == DataTypes.STRING) {
       defaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY;
     }
-    if (dimensionColumnPage.isNoDicitionaryColumn()) {
-      FilterUtil.removeNullValues(dimensionColumnPage, bitSet, defaultValue);
-    }
+    FilterUtil.removeNullValues(dimensionColumnPage, bitSet, defaultValue);
+
     return bitSet;
   }
 

diff --git a/integration/spark-common-test/src/test/resources/data_timestamp.csv b/integration/spark-common-test/src/test/resources/data_timestamp.csv
@@ -0,0 +1,10 @@
+col
+2014-01-01 18:00:00
+2014-01-02 18:00:00
+2014-01-03 18:00:00
+
+2014-01-03 18:00:00
+0
+2014-01-03 18:00:00
+
+2014-01-03 18:00:00
diff --git a/...rbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala b/...rbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
@@ -141,9 +141,39 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte
     )
   }
 
+  test("test timestamp with dictionary include and no_inverted index") {
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_PUSH_ROW_FILTERS_FOR_VECTOR, "true")
+    sql("drop table if exists test_timestamp")
+    sql("drop table if exists test_timestamp_hive")
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy-MM-dd HH:mm:ss")
+    sql(
+      "create table test_timestamp(col timestamp) stored by 'carbondata' tblproperties" +
+      "('no_inverted_index'='col','dictionary_include'='col')")
+    val csvFilePath = s"$resourcesPath/data_timestamp.csv"
+    sql(
+      "load data inpath '" + csvFilePath +
+      "' into table test_timestamp options('delimiter'='=','quotechar'=''," +
+      "'bad_records_action'='force','fileheader'='col')")
+    sql(
+      "create table test_timestamp_hive(col timestamp) row format delimited fields terminated by " +
+      "','")
+    sql("load data inpath '" + csvFilePath + "' into table test_timestamp_hive ")
+    checkAnswer(sql(
+      "select col from test_timestamp where col not between '2014-01-01 18:00:00' and '0'"),
+      sql("select col from test_timestamp_hive where col not between '2014-01-01 18:00:00' and " +
+          "'0'"))
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_PUSH_ROW_FILTERS_FOR_VECTOR,
+        CarbonCommonConstants.CARBON_PUSH_ROW_FILTERS_FOR_VECTOR_DEFAULT)
+  }
+
   override def afterAll {
     sql("drop table directDictionaryTable")
     sql("drop table directDictionaryTable_hive")
+    sql("drop table if exists test_timestamp")
+    sql("drop table if exists test_timestamp_hive")
     CarbonProperties.getInstance()
       .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
         CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)