Added SDV test cases for Local Dictionary Support

integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/LoadTableWithLocalDictionaryTestCase.scala

@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.localdictionary
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+class LoadTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAfterAll {
+
+  val file1 = resourcesPath + "/local_dictionary_source1.csv"
+
+  val file2 = resourcesPath + "/local_dictionary_complex_data.csv"
+
+  val storePath = warehouse + "/local2/Fact/Part0/Segment_0"
+
+  override protected def beforeAll(): Unit = {
+    CarbonProperties.getInstance.addProperty(CarbonCommonConstants.BLOCKLET_SIZE, "10000")
+    QueryTest.createFile(file1)
+    QueryTest.createComplexData(file2)
+    sql("drop table if exists local2")
+  }
+
+  test("test LocalDictionary Load For FallBackScenario"){
+    sql("drop table if exists local2")
+    sql(
+      "CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties" +
+      "('local_dictionary_threshold'='2001','local_dictionary_include'='name')")
+    sql("load data inpath '" + file1 + "' into table local2 OPTIONS('header'='false')")
+    assert(!QueryTest.checkForLocalDictionary(QueryTest.getDimRawChunk(0)))
+  }
+
+  test("test successful local dictionary generation"){
+    sql("drop table if exists local2")
+    sql(
+      "CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties" +
+      "('local_dictionary_enable'='true','local_dictionary_threshold'='9001','local_dictionary_include'='name')")
+    sql("load data inpath '" + file1 + "' into table local2 OPTIONS('header'='false')")
+    assert(QueryTest.checkForLocalDictionary(QueryTest.getDimRawChunk(0)))
+  }
+
+  test("test successful local dictionary generation for default configs") {
+    sql("drop table if exists local2")
+    sql("CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true')")
+    sql("load data inpath '" + file1 + "' into table local2 OPTIONS('header'='false')")
+    assert(QueryTest.checkForLocalDictionary(QueryTest.getDimRawChunk(0)))
+  }
+
+  test("test local dictionary generation for local dictioanry include") {
+    sql("drop table if exists local2")
+    sql(
+      "CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties" +
+      "('dictionary_include'='name')")
+    sql("load data inpath '" + file1 + "' into table local2 OPTIONS('header'='false')")
+    assert(!QueryTest.checkForLocalDictionary(QueryTest.getDimRawChunk(0)))
+  }
+
+  test("test local dictionary generation for local dictioanry exclude"){
+    sql("drop table if exists local2")
+    sql(
+      "CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties" +
+      "('local_dictionary_enable'='true','dictionary_exclude'='name')")
+    sql("load data inpath '" + file1 + "' into table local2 OPTIONS('header'='false')")
+    assert(QueryTest.checkForLocalDictionary(QueryTest.getDimRawChunk(0)))
+  }
+
+  test("test local dictionary generation when it is disabled"){
+    sql("drop table if exists local2")
+    sql(
+      "CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties" +
+      "('local_dictionary_enable'='false','local_dictionary_include'='name')")
+    sql("load data inpath '" + file1 + "' into table local2 OPTIONS('header'='false')")
+    assert(!QueryTest.checkForLocalDictionary(QueryTest.getDimRawChunk(0)))
+  }
+
+  test("test local dictionary generation for invalid threshold configurations"){
+    sql("drop table if exists local2")
+    sql(
+      "CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties" +
+      "('local_dictionary_enable'='true','local_dictionary_include'='name','local_dictionary_threshold'='300000')")
+    sql("load data inpath '" + file1 + "' into table local2 OPTIONS('header'='false')")
+    assert(QueryTest.checkForLocalDictionary(QueryTest.getDimRawChunk(0)))
+  }
+
+  test("test local dictionary generation for include and exclude"){
+    sql("drop table if exists local2")
+    sql(
+      "CREATE TABLE local2(name string, age string) STORED BY 'carbondata' tblproperties" +
+      "('local_dictionary_enable'='true','local_dictionary_include'='name', 'local_dictionary_exclude'='age')")
+    sql("insert into table local2 values('vishal', '30')")
+    assert(QueryTest.checkForLocalDictionary(QueryTest.getDimRawChunk(0)))
+    assert(!QueryTest.checkForLocalDictionary(QueryTest.getDimRawChunk(1)))
+  }
+
+  test("test local dictionary generation for complex type"){
+    sql("drop table if exists local2")
+    sql(
+      "CREATE TABLE local2(name struct<i:string,s:string>) STORED BY 'carbondata' tblproperties" +
+      "('local_dictionary_enable'='true','local_dictionary_include'='name')")
+    sql("load data inpath '" + file2 +
+        "' into table local2 OPTIONS('header'='false','COMPLEX_DELIMITER_LEVEL_1'='$', " +
+        "'COMPLEX_DELIMITER_LEVEL_2'=':')")
+    assert(!QueryTest.checkForLocalDictionary(QueryTest.getDimRawChunk(0)))
+    assert(QueryTest.checkForLocalDictionary(QueryTest.getDimRawChunk(1)))
+    assert(QueryTest.checkForLocalDictionary(QueryTest.getDimRawChunk(2)))
+  }
+
+  test("test to validate local dictionary values"){
+    sql("drop table if exists local2")
+    sql("CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true')")
+    sql("load data inpath '" + resourcesPath + "/localdictionary.csv" + "' into table local2")
+    val dimRawChunk = QueryTest.getDimRawChunk(0)
+    val dictionaryData = Array("vishal", "kumar", "akash", "praveen", "brijoo")
+    try
+      assert(QueryTest.validateDictionary(dimRawChunk.get(0), dictionaryData))
+    catch {
+      case e: Exception =>
+        assert(false)
+    }
+  }
+
+  override protected def afterAll(): Unit = {
+    sql("drop table if exists local2")
+    QueryTest.deleteFile(file1)
+    QueryTest.deleteFile(file2)
+    CarbonProperties.getInstance
+      .addProperty(CarbonCommonConstants.BLOCKLET_SIZE,
+        CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL)
+  }
+}

integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala

@@ -21,6 +21,7 @@ import org.scalatest.{BeforeAndAfterAll, Suites}
 
 import org.apache.carbondata.cluster.sdv.generated._
 import org.apache.carbondata.cluster.sdv.register.TestRegisterCarbonTable
+import org.apache.carbondata.spark.testsuite.localdictionary.LoadTableWithLocalDictionaryTestCase
 
 /**
  * Suite class for all tests.
@@ -63,7 +64,8 @@ class SDVSuites extends Suites with BeforeAndAfterAll {
                               new TimeSeriesPreAggregateTestCase ::
                               new TestPartitionWithGlobalSort ::
                               new PartitionWithPreAggregateTestCase ::
-                              new CreateTableWithLocalDictionaryTestCase :: Nil
+                              new CreateTableWithLocalDictionaryTestCase ::
+                              new LoadTableWithLocalDictionaryTestCase :: Nil
 
   override val nestedSuites = suites.toIndexedSeq
 
@@ -154,7 +156,8 @@ class SDVSuites3 extends Suites with BeforeAndAfterAll {
                     new SDKwriterTestCase ::
                     new SetParameterTestCase ::
                     new PartitionWithPreAggregateTestCase ::
-                    new CreateTableWithLocalDictionaryTestCase :: Nil
+                    new CreateTableWithLocalDictionaryTestCase ::
+                    new LoadTableWithLocalDictionaryTestCase :: Nil
 
   override val nestedSuites = suites.toIndexedSeq