[CARBONDATA-2509][CARBONDATA-2510][CARBONDATA-2511][32K] Add validate…

… for long string columns Add validate for long string columns 1. long string columns cannot be sort_columns 2. long string columns cannot be dictionary include 3. long string columns cannot be dictionary exclude 4. long string columns can only be string columns This closes #2380
apache · Jun 21, 2018 · 218a8de · 218a8de
1 parent 091a28b
commit 218a8de
Show file tree

Hide file tree

Showing 2 changed files with 67 additions and 1 deletion.
diff --git a/...cala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala b/...cala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
@@ -74,6 +74,63 @@ class VarcharDataTypesBasicTestCase extends QueryTest with BeforeAndAfterEach wi
     sql(s"drop table if exists $longStringTable")
   }
 
+  test("long string columns cannot be dictionary include") {
+    val exceptionCaught = intercept[Exception] {
+      sql(
+        s"""
+           | CREATE TABLE if not exists $longStringTable(
+           | id INT, name STRING, description STRING, address STRING, note STRING
+           | ) STORED BY 'carbondata'
+           | TBLPROPERTIES('LONG_STRING_COLUMNS'='address, note', 'dictionary_include'='address')
+           |""".
+          stripMargin)
+    }
+    assert(exceptionCaught.getMessage.contains("DICTIONARY_INCLUDE is unsupported for long string datatype column: address"))
+  }
+
+  test("long string columns cannot be dictionay exclude") {
+    val exceptionCaught = intercept[Exception] {
+      sql(
+        s"""
+           | CREATE TABLE if not exists $longStringTable(
+           | id INT, name STRING, description STRING, address STRING, note STRING
+           | ) STORED BY 'carbondata'
+           | TBLPROPERTIES('LONG_STRING_COLUMNS'='address, note', 'dictionary_exclude'='address')
+           |""".
+          stripMargin)
+    }
+    assert(exceptionCaught.getMessage.contains("DICTIONARY_EXCLUDE is unsupported for long string datatype column: address"))
+  }
+
+  test("long string columns cannot be sort_columns") {
+    val exceptionCaught = intercept[Exception] {
+      sql(
+        s"""
+           | CREATE TABLE if not exists $longStringTable(
+           | id INT, name STRING, description STRING, address STRING, note STRING
+           | ) STORED BY 'carbondata'
+           | TBLPROPERTIES('LONG_STRING_COLUMNS'='name, note', 'SORT_COLUMNS'='name, address')
+           |""".
+          stripMargin)
+    }
+    assert(exceptionCaught.getMessage.contains("sort_columns is unsupported for long string datatype column: name"))
+  }
+
+  test("long string columns can only be string columns") {
+    val exceptionCaught = intercept[Exception] {
+      sql(
+        s"""
+           | CREATE TABLE if not exists $longStringTable(
+           | id INT, name STRING, description STRING, address STRING, note STRING
+           | ) STORED BY 'carbondata'
+           | TBLPROPERTIES('LONG_STRING_COLUMNS'='id, note')
+           |""".
+          stripMargin)
+    }
+    assert(exceptionCaught.getMessage.contains("long_string_columns: id"))
+    assert(exceptionCaught.getMessage.contains("its data type is not string"))
+  }
+
   private def prepareTable(): Unit = {
     sql(
       s"""

diff --git a/...ration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/...ration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -752,7 +752,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
           }
           if (varcharCols.exists(x => x.equalsIgnoreCase(column))) {
             throw new MalformedCarbonCommandException(
-              s"sort_columns is unsupported for long string datatype column $column")
+              s"sort_columns is unsupported for long string datatype column: $column")
           }
         }
       }
@@ -791,6 +791,10 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
               val errorMsg = "DICTIONARY_EXCLUDE is unsupported for " + dataType.toLowerCase() +
                              " data type column: " + dictExcludeCol
               throw new MalformedCarbonCommandException(errorMsg)
+            } else if (varcharCols.exists(x => x.equalsIgnoreCase(dictExcludeCol))) {
+              throw new MalformedCarbonCommandException(
+                "DICTIONARY_EXCLUDE is unsupported for long string datatype column: " +
+                dictExcludeCol)
             }
           }
         }
@@ -805,6 +809,11 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
                          " does not exist in table. Please check create table statement."
           throw new MalformedCarbonCommandException(errormsg)
         }
+        if (varcharCols.exists(x => x.equalsIgnoreCase(distIncludeCol.trim))) {
+          throw new MalformedCarbonCommandException(
+            "DICTIONARY_INCLUDE is unsupported for long string datatype column: " +
+            distIncludeCol.trim)
+        }
       }
     }