MapDDLSupport

apache · Dec 12, 2018 · b0fa65c · b0fa65c
1 parent fa11871
commit b0fa65c
Show file tree

Hide file tree

Showing 25 changed files with 735 additions and 74 deletions.
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -597,6 +597,10 @@ public static String unquoteChar(String parseStr) {
    */
   public static String delimiterConverter(String delimiter) {
     switch (delimiter) {
+      case "\\001":
+      case "\\002":
+      case "\\003":
+      case "\\004":
       case "|":
       case "*":
       case ".":

diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
@@ -37,6 +37,7 @@
 import org.apache.carbondata.core.util.ObjectSerializationUtil;
 import org.apache.carbondata.core.util.ThreadLocalSessionInfo;
 import org.apache.carbondata.hadoop.internal.ObjectArrayWritable;
+import org.apache.carbondata.processing.loading.ComplexDelimitersEnum;
 import org.apache.carbondata.processing.loading.DataLoadExecutor;
 import org.apache.carbondata.processing.loading.TableProcessingOperations;
 import org.apache.carbondata.processing.loading.iterator.CarbonOutputIteratorWrapper;
@@ -338,11 +339,19 @@ public static CarbonLoadModel getLoadModel(Configuration conf) throws IOExceptio
             SKIP_EMPTY_LINE,
             carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_SKIP_EMPTY_LINE)));
 
-    String complexDelim = conf.get(COMPLEX_DELIMITERS, "\\\001" + "," + "\\\002");
+    String complexDelim = conf.get(COMPLEX_DELIMITERS);
+    if (null == complexDelim) {
+      complexDelim = ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_1.value() + ","
+          + ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_2.value() + ","
+          + ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_3.value();
+    }
     String[] split = complexDelim.split(",");
-    model.setComplexDelimiterLevel1(split[0]);
-    if (split.length > 1) {
-      model.setComplexDelimiterLevel2(split[1]);
+    model.setComplexDelimiter(split[0]);
+    if (split.length > 2) {
+      model.setComplexDelimiter(split[1]);
+      model.setComplexDelimiter(split[2]);
+    } else if (split.length > 1) {
+      model.setComplexDelimiter(split[1]);
     }
     model.setDateFormat(
         conf.get(

diff --git a/...a/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala b/...a/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
@@ -42,9 +42,8 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
   test("test Projection PushDown for Struct - Integer type") {
     sql("DROP TABLE IF EXISTS table1")
     sql(
-      "create table table1 (roll string,person Struct<detail:int>) stored by " +
+      "create table table1 (roll string,person Struct<detail:int,name:string>) stored by " +
       "'carbondata'")
-    sql("insert into table1 values('abc',1)")
     checkAnswer(sql("select roll,person,person.detail from table1"),
       Seq(Row("abc", Row(1), 1)))
     checkAnswer(sql("select person,person.detail from table1"),