Merge 0a84d52 into 6026cb5

conf/dataload.properties.template

@@ -51,10 +51,10 @@ delimiter=,
 #all_dictionary_path=
 
 #complex column's level 1 delimiter
-#complex_delimiter_level_1=\\$
+#complex_delimiter_level_1='\\\001'
 
 #complex column's level 2 delimiter
-#complex_delimiter_level_2=\\:
+#complex_delimiter_level_2='\\\002'
 
 #timestamp type column's data format
 #dateformat=

core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java

@@ -597,6 +597,10 @@ public static String unquoteChar(String parseStr) {
    */
   public static String delimiterConverter(String delimiter) {
     switch (delimiter) {
+      case "\\001":
+      case "\\002":
+      case "\\003":
+      case "\\004":
       case "|":
       case "*":
       case ".":

docs/dml-of-carbondata.md

@@ -132,18 +132,18 @@ CarbonData DML statements are documented here,which includes:
 
   - ##### COMPLEX_DELIMITER_LEVEL_1:
 
-    Split the complex type data column in a row (eg., a$b$c --> Array = {a,b,c}).
+    Split the complex type data column in a row (eg., a\001b\001c --> Array = {a,b,c}).
 
     ```
-    OPTIONS('COMPLEX_DELIMITER_LEVEL_1'='$') 
+    OPTIONS('COMPLEX_DELIMITER_LEVEL_1'='\\\001')
     ```
 
   - ##### COMPLEX_DELIMITER_LEVEL_2:
 
-    Split the complex type nested data column in a row. Applies level_1 delimiter & applies level_2 based on complex data type (eg., a:b$c:d --> Array> = {{a,b},{c,d}}).
+    Split the complex type nested data column in a row. Applies level_1 delimiter & applies level_2 based on complex data type (eg., a\002b\001c\002d --> Array> = {{a,b},{c,d}}).
 
     ```
-    OPTIONS('COMPLEX_DELIMITER_LEVEL_2'=':')
+    OPTIONS('COMPLEX_DELIMITER_LEVEL_2'='\\\002')
     ```
 
   - ##### ALL_DICTIONARY_PATH:
@@ -212,8 +212,8 @@ CarbonData DML statements are documented here,which includes:
    'FILEHEADER'='empno,empname,designation,doj,workgroupcategory,
    workgroupcategoryname,deptno,deptname,projectcode,
    projectjoindate,projectenddate,attendance,utilization,salary',
-   'MULTILINE'='true','ESCAPECHAR'='\','COMPLEX_DELIMITER_LEVEL_1'='$',
-   'COMPLEX_DELIMITER_LEVEL_2'=':',
+   'MULTILINE'='true','ESCAPECHAR'='\','COMPLEX_DELIMITER_LEVEL_1'='\\\001',
+   'COMPLEX_DELIMITER_LEVEL_2'='\\\002',
    'ALL_DICTIONARY_PATH'='/opt/alldictionary/data.dictionary',
    'SINGLE_PASS'='TRUE')
    ```

hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java

@@ -37,6 +37,7 @@
 import org.apache.carbondata.core.util.ObjectSerializationUtil;
 import org.apache.carbondata.core.util.ThreadLocalSessionInfo;
 import org.apache.carbondata.hadoop.internal.ObjectArrayWritable;
+import org.apache.carbondata.processing.loading.ComplexDelimitersEnum;
 import org.apache.carbondata.processing.loading.DataLoadExecutor;
 import org.apache.carbondata.processing.loading.TableProcessingOperations;
 import org.apache.carbondata.processing.loading.iterator.CarbonOutputIteratorWrapper;
@@ -338,11 +339,19 @@ public static CarbonLoadModel getLoadModel(Configuration conf) throws IOExceptio
             SKIP_EMPTY_LINE,
             carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_SKIP_EMPTY_LINE)));
 
-    String complexDelim = conf.get(COMPLEX_DELIMITERS, "$" + "," + ":");
+    String complexDelim = conf.get(COMPLEX_DELIMITERS);
+    if (null == complexDelim) {
+      complexDelim = ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_1.value() + ","
+          + ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_2.value() + ","
+          + ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_3.value();
+    }
     String[] split = complexDelim.split(",");
-    model.setComplexDelimiterLevel1(split[0]);
-    if (split.length > 1) {
-      model.setComplexDelimiterLevel2(split[1]);
+    model.setComplexDelimiter(split[0]);
+    if (split.length > 2) {
+      model.setComplexDelimiter(split[1]);
+      model.setComplexDelimiter(split[2]);
+    } else if (split.length > 1) {
+      model.setComplexDelimiter(split[1]);
     }
     model.setDateFormat(
         conf.get(

integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingForNullValues.scala

@@ -53,7 +53,7 @@ class TestAdaptiveEncodingForNullValues
     sql(
       "create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " +
       "stored by 'carbondata'")
-    sql("insert into adaptive values(1,'null$abc$null:null:null')")
+    sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')")
     checkAnswer(sql("select * from adaptive"),
       Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
   }
@@ -64,7 +64,7 @@ class TestAdaptiveEncodingForNullValues
     sql(
       "create table adaptive(roll int, student struct<id:smallint,name:string," +
       "marks:array<smallint>>) stored by 'carbondata'")
-    sql("insert into adaptive values(1,'null$abc$null:null:null')")
+    sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')")
     checkAnswer(sql("select * from adaptive"),
       Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
   }
@@ -75,7 +75,7 @@ class TestAdaptiveEncodingForNullValues
     sql(
       "create table adaptive(roll int, student struct<id:bigint,name:string," +
       "marks:array<bigint>>) stored by 'carbondata'")
-    sql("insert into adaptive values(1,'null$abc$null:null:null')")
+    sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')")
     checkAnswer(sql("select * from adaptive"),
       Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
   }
@@ -85,7 +85,7 @@ class TestAdaptiveEncodingForNullValues
     sql(
       "create table adaptive(roll int, student struct<id:double,name:string," +
       "marks:array<double>>) stored by 'carbondata'")
-    sql("insert into adaptive values(1,'null$abc$null:null:null')")
+    sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')")
     checkAnswer(sql("select * from adaptive"),
       Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
   }
@@ -96,7 +96,7 @@ class TestAdaptiveEncodingForNullValues
       "create table adaptive(roll int, student struct<id:decimal(3,2),name:string," +
       "marks:array<decimal>>) stored by " +
       "'carbondata'")
-    sql("insert into adaptive values(1,'null$abc$null:null:null')")
+    sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')")
     checkAnswer(sql("select * from adaptive"),
       Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
   }
@@ -108,7 +108,7 @@ class TestAdaptiveEncodingForNullValues
     sql(
       "create table adaptive(roll int, student struct<id:timestamp,name:string>) stored by " +
       "'carbondata'")
-    sql("insert into adaptive values(1,'null$abc')")
+    sql("insert into adaptive values(1,'null\001abc')")
     checkAnswer(sql("select * from adaptive"),
       Seq(Row(1, Row(null, "abc"))))
   }
@@ -120,7 +120,7 @@ class TestAdaptiveEncodingForNullValues
     sql(
       "create table adaptive(roll int, student struct<name:string," +
       "marks:array<timestamp>>) stored by 'carbondata'")
-    sql("insert into adaptive values(1,'abc$null:null:null')")
+    sql("insert into adaptive values(1,'abc\001null\002null\002null')")
     checkAnswer(sql("select * from adaptive"),
       Seq(Row(1, Row("abc", mutable.WrappedArray.make(Array(null, null, null))))))
   }
@@ -130,7 +130,7 @@ class TestAdaptiveEncodingForNullValues
     sql(
       "create table adaptive(roll int, student struct<name:string," +
       "marks:array<date>>) stored by 'carbondata'")
-    sql("insert into adaptive values(1,'abc$null:null:null')")
+    sql("insert into adaptive values(1,'abc\001null\002null\002null')")
     checkAnswer(sql("select * from adaptive"),
       Seq(Row(1, Row("abc", mutable.WrappedArray.make(Array(null, null, null))))))
   }
@@ -140,7 +140,7 @@ class TestAdaptiveEncodingForNullValues
     sql(
       "create table adaptive(roll int, student struct<id:long,name:string,marks:array<long>>) " +
       "stored by 'carbondata'")
-    sql("insert into adaptive values(1,'null$abc$null:null:null')")
+    sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')")
     checkAnswer(sql("select * from adaptive"),
       Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
   }
@@ -150,7 +150,7 @@ class TestAdaptiveEncodingForNullValues
     sql(
       "create table adaptive(roll int, student struct<id:short,name:string,marks:array<short>>) " +
       "stored by 'carbondata'")
-    sql("insert into adaptive values(1,'null$abc$null:null:null')")
+    sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')")
     checkAnswer(sql("select * from adaptive"),
       Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
   }
@@ -161,7 +161,7 @@ class TestAdaptiveEncodingForNullValues
       "create table adaptive(roll int, student struct<id:boolean,name:string," +
       "marks:array<boolean>>) " +
       "stored by 'carbondata'")
-    sql("insert into adaptive values(1,'null$abc$null:null:null')")
+    sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')")
     checkAnswer(sql("select * from adaptive"),
       Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
   }