Change of 2 complex delimiters

apache · Dec 13, 2018 · 9ae4520 · 9ae4520
1 parent 7cfa05f
commit 9ae4520
Show file tree

Hide file tree

Showing 16 changed files with 131 additions and 134 deletions.
diff --git a/conf/dataload.properties.template b/conf/dataload.properties.template
@@ -51,10 +51,10 @@ delimiter=,
 #all_dictionary_path=
 
 #complex column's level 1 delimiter
-#complex_delimiter_level_1=\\'\001'
+#complex_delimiter_level_1='\\\001'
 
 #complex column's level 2 delimiter
-#complex_delimiter_level_2=\\'\002'
+#complex_delimiter_level_2='\\\002'
 
 #timestamp type column's data format
 #dateformat=

diff --git a/docs/dml-of-carbondata.md b/docs/dml-of-carbondata.md
@@ -135,15 +135,15 @@ CarbonData DML statements are documented here,which includes:
     Split the complex type data column in a row (eg., a\001b\001c --> Array = {a,b,c}).
 
     ```
-    OPTIONS('COMPLEX_DELIMITER_LEVEL_1'='\001')
+    OPTIONS('COMPLEX_DELIMITER_LEVEL_1'='\\\001')
     ```
 
   - ##### COMPLEX_DELIMITER_LEVEL_2:
 
     Split the complex type nested data column in a row. Applies level_1 delimiter & applies level_2 based on complex data type (eg., a\002b\001c\002d --> Array> = {{a,b},{c,d}}).
 
     ```
-    OPTIONS('COMPLEX_DELIMITER_LEVEL_2'='\002')
+    OPTIONS('COMPLEX_DELIMITER_LEVEL_2'='\\\002')
     ```
 
   - ##### ALL_DICTIONARY_PATH:
@@ -212,8 +212,8 @@ CarbonData DML statements are documented here,which includes:
    'FILEHEADER'='empno,empname,designation,doj,workgroupcategory,
    workgroupcategoryname,deptno,deptname,projectcode,
    projectjoindate,projectenddate,attendance,utilization,salary',
-   'MULTILINE'='true','ESCAPECHAR'='\','COMPLEX_DELIMITER_LEVEL_1'='\001',
-   'COMPLEX_DELIMITER_LEVEL_2'='\002',
+   'MULTILINE'='true','ESCAPECHAR'='\','COMPLEX_DELIMITER_LEVEL_1'='\\\001',
+   'COMPLEX_DELIMITER_LEVEL_2'='\\\002',
    'ALL_DICTIONARY_PATH'='/opt/alldictionary/data.dictionary',
    'SINGLE_PASS'='TRUE')
    ```

diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
@@ -338,7 +338,7 @@ public static CarbonLoadModel getLoadModel(Configuration conf) throws IOExceptio
             SKIP_EMPTY_LINE,
             carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_SKIP_EMPTY_LINE)));
 
-    String complexDelim = conf.get(COMPLEX_DELIMITERS, "\001" + "," + "\002");
+    String complexDelim = conf.get(COMPLEX_DELIMITERS, "\\\001" + "," + "\\\002");
     String[] split = complexDelim.split(",");
     model.setComplexDelimiterLevel1(split[0]);
     if (split.length > 1) {

diff --git a/...g/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveComplexType.scala b/...g/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveComplexType.scala
diff --git a/...arbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingForNullValues.scala b/...arbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingForNullValues.scala
@@ -53,7 +53,7 @@ class TestAdaptiveEncodingForNullValues
     sql(
       "create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " +
       "stored by 'carbondata'")
-    sql("insert into adaptive values(1,'null$abc$null:null:null')")
+    sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')")
     checkAnswer(sql("select * from adaptive"),
       Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
   }
@@ -64,7 +64,7 @@ class TestAdaptiveEncodingForNullValues
     sql(
       "create table adaptive(roll int, student struct<id:smallint,name:string," +
       "marks:array<smallint>>) stored by 'carbondata'")
-    sql("insert into adaptive values(1,'null$abc$null:null:null')")
+    sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')")
     checkAnswer(sql("select * from adaptive"),
       Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
   }
@@ -75,7 +75,7 @@ class TestAdaptiveEncodingForNullValues
     sql(
       "create table adaptive(roll int, student struct<id:bigint,name:string," +
       "marks:array<bigint>>) stored by 'carbondata'")
-    sql("insert into adaptive values(1,'null$abc$null:null:null')")
+    sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')")
     checkAnswer(sql("select * from adaptive"),
       Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
   }
@@ -85,7 +85,7 @@ class TestAdaptiveEncodingForNullValues
     sql(
       "create table adaptive(roll int, student struct<id:double,name:string," +
       "marks:array<double>>) stored by 'carbondata'")
-    sql("insert into adaptive values(1,'null$abc$null:null:null')")
+    sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')")
     checkAnswer(sql("select * from adaptive"),
       Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
   }
@@ -96,7 +96,7 @@ class TestAdaptiveEncodingForNullValues
       "create table adaptive(roll int, student struct<id:decimal(3,2),name:string," +
       "marks:array<decimal>>) stored by " +
       "'carbondata'")
-    sql("insert into adaptive values(1,'null$abc$null:null:null')")
+    sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')")
     checkAnswer(sql("select * from adaptive"),
       Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
   }
@@ -108,7 +108,7 @@ class TestAdaptiveEncodingForNullValues
     sql(
       "create table adaptive(roll int, student struct<id:timestamp,name:string>) stored by " +
       "'carbondata'")
-    sql("insert into adaptive values(1,'null$abc')")
+    sql("insert into adaptive values(1,'null\001abc')")
     checkAnswer(sql("select * from adaptive"),
       Seq(Row(1, Row(null, "abc"))))
   }
@@ -120,7 +120,7 @@ class TestAdaptiveEncodingForNullValues
     sql(
       "create table adaptive(roll int, student struct<name:string," +
       "marks:array<timestamp>>) stored by 'carbondata'")
-    sql("insert into adaptive values(1,'abc$null:null:null')")
+    sql("insert into adaptive values(1,'abc\001null\002null\002null')")
     checkAnswer(sql("select * from adaptive"),
       Seq(Row(1, Row("abc", mutable.WrappedArray.make(Array(null, null, null))))))
   }
@@ -130,7 +130,7 @@ class TestAdaptiveEncodingForNullValues
     sql(
       "create table adaptive(roll int, student struct<name:string," +
       "marks:array<date>>) stored by 'carbondata'")
-    sql("insert into adaptive values(1,'abc$null:null:null')")
+    sql("insert into adaptive values(1,'abc\001null\002null\002null')")
     checkAnswer(sql("select * from adaptive"),
       Seq(Row(1, Row("abc", mutable.WrappedArray.make(Array(null, null, null))))))
   }
@@ -140,7 +140,7 @@ class TestAdaptiveEncodingForNullValues
     sql(
       "create table adaptive(roll int, student struct<id:long,name:string,marks:array<long>>) " +
       "stored by 'carbondata'")
-    sql("insert into adaptive values(1,'null$abc$null:null:null')")
+    sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')")
     checkAnswer(sql("select * from adaptive"),
       Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
   }
@@ -150,7 +150,7 @@ class TestAdaptiveEncodingForNullValues
     sql(
       "create table adaptive(roll int, student struct<id:short,name:string,marks:array<short>>) " +
       "stored by 'carbondata'")
-    sql("insert into adaptive values(1,'null$abc$null:null:null')")
+    sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')")
     checkAnswer(sql("select * from adaptive"),
       Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
   }
@@ -161,7 +161,7 @@ class TestAdaptiveEncodingForNullValues
       "create table adaptive(roll int, student struct<id:boolean,name:string," +
       "marks:array<boolean>>) " +
       "stored by 'carbondata'")
-    sql("insert into adaptive values(1,'null$abc$null:null:null')")
+    sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')")
     checkAnswer(sql("select * from adaptive"),
       Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
   }