Delimiters changed

apache · Dec 7, 2018 · 7cfa05f · 7cfa05f
1 parent 382ce43
commit 7cfa05f
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 11 deletions.
diff --git a/conf/dataload.properties.template b/conf/dataload.properties.template
@@ -51,10 +51,10 @@ delimiter=,
 #all_dictionary_path=
 
 #complex column's level 1 delimiter
-#complex_delimiter_level_1=\\$
+#complex_delimiter_level_1=\\'\001'
 
 #complex column's level 2 delimiter
-#complex_delimiter_level_2=\\:
+#complex_delimiter_level_2=\\'\002'
 
 #timestamp type column's data format
 #dateformat=

diff --git a/docs/dml-of-carbondata.md b/docs/dml-of-carbondata.md
@@ -132,18 +132,18 @@ CarbonData DML statements are documented here,which includes:
 
   - ##### COMPLEX_DELIMITER_LEVEL_1:
 
-    Split the complex type data column in a row (eg., a$b$c --> Array = {a,b,c}).
+    Split the complex type data column in a row (eg., a\001b\001c --> Array = {a,b,c}).
 
     ```
-    OPTIONS('COMPLEX_DELIMITER_LEVEL_1'='$') 
+    OPTIONS('COMPLEX_DELIMITER_LEVEL_1'='\001')
     ```
 
   - ##### COMPLEX_DELIMITER_LEVEL_2:
 
-    Split the complex type nested data column in a row. Applies level_1 delimiter & applies level_2 based on complex data type (eg., a:b$c:d --> Array> = {{a,b},{c,d}}).
+    Split the complex type nested data column in a row. Applies level_1 delimiter & applies level_2 based on complex data type (eg., a\002b\001c\002d --> Array> = {{a,b},{c,d}}).
 
     ```
-    OPTIONS('COMPLEX_DELIMITER_LEVEL_2'=':')
+    OPTIONS('COMPLEX_DELIMITER_LEVEL_2'='\002')
     ```
 
   - ##### ALL_DICTIONARY_PATH:
@@ -212,8 +212,8 @@ CarbonData DML statements are documented here,which includes:
    'FILEHEADER'='empno,empname,designation,doj,workgroupcategory,
    workgroupcategoryname,deptno,deptname,projectcode,
    projectjoindate,projectenddate,attendance,utilization,salary',
-   'MULTILINE'='true','ESCAPECHAR'='\','COMPLEX_DELIMITER_LEVEL_1'='$',
-   'COMPLEX_DELIMITER_LEVEL_2'=':',
+   'MULTILINE'='true','ESCAPECHAR'='\','COMPLEX_DELIMITER_LEVEL_1'='\001',
+   'COMPLEX_DELIMITER_LEVEL_2'='\002',
    'ALL_DICTIONARY_PATH'='/opt/alldictionary/data.dictionary',
    'SINGLE_PASS'='TRUE')
    ```

diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
@@ -338,7 +338,7 @@ public static CarbonLoadModel getLoadModel(Configuration conf) throws IOExceptio
             SKIP_EMPTY_LINE,
             carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_SKIP_EMPTY_LINE)));
 
-    String complexDelim = conf.get(COMPLEX_DELIMITERS, "$" + "," + ":");
+    String complexDelim = conf.get(COMPLEX_DELIMITERS, "\001" + "," + "\002");
     String[] split = complexDelim.split(",");
     model.setComplexDelimiterLevel1(split[0]);
     if (split.length > 1) {

diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
@@ -166,8 +166,8 @@ public CarbonWriterBuilder uniqueIdentifier(long timestamp) {
    * c. bad_record_path -- ""
    * d. dateformat -- "" , uses from carbon.properties file
    * e. timestampformat -- "", uses from carbon.properties file
-   * f. complex_delimiter_level_1 -- "$"
-   * g. complex_delimiter_level_2 -- ":"
+   * f. complex_delimiter_level_1 -- "\001"
+   * g. complex_delimiter_level_2 -- "\002"
    * h. quotechar -- "\""
    * i. escapechar -- "\\"
    *