Merge db02b72 into 82adc50

apache · Dec 14, 2018 · ff70ff3 · ff70ff3
2 parents 82adc50 + db02b72
commit ff70ff3
Show file tree

Hide file tree

Showing 25 changed files with 709 additions and 74 deletions.
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -597,6 +597,10 @@ public static String unquoteChar(String parseStr) {
    */
   public static String delimiterConverter(String delimiter) {
     switch (delimiter) {
+      case "\\001":
+      case "\\002":
+      case "\\003":
+      case "\\004":
       case "|":
       case "*":
       case ".":

diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
@@ -37,6 +37,7 @@
 import org.apache.carbondata.core.util.ObjectSerializationUtil;
 import org.apache.carbondata.core.util.ThreadLocalSessionInfo;
 import org.apache.carbondata.hadoop.internal.ObjectArrayWritable;
+import org.apache.carbondata.processing.loading.ComplexDelimitersEnum;
 import org.apache.carbondata.processing.loading.DataLoadExecutor;
 import org.apache.carbondata.processing.loading.TableProcessingOperations;
 import org.apache.carbondata.processing.loading.iterator.CarbonOutputIteratorWrapper;
@@ -338,11 +339,19 @@ public static CarbonLoadModel getLoadModel(Configuration conf) throws IOExceptio
             SKIP_EMPTY_LINE,
             carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_SKIP_EMPTY_LINE)));
 
-    String complexDelim = conf.get(COMPLEX_DELIMITERS, "\\\001" + "," + "\\\002");
+    String complexDelim = conf.get(COMPLEX_DELIMITERS);
+    if (null == complexDelim) {
+      complexDelim = ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_1.value() + ","
+          + ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_2.value() + ","
+          + ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_3.value();
+    }
     String[] split = complexDelim.split(",");
-    model.setComplexDelimiterLevel1(split[0]);
-    if (split.length > 1) {
-      model.setComplexDelimiterLevel2(split[1]);
+    model.setComplexDelimiter(split[0]);
+    if (split.length > 2) {
+      model.setComplexDelimiter(split[1]);
+      model.setComplexDelimiter(split[2]);
+    } else if (split.length > 1) {
+      model.setComplexDelimiter(split[1]);
     }
     model.setDateFormat(
         conf.get(