[CARBONDATA-2683][32K] fix data convertion problem for Varchar

Spark uses org.apache.spark.unsafe.types.UTF8String for string datatype internally. In carbon, varchar datatype should do the same convertion as string datatype. Or it may throw exception This closes #2438
apache · Jul 6, 2018 · 3e88858 · 3e88858
1 parent 5159abf
commit 3e88858
Showing 1 changed file with 9 additions and 11 deletions.
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -323,8 +323,6 @@ public static byte[] getBytesBasedOnDataTypeForNoDictionaryColumn(String dimensi
       DataType actualDataType, String dateFormat) {
     if (actualDataType == DataTypes.BOOLEAN) {
       return ByteUtil.toBytes(BooleanConvert.parseBoolean(dimensionValue));
-    } else if (actualDataType == DataTypes.STRING) {
-      return ByteUtil.toBytes(dimensionValue);
     } else if (actualDataType == DataTypes.SHORT) {
       return ByteUtil.toBytes(Short.parseShort(dimensionValue));
     } else if (actualDataType == DataTypes.INT) {
@@ -351,6 +349,7 @@ public static byte[] getBytesBasedOnDataTypeForNoDictionaryColumn(String dimensi
         throw new NumberFormatException(e.getMessage());
       }
     } else {
+      // Default action for String/Varchar
       return ByteUtil.toBytes(dimensionValue);
     }
   }
@@ -359,8 +358,6 @@ public static Object getDataDataTypeForNoDictionaryColumn(String dimensionValue,
       DataType actualDataType, String dateFormat) {
     if (actualDataType == DataTypes.BOOLEAN) {
       return BooleanConvert.parseBoolean(dimensionValue);
-    } else if (actualDataType == DataTypes.STRING) {
-      return converter.convertFromStringToUTF8String(dimensionValue);
     } else if (actualDataType == DataTypes.SHORT) {
       return Short.parseShort(dimensionValue);
     } else if (actualDataType == DataTypes.INT) {
@@ -387,6 +384,7 @@ public static Object getDataDataTypeForNoDictionaryColumn(String dimensionValue,
         throw new NumberFormatException(e.getMessage());
       }
     } else {
+      // Default action for String/Varchar
       return converter.convertFromStringToUTF8String(dimensionValue);
     }
   }
@@ -402,8 +400,6 @@ public static byte[] getBytesDataDataTypeForNoDictionaryColumn(Object dimensionV
     }
     if (actualDataType == DataTypes.BOOLEAN) {
       return ByteUtil.toBytes((Boolean) dimensionValue);
-    } else if (actualDataType == DataTypes.STRING) {
-      return ByteUtil.toBytes(dimensionValue.toString());
     } else if (actualDataType == DataTypes.SHORT) {
       return ByteUtil.toBytes((Short) dimensionValue);
     } else if (actualDataType == DataTypes.INT) {
@@ -413,6 +409,7 @@ public static byte[] getBytesDataDataTypeForNoDictionaryColumn(Object dimensionV
     } else if (actualDataType == DataTypes.TIMESTAMP) {
       return ByteUtil.toBytes((Long)dimensionValue);
     } else {
+      // Default action for String/Varchar
       return ByteUtil.toBytes(dimensionValue.toString());
     }
   }
@@ -423,7 +420,9 @@ public static byte[] getBytesDataDataTypeForNoDictionaryColumn(Object dimensionV
    * @return
    */
   public static boolean isFixedSizeDataType(DataType dataType) {
-    if (dataType == DataTypes.STRING || DataTypes.isDecimal(dataType)) {
+    if (dataType == DataTypes.STRING ||
+        dataType == DataTypes.VARCHAR ||
+        DataTypes.isDecimal(dataType)) {
       return false;
     } else {
       return true;
@@ -447,8 +446,6 @@ public static Object getDataBasedOnDataTypeForNoDictionaryColumn(byte[] dataInBy
     try {
       if (actualDataType == DataTypes.BOOLEAN) {
         return ByteUtil.toBoolean(dataInBytes);
-      } else if (actualDataType == DataTypes.STRING) {
-        return getDataTypeConverter().convertFromByteToUTF8String(dataInBytes);
       } else if (actualDataType == DataTypes.SHORT) {
         // for non string type no dictionary column empty byte array is empty value
         // so no need to parse
@@ -480,9 +477,10 @@ public static Object getDataBasedOnDataTypeForNoDictionaryColumn(byte[] dataInBy
         if (isEmptyByteArray(dataInBytes)) {
           return null;
         }
-        return converter.convertFromBigDecimalToDecimal(byteToBigDecimal(dataInBytes));
+        return getDataTypeConverter().convertFromBigDecimalToDecimal(byteToBigDecimal(dataInBytes));
       } else {
-        return ByteUtil.toString(dataInBytes, 0, dataInBytes.length);
+        // Default action for String/Varchar
+        return getDataTypeConverter().convertFromByteToUTF8String(dataInBytes);
       }
     } catch (Throwable ex) {
       String data = new String(dataInBytes, CarbonCommonConstants.DEFAULT_CHARSET_CLASS);