[CARBONDATA-3174]varchar column trailing space issue fixed

apache · Dec 17, 2018 · 45520c1 · 45520c1
1 parent 0e467eb
commit 45520c1
Show file tree

Hide file tree

Showing 2 changed files with 53 additions and 5 deletions.
diff --git a/...a/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala b/...a/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
@@ -2490,6 +2490,54 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
     FileUtils.deleteDirectory(new File(writerPath))
   }
 
+  test("check varchar with trailing space") {
+    FileUtils.deleteDirectory(new File(writerPath))
+    val fields: Array[Field] = new Array[Field](8)
+
+    fields(0) = new Field("Event_ID", DataTypes.STRING)
+    fields(1) = new Field("Event_Time", DataTypes.TIMESTAMP)
+    fields(2) = new Field("subject", DataTypes.VARCHAR)
+    fields(3) = new Field("From_Email", DataTypes.STRING)
+    fields(4) = new Field("To_Email", DataTypes.createArrayType(DataTypes.STRING))
+    fields(5) = new Field("CC_Email", DataTypes.createArrayType(DataTypes.STRING))
+    fields(6) = new Field("BCC_Email", DataTypes.createArrayType(DataTypes.STRING))
+    fields(7) = new Field("messagebody ", DataTypes.VARCHAR)
+
+    var options = Map("bAd_RECords_action" -> "FORCE").asJava
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "EEE, d MMM yyyy HH:mm:ss Z")
+
+    val writer: CarbonWriter = CarbonWriter.builder
+      .outputPath(writerPath)
+      .withCsvInput(new Schema(fields)).writtenBy("TestNonTransactionalCarbonTable").build()
+
+    writer
+      .write(Array("aaa",
+        "Fri, 4 May 2001 13:51:00 -0700 (PDT)",
+        "Re",
+        "cdn@ws.com",
+        "sd#er",
+        "sd",
+        "sds",
+        "ew"))
+    writer.close()
+
+    sql("drop table if exists test")
+    sql(
+      s"""CREATE TABLE test using carbon options('long_string_columns'='subject,messagebody')
+         |LOCATION '$writerPath'"""
+        .stripMargin)
+    checkAnswer(sql("select * from test"), Seq(Row("aaa",
+      Timestamp.valueOf("2001-05-04 13:51:00.0"),
+      "Re",
+      "cdn@ws.com",
+      mutable.WrappedArray.make(Array("sd#er")),
+      mutable.WrappedArray.make(Array("sd")),
+      mutable.WrappedArray.make(Array("sds")),
+      "ew")))
+    FileUtils.deleteDirectory(new File(writerPath))
+  }
+
   def generateCarbonData(builder :CarbonWriterBuilder): Unit ={
     val fields = new Array[Field](3)
     fields(0) = new Field("name", DataTypes.STRING)

diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java
@@ -55,7 +55,7 @@ public class Field {
    * @param type datatype of field, specified in strings.
    */
   public Field(String name, String type) {
-    this.name = name;
+    this.name = name.toLowerCase().trim();
     if (type.equalsIgnoreCase("string")) {
       this.type = DataTypes.STRING;
     } else if (type.equalsIgnoreCase("varchar")) {
@@ -90,7 +90,7 @@ public Field(String name, String type) {
   }
 
   public Field(String name, String type, List<StructField> fields) {
-    this.name = name;
+    this.name = name.toLowerCase().trim();
     this.children = fields;
     if (type.equalsIgnoreCase("string")) {
       this.type = DataTypes.STRING;
@@ -126,13 +126,13 @@ public Field(String name, String type, List<StructField> fields) {
 
 
   public Field(String name, DataType type, List<StructField> fields) {
-    this.name = name;
+    this.name = name.toLowerCase().trim();
     this.type = type;
     this.children = fields;
   }
 
   public Field(String name, DataType type) {
-    this.name = name;
+    this.name = name.toLowerCase().trim();
     this.type = type;
     initComplexTypeChildren();
   }
@@ -143,7 +143,7 @@ public Field(String name, DataType type) {
    * @param columnSchema ColumnSchema, Store the information about the column meta data
    */
   public Field(ColumnSchema columnSchema) {
-    this.name = columnSchema.getColumnName();
+    this.name = columnSchema.getColumnName().toLowerCase().trim();
     this.type = columnSchema.getDataType();
     children = new LinkedList<>();
     schemaOrdinal = columnSchema.getSchemaOrdinal();