[CARBONDATA-3174]varchar column trailing space issue fixed

apache · Dec 17, 2018 · 5b13c15 · 5b13c15
1 parent 8fd449c
commit 5b13c15
Show file tree

Hide file tree

Showing 3 changed files with 45 additions and 3 deletions.
diff --git a/...a/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala b/...a/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
@@ -2490,6 +2490,48 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
     FileUtils.deleteDirectory(new File(writerPath))
   }
 
+  test("check varchar with trailing space") {
+    FileUtils.deleteDirectory(new File(writerPath))
+    val fields: Array[Field] = new Array[Field](6)
+
+    fields(0) = new Field("event_id", DataTypes.STRING)
+    fields(1) = new Field("time ", DataTypes.TIMESTAMP)
+    fields(2) = new Field("subject", DataTypes.VARCHAR)
+    fields(3) = new Field("from", DataTypes.STRING)
+    fields(4) = new Field("to", DataTypes.createArrayType(DataTypes.STRING))
+    fields(5) = new Field("message ", DataTypes.VARCHAR)
+
+    var options = Map("bAd_RECords_action" -> "FORCE").asJava
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "EEE, d MMM yyyy HH:mm:ss Z")
+
+    val writer: CarbonWriter = CarbonWriter.builder
+      .outputPath(writerPath)
+      .withCsvInput(new Schema(fields)).writtenBy("TestNonTransactionalCarbonTable").build()
+
+    writer
+      .write(Array("aaa",
+        "Fri, 4 May 2001 13:51:00 -0700 (PDT)",
+        "Re",
+        "dws.cm",
+        "ser",
+        "ew"))
+    writer.close()
+
+    sql("drop table if exists test")
+    sql(
+      s"""CREATE TABLE test using carbon options('long_string_columns'='subject,message')
+         |LOCATION '$writerPath'"""
+        .stripMargin)
+    checkAnswer(sql("select * from test"), Seq(Row("aaa",
+      Timestamp.valueOf("2001-05-04 13:51:00.0"),
+      "Re",
+      "dws.cm",
+      mutable.WrappedArray.make(Array("ser")),
+      "ew")))
+    FileUtils.deleteDirectory(new File(writerPath))
+  }
+
   def generateCarbonData(builder :CarbonWriterBuilder): Unit ={
     val fields = new Array[Field](3)
     fields(0) = new Field("name", DataTypes.STRING)

diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
@@ -709,7 +709,7 @@ private Schema updateSchemaFields(Schema schema, Set<String> longStringColumns)
     Field[] fields =  schema.getFields();
     for (int i = 0; i < fields.length; i++) {
       if (fields[i] != null) {
-        fields[i].updateNameToLowerCase();
+        fields[i].updateName();
         if (longStringColumns != null) {
           /* Also update the string type to varchar */
           if (longStringColumns.contains(fields[i].getFieldName())) {

diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java
@@ -225,8 +225,8 @@ public void updateDataTypeToVarchar() {
   }
 
   /*can use to change the case of the schema */
-  public void updateNameToLowerCase() {
-    this.name = name.toLowerCase();
+  public void updateName() {
+    this.name = name.toLowerCase().trim();
   }
 
   private void initComplexTypeChildren() {