Skip to content

Commit

Permalink
[CARBONDATA-2401] Date and Timestamp options are not working in SDK
Browse files Browse the repository at this point in the history
Issue:- Date and Timestamp format is passed in options of SDK but data
load is failed even correct data is set as per format .
Cause:- Load model is getting overwritten with default .
Fix :- if user has passed the options then Load model should use from
Options otherwise from Default.

This closes #2227
  • Loading branch information
BJangir authored and kumarvishal09 committed May 10, 2018
1 parent ceb7c8d commit b2060c6
Show file tree
Hide file tree
Showing 2 changed files with 39 additions and 4 deletions.
Expand Up @@ -17,6 +17,8 @@

package org.apache.carbondata.spark.testsuite.createTable

import java.sql.Timestamp
import java.io.{File, FileFilter, IOException}
import java.io.{File, FileFilter}
import java.util

Expand All @@ -31,6 +33,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.filesystem.CarbonFile
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.util.CarbonUtil
import org.apache.carbondata.sdk.file.{CarbonWriter, CarbonWriterBuilder, Field, Schema}
import org.apache.carbondata.sdk.file.{AvroCarbonWriter, CarbonWriter, Field, Schema}
import scala.collection.JavaConverters._
import scala.collection.mutable
Expand All @@ -39,6 +42,10 @@ import org.apache.avro
import org.apache.commons.lang.CharEncoding
import tech.allegro.schema.json2avro.converter.JsonAvroConverter

import org.apache.carbondata.core.metadata.datatype.{DataTypes, StructField}
import org.apache.carbondata.sdk.file.{CarbonWriter, CarbonWriterBuilder, Field, Schema}


class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {

var writerPath = new File(this.getClass.getResource("/").getPath
Expand Down Expand Up @@ -669,6 +676,38 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
cleanTestData()
}

test("test custom format for date and timestamp in sdk") {

cleanTestData()
var options = Map("dateformat" -> "dd-MM-yyyy" ,"timestampformat" -> "dd-MM-yyyy HH:mm:ss").asJava

val fields: Array[Field] = new Array[Field](4)
fields(0) = new Field("stringField", DataTypes.STRING)
fields(1) = new Field("intField", DataTypes.INT)
fields(2) = new Field("mydate", DataTypes.DATE)
fields(3) = new Field("mytime", DataTypes.TIMESTAMP)

val builder: CarbonWriterBuilder = CarbonWriter.builder.withSchema(new Schema(fields))
.outputPath(writerPath).isTransactionalTable(false).withLoadOptions(options)

val writer: CarbonWriter = builder.buildWriterForCSVInput
writer.write(Array("babu","1","02-01-2002","02-01-2002 01:01:00"));
writer.close()

assert(new File(writerPath).exists())

sql("DROP TABLE IF EXISTS sdkOutputTable")
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY 'carbondata' LOCATION
|'$writerPath' """.stripMargin)

checkAnswer(sql("select * from sdkOutputTable"), Seq(
Row("babu", 1, java.sql.Date.valueOf("2002-01-02"),Timestamp.valueOf("2002-01-02 01:01:00.0"))))
sql("DROP TABLE sdkOutputTable")
cleanTestData()

}

test("test huge data write with one batch having bad record") {

val exception =
Expand Down
Expand Up @@ -80,10 +80,6 @@ public CarbonLoadModel build(Map<String, String> options, long UUID, String task
// we have provided 'fileheader', so it hadoopConf can be null
build(options, optionsFinal, model, null);


// set default values
model.setTimestampformat(CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
model.setDateFormat(CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT);
model.setUseOnePass(Boolean.parseBoolean(Maps.getOrDefault(options, "onepass", "false")));
model.setDictionaryServerHost(Maps.getOrDefault(options, "dicthost", null));
try {
Expand Down

0 comments on commit b2060c6

Please sign in to comment.