diff --git a/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala b/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala index fa37548ca77..8303c54346f 100644 --- a/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala +++ b/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala @@ -1760,6 +1760,16 @@ class SparkCarbonDataSourceTest extends FunSuite with BeforeAndAfterAll { spark.sql("drop table if exists fileformat_drop_hive") } + test("test complexdatype for date and timestamp datatype") { + spark.sql("drop table if exists fileformat_date") + spark.sql("drop table if exists fileformat_date_hive") + spark.sql("create table fileformat_date_hive(name string, age int, dob array, joinTime array) using parquet") + spark.sql("create table fileformat_date(name string, age int, dob array, joinTime array) using carbon") + spark.sql("insert into fileformat_date_hive select 'joey', 32, array('1994-04-06','1887-05-06'), array('1994-04-06 00:00:05','1887-05-06 00:00:08')") + spark.sql("insert into fileformat_date select 'joey', 32, array('1994-04-06','1887-05-06'), array('1994-04-06 00:00:05','1887-05-06 00:00:08')") + checkAnswer(spark.sql("select * from fileformat_date_hive"), spark.sql("select * from fileformat_date")) + } + test("validate the columns not present in schema") { spark.sql("drop table if exists validate") spark.sql("create table validate (name string, age int, address string) using carbon options('inverted_index'='abc')") diff --git a/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java b/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java index cfbaa116524..18dc89d9f23 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java +++ b/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java @@ -344,7 +344,7 @@ public int getSurrogateIndex() { byte[] value = null; if (isDirectDictionary) { int surrogateKey; - if (!(input instanceof Long)) { + if (!(input instanceof Long) && !(input instanceof Integer)) { SimpleDateFormat parser = new SimpleDateFormat(getDateFormat(carbonDimension)); parser.parse(parsedValue); } @@ -353,6 +353,11 @@ public int getSurrogateIndex() { // using dictionaryGenerator. if (dictionaryGenerator instanceof DirectDictionary && input instanceof Long) { surrogateKey = ((DirectDictionary) dictionaryGenerator).generateKey((long) input); + } else if (dictionaryGenerator instanceof DirectDictionary + && input instanceof Integer) { + // In case of file format, for complex type date or time type, input data comes as a + // Integer object, so just assign the surrogate key with the input object value + surrogateKey = (int) input; } else { surrogateKey = dictionaryGenerator.getOrGenerateKey(parsedValue); }