Skip to content

Commit

Permalink
[CARBONDATA-4214] inserting NULL value when timestamp value received …
Browse files Browse the repository at this point in the history
…from FROM_UNIXTIME(0)

Why is this PR needed?
Filling null in case of timestamp value is received from FROM_UNIXTIME(0) as spark original
insert rdd value[internalRow] received in this case zero. if the original column
value[internalRow] is zero then in insert flow adding NULL and giving NULL to spark.
When query happens on the same column received NULL value instead of timestamp value.
Problem code: if (internalRow.getLong(index) == 0) { internalRow.setNullAt(index) }

What changes were proposed in this PR?
Removed the null filling check for zero value case and if internalRow value is non
null/empty then only set the internalRow timestamp value.

Does this PR introduce any user interface change?
No

Is any new testcase added?
Yes

This closes #4154
  • Loading branch information
maheshrajus authored and Indhumathi27 committed Jun 23, 2021
1 parent d5cb011 commit 18665cc
Show file tree
Hide file tree
Showing 2 changed files with 31 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -781,9 +781,8 @@ object CommonLoadUtils {
internalRowOriginal
}
for (index <- timeStampIndex) {
if (internalRow.getLong(index) == 0) {
internalRow.setNullAt(index)
} else {
// timestmap value can be set other than null/empty case
if (!internalRow.isNullAt(index)) {
internalRow.setLong(
index,
internalRow.getLong(index) / TimeStampGranularityTypeValue.MILLIS_SECONDS.getValue)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

package org.apache.carbondata.spark.testsuite.allqueries

import java.sql.Timestamp

import org.apache.spark.sql.{Row, SaveMode}
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
Expand Down Expand Up @@ -73,6 +75,33 @@ class AllDataTypesTestCase extends QueryTest with BeforeAndAfterAll {
}
}

test("insert data with from_unixtime(0) and query") {
sql("drop table if exists time_carbon1")
sql("drop table if exists time_parquet")
sql("create table if not exists time_carbon1(time1 timestamp) stored as carbondata")
sql("create table if not exists time_parquet(time1 timestamp) stored as parquet")
sql("insert into time_carbon1 select from_unixtime(0)")
sql("insert into time_parquet select from_unixtime(0)")
sql("insert into time_carbon1 select from_unixtime(123456)")
sql("insert into time_parquet select from_unixtime(123456)")
checkAnswer(sql("select * from time_carbon1"),
sql("select * from time_parquet"))
sql("drop table if exists time_carbon1")
sql("drop table if exists time_parquet")
}

test("insert data with empty/null and query") {
sql("drop table if exists time_carbon2")
sql("create table if not exists time_carbon2(time1 timestamp) stored as carbondata")
sql("insert into time_carbon2 select null")
checkAnswer(sql("select count(*) from time_carbon2"), Seq(Row(1)))
checkAnswer(sql("select * from time_carbon2"), Seq(Row(null)))
sql("insert into time_carbon2 select ''")
checkAnswer(sql("select count(*) from time_carbon2"), Seq(Row(2)))
checkAnswer(sql("select * from time_carbon2"), Seq(Row(null), Row(null)))
sql("drop table if exists time_carbon2")
}

// Test-24
test("select channelsId, sum(channelsId+ 10) Total from Carbon_automation_test group by channelsId order by Total") {
checkAnswer(
Expand Down

0 comments on commit 18665cc

Please sign in to comment.