Skip to content
Permalink
Browse files
[CARBONDATA-4243] Fixed si with column meta cache on same column
Why is this PR needed?
Currently, the select query fails when table contains SI and column_meta_cache
on the same columns with to date() UDF. This is happening because pushdownfilters
is null in CarbonDataSourceScanHelper and it is causing null pointer exception.

What changes were proposed in this PR?
At place of passing null value for pushdownfilters in CarbonDataSourceScan.doCanonicalize passed Seq.empty.

This closes #4225
  • Loading branch information
nihal0107 authored and Indhumathi27 committed Oct 7, 2021
1 parent ce860d0 commit 9944936d684f241c5e81ca809507f97d1ef82f26
Showing 2 changed files with 36 additions and 2 deletions.
@@ -16,6 +16,8 @@
*/
package org.apache.carbondata.spark.testsuite.secondaryindex

import java.sql.Date

import mockit.{Mock, MockUp}
import org.apache.commons.lang3.StringUtils
import org.apache.spark.sql.{CarbonEnv, Row}
@@ -29,7 +31,6 @@ import org.apache.carbondata.core.locks.AbstractCarbonLock
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatus, SegmentStatusManager}
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.spark.exception.ProcessMetaDataException
import org.apache.carbondata.spark.testsuite.secondaryindex.TestSecondaryIndexUtils.isFilterPushedDownToSI

class TestSIWithSecondaryIndex extends QueryTest with BeforeAndAfterAll {
@@ -114,6 +115,39 @@ class TestSIWithSecondaryIndex extends QueryTest with BeforeAndAfterAll {
.contains("Table property global_sort_partitions : -1 is invalid"))
}

test ("test SI with column meta cache") {
val timeStampFormat = CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT)
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
.addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "dd-MM-yyyy")
try {
sql("drop table if exists uniqdataTable")
sql("CREATE table uniqdataTable (empno int, empname String, designation String, " +
"doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, " +
"deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp," +
" attendance int, utilization int,salary int) STORED AS carbondata " +
"TBLPROPERTIES('COLUMN_META_CACHE'='projectjoindate')")
sql("create index uniqdataindex1 on table uniqdataTable (projectjoindate) AS 'carbondata'")
sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE uniqdataTable " +
"OPTIONS('DELIMITER'=',')")

checkAnswer(sql("select max(to_date(projectjoindate)),min(to_date(projectjoindate))," +
"count(to_date(projectjoindate)) from uniqdataTable where to_date(projectjoindate)=" +
"'2007-02-17' or to_date(projectjoindate)='2011-01-29' union select " +
"max(to_date(projectjoindate)), min(to_date(projectjoindate))," +
"count(to_date(projectjoindate)) from uniqdataTable where to_date(projectjoindate)=" +
"'2007-02-17' or to_date(projectjoindate)='2011-01-29'"), Seq(Row(Date
.valueOf("2011-01-29"), Date.valueOf("2007-02-17"), 2)))
sql("drop table if exists uniqdataTable")
} finally {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, timeStampFormat)
.addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT,
CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
}
}

test("test create secondary index global sort before insert") {
sql("create table table11 (name string, id string, country string) stored as carbondata")
sql("create index table11_index on table table11(id, country) as 'carbondata' properties" +
@@ -144,7 +144,7 @@ case class CarbonDataSourceScan(
QueryPlan.normalizePredicates(dataFilters, output),
null,
null,
null,
Seq.empty,
directScanSupport,
extraRDD,
tableIdentifier,

0 comments on commit 9944936

Please sign in to comment.