Skip to content

Commit

Permalink
Merge d635431 into 5d17ff4
Browse files Browse the repository at this point in the history
  • Loading branch information
@ravipesala
ravipesala committed Sep 27, 2018
2 parents 5d17ff4 + d635431 commit 3a1bb5a
Show file tree
Hide file tree
Showing 3 changed files with 84 additions and 35 deletions.
49 changes: 18 additions & 31 deletions core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
View file
Expand Up @@ -123,9 +123,11 @@ private DataMapExprWrapper chooseDataMap(DataMapLevel level, FilterResolverIntf
if (resolverIntf != null) {
Expression expression = resolverIntf.getFilterExpression();
List<TableDataMap> datamaps = level == DataMapLevel.CG ? cgDataMaps : fgDataMaps;
ExpressionTuple tuple = selectDataMap(expression, datamaps, resolverIntf);
if (tuple.dataMapExprWrapper != null) {
return tuple.dataMapExprWrapper;
if (datamaps.size() > 0) {
ExpressionTuple tuple = selectDataMap(expression, datamaps, resolverIntf);
if (tuple.dataMapExprWrapper != null) {
return tuple.dataMapExprWrapper;
}
}
}
return null;
Expand Down Expand Up @@ -177,12 +179,8 @@ private ExpressionTuple selectDataMap(Expression expression, List<TableDataMap>
// If both left and right has datamap then we can either merge both datamaps to single
// datamap if possible. Otherwise apply AND expression.
if (left.dataMapExprWrapper != null && right.dataMapExprWrapper != null) {
filterExpressionTypes.add(
left.dataMapExprWrapper.getFilterResolverIntf().getFilterExpression()
.getFilterExpressionType());
filterExpressionTypes.add(
right.dataMapExprWrapper.getFilterResolverIntf().getFilterExpression()
.getFilterExpressionType());
filterExpressionTypes.addAll(left.filterExpressionTypes);
filterExpressionTypes.addAll(right.filterExpressionTypes);
List<ColumnExpression> columnExpressions = new ArrayList<>();
columnExpressions.addAll(left.columnExpressions);
columnExpressions.addAll(right.columnExpressions);
Expand Down Expand Up @@ -218,33 +216,17 @@ private ExpressionTuple selectDataMap(Expression expression, List<TableDataMap>
filterResolverIntf.getLeft());
ExpressionTuple right = selectDataMap(orExpression.getRight(), allDataMap,
filterResolverIntf.getRight());
Set<ExpressionType> filterExpressionTypes = new HashSet<>();
// If both left and right has datamap then we can either merge both datamaps to single
// datamap if possible. Otherwise apply OR expression.
if (left.dataMapExprWrapper != null && right.dataMapExprWrapper != null) {
filterExpressionTypes.add(
left.dataMapExprWrapper.getFilterResolverIntf().getFilterExpression()
.getFilterExpressionType());
filterExpressionTypes.add(
right.dataMapExprWrapper.getFilterResolverIntf().getFilterExpression()
.getFilterExpressionType());
List<ColumnExpression> columnExpressions = new ArrayList<>();
columnExpressions.addAll(left.columnExpressions);
columnExpressions.addAll(right.columnExpressions);
TableDataMap dataMap =
chooseDataMap(allDataMap, columnExpressions, filterExpressionTypes);
if (dataMap != null) {
ExpressionTuple tuple = new ExpressionTuple();
tuple.columnExpressions = columnExpressions;
tuple.dataMapExprWrapper = new DataMapExprWrapperImpl(dataMap, filterResolverIntf);
return tuple;
} else {
ExpressionTuple tuple = new ExpressionTuple();
tuple.columnExpressions = columnExpressions;
tuple.dataMapExprWrapper = new OrDataMapExprWrapper(left.dataMapExprWrapper,
right.dataMapExprWrapper, filterResolverIntf);
return tuple;
}
ExpressionTuple tuple = new ExpressionTuple();
tuple.columnExpressions = columnExpressions;
tuple.dataMapExprWrapper = new OrDataMapExprWrapper(left.dataMapExprWrapper,
right.dataMapExprWrapper, filterResolverIntf);
return tuple;
} else {
left.dataMapExprWrapper = null;
return left;
Expand All @@ -260,6 +242,7 @@ private ExpressionTuple selectDataMap(Expression expression, List<TableDataMap>
chooseDataMap(allDataMap, tuple.columnExpressions, filterExpressionTypes);
if (dataMap != null) {
tuple.dataMapExprWrapper = new DataMapExprWrapperImpl(dataMap, filterResolverIntf);
tuple.filterExpressionTypes.addAll(filterExpressionTypes);
}
return tuple;
}
Expand All @@ -282,7 +265,9 @@ private void extractColumnExpression(Expression expression,
List<Expression> children = expression.getChildren();
if (children != null && children.size() > 0) {
for (Expression exp : children) {
extractColumnExpression(exp, columnExpressions);
if (exp != null && exp.getFilterExpressionType() != ExpressionType.UNKNOWN) {
extractColumnExpression(exp, columnExpressions);
}
}
}
}
Expand Down Expand Up @@ -332,6 +317,8 @@ private static class ExpressionTuple {

List<ColumnExpression> columnExpressions = new ArrayList<>();

Set<ExpressionType> filterExpressionTypes = new HashSet<>();

}

private static class DataMapTuple implements Comparable<DataMapTuple> {
Expand Down
8 changes: 5 additions & 3 deletions datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
View file
Expand Up @@ -59,6 +59,7 @@
import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression;
import org.apache.carbondata.core.scan.expression.conditional.InExpression;
import org.apache.carbondata.core.scan.expression.conditional.ListExpression;
import org.apache.carbondata.core.scan.expression.logical.AndExpression;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
Expand Down Expand Up @@ -264,11 +265,12 @@ private List<BloomQueryModel> createQueryModel(Expression expression)
LOGGER.warn(errorMsg);
throw new RuntimeException(errorMsg);
}
} else if (expression instanceof AndExpression) {
queryModels.addAll(createQueryModel(((AndExpression) expression).getLeft()));
queryModels.addAll(createQueryModel(((AndExpression) expression).getRight()));
return queryModels;
}

for (Expression child : expression.getChildren()) {
queryModels.addAll(createQueryModel(child));
}
return queryModels;
}

Expand Down
62 changes: 61 additions & 1 deletion ...rk2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
View file
Expand Up @@ -22,7 +22,7 @@ import java.util.UUID

import scala.util.Random

import org.apache.spark.sql.{CarbonSession, DataFrame}
import org.apache.spark.sql.{CarbonSession, DataFrame, Row}
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}

Expand Down Expand Up @@ -923,6 +923,66 @@ class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll with
CarbonCommonConstants.CARBON_DATE_FORMAT, originDateFormat)
}

test("test bloom datamap on multiple columns") {
sql("drop table if exists store")
sql(
s"""
|CREATE TABLE IF NOT EXISTS store(
| market_code STRING,
| device_code STRING,
| country_code STRING,
| category_id INTEGER,
| product_id string,
| date date,
| est_free_app_download LONG,
| est_paid_app_download LONG,
| est_revenue LONG
| )
| STORED BY 'carbondata'
| TBLPROPERTIES(
| 'SORT_COLUMNS'='market_code, device_code, country_code, category_id, date,product_id',
| 'NO_INVERTED_INDEX'='est_free_app_download, est_paid_app_download,est_revenue',
| 'DICTIONARY_INCLUDE' = 'market_code, device_code, country_code,category_id, product_id',
| 'SORT_SCOPE'='GLOBAL_SORT',
| 'CACHE_LEVEL'='BLOCKLET', 'TABLE_BLOCKSIZE'='256',
| 'GLOBAL_SORT_PARTITIONS'='2'
| )""".stripMargin)

sql(s"""insert into store values('a', 'ios-phone', 'EE', 100021, 590416158, '2016-09-01', 100, 200, 300)""")
sql(s"""insert into store values('b', 'ios-phone', 'EE', 100021, 590437560, '2016-09-03', 100, 200, 300)""")
sql(s"""insert into store values('a', 'ios-phone', 'EF', 100022, 590416159, '2016-09-04', 100, 200, 300)""")

sql(
s"""
|CREATE DATAMAP IF NOT EXISTS bloomfilter_all_dimensions ON TABLE store
| USING 'bloomfilter'
| DMPROPERTIES (
| 'INDEX_COLUMNS'='market_code, device_code, country_code, category_id, date,product_id',
| 'BLOOM_SIZE'='640000',
| 'BLOOM_FPP'='0.000001',
| 'BLOOM_COMPRESS'='true'
| )
""".stripMargin).show()

checkAnswer(sql(
s"""SELECT market_code, device_code, country_code,
|category_id, sum(est_free_app_download) FROM store WHERE date
|BETWEEN '2016-09-01' AND '2016-09-03' AND device_code='ios-phone'
|AND country_code='EE' AND category_id=100021 AND product_id IN (590416158, 590437560)
|GROUP BY date, market_code, device_code, country_code, category_id""".stripMargin),
Seq(Row("a", "ios-phone", "EE", 100021, 100), Row("b", "ios-phone", "EE", 100021, 100)))

assert(sql(
s"""SELECT market_code, device_code, country_code,
|category_id, sum(est_free_app_download) FROM store WHERE (device_code='ios-phone'
|AND country_code='EF') or (category_id=100021 AND product_id IN (590416158, 590437560))
|GROUP BY date, market_code, device_code, country_code, category_id""".stripMargin).collect().length == 3)

checkAnswer(sql("select device_code from store where product_id=590416158"), Seq(Row("ios-phone")))

sql("drop table if exists store")
}

override protected def afterAll(): Unit = {
// in case of search mode test case failed, stop search mode again
if (carbonSession.isSearchModeEnabled) {
Expand Down

0 comments on commit 3a1bb5a

Please sign in to comment.