Skip to content

Commit

Permalink
[CARBONDATA-2277] Fix for filter of default values on all datatypes
Browse files Browse the repository at this point in the history 
1. Added solution to handle filter keys for the direct dictionary on default values.
2. For no dictionary columns changes code to get correct bytes value of default values.

This closes apache#2102
  • Loading branch information
@jatin9896 @geetikagupta16
jatin9896 authored and geetikagupta16 committed Apr 8, 2018
1 parent 464d6fb commit 952586b
Show file tree
Hide file tree
Showing 11 changed files with 203 additions and 43 deletions.
2 changes: 1 addition & 1 deletion core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
View file
Expand Up @@ -194,7 +194,7 @@ private static Object getDictionaryDefaultValue(byte[] defaultValue) {
* @param defaultValue
* @return
*/
private static Object getDirectDictionaryDefaultValue(DataType dataType, byte[] defaultValue) {
public static Object getDirectDictionaryDefaultValue(DataType dataType, byte[] defaultValue) {
Object directDictionaryDefaultValue = null;
if (!isDefaultValueNull(defaultValue)) {
DirectDictionaryGenerator directDictionaryGenerator =
Expand Down
75 changes: 52 additions & 23 deletions core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
View file
Expand Up @@ -52,6 +52,7 @@
import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
Expand Down Expand Up @@ -88,7 +89,7 @@
import org.apache.carbondata.core.scan.filter.executer.RestructureExcludeFilterExecutorImpl;
import org.apache.carbondata.core.scan.filter.executer.RestructureIncludeFilterExecutorImpl;
import org.apache.carbondata.core.scan.filter.executer.RowLevelFilterExecuterImpl;
import org.apache.carbondata.core.scan.filter.executer.RowLevelRangeTypeExecuterFacory;
import org.apache.carbondata.core.scan.filter.executer.RowLevelRangeTypeExecuterFactory;
import org.apache.carbondata.core.scan.filter.executer.TrueFilterExecutor;
import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
Expand Down Expand Up @@ -167,7 +168,7 @@ private static FilterExecuter createFilterExecuterTree(
case ROWLEVEL_LESSTHAN_EQUALTO:
case ROWLEVEL_GREATERTHAN_EQUALTO:
case ROWLEVEL_GREATERTHAN:
return RowLevelRangeTypeExecuterFacory
return RowLevelRangeTypeExecuterFactory
.getRowLevelRangeTypeExecuter(filterExecuterType, filterExpressionResolverTree,
segmentProperties);
case RANGE:
Expand Down Expand Up @@ -834,27 +835,9 @@ public static ColumnFilterInfo getFilterListForAllMembersRS(Expression expressio
return columnFilterInfo;
}

/**
* Below method will be used to covert the filter surrogate keys
* to mdkey
*
* @param columnFilterInfo
* @param carbonDimension
* @param segmentProperties
* @return
*/
public static byte[][] getKeyArray(ColumnFilterInfo columnFilterInfo,
CarbonDimension carbonDimension, SegmentProperties segmentProperties, boolean isExclude) {
if (!carbonDimension.hasEncoding(Encoding.DICTIONARY)) {
return columnFilterInfo.getNoDictionaryFilterValuesList()
.toArray((new byte[columnFilterInfo.getNoDictionaryFilterValuesList().size()][]));
}
KeyGenerator blockLevelKeyGenerator = segmentProperties.getDimensionKeyGenerator();
int[] dimColumnsCardinality = segmentProperties.getDimColumnsCardinality();
int[] keys = new int[blockLevelKeyGenerator.getDimCount()];
List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
Arrays.fill(keys, 0);
int keyOrdinalOfDimensionFromCurrentBlock = carbonDimension.getKeyOrdinal();
private static byte[][] getFilterValuesInBytes(ColumnFilterInfo columnFilterInfo,
boolean isExclude, KeyGenerator blockLevelKeyGenerator, int[] dimColumnsCardinality,
int[] keys, List<byte[]> filterValuesList, int keyOrdinalOfDimensionFromCurrentBlock) {
if (null != columnFilterInfo) {
int[] rangesForMaskedByte =
getRangesForMaskedByte(keyOrdinalOfDimensionFromCurrentBlock, blockLevelKeyGenerator);
Expand All @@ -881,7 +864,53 @@ public static byte[][] getKeyArray(ColumnFilterInfo columnFilterInfo,
}
}
return filterValuesList.toArray(new byte[filterValuesList.size()][]);
}

/**
* This method will be used to get the Filter key array list for blocks which do not contain
* filter column and the column Encoding is Direct Dictionary
*
* @param columnFilterInfo
* @param isExclude
* @return
*/
public static byte[][] getKeyArray(ColumnFilterInfo columnFilterInfo, boolean isExclude) {
int[] dimColumnsCardinality = new int[] { Integer.MAX_VALUE };
int[] dimensionBitLength =
CarbonUtil.getDimensionBitLength(dimColumnsCardinality, new int[] { 1 });
KeyGenerator blockLevelKeyGenerator = new MultiDimKeyVarLengthGenerator(dimensionBitLength);
int[] keys = new int[blockLevelKeyGenerator.getDimCount()];
Arrays.fill(keys, 0);
int keyOrdinalOfDimensionFromCurrentBlock = 0;
List<byte[]> filterValuesList =
new ArrayList<byte[]>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
return getFilterValuesInBytes(columnFilterInfo, isExclude, blockLevelKeyGenerator,
dimColumnsCardinality, keys, filterValuesList, keyOrdinalOfDimensionFromCurrentBlock);
}

/**
* Below method will be used to covert the filter surrogate keys
* to mdkey
*
* @param columnFilterInfo
* @param carbonDimension
* @param segmentProperties
* @return
*/
public static byte[][] getKeyArray(ColumnFilterInfo columnFilterInfo,
CarbonDimension carbonDimension, SegmentProperties segmentProperties, boolean isExclude) {
if (!carbonDimension.hasEncoding(Encoding.DICTIONARY)) {
return columnFilterInfo.getNoDictionaryFilterValuesList()
.toArray((new byte[columnFilterInfo.getNoDictionaryFilterValuesList().size()][]));
}
KeyGenerator blockLevelKeyGenerator = segmentProperties.getDimensionKeyGenerator();
int[] dimColumnsCardinality = segmentProperties.getDimColumnsCardinality();
int[] keys = new int[blockLevelKeyGenerator.getDimCount()];
List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
Arrays.fill(keys, 0);
int keyOrdinalOfDimensionFromCurrentBlock = carbonDimension.getKeyOrdinal();
return getFilterValuesInBytes(columnFilterInfo, isExclude, blockLevelKeyGenerator,
dimColumnsCardinality, keys, filterValuesList, keyOrdinalOfDimensionFromCurrentBlock);
}

/**
Expand Down
20 changes: 16 additions & 4 deletions ...c/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java
View file
Expand Up @@ -21,15 +21,17 @@
import java.util.List;

import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
import org.apache.carbondata.core.scan.filter.ColumnFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.core.util.comparator.Comparator;
import org.apache.carbondata.core.util.comparator.SerializableComparator;

Expand Down Expand Up @@ -74,7 +76,17 @@ protected boolean isDimensionDefaultValuePresentInFilterValues(
// 3 cases: is NUll, is Not Null and filter on default value of newly added column
int defaultSurrogateValueToCompare = CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY;
if (null != defaultValue) {
defaultSurrogateValueToCompare++;
if (dimension.hasEncoding(Encoding.DIRECT_DICTIONARY)) {
DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(dimension.getDataType());
if (directDictionaryGenerator != null) {
String value =
new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
defaultSurrogateValueToCompare = Integer.parseInt(value);
}
} else {
defaultSurrogateValueToCompare++;
}
}
List<Integer> filterList = null;
if (filterValues.isIncludeFilter() && !filterValues.isOptimized()) {
Expand Down Expand Up @@ -109,8 +121,8 @@ protected boolean isMeasureDefaultValuePresentInFilterValues(
Object defaultValue = null;
if (null != measure.getDefaultValue()) {
// default value for case where user gives is Null condition
defaultValue = DataTypeUtil
.getMeasureObjectFromDataType(measure.getDefaultValue(), measure.getDataType());
defaultValue = RestructureUtil
.getMeasureDefaultValue(measure.getColumnSchema(), measure.getDefaultValue());
}
List<Object> measureFilterValuesList = filterValues.getMeasuresFilterValuesList();
for (Object filterValue : measureFilterValuesList) {
Expand Down
7 changes: 5 additions & 2 deletions ...rg/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
View file
Expand Up @@ -31,6 +31,7 @@
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
Expand Down Expand Up @@ -66,7 +67,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
this.filterRangeValues = filterRangeValues;
this.msrFilterRangeValues = msrFilterRangeValues;
lastDimensionColOrdinal = segmentProperties.getLastDimensionColOrdinal();
if (isMeasurePresentInCurrentBlock[0]) {
if (!msrColEvalutorInfoList.isEmpty()) {
CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure();
comparator = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType());
}
Expand Down Expand Up @@ -100,7 +101,9 @@ private void ifDefaultValueMatchesFilter() {
if (null != defaultValue) {
for (int k = 0; k < msrFilterRangeValues.length; k++) {
int maxCompare = comparator.compare(msrFilterRangeValues[k],
DataTypeUtil.getMeasureObjectFromDataType(defaultValue, measure.getDataType()));
RestructureUtil.getMeasureDefaultValue(measure.getColumnSchema(),
measure.getDefaultValue()));

if (maxCompare < 0) {
isDefaultValuePresentInFilter = true;
break;
Expand Down
6 changes: 4 additions & 2 deletions ...e/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
View file
Expand Up @@ -31,6 +31,7 @@
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
Expand Down Expand Up @@ -66,7 +67,7 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
this.filterRangeValues = filterRangeValues;
this.msrFilterRangeValues = msrFilterRangeValues;
lastDimensionColOrdinal = segmentProperties.getLastDimensionColOrdinal();
if (isMeasurePresentInCurrentBlock[0]) {
if (!msrColEvalutorInfoList.isEmpty()) {
CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure();
comparator = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType());
}
Expand Down Expand Up @@ -100,7 +101,8 @@ private void ifDefaultValueMatchesFilter() {
if (null != defaultValue) {
for (int k = 0; k < msrFilterRangeValues.length; k++) {
int maxCompare = comparator.compare(msrFilterRangeValues[k],
DataTypeUtil.getMeasureObjectFromDataType(defaultValue, measure.getDataType()));
RestructureUtil.getMeasureDefaultValue(measure.getColumnSchema(),
measure.getDefaultValue()));
if (maxCompare <= 0) {
isDefaultValuePresentInFilter = true;
break;
Expand Down
7 changes: 5 additions & 2 deletions ...he/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
View file
Expand Up @@ -34,6 +34,7 @@
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
Expand Down Expand Up @@ -68,7 +69,7 @@ public RowLevelRangeLessThanEqualFilterExecuterImpl(
lastDimensionColOrdinal = segmentProperties.getLastDimensionColOrdinal();
this.filterRangeValues = filterRangeValues;
this.msrFilterRangeValues = msrFilterRangeValues;
if (isMeasurePresentInCurrentBlock[0]) {
if (!msrColEvalutorInfoList.isEmpty()) {
CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure();
comparator = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType());
}
Expand Down Expand Up @@ -102,7 +103,9 @@ private void ifDefaultValueMatchesFilter() {
if (null != defaultValue) {
for (int k = 0; k < msrFilterRangeValues.length; k++) {
int maxCompare = comparator.compare(msrFilterRangeValues[k],
DataTypeUtil.getMeasureObjectFromDataType(defaultValue, measure.getDataType()));
RestructureUtil.getMeasureDefaultValue(measure.getColumnSchema(),
measure.getDefaultValue()));

if (maxCompare >= 0) {
isDefaultValuePresentInFilter = true;
break;
Expand Down
11 changes: 6 additions & 5 deletions ...wLevelRangeLessThanFiterExecuterImpl.java → ...LevelRangeLessThanFilterExecuterImpl.java
View file
Expand Up @@ -34,6 +34,7 @@
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
Expand All @@ -48,7 +49,7 @@
import org.apache.carbondata.core.util.comparator.Comparator;
import org.apache.carbondata.core.util.comparator.SerializableComparator;

public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecuterImpl {
public class RowLevelRangeLessThanFilterExecuterImpl extends RowLevelFilterExecuterImpl {
private byte[][] filterRangeValues;
private Object[] msrFilterRangeValues;
private SerializableComparator comparator;
Expand All @@ -58,7 +59,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
*/
private boolean isDefaultValuePresentInFilter;
private int lastDimensionColOrdinal = 0;
public RowLevelRangeLessThanFiterExecuterImpl(
public RowLevelRangeLessThanFilterExecuterImpl(
List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues,
Expand All @@ -68,7 +69,7 @@ public RowLevelRangeLessThanFiterExecuterImpl(
this.filterRangeValues = filterRangeValues;
this.msrFilterRangeValues = msrFilterRangeValues;
lastDimensionColOrdinal = segmentProperties.getLastDimensionColOrdinal();
if (isMeasurePresentInCurrentBlock[0]) {
if (!msrColEvalutorInfoList.isEmpty()) {
CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure();
comparator = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType());
}
Expand Down Expand Up @@ -101,8 +102,8 @@ private void ifDefaultValueMatchesFilter() {
byte[] defaultValue = measure.getDefaultValue();
if (null != defaultValue) {
for (int k = 0; k < msrFilterRangeValues.length; k++) {
Object convertedValue =
DataTypeUtil.getMeasureObjectFromDataType(defaultValue, measure.getDataType());
Object convertedValue = RestructureUtil
.getMeasureDefaultValue(measure.getColumnSchema(), measure.getDefaultValue());
int maxCompare =
comparator.compare(msrFilterRangeValues[k], convertedValue);
if (maxCompare > 0) {
Expand Down
6 changes: 3 additions & 3 deletions ...uter/RowLevelRangeTypeExecuterFacory.java → ...ter/RowLevelRangeTypeExecuterFactory.java
View file
Expand Up @@ -21,9 +21,9 @@
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.core.scan.filter.resolver.RowLevelRangeFilterResolverImpl;

public class RowLevelRangeTypeExecuterFacory {
public class RowLevelRangeTypeExecuterFactory {

private RowLevelRangeTypeExecuterFacory() {
private RowLevelRangeTypeExecuterFactory() {

}

Expand All @@ -41,7 +41,7 @@ public static RowLevelFilterExecuterImpl getRowLevelRangeTypeExecuter(
switch (filterExecuterType) {

case ROWLEVEL_LESSTHAN:
return new RowLevelRangeLessThanFiterExecuterImpl(
return new RowLevelRangeLessThanFilterExecuterImpl(
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
.getDimColEvaluatorInfoList(),
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
Expand Down
2 changes: 2 additions & 0 deletions ...java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
View file
Expand Up @@ -91,6 +91,8 @@ public byte[][] getFilterRangeValues(SegmentProperties segmentProperties) {
if (null != dimensionFromCurrentBlock) {
return FilterUtil.getKeyArray(this.dimColEvaluatorInfoList.get(0).getFilterValues(),
dimensionFromCurrentBlock, segmentProperties, false);
} else {
return FilterUtil.getKeyArray(this.dimColEvaluatorInfoList.get(0).getFilterValues(), false);
}
}
return null;
Expand Down
1 change: 1 addition & 0 deletions core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
View file
Expand Up @@ -729,6 +729,7 @@ public static byte[] convertDataToBytesBasedOnDataType(String data, ColumnSchema
.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
} else {
try {
timeStampformatter.remove();
Date dateToStr = timeStampformatter.get().parse(data);
return ByteUtil.toBytes(dateToStr.getTime());
} catch (ParseException e) {
Expand Down

0 comments on commit 952586b

Please sign in to comment.