Skip to content

Commit

Permalink
fixed filter mismatch issue
Browse files Browse the repository at this point in the history
  • Loading branch information
@kumarvishal09
kumarvishal09 committed Oct 13, 2018
1 parent 9491529 commit 1060f4f
Show file tree
Hide file tree
Showing 4 changed files with 91 additions and 82 deletions.
17 changes: 15 additions & 2 deletions ...n/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLenAdaptiveDimColumnPage.java
View file
Expand Up @@ -16,6 +16,7 @@
*/
package org.apache.carbondata.core.datastore.chunk.impl;

import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.blocklet.PresenceMeta;
Expand Down Expand Up @@ -68,7 +69,12 @@ public FixedLenAdaptiveDimColumnPage(ColumnPage columnPage, int[] invertedIndex,
}

@Override public int fillRawData(int rowId, int offset, byte[] data) {
fillDataInternal(getDataBasedOnActualRowId(rowId), data, offset);
if ((presenceMeta.isNullBitset() && presenceMeta.getBitSet().get(rowId))
|| (!presenceMeta.isNullBitset() && !presenceMeta.getBitSet().get(rowId))) {
fillDataInternal(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY, data, offset);
} else {
fillDataInternal(getDataBasedOnActualRowId(rowId), data, offset);
}
return eachRowSize;
}

Expand Down Expand Up @@ -194,7 +200,14 @@ public int fillVector(int[] filteredRowId, ColumnVectorInfo[] vectorInfo, int ch
}

@Override public byte[] getChunkData(int rowId) {
return new byte[0];
byte[] data = new byte[eachRowSize];
if ((presenceMeta.isNullBitset() && presenceMeta.getBitSet().get(rowId))
|| (!presenceMeta.isNullBitset() && !presenceMeta.getBitSet().get(rowId))) {
fillDataInternal(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY, data, 0);
} else {
fillDataInternal(getDataBasedOnActualRowId(rowId), data, 0);
}
return data;
}

@Override public int getInvertedIndex(int rowId) {
Expand Down
12 changes: 6 additions & 6 deletions core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
View file
Expand Up @@ -545,17 +545,17 @@ protected void putNull(int rowId) {
if (dataType == DataTypes.BOOLEAN) {
putBoolean(rowId, false);
} else if (dataType == DataTypes.BYTE) {
putByte(rowId, (byte) 1);
putByte(rowId, (byte) 0);
} else if (dataType == DataTypes.SHORT) {
putShort(rowId, (short) 1);
putShort(rowId, (short) 0);
} else if (dataType == DataTypes.INT) {
putInt(rowId, 1);
putInt(rowId, 0);
} else if (dataType == DataTypes.LONG) {
putLong(rowId, 1L);
putLong(rowId, 0L);
} else if (dataType == DataTypes.DOUBLE) {
putDouble(rowId, 1.0);
putDouble(rowId, 0.0);
} else if (dataType == DataTypes.FLOAT) {
putFloat(rowId, 1.0f);
putFloat(rowId, 0.0f);
} else if (DataTypes.isDecimal(dataType)) {
putDecimal(rowId, BigDecimal.ZERO);
} else {
Expand Down
6 changes: 3 additions & 3 deletions ...ava/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
View file
Expand Up @@ -164,7 +164,7 @@ private PrimitivePageStatsCollector(DataType dataType) {

@Override
public void updateNull(int rowId) {
long value = 1;
long value = 0;
if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
update((byte) value);
} else if (dataType == DataTypes.SHORT) {
Expand All @@ -174,9 +174,9 @@ public void updateNull(int rowId) {
} else if (dataType == DataTypes.LONG || dataType == DataTypes.TIMESTAMP) {
update(value);
} else if (dataType == DataTypes.DOUBLE) {
update(1d);
update(0d);
} else if (dataType == DataTypes.FLOAT) {
update(1f);
update(0f);
} else if (DataTypes.isDecimal(dataType)) {
if (isFirst) {
maxDecimal = zeroDecimal;
Expand Down
138 changes: 67 additions & 71 deletions ...rk2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
View file
Expand Up @@ -844,77 +844,73 @@ class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll with
| (true,3,14,160,43.4,'hive','2015-7-26 12:01:06',3454.32,'2015-7-26','ff',5.5),
| (NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)
""".stripMargin)
sql(s"SELECT * FROM $bloomDMSampleTable WHERE booleanField is null").show()
// sql(s"SELECT * FROM $normalTable WHERE booleanField is null").show()
// checkAnswer(,
// )


// // check simply query
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE booleanField = true"),
// sql(s"SELECT * FROM $normalTable WHERE booleanField = true"))
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE shortField = 3"),
// sql(s"SELECT * FROM $normalTable WHERE shortField = 3"))
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE intField = 14"),
// sql(s"SELECT * FROM $normalTable WHERE intField = 14"))
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE bigintField = 100"),
// sql(s"SELECT * FROM $normalTable WHERE bigintField = 100"))
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE doubleField = 43.4"),
// sql(s"SELECT * FROM $normalTable WHERE doubleField = 43.4"))
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE stringField = 'spark'"),
// sql(s"SELECT * FROM $normalTable WHERE stringField = 'spark'"))
// checkAnswer(
// sql(s"SELECT * FROM $bloomDMSampleTable WHERE timestampField = '2015-7-26 12:01:06'"),
// sql(s"SELECT * FROM $normalTable WHERE timestampField = '2015-7-26 12:01:06'"))
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE decimalField = 23.23"),
// sql(s"SELECT * FROM $normalTable WHERE decimalField = 23.23"))
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE dateField = '2015-4-23'"),
// sql(s"SELECT * FROM $normalTable WHERE dateField = '2015-4-23'"))
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE charField = 'ccc'"),
// sql(s"SELECT * FROM $normalTable WHERE charField = 'ccc'"))
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE floatField = 2.5"),
// sql(s"SELECT * FROM $normalTable WHERE floatField = 2.5"))
//
// // check query using null
//
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE shortField is null"),
// sql(s"SELECT * FROM $normalTable WHERE shortField is null"))
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE intField is null"),
// sql(s"SELECT * FROM $normalTable WHERE intField is null"))
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE bigintField is null"),
// sql(s"SELECT * FROM $normalTable WHERE bigintField is null"))
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE doubleField is null"),
// sql(s"SELECT * FROM $normalTable WHERE doubleField is null"))
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE stringField is null"),
// sql(s"SELECT * FROM $normalTable WHERE stringField is null"))
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE timestampField is null"),
// sql(s"SELECT * FROM $normalTable WHERE timestampField is null"))
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE decimalField is null"),
// sql(s"SELECT * FROM $normalTable WHERE decimalField is null"))
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE dateField is null"),
// sql(s"SELECT * FROM $normalTable WHERE dateField is null"))
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE charField is null"),
// sql(s"SELECT * FROM $normalTable WHERE charField is null"))
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE floatField is null"),
// sql(s"SELECT * FROM $normalTable WHERE floatField is null"))
//
// // check default `NullValue` of measure does not affect result
// // Note: Test data has row contains NULL for each column but no corresponding `NullValue`,
// // so we should get 0 row if query uses the `NullValue`
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE booleanField = false"),
// sql(s"SELECT * FROM $normalTable WHERE booleanField = false"))
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE shortField = 0"),
// sql(s"SELECT * FROM $normalTable WHERE shortField = 0"))
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE intField = 0"),
// sql(s"SELECT * FROM $normalTable WHERE intField = 0"))
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE bigintField = 0"),
// sql(s"SELECT * FROM $normalTable WHERE bigintField = 0"))
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE doubleField = 0"),
// sql(s"SELECT * FROM $normalTable WHERE doubleField = 0"))
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE decimalField = 0"),
// sql(s"SELECT * FROM $normalTable WHERE decimalField = 0"))
// checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE floatField = 0"),
// sql(s"SELECT * FROM $normalTable WHERE floatField = 0"))

// check simply query
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE booleanField = true"),
sql(s"SELECT * FROM $normalTable WHERE booleanField = true"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE shortField = 3"),
sql(s"SELECT * FROM $normalTable WHERE shortField = 3"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE intField = 14"),
sql(s"SELECT * FROM $normalTable WHERE intField = 14"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE bigintField = 100"),
sql(s"SELECT * FROM $normalTable WHERE bigintField = 100"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE doubleField = 43.4"),
sql(s"SELECT * FROM $normalTable WHERE doubleField = 43.4"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE stringField = 'spark'"),
sql(s"SELECT * FROM $normalTable WHERE stringField = 'spark'"))
checkAnswer(
sql(s"SELECT * FROM $bloomDMSampleTable WHERE timestampField = '2015-7-26 12:01:06'"),
sql(s"SELECT * FROM $normalTable WHERE timestampField = '2015-7-26 12:01:06'"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE decimalField = 23.23"),
sql(s"SELECT * FROM $normalTable WHERE decimalField = 23.23"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE dateField = '2015-4-23'"),
sql(s"SELECT * FROM $normalTable WHERE dateField = '2015-4-23'"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE charField = 'ccc'"),
sql(s"SELECT * FROM $normalTable WHERE charField = 'ccc'"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE floatField = 2.5"),
sql(s"SELECT * FROM $normalTable WHERE floatField = 2.5"))

// check query using null
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE booleanField is null"),
sql(s"SELECT * FROM $normalTable WHERE booleanField is null"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE shortField is null"),
sql(s"SELECT * FROM $normalTable WHERE shortField is null"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE intField is null"),
sql(s"SELECT * FROM $normalTable WHERE intField is null"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE bigintField is null"),
sql(s"SELECT * FROM $normalTable WHERE bigintField is null"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE doubleField is null"),
sql(s"SELECT * FROM $normalTable WHERE doubleField is null"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE stringField is null"),
sql(s"SELECT * FROM $normalTable WHERE stringField is null"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE timestampField is null"),
sql(s"SELECT * FROM $normalTable WHERE timestampField is null"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE decimalField is null"),
sql(s"SELECT * FROM $normalTable WHERE decimalField is null"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE dateField is null"),
sql(s"SELECT * FROM $normalTable WHERE dateField is null"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE charField is null"),
sql(s"SELECT * FROM $normalTable WHERE charField is null"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE floatField is null"),
sql(s"SELECT * FROM $normalTable WHERE floatField is null"))

// check default `NullValue` of measure does not affect result
// Note: Test data has row contains NULL for each column but no corresponding `NullValue`,
// so we should get 0 row if query uses the `NullValue`
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE booleanField = false"),
sql(s"SELECT * FROM $normalTable WHERE booleanField = false"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE shortField = 0"),
sql(s"SELECT * FROM $normalTable WHERE shortField = 0"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE intField = 0"),
sql(s"SELECT * FROM $normalTable WHERE intField = 0"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE bigintField = 0"),
sql(s"SELECT * FROM $normalTable WHERE bigintField = 0"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE doubleField = 0"),
sql(s"SELECT * FROM $normalTable WHERE doubleField = 0"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE decimalField = 0"),
sql(s"SELECT * FROM $normalTable WHERE decimalField = 0"))
checkAnswer(sql(s"SELECT * FROM $bloomDMSampleTable WHERE floatField = 0"),
sql(s"SELECT * FROM $normalTable WHERE floatField = 0"))

}

Expand Down

0 comments on commit 1060f4f

Please sign in to comment.