From 94a4f8314068ffd4c0743907752b58879578749b Mon Sep 17 00:00:00 2001 From: ajantha-bhat Date: Mon, 29 Oct 2018 12:33:55 +0530 Subject: [PATCH] [HOTFIX-compatibility] Handle Lazy loading with inverted index for ColumnarVectorWrapperDirectWithInvertedIndex Problem: Create a store with 1.4 code with inverted index and read it with vector filling (latest master code). below exception will be thrown from AbstractCarbonColumnarVector. UnsupportedOperationException("Not allowed from here " + getClass().getName()); cause: when the lazy loading with an inverted index, getBlockDataType() was not implemented for ColumnarVectorWrapperDirectWithInvertedIndex. So, Added implementation. This closes #2870 --- .../encoding/adaptive/AdaptiveDeltaFloatingCodec.java | 10 ++++++++++ .../page/encoding/adaptive/AdaptiveFloatingCodec.java | 10 ++++++++++ .../ColumnarVectorWrapperDirectWithInvertedIndex.java | 6 ++++++ 3 files changed, 26 insertions(+) diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java index d73318d7de7..f91ede58163 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java @@ -272,6 +272,11 @@ public void decodeAndFillVector(ColumnPage columnPage, ColumnVectorInfo vectorIn int shortInt = ByteUtil.valueOf3Bytes(shortIntPage, i * 3); vector.putFloat(i, (max - shortInt) / floatFactor); } + } else if (pageDataType == DataTypes.INT) { + int[] intData = columnPage.getIntPage(); + for (int i = 0; i < pageSize; i++) { + vector.putFloat(i, (max - intData[i]) / floatFactor); + } } else { throw new RuntimeException("internal error: " + this.toString()); } @@ -298,6 +303,11 @@ public void decodeAndFillVector(ColumnPage columnPage, ColumnVectorInfo vectorIn for (int i = 0; i < pageSize; i++) { vector.putDouble(i, (max - intData[i]) / factor); } + } else if (pageDataType == DataTypes.LONG) { + long[] longData = columnPage.getLongPage(); + for (int i = 0; i < pageSize; i++) { + vector.putDouble(i, (max - longData[i]) / factor); + } } else { throw new RuntimeException("Unsupported datatype : " + pageDataType); } diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java index b300ee16b47..49696eb7c18 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java @@ -274,6 +274,11 @@ public void decodeAndFillVector(ColumnPage columnPage, ColumnVectorInfo vectorIn int shortInt = ByteUtil.valueOf3Bytes(shortIntPage, i * 3); vector.putFloat(i, (shortInt / floatFactor)); } + } else if (pageDataType == DataTypes.INT) { + int[] intData = columnPage.getIntPage(); + for (int i = 0; i < pageSize; i++) { + vector.putFloat(i, (intData[i] / floatFactor)); + } } else { throw new RuntimeException("internal error: " + this.toString()); } @@ -300,6 +305,11 @@ public void decodeAndFillVector(ColumnPage columnPage, ColumnVectorInfo vectorIn for (int i = 0; i < pageSize; i++) { vector.putDouble(i, (intData[i] / factor)); } + } else if (pageDataType == DataTypes.LONG) { + long[] longData = columnPage.getLongPage(); + for (int i = 0; i < pageSize; i++) { + vector.putDouble(i, (longData[i] / factor)); + } } else { throw new RuntimeException("Unsupported datatype : " + pageDataType); } diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/ColumnarVectorWrapperDirectWithInvertedIndex.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/ColumnarVectorWrapperDirectWithInvertedIndex.java index d95267f77b7..f190d7d76ec 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/ColumnarVectorWrapperDirectWithInvertedIndex.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/ColumnarVectorWrapperDirectWithInvertedIndex.java @@ -19,6 +19,7 @@ import java.math.BigDecimal; +import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector; /** @@ -141,4 +142,9 @@ public void putBytes(int rowId, int count, byte[] src, int srcIndex) { columnVector.putByte(invertedIndex[rowId++], src[i]); } } + + @Override + public DataType getBlockDataType() { + return columnVector.getBlockDataType(); + } }