Added validation for supported format version and Encoding type to th…

…row proper exception to the user while reading a file
apache · Nov 2, 2018 · 838c397 · 838c397
1 parent 9de9466
commit 838c397
Show file tree

Hide file tree

Showing 9 changed files with 56 additions and 80 deletions.
diff --git a/.../main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java b/.../main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
@@ -74,14 +74,15 @@ public DimensionColumnChunkReader getDimensionColumnChunkReader(ColumnarFormatVe
         return new CompressedDimensionChunkFileBasedReaderV2(blockletInfo, eachColumnValueSize,
             filePath);
       case V3:
-      default:
         if (readPagebyPage) {
           return new CompressedDimChunkFileBasedPageLevelReaderV3(blockletInfo, eachColumnValueSize,
               filePath);
         } else {
           return new CompressedDimensionChunkFileBasedReaderV3(blockletInfo, eachColumnValueSize,
               filePath);
         }
+      default:
+        throw new UnsupportedOperationException("Unsupported columnar format version: " + version);
     }
   }
 
@@ -101,14 +102,13 @@ public MeasureColumnChunkReader getMeasureColumnChunkReader(ColumnarFormatVersio
       case V2:
         return new CompressedMeasureChunkFileBasedReaderV2(blockletInfo, filePath);
       case V3:
-      default:
         if (readPagebyPage) {
           return new CompressedMsrChunkFileBasedPageLevelReaderV3(blockletInfo, filePath);
         } else {
           return new CompressedMeasureChunkFileBasedReaderV3(blockletInfo, filePath);
         }
-
+      default:
+        throw new UnsupportedOperationException("Unsupported columnar format version: " + version);
     }
-
   }
 }
diff --git a/...a/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java b/...a/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
@@ -264,6 +264,7 @@ protected DimensionColumnPage decodeDimension(DimensionRawColumnChunk rawColumnP
       ByteBuffer pageData, DataChunk2 pageMetadata, int offset, ColumnVectorInfo vectorInfo)
       throws IOException, MemoryException {
     List<Encoding> encodings = pageMetadata.getEncoders();
+    org.apache.carbondata.core.metadata.encoder.Encoding.validateEncodingTypes(encodings);
     if (CarbonUtil.isEncodedWithMeta(encodings)) {
       int[] invertedIndexes = new int[0];
       int[] invertedIndexesReverse = new int[0];

diff --git a/...ndata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java b/...ndata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
@@ -233,6 +233,7 @@ private ColumnPage decodeColumnPage(
   protected ColumnPage decodeMeasure(DataChunk2 pageMetadata, ByteBuffer pageData, int offset,
       ColumnVectorInfo vectorInfo, BitSet nullBitSet) throws MemoryException, IOException {
     List<Encoding> encodings = pageMetadata.getEncoders();
+    org.apache.carbondata.core.metadata.encoder.Encoding.validateEncodingTypes(encodings);
     List<ByteBuffer> encoderMetas = pageMetadata.getEncoder_meta();
     String compressorName =
         CarbonMetadataUtil.getCompressorNameFromChunkMeta(pageMetadata.getChunk_meta());

diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
@@ -192,7 +192,7 @@ private BlockletDataMapIndexWrapper get(TableBlockIndexUniqueIdentifierWrapper i
           dataMap.clear();
         }
       }
-      throw new IOException("Problem in loading segment blocks.", e);
+      throw new IOException("Problem in loading segment blocks: " + e.getMessage(), e);
     }
 
     return blockletDataMapIndexWrappers;

diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/ColumnarFormatVersion.java b/core/src/main/java/org/apache/carbondata/core/metadata/ColumnarFormatVersion.java
@@ -49,7 +49,7 @@ public static ColumnarFormatVersion valueOf(short version) {
       case 3:
         return V3;
       default:
-        return V3;
+        throw new UnsupportedOperationException("Unsupported columnar format version: " + version);
     }
   }
 }
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/encoder/Encoding.java b/core/src/main/java/org/apache/carbondata/core/metadata/encoder/Encoding.java
@@ -16,6 +16,8 @@
  */
 package org.apache.carbondata.core.metadata.encoder;
 
+import java.util.List;
+
 /**
  * Encoding type supported in carbon
  */
@@ -27,11 +29,14 @@ public enum Encoding {
   BIT_PACKED,
   DIRECT_DICTIONARY,
   IMPLICIT,
-
   DIRECT_COMPRESS,
   ADAPTIVE_INTEGRAL,
   ADAPTIVE_DELTA_INTEGRAL,
   RLE_INTEGRAL,
+  DIRECT_STRING,
+  ADAPTIVE_FLOATING,
+  BOOL_BYTE,
+  ADAPTIVE_DELTA_FLOATING,
   DIRECT_COMPRESS_VARCHAR;
 
   public static Encoding valueOf(int ordinal) {
@@ -57,10 +62,46 @@ public static Encoding valueOf(int ordinal) {
       return ADAPTIVE_DELTA_INTEGRAL;
     } else if (ordinal == RLE_INTEGRAL.ordinal()) {
       return RLE_INTEGRAL;
+    } else if (ordinal == DIRECT_STRING.ordinal()) {
+      return DIRECT_STRING;
+    } else if (ordinal == ADAPTIVE_FLOATING.ordinal()) {
+      return ADAPTIVE_FLOATING;
+    } else if (ordinal == BOOL_BYTE.ordinal()) {
+      return BOOL_BYTE;
+    } else if (ordinal == ADAPTIVE_DELTA_FLOATING.ordinal()) {
+      return ADAPTIVE_DELTA_FLOATING;
     } else if (ordinal == DIRECT_COMPRESS_VARCHAR.ordinal()) {
       return DIRECT_COMPRESS_VARCHAR;
     } else {
       throw new RuntimeException("create Encoding with invalid ordinal: " + ordinal);
     }
   }
+
+  /**
+   * Method to validate for supported encoding types that can be read using the current version
+   *
+   * @param encodings
+   */
+  public static void validateEncodingTypes(List<org.apache.carbondata.format.Encoding> encodings) {
+    if (null != encodings && !encodings.isEmpty()) {
+      for (org.apache.carbondata.format.Encoding encoder : encodings) {
+        // if case is handle unsupported encoding type. An encoding not supported for read will
+        // be added as null by thrift during deserialization
+        // if given encoding name is not supported exception will be thrown
+        if (null == encoder) {
+          throw new UnsupportedOperationException(
+              "There is mismatch between the encodings in data file and the encodings supported"
+                  + " for read in the current version");
+        } else {
+          try {
+            Encoding.valueOf(encoder.name());
+          } catch (IllegalArgumentException ex) {
+            throw new UnsupportedOperationException(
+                "There is mismatch between the encodings in data file and the encodings supported"
+                    + " for read in the current version. Encoding: " + encoder.name());
+          }
+        }
+      }
+    }
+  }
 }
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -686,7 +686,8 @@ private void validateCarbonDataFileVersion() {
           CarbonCommonConstants.CARBON_DATA_FILE_DEFAULT_VERSION);
     } else {
       try {
-        ColumnarFormatVersion.valueOf(carbondataFileVersionString);
+        carbonProperties.setProperty(CARBON_DATA_FILE_VERSION,
+            ColumnarFormatVersion.valueOf(carbondataFileVersionString).name());
       } catch (IllegalArgumentException e) {
         // use default property if user specifies an invalid version property
         LOGGER.warn("Specified file version property is invalid: " + carbondataFileVersionString
@@ -696,9 +697,8 @@ private void validateCarbonDataFileVersion() {
             CarbonCommonConstants.CARBON_DATA_FILE_DEFAULT_VERSION);
       }
     }
-    LOGGER.info("Carbon Current data file version: " + carbonProperties
-        .setProperty(CARBON_DATA_FILE_VERSION,
-            CarbonCommonConstants.CARBON_DATA_FILE_DEFAULT_VERSION));
+    LOGGER.info(
+        "Considered file format is: " + carbonProperties.getProperty(CARBON_DATA_FILE_VERSION));
   }
 
   /**
@@ -845,8 +845,7 @@ public ColumnarFormatVersion getFormatVersion() {
       return getDefaultFormatVersion();
     } else {
       try {
-        short version = Short.parseShort(versionStr);
-        return ColumnarFormatVersion.valueOf(version);
+        return ColumnarFormatVersion.valueOf(versionStr);
       } catch (IllegalArgumentException e) {
         return getDefaultFormatVersion();
       }

diff --git a/...ala/org/apache/carbondata/spark/testsuite/allqueries/TestQueryWithOldCarbonDataFile.scala b/...ala/org/apache/carbondata/spark/testsuite/allqueries/TestQueryWithOldCarbonDataFile.scala
diff --git a/...a/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala b/...a/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
@@ -1076,7 +1076,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
         sql("select * from sdkOutputTable").show(false)
       }
     assert(exception.getMessage()
-      .contains("Problem in loading segment blocks."))
+      .contains("Problem in loading segment blocks"))
 
 
     sql("DROP TABLE sdkOutputTable")