Fixed Wrong Datasize and Indexsize calculation for old store

apache · Jan 4, 2019 · 49bd919 · 49bd919
1 parent 7477527
commit 49bd919
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 10 deletions.
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
@@ -276,12 +276,7 @@ public static void setSegmentsToAccess(Configuration configuration, List<Segment
   public static void setQuerySegment(Configuration conf, AbsoluteTableIdentifier identifier) {
     String dbName = identifier.getCarbonTableIdentifier().getDatabaseName().toLowerCase();
     String tbName = identifier.getCarbonTableIdentifier().getTableName().toLowerCase();
-    String segmentNumbersFromProperty = CarbonProperties.getInstance()
-        .getProperty(CarbonCommonConstants.CARBON_INPUT_SEGMENTS + dbName + "." + tbName, "*");
-    if (!segmentNumbersFromProperty.trim().equals("*")) {
-      CarbonInputFormat.setSegmentsToAccess(conf,
-          Segment.toSegmentList(segmentNumbersFromProperty.split(","), null));
-    }
+    getQuerySegmentToAccess(conf, dbName, tbName);
   }
 
   /**
@@ -788,4 +783,22 @@ public String[] projectAllColumns(CarbonTable carbonTable) {
     }
     return projectColumns.toArray(new String[projectColumns.size()]);
   }
+
+  private static void getQuerySegmentToAccess(Configuration conf, String dbName, String tableName) {
+    String segmentNumbersFromProperty = CarbonProperties.getInstance()
+        .getProperty(CarbonCommonConstants.CARBON_INPUT_SEGMENTS + dbName + "." + tableName, "*");
+    if (!segmentNumbersFromProperty.trim().equals("*")) {
+      CarbonInputFormat.setSegmentsToAccess(conf,
+          Segment.toSegmentList(segmentNumbersFromProperty.split(","), null));
+    }
+  }
+
+  /**
+   * Set `CARBON_INPUT_SEGMENTS` from property to configuration
+   */
+  public static void setQuerySegment(Configuration conf, CarbonTable carbonTable) {
+    String tableName = carbonTable.getTableName();
+    getQuerySegmentToAccess(conf, carbonTable.getDatabaseName(), tableName);
+  }
+
 }
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
@@ -101,14 +101,15 @@ object CarbonStore {
           val (dataSize, indexSize) = if (load.getFileFormat == FileFormat.ROW_V1) {
             // for streaming segment, we should get the actual size from the index file
             // since it is continuously inserting data
-            val segmentDir = CarbonTablePath.getSegmentPath(tablePath, load.getLoadName)
+            val segmentDir = CarbonTablePath
+              .getSegmentPath(tablePath, load.getLoadName)
             val indexPath = CarbonTablePath.getCarbonStreamIndexFilePath(segmentDir)
             val indices = StreamSegment.readIndexFile(indexPath, FileFactory.getFileType(indexPath))
             (indices.asScala.map(_.getFile_size).sum, FileFactory.getCarbonFile(indexPath).getSize)
           } else {
             // for batch segment, we can get the data size from table status file directly
-            (if (load.getDataSize == null) 0L else load.getDataSize.toLong,
-              if (load.getIndexSize == null) 0L else load.getIndexSize.toLong)
+            (if (load.getDataSize == null) -1L else load.getDataSize.toLong,
+              if (load.getIndexSize == null) -1L else load.getIndexSize.toLong)
           }
 
           if (showHistory) {

diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonCountStar.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonCountStar.scala
@@ -52,7 +52,7 @@ case class CarbonCountStar(
       .setConfigurationToCurrentThread(sparkSession.sessionState.newHadoopConf())
     val absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier
     val (job, tableInputFormat) = createCarbonInputFormat(absoluteTableIdentifier)
-    CarbonInputFormat.setQuerySegment(job.getConfiguration, absoluteTableIdentifier)
+    CarbonInputFormat.setQuerySegment(job.getConfiguration, carbonTable)
 
     // get row count
     val rowCount = CarbonUpdateUtil.getRowCount(