Skip to content

Commit

Permalink
[CARBONDATA-3223] Fixed Wrong Datasize and Indexsize calculation for …
Browse files Browse the repository at this point in the history
…old store using Show Segments

Problem: Table Created and Loading on older version(1.1) was showing data-size and index-size 0B when refreshed on new version. This was
because when the data-size was coming as "null" we were not computing it, directly assigning 0 value to it.

Solution: Showing the old datasize and indexsize as NA.

Also refactored SetQuerySegment code for better understandability.

This closes #3047
  • Loading branch information
manishnalla1994 authored and manishgupta88 committed Jan 7, 2019
1 parent 923dab1 commit 72da334
Show file tree
Hide file tree
Showing 3 changed files with 22 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -277,12 +277,7 @@ public static void setSegmentsToAccess(Configuration configuration, List<Segment
public static void setQuerySegment(Configuration conf, AbsoluteTableIdentifier identifier) {
String dbName = identifier.getCarbonTableIdentifier().getDatabaseName().toLowerCase();
String tbName = identifier.getCarbonTableIdentifier().getTableName().toLowerCase();
String segmentNumbersFromProperty = CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.CARBON_INPUT_SEGMENTS + dbName + "." + tbName, "*");
if (!segmentNumbersFromProperty.trim().equals("*")) {
CarbonInputFormat.setSegmentsToAccess(conf,
Segment.toSegmentList(segmentNumbersFromProperty.split(","), null));
}
getQuerySegmentToAccess(conf, dbName, tbName);
}

/**
Expand Down Expand Up @@ -827,4 +822,22 @@ public String[] projectAllColumns(CarbonTable carbonTable) {
}
return projectColumns.toArray(new String[projectColumns.size()]);
}

private static void getQuerySegmentToAccess(Configuration conf, String dbName, String tableName) {
String segmentNumbersFromProperty = CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.CARBON_INPUT_SEGMENTS + dbName + "." + tableName, "*");
if (!segmentNumbersFromProperty.trim().equals("*")) {
CarbonInputFormat.setSegmentsToAccess(conf,
Segment.toSegmentList(segmentNumbersFromProperty.split(","), null));
}
}

/**
* Set `CARBON_INPUT_SEGMENTS` from property to configuration
*/
public static void setQuerySegment(Configuration conf, CarbonTable carbonTable) {
String tableName = carbonTable.getTableName();
getQuerySegmentToAccess(conf, carbonTable.getDatabaseName(), tableName);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -107,8 +107,8 @@ object CarbonStore {
(indices.asScala.map(_.getFile_size).sum, FileFactory.getCarbonFile(indexPath).getSize)
} else {
// for batch segment, we can get the data size from table status file directly
(if (load.getDataSize == null) 0L else load.getDataSize.toLong,
if (load.getIndexSize == null) 0L else load.getIndexSize.toLong)
(if (load.getDataSize == null) -1L else load.getDataSize.toLong,
if (load.getIndexSize == null) -1L else load.getIndexSize.toLong)
}

if (showHistory) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ case class CarbonCountStar(
.setConfigurationToCurrentThread(sparkSession.sessionState.newHadoopConf())
val absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier
val (job, tableInputFormat) = createCarbonInputFormat(absoluteTableIdentifier)
CarbonInputFormat.setQuerySegment(job.getConfiguration, absoluteTableIdentifier)
CarbonInputFormat.setQuerySegment(job.getConfiguration, carbonTable)

// get row count
val rowCount = CarbonUpdateUtil.getRowCount(
Expand Down

0 comments on commit 72da334

Please sign in to comment.