Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[CARBONDATA-3620][CARBONDATA-3622]: Update does not load cache in memory, behavior inconsistent with scenario when index server is not running #3511

Closed
wants to merge 1 commit into from
Closed

[CARBONDATA-3620][CARBONDATA-3622]: Update does not load cache in memory, behavior inconsistent with scenario when index server is not running #3511

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
10 changes: 5 additions & 5 deletions ...gration/spark2/src/main/scala/org/apache/carbondata/indexserver/DistributedCountRDD.scala
View file
Open in desktop
Expand Up @@ -64,6 +64,11 @@ class DistributedCountRDD(@transient ss: SparkSession, dataMapFormat: Distributa
.newFixedThreadPool(numOfThreads, new CarbonThreadFactory("IndexPruningPool", true))
implicit val ec: ExecutionContextExecutor = ExecutionContext
.fromExecutor(service)
if (dataMapFormat.ifAsyncCall()) {
// to clear cache of invalid segments during pre-priming in index server
DataMapStoreManager.getInstance().clearInvalidSegments(dataMapFormat.getCarbonTable,
dataMapFormat.getInvalidSegments)
}
val futures = if (inputSplits.length <= numOfThreads) {
inputSplits.map {
split => generateFuture(Seq(split))
Expand All @@ -84,11 +89,6 @@ class DistributedCountRDD(@transient ss: SparkSession, dataMapFormat: Distributa
} else {
0L
}
if (dataMapFormat.ifAsyncCall()) {
// to clear cache of invalid segments during pre-priming in index server
DataMapStoreManager.getInstance().clearInvalidSegments(dataMapFormat.getCarbonTable,
dataMapFormat.getInvalidSegments)
}
Iterator((executorIP + "_" + cacheSize.toString, results.map(_._2.toLong).sum.toString))
}

Expand Down
10 changes: 8 additions & 2 deletions integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
View file
Open in desktop
Expand Up @@ -455,20 +455,26 @@ object CarbonDataRDDFactory {
if (resultSize == 0) {
return null
}
if (CarbonUpdateUtil.updateTableMetadataStatus(
if (!CarbonUpdateUtil.updateTableMetadataStatus(
segmentDetails,
carbonTable,
updateModel.get.updatedTimeStamp + "",
true,
new util.ArrayList[Segment](0),
new util.ArrayList[Segment](segmentFiles), "")) {
} else {
LOGGER.error("Data update failed due to failure in table status updation.")
updateModel.get.executorErrors.errorMsg = errorMessage
updateModel.get.executorErrors.failureCauses = FailureCauses
.STATUS_FILE_UPDATION_FAILURE
return null
}
// code to handle Pre-Priming cache for update command
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

may be you can remove the empty if block above this

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

if (!segmentFiles.isEmpty) {
val segmentsToPrePrime = segmentFiles.asScala.map(iterator => iterator.getSegmentNo).toSeq
DistributedRDDUtils
.triggerPrepriming(sqlContext.sparkSession, carbonTable, segmentsToPrePrime,
operationContext, hadoopConf, segmentsToPrePrime.toList)
}
}
return null
}
Expand Down
3 changes: 2 additions & 1 deletion integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
View file
Open in desktop
Expand Up @@ -454,11 +454,12 @@ class CarbonFileMetastore extends CarbonMetaStore {
def dropTable(absoluteTableIdentifier: AbsoluteTableIdentifier)(sparkSession: SparkSession) {
val dbName = absoluteTableIdentifier.getCarbonTableIdentifier.getDatabaseName
val tableName = absoluteTableIdentifier.getCarbonTableIdentifier.getTableName
// Clear both driver and executor cache.
DataMapStoreManager.getInstance().clearDataMaps(absoluteTableIdentifier)
CarbonHiveMetadataUtil.invalidateAndDropTable(dbName, tableName, sparkSession)
// discard cached table info in cachedDataSourceTables
val tableIdentifier = TableIdentifier(tableName, Option(dbName))
sparkSession.sessionState.catalog.refreshTable(tableIdentifier)
DataMapStoreManager.getInstance().clearDataMaps(absoluteTableIdentifier)
SegmentPropertiesAndSchemaHolder.getInstance().invalidate(absoluteTableIdentifier)
removeTableFromMetadata(dbName, tableName)
}
Expand Down