add some comment

apache · Sep 20, 2019 · 12f6300 · 12f6300
1 parent 1c36bfe
commit 12f6300
Showing 1 changed file with 4 additions and 2 deletions.
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -160,6 +160,8 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
     new HistoryServerDiskManager(conf, path, listing, clock)
   }
 
+  // Used to store the paths, which are being processed. This enable the replay log tasks execute
+  // asynchronously and make sure that checkForLogs would not process a path repeatedly.
   private val processing = ConcurrentHashMap.newKeySet[String]
 
   private def isProcessing(path: Path): Boolean = {
@@ -170,7 +172,7 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
     processing.add(path.getName)
   }
 
-  private def endProcess(path: Path): Unit = {
+  private def endProcessing(path: Path): Unit = {
     processing.remove(path.getName)
   }
 
@@ -685,7 +687,7 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
       case e: Exception =>
         logError("Exception while merging application listings", e)
     } finally {
-      endProcess(fileStatus.getPath)
+      endProcessing(fileStatus.getPath)
       pendingReplayTasksCount.decrementAndGet()
     }
   }