check and delete expired log when complete process

apache · Nov 6, 2019 · 832dd7a · 832dd7a
1 parent 611ea30
commit 832dd7a
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -682,6 +682,12 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
     } finally {
       endProcessing(reader.rootPath)
       pendingReplayTasksCount.decrementAndGet()
+
+      val isExpired = scanTime + conf.get(MAX_LOG_AGE_S) * 1000 > clock.getTimeMillis()
+      if (isExpired) {
+        listing.delete(classOf[LogInfo], reader.rootPath.toString)
+        deleteLog(fs, reader.rootPath)
+      }
     }
   }