diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala index c38afae2ea544..94cafd070a538 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala @@ -682,6 +682,12 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock) } finally { endProcessing(reader.rootPath) pendingReplayTasksCount.decrementAndGet() + + val isExpired = scanTime + conf.get(MAX_LOG_AGE_S) * 1000 > clock.getTimeMillis() + if (isExpired) { + listing.delete(classOf[LogInfo], reader.rootPath.toString) + deleteLog(fs, reader.rootPath) + } } }