From 4c7b646ac1aa5b4ae6e4d594d678e515371a0c9b Mon Sep 17 00:00:00 2001 From: zuotingbing Date: Sat, 30 Sep 2017 10:05:27 +0800 Subject: [PATCH] [SPARK-22174][CORE]Support to automatically create the directory where the event logs go (`spark.eventLog.dir`) --- .../org/apache/spark/scheduler/EventLoggingListener.scala | 5 ++++- .../apache/spark/scheduler/EventLoggingListenerSuite.scala | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala index 9dafa0b7646bf..a07381020e441 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala @@ -94,7 +94,10 @@ private[spark] class EventLoggingListener( * Creates the log file in the configured log directory. */ def start() { - if (!fileSystem.getFileStatus(new Path(logBaseDir)).isDirectory) { + val logBasePath = new Path(logBaseDir) + if(!fileSystem.exists(logBasePath) && !fileSystem.mkdirs(logBasePath)) { + throw new IOException(s"Failed to create log directory $logBaseDir." ) + } else if (!fileSystem.getFileStatus(logBasePath).isDirectory) { throw new IllegalArgumentException(s"Log directory $logBaseDir is not a directory.") } diff --git a/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala index 6b42775ccb0f6..6632ca6aae178 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala @@ -51,7 +51,7 @@ class EventLoggingListenerSuite extends SparkFunSuite with LocalSparkContext wit private var testDirPath: Path = _ before { - testDir = Utils.createTempDir(namePrefix = s"history log") + testDir = new File(System.getProperty("java.io.tmpdir") + "/history log") testDir.deleteOnExit() testDirPath = new Path(testDir.getAbsolutePath()) } @@ -109,6 +109,9 @@ class EventLoggingListenerSuite extends SparkFunSuite with LocalSparkContext wit } test("Log overwriting") { + val conf = getLoggingConf(testDirPath) + val eventLogger = new EventLoggingListener("test", None, testDirPath.toUri(), conf) + eventLogger.start() val logUri = EventLoggingListener.getLogPath(testDir.toURI, "test", None) val logPath = new Path(logUri).toUri.getPath // Create file before writing the event log