Skip to content

Commit

Permalink
Warn instead of throw
Browse files Browse the repository at this point in the history
... in case the user provides a NFS mounted directory.
  • Loading branch information
Andrew Or committed Jun 24, 2015
1 parent 5484293 commit b1437ad
Showing 1 changed file with 6 additions and 3 deletions.
9 changes: 6 additions & 3 deletions core/src/main/scala/org/apache/spark/SparkContext.scala
Original file line number Diff line number Diff line change
Expand Up @@ -1898,10 +1898,13 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
*/
def setCheckpointDir(directory: String) {

// If we are running on a cluster, require the path to be non-local
// If we are running on a cluster, log a warning if the directory is local.
// Otherwise, the driver may attempt to reconstruct the checkpointed RDD from
// its own local file system, which is incorrect because the checkpoint files
// are actually on the executor machines.
if (!isLocal && Utils.nonLocalPaths(directory).isEmpty) {
throw new IllegalArgumentException(
"Checkpoint directory must be non-local if Spark is running on a cluster: " + directory)
logWarning("Checkpoint directory must be non-local " +
"if Spark is running on a cluster: " + directory)
}

checkpointDir = Option(directory).map { dir =>
Expand Down

0 comments on commit b1437ad

Please sign in to comment.