Skip to content

Commit

Permalink
[SPARK-13265][ML] Refactoring of basic ML import/export for other fil…
Browse files Browse the repository at this point in the history
…e system besides HDFS
  • Loading branch information
yu-iskw committed Feb 10, 2016
1 parent c0b71e0 commit 5f0092f
Showing 1 changed file with 7 additions and 6 deletions.
13 changes: 7 additions & 6 deletions mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,10 @@ package org.apache.spark.ml.util

import java.io.IOException

import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.hadoop.fs.Path
import org.json4s._
import org.json4s.JsonDSL._
import org.json4s.jackson.JsonMethods._
import org.json4s.JsonDSL._

import org.apache.spark.{Logging, SparkContext}
import org.apache.spark.annotation.{Experimental, Since}
Expand Down Expand Up @@ -75,13 +75,14 @@ abstract class MLWriter extends BaseReadWrite with Logging {
@throws[IOException]("If the input path already exists but overwrite is not enabled.")
def save(path: String): Unit = {
val hadoopConf = sc.hadoopConfiguration
val fs = FileSystem.get(hadoopConf)
val p = new Path(path)
if (fs.exists(p)) {
val outputPath = new Path(path)
val fs = outputPath.getFileSystem(hadoopConf)
val qualifiedOutputPath = outputPath.makeQualified(fs.getUri, fs.getWorkingDirectory)
if (fs.exists(qualifiedOutputPath)) {
if (shouldOverwrite) {
logInfo(s"Path $path already exists. It will be overwritten.")
// TODO: Revert back to the original content if save is not successful.
fs.delete(p, true)
fs.delete(qualifiedOutputPath, true)
} else {
throw new IOException(
s"Path $path already exists. Please use write.overwrite().save(path) to overwrite it.")
Expand Down

0 comments on commit 5f0092f

Please sign in to comment.