Skip to content

Commit

Permalink
fixed to use File.toURI()
Browse files Browse the repository at this point in the history
  • Loading branch information
tsudukim committed Apr 17, 2015
1 parent 2c62e3b commit 016128d
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 27 deletions.
28 changes: 8 additions & 20 deletions core/src/main/scala/org/apache/spark/util/Utils.scala
Original file line number Diff line number Diff line change
Expand Up @@ -1766,27 +1766,15 @@ private[spark] object Utils extends Logging {
* converted into an absolute path with a file:// scheme.
*/
def resolveURI(path: String, testWindows: Boolean = false): URI = {

val windows = isWindows || testWindows
val formattedPath = formatPath(path, windows)

val uri = new URI(formattedPath)
if (uri.getPath == null) {
throw new IllegalArgumentException(s"Given path is malformed: $uri")
}

Option(uri.getScheme) match {
case Some(windowsDrive(d)) if windows =>
new URI("file:/" + uri.toString.stripPrefix("/"))
case None =>
// Preserve fragments for HDFS file name substitution (denoted by "#")
// For instance, in "abc.py#xyz.py", "xyz.py" is the name observed by the application
val fragment = uri.getFragment
val part = new File(uri.getPath).toURI
new URI(part.getScheme, part.getPath, fragment)
case Some(other) =>
uri
try {
val uri = new URI(path)
if (uri.getScheme() != null) {
return uri
}
} catch {
case e: URISyntaxException =>
}
new File(path).getAbsoluteFile().toURI()
}

/** Resolve a comma-separated list of paths. */
Expand Down
13 changes: 6 additions & 7 deletions core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
Original file line number Diff line number Diff line change
Expand Up @@ -239,16 +239,15 @@ class UtilsSuite extends FunSuite with ResetSystemProperties {
assertResolves("hdfs:/root/spark.jar", "hdfs:/root/spark.jar")
assertResolves("hdfs:///root/spark.jar#app.jar", "hdfs:/root/spark.jar#app.jar")
assertResolves("spark.jar", s"file:$cwd/spark.jar")
assertResolves("spark.jar#app.jar", s"file:$cwd/spark.jar#app.jar")
assertResolves("spark.jar#app.jar", s"file:$cwd/spark.jar%23app.jar")
assertResolves("path to/file.txt", s"file:$cwd/path%20to/file.txt")
assertResolves("C:/path/to/file.txt", "file:/C:/path/to/file.txt", testWindows = true)
assertResolves("C:\\path\\to\\file.txt", "file:/C:/path/to/file.txt", testWindows = true)
assertResolves("C:/path to/file.txt", "file:/C:/path%20to/file.txt", testWindows = true)
assertResolves("C:\\path to\\file.txt", "file:/C:/path%20to/file.txt", testWindows = true)
assertResolves("file:/C:/path/to/file.txt", "file:/C:/path/to/file.txt", testWindows = true)
assertResolves("file:///C:/path/to/file.txt", "file:/C:/path/to/file.txt", testWindows = true)
assertResolves("file:/C:/file.txt#alias.txt", "file:/C:/file.txt#alias.txt", testWindows = true)
intercept[IllegalArgumentException] { Utils.resolveURI("file:foo") }
intercept[IllegalArgumentException] { Utils.resolveURI("file:foo:baby") }
assertResolves("file:foo", s"file:foo")
assertResolves("file:foo:baby", s"file:foo:baby")
}

test("resolveURIs with multiple paths") {
Expand All @@ -268,9 +267,9 @@ class UtilsSuite extends FunSuite with ResetSystemProperties {
assertResolves("file:/jar1,file:/jar2", "file:/jar1,file:/jar2")
assertResolves("hdfs:/jar1,file:/jar2,jar3", s"hdfs:/jar1,file:/jar2,file:$cwd/jar3")
assertResolves("hdfs:/jar1,file:/jar2,jar3,jar4#jar5,path to/jar6",
s"hdfs:/jar1,file:/jar2,file:$cwd/jar3,file:$cwd/jar4#jar5,file:$cwd/path%20to/jar6")
s"hdfs:/jar1,file:/jar2,file:$cwd/jar3,file:$cwd/jar4%23jar5,file:$cwd/path%20to/jar6")
assertResolves("""hdfs:/jar1,file:/jar2,jar3,C:\pi.py#py.pi,C:\path to\jar4""",
s"hdfs:/jar1,file:/jar2,file:$cwd/jar3,file:/C:/pi.py#py.pi,file:/C:/path%20to/jar4",
s"hdfs:/jar1,file:/jar2,file:$cwd/jar3,file:/C:/pi.py%23py.pi,file:/C:/path%20to/jar4",
testWindows = true)
}

Expand Down

0 comments on commit 016128d

Please sign in to comment.