only modify hdfs paths

apache · Sep 26, 2017 · c6acdf2 · c6acdf2
1 parent c2e125e
commit c6acdf2
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 9 deletions.
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
@@ -1404,14 +1404,18 @@ object HiveExternalCatalog extends Logging {
     // With HDFS HA, the user really *should* update their DB locations to list the
     // nameservice, not one specific namenode, since that won't work if that namenode happens
     // to be in standby.   But, users mess this up, so we can try to fix this for them.
-    val prefix = new URI(uri.getScheme(), uri.getAuthority(), null, null, null).toString()
-    namenodeToNameservice.get(prefix).map { nameservicePrefix =>
-      val s = uri.toString()
-      val fixedPath = new Path(s"${nameservicePrefix}${s.substring(prefix.length())}").toUri()
-      logWarning(s"Database $db has location $s, but this references only one namenode in " +
-        s"an HA setup.  Converting to reference nameservice $fixedPath")
-      fixedPath
-    }.getOrElse(uri)
+    if (uri.getScheme() == "hdfs") {
+      val prefix = new URI(uri.getScheme(), uri.getAuthority(), null, null, null).toString()
+      namenodeToNameservice.get(prefix).map { nameservicePrefix =>
+        val s = uri.toString()
+        val fixedPath = new Path(s"${nameservicePrefix}${s.substring(prefix.length())}").toUri()
+        logWarning(s"Database $db has location $s, but this references only one namenode in " +
+          s"an HA setup.  Converting to reference nameservice $fixedPath")
+        fixedPath
+      }.getOrElse(uri)
+    } else {
+      uri
+    }
   }
 
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala
@@ -119,7 +119,10 @@ class HiveExternalCatalogSuite extends ExternalCatalogSuite {
       "hdfs://foo-1.xyz.com/some/path" -> "hdfs://foo-1.xyz.com/some/path",
       "hdfs://foo-2.xyz.com:1234/some/path" -> "hdfs://ns1/some/path",
       "hdfs://blah-1.bar.com:8020/another/path" -> "hdfs://ns2/another/path",
-      "hdfs://another.cluster.com:8020/my/path" -> "hdfs://another.cluster.com:8020/my/path"
+      "hdfs://another.cluster.com:8020/my/path" -> "hdfs://another.cluster.com:8020/my/path",
+      "file:/some/local/path/spark-warehouse" ->
+        "file:/some/local/path/spark-warehouse",
+      "/bare/path" -> "/bare/path"
     ).foreach { case (orig, exp) =>
       val convertedName = HiveExternalCatalog.convertNamenodeToNameservice(
           namenodeToNameservice,