From 453e104c3e6ed6a5ca310f599c274d6c66a3d3c8 Mon Sep 17 00:00:00 2001 From: Stefan Obermeier Date: Sun, 1 Oct 2017 16:28:58 +0200 Subject: [PATCH 1/4] [SPARK-22180][CORE] Allow IPv6 address in org.apache.spark.util.Utils.parseHostPort ## What changes were proposed in this pull request? Take ```hostPort``` as literal IPv6 address if it contains tow ore more colons. If IPv6 addresses are enclosed in square brackets port definition is still possible. ## How was this patch tested? Added a new test case into UtilsSuite Remove comment --- .../main/scala/org/apache/spark/util/Utils.scala | 16 +++++++++++++--- .../scala/org/apache/spark/util/UtilsSuite.scala | 15 +++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 930e09d90c2f5..c8981202db90d 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -982,9 +982,19 @@ private[spark] object Utils extends Logging { return cached } - val indx: Int = hostPort.lastIndexOf(':') - // This is potentially broken - when dealing with ipv6 addresses for example, sigh ... - // but then hadoop does not support ipv6 right now. + val indx: Int = + // Interpret hostPort as literal IPv6 address if it contains multiple colons. + // IPv6 addresses enclosed in square brackets like [::1]:123 are not included. + // scalastyle:off SingleSpaceBetweenRParenAndLCurlyBrace + if (hostPort.matches("(([0-9a-fA-F]*):([0-9a-fA-F]*)){2,}")) { + // scalastyle:on SingleSpaceBetweenRParenAndLCurlyBrace + -1 + } else { + // Else last colon defines start of port definition + hostPort.lastIndexOf(':') + } + + // For now, we assume that if port exists, then it is valid - not check if it is an int > 0 if (-1 == indx) { val retval = (hostPort, 0) diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala index 4d3adeb968e84..6f16748307b03 100644 --- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala @@ -1146,6 +1146,20 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging { } } + test("parseHostPort") { + assert(Utils.parseHostPort("abc:123") === (("abc", 123))) + assert(Utils.parseHostPort("example.com") === (("example.com", 0))) + assert(Utils.parseHostPort("example.com:123") === (("example.com", 123))) + assert(Utils.parseHostPort("127.0.0.1") === (("127.0.0.1", 0))) + assert(Utils.parseHostPort("127.0.0.1:123") === (("127.0.0.1", 123))) + assert(Utils.parseHostPort("2001:db8::1") === (("2001:db8::1", 0))) + assert(Utils.parseHostPort("2001:DB8::1") === (("2001:DB8::1", 0))) + assert(Utils.parseHostPort("2001:dB8::1") === (("2001:dB8::1", 0))) + assert(Utils.parseHostPort("0:0:0:0:0:0:0:0") === (("0:0:0:0:0:0:0:0", 0))) + assert(Utils.parseHostPort("::1") === (("::1", 0))) + assert(Utils.parseHostPort("[::1]:123") === (("[::1]", 123))) + assert(Utils.parseHostPort("[2001:db8:42::1]:123") === (("[2001:db8:42::1]", 123))) + } } private class SimpleExtension @@ -1165,3 +1179,4 @@ private class ExtensionWithError { } private class ListenerImpl extends SparkListener + From 1400299808631da0196a61f3588ede786dd0b041 Mon Sep 17 00:00:00 2001 From: Stefan Obermeier Date: Fri, 15 Dec 2017 08:39:45 +0100 Subject: [PATCH 2/4] Fix build problem --- core/src/test/scala/org/apache/spark/util/UtilsSuite.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala index 19e51ddf6fac9..ab0be06737f04 100644 --- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala @@ -1159,6 +1159,7 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging { assert(Utils.parseHostPort("::1") === (("::1", 0))) assert(Utils.parseHostPort("[::1]:123") === (("[::1]", 123))) assert(Utils.parseHostPort("[2001:db8:42::1]:123") === (("[2001:db8:42::1]", 123))) + } test("check Kubernetes master URL") { val k8sMasterURLHttps = Utils.checkAndGetK8sMasterUrl("k8s://https://host:port") From 68c322129305a35c9d3e04f8cacc011be5fbaec4 Mon Sep 17 00:00:00 2001 From: Stefan Obermeier Date: Mon, 18 Dec 2017 14:51:04 +0100 Subject: [PATCH 3/4] Fix style checks violation. Remove whitespace at end of line. --- core/src/test/scala/org/apache/spark/util/UtilsSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala index ab0be06737f04..d3e9cb5953eb2 100644 --- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala @@ -1145,7 +1145,7 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging { Utils.loadExtensions(classOf[Seq[_]], wrongType, conf) } } - + test("parseHostPort") { assert(Utils.parseHostPort("abc:123") === (("abc", 123))) assert(Utils.parseHostPort("example.com") === (("example.com", 0))) From 8220d95a99f4564e735f22947cb1cb698613efa5 Mon Sep 17 00:00:00 2001 From: Stefan Obermeier Date: Tue, 9 Oct 2018 23:54:20 +0200 Subject: [PATCH 4/4] Add log message if hostname:port is not valid --- core/src/main/scala/org/apache/spark/util/Utils.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index bee8440b62609..8189ff86c82de 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -1001,6 +1001,7 @@ private[spark] object Utils extends Logging { // scalastyle:off SingleSpaceBetweenRParenAndLCurlyBrace if (hostPort.matches("(([0-9a-fA-F]*):([0-9a-fA-F]*)){2,}")) { // scalastyle:on SingleSpaceBetweenRParenAndLCurlyBrace + log.warn(s"Invalid hostname ${hostPort}. IPv6 addresses are currently not supported") -1 } else { // Else last colon defines start of port definition