[SW-683] Fix starting H2OContext on Databricks

core/src/main/scala/org/apache/spark/h2o/H2OContext.scala

@@ -127,8 +127,8 @@ class H2OContext private(val sparkSession: SparkSession, conf: H2OConf) extends
     h2oNodes.append(nodes: _*)
     localClientIp = sys.env.getOrElse("SPARK_PUBLIC_DNS", sparkContext.env.rpcEnv.address.host)
     localClientPort = H2O.API_PORT
-    // Register UI
-    if (conf.getBoolean("spark.ui.enabled", true)) {
+    // Register UI, but not in Databricks as Databricks is not using standard Spark UI API
+    if (conf.getBoolean("spark.ui.enabled", true) && !isRunningOnDatabricks()) {
       new SparklingWaterUITab(sparklingWaterListener, sparkContext.ui.get)
     }
 

core/src/main/scala/org/apache/spark/h2o/utils/H2OContextUtils.scala

@@ -77,4 +77,13 @@ private[spark] trait H2OContextUtils extends Logging {
     }
   }
 
+  def isRunningOnDatabricks(): Boolean = {
+    try {
+      Class.forName("com.databricks.backend.daemon.driver.DriverLocal")
+      true
+    } catch {
+      case _: ClassNotFoundException => false
+    }
+  }
+
 }

doc/tutorials/pysparkling_azure_dbc.rst

@@ -29,7 +29,6 @@ To start Sparkling Water ``H2OContext`` on Databricks Azure, the steps are:
 4.  Create the cluster
 
     - Make sure the library is attached to the cluster
-    - Put ``spark.ui.enabled false`` line in the spark configuration field. This is currently required due to limitation in our code and will be fixed as soon as possible.
 
     - Select Spark version according to you Sparkling Water version:
         - for the latest PySparkling 2.2, select Spark 2.2.1.

doc/tutorials/rsparkling_azure_dbc.rst

@@ -26,7 +26,6 @@ To start Sparkling Water ``H2OContext`` on Databricks Azure, the steps are:
 4.  Create the cluster
 
     - Make sure the assembly JAR is attached to the cluster
-    - Put ``spark.ui.enabled false`` line in the spark configuration field. This is currently required due to limitation in our code and will be fixed as soon as possible.
 
     - Select Spark version according to your Sparkling Water version:
         - for the latest Sparkling Water 2.2.7, select Spark 2.2.0. Spark 2.2.1 is not yet supported due to missing SparklyR support.

doc/tutorials/sw_azure_dbc.rst

@@ -27,7 +27,6 @@ To start Sparkling Water ``H2OContext`` on Databricks Azure, the steps are:
 4.  Create the cluster
 
     - Make sure the assembly JAR is attached to the cluster
-    - Put ``spark.ui.enabled false`` line in the spark configuration field. This is currently required due to limitation in our code and will be fixed as soon as possible.
 
     - Select Spark version according to your Sparkling Water version:
         - for the latest Sparkling Water 2.2.7, select Spark 2.2.1.