Merge pull request #1921 from MrBago/session-for-hive

Databricks connection should be using SparkSession not HiveContext
sparklyr · Feb 20, 2019 · 90466eb · 90466eb
2 parents f69ce12 + 79b571a
commit 90466eb
Showing 1 changed file with 5 additions and 2 deletions.
diff --git a/R/databricks_connection.R b/R/databricks_connection.R
@@ -66,7 +66,10 @@ new_databricks_connection <- function(scon, guid) {
   # In databricks, sparklyr should use the SqlContext associated with the RDriverLocal instance for
   # this guid.
   r_driver_local <- "com.databricks.backend.daemon.driver.RDriverLocal"
-  hive_context <- invoke_static(sc, r_driver_local, "getDriver", guid) %>% invoke("sqlContext")
-  sc$state$hive_context <- hive_context
+  session <- invoke_static(sc, r_driver_local, "getDriver", guid) %>%
+    invoke("sqlContext") %>%
+    invoke("sparkSession")
+  # This is called hive_context but for spark > 2.0, it should actually be a spark session
+  sc$state$hive_context <- session
   sc
 }