From b7e0100b85a1cbcbe9e6cabafb4ba76ce4049171 Mon Sep 17 00:00:00 2001 From: Max Baak Date: Tue, 9 Feb 2021 15:18:15 +0100 Subject: [PATCH] Initialize spark with both histogrammar jar files --- README.rst | 4 ++-- docs/source/configuration.rst | 2 +- popmon/notebooks/popmon_tutorial_advanced.ipynb | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 0601bb4f..6898d916 100644 --- a/README.rst +++ b/README.rst @@ -25,11 +25,11 @@ Announcements Spark 3.0 --------- -With Spark 3.0, based on Scala 2.12, make sure to pick up the correct `histogrammar` jar file: +With Spark 3.0, based on Scala 2.12, make sure to pick up the correct `histogrammar` jar files: .. code-block:: python - spark = SparkSession.builder.config("spark.jars.packages", "io.github.histogrammar:histogrammar-sparksql_2.12:1.0.11").getOrCreate() + spark = SparkSession.builder.config("spark.jars.packages", "io.github.histogrammar:histogrammar_2.12:1.0.11,io.github.histogrammar:histogrammar-sparksql_2.12:1.0.11").getOrCreate() For Spark 2.X compiled against scala 2.11, in the string above simply replace 2.12 with 2.11. diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index ce976624..489fabb7 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -203,7 +203,7 @@ Spark usage from pyspark.sql import SparkSession # downloads histogrammar jar files if not already installed, used for histogramming of spark dataframe - spark = SparkSession.builder.config('spark.jars.packages','io.github.histogrammar:histogrammar-sparksql_2.12:1.0.11').getOrCreate() + spark = SparkSession.builder.config("spark.jars.packages", "io.github.histogrammar:histogrammar_2.12:1.0.11,io.github.histogrammar:histogrammar-sparksql_2.12:1.0.11").getOrCreate() # load a dataframe spark_df = spark.read.format('csv').options(header='true').load('file.csv') diff --git a/popmon/notebooks/popmon_tutorial_advanced.ipynb b/popmon/notebooks/popmon_tutorial_advanced.ipynb index eb11b3ce..315af09f 100644 --- a/popmon/notebooks/popmon_tutorial_advanced.ipynb +++ b/popmon/notebooks/popmon_tutorial_advanced.ipynb @@ -162,7 +162,7 @@ "source": [ "if pyspark_installed:\n", " spark = SparkSession.builder.config(\n", - " \"spark.jars.packages\", \"io.github.histogrammar:histogrammar-sparksql_2.12:1.0.11\"\n", + " \"spark.jars.packages\", \"io.github.histogrammar:histogrammar_2.12:1.0.11,io.github.histogrammar:histogrammar-sparksql_2.12:1.0.11\"\n", " ).getOrCreate()\n", "\n", " sdf = spark.createDataFrame(df)\n",