diff --git a/CHANGELOG b/CHANGELOG index f5c88a06..986f822c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,9 @@ +2023-07-18 Version 4.1.4 + * Added support of Spark 3.4 + * Added connection attributes + * Fixed conflicts of result table names during parallel read + * Updated version of the SingleStore JDBC driver + 2023-03-31 Version 4.1.3 * Updated version of the SingleStore JDBC driver * Fixed error handling when `onDuplicateKeySQL` option is used diff --git a/README.md b/README.md index 535cf4da..44ee1a1f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # SingleStoreDB Spark Connector -## Version: 4.1.3 [![License](http://img.shields.io/:license-Apache%202-brightgreen.svg)](http://www.apache.org/licenses/LICENSE-2.0.txt) +## Version: 4.1.4 [![License](http://img.shields.io/:license-Apache%202-brightgreen.svg)](http://www.apache.org/licenses/LICENSE-2.0.txt) ## Getting Started @@ -13,14 +13,14 @@ spark-packages.org. The group is `com.singlestore` and the artifact is You can add the connector to your Spark application using: spark-shell, pyspark, or spark-submit ``` -$SPARK_HOME/bin/spark-shell --packages com.singlestore:singlestore-spark-connector_2.12:4.1.3-spark-3.1.0 +$SPARK_HOME/bin/spark-shell --packages com.singlestore:singlestore-spark-connector_2.12:4.1.4-spark-3.4.0 ``` We release multiple versions of the `singlestore-spark-connector`, one for each supported Spark version. The connector follows the `x.x.x-spark-y.y.y` naming convention, where `x.x.x` represents the connector version and `y.y.y` represents the corresponding Spark version. -For example, in connector `4.1.3-spark-3.2.1`, 4.1.3 is the version of the connector, -compiled and tested against Spark version 3.2.1. +For example, in connector `4.1.4-spark-3.4.0`, 4.1.4 is the version of the connector, +compiled and tested against Spark version 3.4.0. It is critical to select the connector version that corresponds to the Spark version in use. In addition to adding the `singlestore-spark-connector`, you will also need to have the diff --git a/build.sbt b/build.sbt index 5d3f8846..3139fa41 100644 --- a/build.sbt +++ b/build.sbt @@ -23,7 +23,7 @@ lazy val root = project case "3.3.2" => "scala-sparkv3.3" case "3.4.0" => "scala-sparkv3.4" }), - version := s"4.1.3-spark-${sparkVersion}", + version := s"4.1.4-spark-${sparkVersion}", licenses += "Apache-2.0" -> url( "http://opensource.org/licenses/Apache-2.0" ), @@ -35,7 +35,7 @@ lazy val root = project "org.apache.avro" % "avro" % "1.8.2", "org.apache.commons" % "commons-dbcp2" % "2.7.0", "org.scala-lang.modules" %% "scala-java8-compat" % "0.9.0", - "com.singlestore" % "singlestore-jdbc-client" % "1.1.5", + "com.singlestore" % "singlestore-jdbc-client" % "1.1.8", "io.spray" %% "spray-json" % "1.3.5", "io.netty" % "netty-buffer" % "4.1.70.Final", "org.apache.commons" % "commons-dbcp2" % "2.9.0", diff --git a/demo/Dockerfile b/demo/Dockerfile index a1c20896..84bf1a8b 100644 --- a/demo/Dockerfile +++ b/demo/Dockerfile @@ -1,6 +1,6 @@ FROM apache/zeppelin:0.9.0 -ENV SPARK_VERSION=3.0.3 +ENV SPARK_VERSION=3.4.0 USER root diff --git a/demo/notebook/pyspark-singlestore-demo_2F8XQUKFG.zpln b/demo/notebook/pyspark-singlestore-demo_2F8XQUKFG.zpln index 1ccf84c9..35752c06 100644 --- a/demo/notebook/pyspark-singlestore-demo_2F8XQUKFG.zpln +++ b/demo/notebook/pyspark-singlestore-demo_2F8XQUKFG.zpln @@ -45,7 +45,7 @@ }, { "title": "Configure Spark", - "text": "%spark.conf\n\n// Comma-separated list of Maven coordinates of jars to include on the driver and executor classpaths\nspark.jars.packages com.singlestore:singlestore-spark-connector_2.12:4.1.3-spark-3.0.0\n\n// The hostname or IP address of the SingleStore Master Aggregator in the `host[:port]` format, where port is an optional parameter\n// singlestore-ciab-for-zeppelin - hostname of the docker created by https://hub.docker.com/r/singlestore/cluster-in-a-box\n// 3306 - port on which SingleStore Master Aggregator is started\nspark.datasource.singlestore.ddlEndpoint singlestore-ciab-for-zeppelin:3306\n\n// The hostname or IP address of SingleStore Aggregator nodes to run queries against in the `host[:port],host[:port],...` format, \n// where :port is an optional parameter (multiple hosts separated by comma) (default: ddlEndpoint)\n// Example\n// spark.datasource.singlestore.dmlEndpoints child-agg:3308,child-agg2\nspark.datasource.singlestore.dmlEndpoints singlestore-ciab-for-zeppelin:3306\n\n// SingleStore username (default: root)\nspark.datasource.singlestore.user root\n\n// SingleStore password (default: no password)\nspark.datasource.singlestore.password my_password", + "text": "%spark.conf\n\n// Comma-separated list of Maven coordinates of jars to include on the driver and executor classpaths\nspark.jars.packages com.singlestore:singlestore-spark-connector_2.12:4.1.4-spark-3.4.0\n\n// The hostname or IP address of the SingleStore Master Aggregator in the `host[:port]` format, where port is an optional parameter\n// singlestore-ciab-for-zeppelin - hostname of the docker created by https://hub.docker.com/r/singlestore/cluster-in-a-box\n// 3306 - port on which SingleStore Master Aggregator is started\nspark.datasource.singlestore.ddlEndpoint singlestore-ciab-for-zeppelin:3306\n\n// The hostname or IP address of SingleStore Aggregator nodes to run queries against in the `host[:port],host[:port],...` format, \n// where :port is an optional parameter (multiple hosts separated by comma) (default: ddlEndpoint)\n// Example\n// spark.datasource.singlestore.dmlEndpoints child-agg:3308,child-agg2\nspark.datasource.singlestore.dmlEndpoints singlestore-ciab-for-zeppelin:3306\n\n// SingleStore username (default: root)\nspark.datasource.singlestore.user root\n\n// SingleStore password (default: no password)\nspark.datasource.singlestore.password my_password", "user": "anonymous", "dateUpdated": "2022-07-06 11:26:15.232", "progress": 0, diff --git a/demo/notebook/scala-singlestore-demo_2F6Y3APTX.zpln b/demo/notebook/scala-singlestore-demo_2F6Y3APTX.zpln index 8f57a7f0..2e82cc68 100644 --- a/demo/notebook/scala-singlestore-demo_2F6Y3APTX.zpln +++ b/demo/notebook/scala-singlestore-demo_2F6Y3APTX.zpln @@ -45,7 +45,7 @@ }, { "title": "Configure Spark", - "text": "%spark.conf\n\n// Comma-separated list of Maven coordinates of jars to include on the driver and executor classpaths\nspark.jars.packages com.singlestore:singlestore-spark-connector_2.12:4.1.3-spark-3.0.0\n\n// The hostname or IP address of the SingleStore Master Aggregator in the `host[:port]` format, where port is an optional parameter\n// singlestore-ciab-for-zeppelin - hostname of the docker created by https://hub.docker.com/r/singlestore/cluster-in-a-box\n// 3306 - port on which SingleStore Master Aggregator is started\nspark.datasource.singlestore.ddlEndpoint singlestore-ciab-for-zeppelin:3306\n\n// The hostname or IP address of SingleStore Aggregator nodes to run queries against in the `host[:port],host[:port],...` format, \n// where :port is an optional parameter (multiple hosts separated by comma) (default: ddlEndpoint)\n// Example\n// spark.datasource.singlestore.dmlEndpoints child-agg:3308,child-agg2\nspark.datasource.singlestore.dmlEndpoints singlestore-ciab-for-zeppelin:3306\n\n// SingleStore username (default: root)\nspark.datasource.singlestore.user root\n\n// SingleStore password (default: no password)\nspark.datasource.singlestore.password my_password", + "text": "%spark.conf\n\n// Comma-separated list of Maven coordinates of jars to include on the driver and executor classpaths\nspark.jars.packages com.singlestore:singlestore-spark-connector_2.12:4.1.4-spark-3.4.0\n\n// The hostname or IP address of the SingleStore Master Aggregator in the `host[:port]` format, where port is an optional parameter\n// singlestore-ciab-for-zeppelin - hostname of the docker created by https://hub.docker.com/r/singlestore/cluster-in-a-box\n// 3306 - port on which SingleStore Master Aggregator is started\nspark.datasource.singlestore.ddlEndpoint singlestore-ciab-for-zeppelin:3306\n\n// The hostname or IP address of SingleStore Aggregator nodes to run queries against in the `host[:port],host[:port],...` format, \n// where :port is an optional parameter (multiple hosts separated by comma) (default: ddlEndpoint)\n// Example\n// spark.datasource.singlestore.dmlEndpoints child-agg:3308,child-agg2\nspark.datasource.singlestore.dmlEndpoints singlestore-ciab-for-zeppelin:3306\n\n// SingleStore username (default: root)\nspark.datasource.singlestore.user root\n\n// SingleStore password (default: no password)\nspark.datasource.singlestore.password my_password", "user": "anonymous", "dateUpdated": "2022-07-06 11:31:08.311", "progress": 0, diff --git a/demo/notebook/spark-sql-singlestore-demo_2F7PZ81H6.zpln b/demo/notebook/spark-sql-singlestore-demo_2F7PZ81H6.zpln index e47ea303..ca8ec33c 100644 --- a/demo/notebook/spark-sql-singlestore-demo_2F7PZ81H6.zpln +++ b/demo/notebook/spark-sql-singlestore-demo_2F7PZ81H6.zpln @@ -45,7 +45,7 @@ }, { "title": "Configure Spark", - "text": "%spark.conf\n\n// Comma-separated list of Maven coordinates of jars to include on the driver and executor classpaths\nspark.jars.packages com.singlestore:singlestore-spark-connector_2.12:4.1.3-spark-3.0.0\n\n// The hostname or IP address of the SingleStore Master Aggregator in the `host[:port]` format, where port is an optional parameter\n// singlestore-ciab-for-zeppelin - hostname of the docker created by https://hub.docker.com/r/singlestore/cluster-in-a-box\n// 3306 - port on which SingleStore Master Aggregator is started\nspark.datasource.singlestore.ddlEndpoint singlestore-ciab-for-zeppelin:3306\n\n// The hostname or IP address of SingleStore Aggregator nodes to run queries against in the `host[:port],host[:port],...` format, \n// where :port is an optional parameter (multiple hosts separated by comma) (default: ddlEndpoint)\n// Example\n// spark.datasource.singlestore.dmlEndpoints child-agg:3308,child-agg2\nspark.datasource.singlestore.dmlEndpoints singlestore-ciab-for-zeppelin:3306\n\n// SingleStore username (default: root)\nspark.datasource.singlestore.user root\n\n// SingleStore password (default: no password)\nspark.datasource.singlestore.password my_password", + "text": "%spark.conf\n\n// Comma-separated list of Maven coordinates of jars to include on the driver and executor classpaths\nspark.jars.packages com.singlestore:singlestore-spark-connector_2.12:4.1.4-spark-3.4.0\n\n// The hostname or IP address of the SingleStore Master Aggregator in the `host[:port]` format, where port is an optional parameter\n// singlestore-ciab-for-zeppelin - hostname of the docker created by https://hub.docker.com/r/singlestore/cluster-in-a-box\n// 3306 - port on which SingleStore Master Aggregator is started\nspark.datasource.singlestore.ddlEndpoint singlestore-ciab-for-zeppelin:3306\n\n// The hostname or IP address of SingleStore Aggregator nodes to run queries against in the `host[:port],host[:port],...` format, \n// where :port is an optional parameter (multiple hosts separated by comma) (default: ddlEndpoint)\n// Example\n// spark.datasource.singlestore.dmlEndpoints child-agg:3308,child-agg2\nspark.datasource.singlestore.dmlEndpoints singlestore-ciab-for-zeppelin:3306\n\n// SingleStore username (default: root)\nspark.datasource.singlestore.user root\n\n// SingleStore password (default: no password)\nspark.datasource.singlestore.password my_password", "user": "anonymous", "dateUpdated": "2022-07-06 11:32:22.885", "progress": 0,