DSP-13218 DSE 5.1.1 support

README.md

@@ -51,7 +51,7 @@ run (Java)   | `dse spark-submit --class com.datastax.spark.example.WriteRead ta
 
 Task                | Command
 --------------------|------------
-build               | `gradle build`
+build               | `gradle shadowJar`
 run (Scala, Java)   | `dse spark-submit --class com.datastax.spark.example.WriteRead build/libs/writeRead-0.1-all.jar`
 
 ### Maven

java/gradle/dse/build.gradle

@@ -13,7 +13,7 @@ repositories {
     }
 }
 
-def dseVersion = "5.1.0"
+def dseVersion = "5.1.1"
 
 // The assembly configuration will cause jar to be included in assembled fat-jar
 configurations {
@@ -31,14 +31,9 @@ configurations {
 // Please make sure that following dependencies have versions corresponding to the ones in your cluster.
 // Note that spark-cassandra-connector should be provided with '--packages' flag to spark-submit command.
 dependencies {
-    provided("com.datastax.dse:dse-spark-dependencies:$dseVersion") {
-        exclude group: 'com.datastax.dse', module: 'dse-java-driver-core'
-        exclude group: 'org.apache.solr', module: 'solr-solrj'
-    }
+    provided "com.datastax.dse:dse-spark-dependencies:$dseVersion"
 //    assembly "org.apache.commons:commons-math3:3.6.1"
 //    assembly "org.apache.commons:commons-csv:1.0"
-    provided "com.datastax.dse:dse-java-driver-core:1.2.3"
-    provided "org.apache.solr:solr-solrj:6.0.1"
 }
 
 shadowJar {

java/gradle/oss/build.gradle

@@ -10,8 +10,8 @@ repositories {
     mavenCentral()
 }
 
-def sparkVersion = "2.0.0"
-def connectorVersion = "2.0.1"
+def sparkVersion = "2.0.2"
+def connectorVersion = "2.0.2"
 
 // The assembly configuration will cause jar to be included in assembled fat-jar
 configurations {

java/maven/dse/pom.xml

@@ -9,7 +9,7 @@
 
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    <dse.version>5.1.0</dse.version>
+    <dse.version>5.1.1</dse.version>
   </properties>
 
   <dependencies>
@@ -18,16 +18,6 @@
       <artifactId>dse-spark-dependencies</artifactId>
       <version>${dse.version}</version>
       <scope>provided</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>com.datastax.dse</groupId>
-          <artifactId>dse-java-driver-core</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.solr</groupId>
-          <artifactId>solr-solrj</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <!-- Your dependencies, 'provided' are not included in jar -->
     <!--<dependency>-->
@@ -40,16 +30,6 @@
       <!--<artifactId>commons-csv</artifactId>-->
       <!--<version>1.0</version>-->
     <!--</dependency>-->
-    <dependency>
-      <groupId>com.datastax.dse</groupId>
-      <artifactId>dse-java-driver-core</artifactId>
-      <version>1.2.3</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.solr</groupId>
-      <artifactId>solr-solrj</artifactId>
-      <version>6.0.1</version>
-    </dependency>
   </dependencies>
 
   <repositories>

java/maven/oss/pom.xml

@@ -9,8 +9,8 @@
 
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    <spark.version>2.0.0</spark.version>
-    <connector.version>2.0.1</connector.version>
+    <spark.version>2.0.2</spark.version>
+    <connector.version>2.0.2</connector.version>
   </properties>
 
   <!--

java/sbt/dse/build.sbt

@@ -10,28 +10,17 @@ scalaVersion := "2.11.8"
 
 resolvers += "DataStax Repo" at "https://repo.datastax.com/public-repos/"
 
-val dseVersion = "5.1.0"
+val dseVersion = "5.1.1"
 
 // Please make sure that following DSE version matches your DSE cluster version.
 // SBT 0.13.13 or greater required because of a dependency resolution bug
-libraryDependencies ++= Seq(
-  "com.datastax.dse" % "dse-spark-dependencies" % dseVersion % "provided"  excludeAll (
-    ExclusionRule("com.datastax.dse", "dse-java-driver-core"),
-    ExclusionRule("org.apache.solr", "solr-solrj")
-  ),
-  "com.datastax.dse" % "dse-java-driver-core" % "1.2.3",
-  "org.apache.solr" % "solr-solrj" % "6.0.1"
-)
+libraryDependencies += "com.datastax.dse" % "dse-spark-dependencies" % dseVersion % "provided"
 
 //Your dependencies
 //libraryDependencies += "org.apache.commons" % "commons-math3" % "3.6.1"
 //libraryDependencies += "org.apache.commons" % "commons-csv" % "1.0"
 
-assemblyMergeStrategy in assembly := {
-  case "META-INF/io.netty.versions.properties" => MergeStrategy.discard
-  case x => (assemblyMergeStrategy in assembly).value(x)
-}
-
+assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false)
 //assemblyShadeRules in assembly := Seq(
 //  ShadeRule.rename("org.apache.commons.csv.**" -> "shaded.org.apache.commons.csv.@1").inAll
 //)

java/sbt/dse/project/assembly.sbt

@@ -1 +1 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.3")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.5")

java/sbt/oss/build.sbt

@@ -8,8 +8,8 @@ autoScalaLibrary := false
 
 scalaVersion := "2.11.8"
 
-val sparkVersion = "2.0.0"
-val connectorVersion = "2.0.1"
+val sparkVersion = "2.0.2"
+val connectorVersion = "2.0.2"
 
 // Please make sure that following dependencies have versions corresponding to the ones in your cluster.
 // Note that spark-cassandra-connector should be provided with '--packages' flag to spark-submit command.
@@ -24,6 +24,7 @@ libraryDependencies ++= Seq(
 //libraryDependencies += "org.apache.commons" % "commons-math3" % "3.6.1"
 //libraryDependencies += "org.apache.commons" % "commons-csv" % "1.0"
 
+assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false)
 //assemblyShadeRules in assembly := Seq(
 //  ShadeRule.rename("org.apache.commons.csv.**" -> "shaded.org.apache.commons.csv.@1").inAll
 //)

java/sbt/oss/project/assembly.sbt

@@ -1 +1 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.3")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.5")

scala/gradle/dse/build.gradle

@@ -28,11 +28,11 @@ configurations {
     testCompile.exclude group: 'org.slf4j', module: 'log4j-over-slf4j'
 }
 
-def dseVersion = "5.1.0"
+def dseVersion = "5.1.1"
 
 def scalaVersion = "2.11"
 def scalaTestVersion = "3.0.0"
-def connectorVersion = "2.0.1"
+def connectorVersion = "2.0.2"
 def cassandraVersion = "3.2"
 def jUnitVersion = "4.12"
 
@@ -41,13 +41,9 @@ dependencies {
     provided("com.datastax.dse:dse-spark-dependencies:$dseVersion") {
         exclude group: 'org.slf4j', module: 'slf4j-log4j12'
         exclude group: 'org.apache.cassandra'
-        exclude group: 'com.datastax.dse', module: 'dse-java-driver-core'
-        exclude group: 'org.apache.solr', module: 'solr-solrj'
     }
 //  assembly "org.apache.commons:commons-math3:3.6.1"
 //  assembly "org.apache.commons:commons-csv:1.0"
-    provided "com.datastax.dse:dse-java-driver-core:1.2.3"
-    provided "org.apache.solr:solr-solrj:6.0.1"
 
     // Test Dependencies
     testCompile "com.datastax.spark:spark-cassandra-connector-embedded_$scalaVersion:$connectorVersion"

scala/gradle/oss/build.gradle

@@ -10,8 +10,8 @@ repositories {
     mavenCentral()
 }
 
-def sparkVersion = "2.0.0"
-def connectorVersion = "2.0.1"
+def sparkVersion = "2.0.2"
+def connectorVersion = "2.0.2"
 def scalaVersion = "2.11"
 def scalaTestVersion = "3.0.0"
 def cassandraVersion = "3.2"

scala/maven/dse/pom.xml

@@ -9,11 +9,11 @@
 
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    <dse.version>5.1.0</dse.version>
+    <dse.version>5.1.1</dse.version>
     <scala.version>2.11.8</scala.version>
     <scala.main.version>2.11</scala.main.version>
     <scalatest.version>3.0.0</scalatest.version>
-    <connector.version>2.0.1</connector.version>
+    <connector.version>2.0.2</connector.version>
     <cassandra.version>3.2</cassandra.version>
     <junit.version>4.12</junit.version>
   </properties>
@@ -33,14 +33,6 @@
           <groupId>org.apache.cassandra</groupId>
           <artifactId>*</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>com.datastax.dse</groupId>
-          <artifactId>dse-java-driver-core</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.solr</groupId>
-          <artifactId>solr-solrj</artifactId>
-        </exclusion>
       </exclusions>
     </dependency>
     <!-- Your dependencies, 'provided' are not included in jar -->
@@ -54,16 +46,7 @@
       <!--<artifactId>commons-csv</artifactId>-->
       <!--<version>1.0</version>-->
     <!--</dependency>-->
-    <dependency>
-      <groupId>com.datastax.dse</groupId>
-      <artifactId>dse-java-driver-core</artifactId>
-      <version>1.2.3</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.solr</groupId>
-      <artifactId>solr-solrj</artifactId>
-      <version>6.0.1</version>
-    </dependency>
+
     <!-- Test Dependencies -->
     <dependency>
       <groupId>com.datastax.spark</groupId>

scala/maven/oss/pom.xml

@@ -11,9 +11,9 @@
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <scala.version>2.11.8</scala.version>
     <scala.main.version>2.11</scala.main.version>
-    <spark.version>2.0.0</spark.version>
+    <spark.version>2.0.2</spark.version>
     <scalatest.version>3.0.0</scalatest.version>
-    <connector.version>2.0.1</connector.version>
+    <connector.version>2.0.2</connector.version>
     <cassandra.version>3.2</cassandra.version>
     <junit.version>4.12</junit.version>
   </properties>
@@ -27,6 +27,7 @@
       <groupId>org.scala-lang</groupId>
       <artifactId>scala-library</artifactId>
       <version>${scala.version}</version>
+      <scope>provided</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>

scala/sbt/dse/build.sbt

@@ -7,35 +7,33 @@ scalaVersion := "2.11.8"
 
 resolvers += "DataStax Repo" at "https://repo.datastax.com/public-repos/"
 
-val dseVersion = "5.1.0"
+val dseVersion = "5.1.1"
 
 // Please make sure that following DSE version matches your DSE cluster version.
 // Exclusions are solely for running integrated testing
 // Warning Sbt 0.13.13 or greater is required due to a bug with dependency resolution
-libraryDependencies += "com.datastax.dse" % "dse-spark-dependencies" % dseVersion % "provided" excludeAll (
-    ExclusionRule("org.slf4j","slf4j-log4j12"),
-    ExclusionRule("org.mortbay.jetty"),
-    ExclusionRule("javax.servlet"),
-    ExclusionRule("org.apache.cassandra"),
-    ExclusionRule("com.datastax.dse", "dse-java-driver-core"),
-    ExclusionRule("org.apache.solr", "solr-solrj")
+libraryDependencies += (
+  "com.datastax.dse" % "dse-spark-dependencies" % dseVersion % "provided" excludeAll (
+      ExclusionRule("org.slf4j","slf4j-log4j12"),
+      ExclusionRule("org.mortbay.jetty"),
+      ExclusionRule("javax.servlet"),
+      ExclusionRule("org.apache.cassandra")
+    )
   )
 
 // Test Dependencies
 // The 'test/resources' Directory in should match the resources directory in the `it` directory
 // for the version of the Spark Cassandra Connector in use.
 val scalaTestVersion = "3.0.0"
-val connectorVersion = "2.0.1"
+val connectorVersion = "2.0.2"
 val jUnitVersion = "4.12"
 val cassandraVersion = "3.2"
 
 libraryDependencies ++= Seq(
-  "com.datastax.dse" % "dse-java-driver-core" % "1.2.3",
-  "org.apache.solr" % "solr-solrj" % "6.0.1",
   "com.datastax.spark" %% "spark-cassandra-connector-embedded" % connectorVersion % "test",
   "org.apache.cassandra" % "cassandra-all" % cassandraVersion % "test",
   "org.scalatest" %% "scalatest" % scalaTestVersion % "test",
-  "junit" % "junit" % "4.12"
+  "junit" % "junit" % "4.12" % "test"
 ).map(_.excludeAll(
   ExclusionRule("org.slf4j","log4j-over-slf4j"),
   ExclusionRule("org.slf4j","slf4j-log4j12"))
@@ -44,15 +42,12 @@ libraryDependencies ++= Seq(
 //Forking is required for the Embedded Cassandra
 fork in Test := true
 
-assemblyMergeStrategy in assembly := {
-  case "META-INF/io.netty.versions.properties" => MergeStrategy.discard
-  case x => (assemblyMergeStrategy in assembly).value(x)
-}
 
 //Your dependencies
 //libraryDependencies += "org.apache.commons" % "commons-math3" % "3.6.1"
 //libraryDependencies += "org.apache.commons" % "commons-csv" % "1.0"
 
+assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false)
 //assemblyShadeRules in assembly := Seq(
 //  ShadeRule.rename("org.apache.commons.csv.**" -> "shaded.org.apache.commons.csv.@1").inAll
 //)

scala/sbt/dse/project/assembly.sbt

@@ -1 +1 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.3")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.5")

scala/sbt/oss/build.sbt

@@ -4,8 +4,8 @@ version := "0.1"
 
 scalaVersion := "2.11.8"
 
-val sparkVersion = "2.0.0"
-val connectorVersion = "2.0.1"
+val sparkVersion = "2.0.2"
+val connectorVersion = "2.0.2"
 //The 'test/resources' Directory in  should match the resources directory in the `it` directory
 //for the version of the Spark Cassandra Connector in use.
 
@@ -38,6 +38,7 @@ fork in Test := true
 //libraryDependencies += "org.apache.commons" % "commons-math3" % "3.6.1"
 //libraryDependencies += "org.apache.commons" % "commons-csv" % "1.0"
 
+assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false)
 //assemblyShadeRules in assembly := Seq(
 //  ShadeRule.rename("org.apache.commons.csv.**" -> "shaded.org.apache.commons.csv.@1").inAll
 //)

scala/sbt/oss/project/assembly.sbt

@@ -1 +1 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.3")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.5")