Boosting Spark to 2.1.0. Adding tests that include Java types encoded …

build.sbt

@@ -1,4 +1,4 @@
-val sparkVersion = "2.0.2"
+val sparkVersion = "2.1.0"
 val catsv = "0.9.0"
 val scalatest = "3.0.1"
 val shapeless = "2.3.2"

core/src/main/scala/frameless/Injection.scala

@@ -1,17 +1,21 @@
 package frameless
 
-/** An Injection[A, B] is a reverible function from A to B.
+import scala.annotation.implicitNotFound
+
+/** An Injection[A, B] is a reversible function from A to B.
   *
   * Must obey `forAll { a: A => invert(apply(a)) == a }`.
   */
+@implicitNotFound(msg = "Cannot find Injection from ${A} to ${B}. Try bringing Frameless' implicit Injections in scope via 'import frameless.implicits.injections._'")
 trait Injection[A, B] extends Serializable {
   def apply(a: A): B
   def invert(b: B): A
 }
 
 object Injection {
+  def apply[A, B]()(implicit i: Injection[A,B]): Injection[A, B] = i
   def apply[A, B](f: A => B, g: B => A): Injection[A, B] = new Injection[A, B] {
     def apply(a: A): B = f(a)
     def invert(b: B): A = g(b)
   }
-}
+}

dataset/src/main/scala/frameless/implicits.scala

@@ -32,4 +32,12 @@ object implicits {
     // implicit def floatToDouble[T](col: TypedColumn[T, Float]): TypedColumn[T, Double] = col.cast[Double]
     // implicit def floatToBigDecimal[T](col: TypedColumn[T, Float]): TypedColumn[T, BigDecimal] = col.cast[BigDecimal]
   }
+
+  object injections {
+    implicit val javaBoolean: Injection[java.lang.Boolean, Boolean] = Injection(a => a, b => b)
+    implicit val javaInt: Injection[java.lang.Integer, Int] = Injection(a => a, b => b)
+    implicit val javaDouble: Injection[java.lang.Double, Double] = Injection(a => a, b => b)
+    implicit val javaFloat: Injection[java.lang.Float, Float] = Injection(a => a, b => b)
+    implicit val javaLong: Injection[java.lang.Long, Long] = Injection(a => a, b => b)
+  }
 }

dataset/src/test/scala/frameless/CollectTests.scala

@@ -7,6 +7,8 @@ import org.scalacheck.Prop._
 import scala.reflect.ClassTag
 
 class CollectTests extends TypedDatasetSuite {
+  import implicits.injections._
+
   test("collect()") {
     check(forAll(prop[X2[Int, Int]] _))
     check(forAll(prop[X2[String, String]] _))
@@ -38,7 +40,9 @@ class CollectTests extends TypedDatasetSuite {
     check(forAll(prop[Option[SQLDate]] _))
     check(forAll(prop[Option[SQLTimestamp]] _))
 
-    check(forAll(prop[Vector[Int]] _))
+    check(forAll(prop[Vector[Boolean]] _))
+    check(forAll(prop[Vector[java.lang.Boolean]] _))
+    check(forAll(prop[Vector[(java.lang.Float, Option[java.lang.Long])]] _))
     check(forAll(prop[Option[Int]] _))
     check(forAll(prop[Vector[X2[Int, Int]]] _))
 

dataset/src/test/scala/frameless/InjectionTests.scala

@@ -41,7 +41,7 @@ object Food {
     )
 }
 
-// Supposingly coming from a java lib, shapeless can't derive stuff for this one :(
+// Supposedly coming from a java lib, shapeless can't derive stuff for this one :(
 class LocalDateTime {
   var instant: Long = _
 

dataset/src/test/scala/frameless/TypedDatasetSuite.scala

@@ -3,10 +3,11 @@ package frameless
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.SparkSession
 import org.scalactic.anyvals.PosZInt
+import org.scalacheck.Arbitrary
 import org.scalatest.FunSuite
 import org.scalatest.prop.Checkers
 
-class TypedDatasetSuite extends FunSuite with Checkers {
+class TypedDatasetSuite extends FunSuite with Checkers with JavaTypeArbitraries {
   // Limit size of generated collections and number of checks because Travis
   implicit override val generatorDrivenConfig =
     PropertyCheckConfiguration(sizeRange = PosZInt(10), minSize = PosZInt(10))
@@ -23,3 +24,19 @@ class TypedDatasetSuite extends FunSuite with Checkers {
   implicit def sc = session.sparkContext
   implicit def sqlContext = session.sqlContext
 }
+
+trait JavaTypeArbitraries {
+  import implicits.injections._
+
+  implicit val arbitraryJavaBoolean =
+    Arbitrary(Arbitrary.arbitrary[Boolean].map(Injection[java.lang.Boolean, Boolean].invert))
+  implicit val arbitraryJavaInteger =
+      Arbitrary(Arbitrary.arbitrary[Int].map(Injection[java.lang.Integer, Int].invert))
+  implicit val arbitraryJavaLong =
+      Arbitrary(Arbitrary.arbitrary[Long].map(Injection[java.lang.Long, Long].invert))
+  implicit val arbitraryJavaDouble =
+    Arbitrary(Arbitrary.arbitrary[Double].map(Injection[java.lang.Double, Double].invert))
+  implicit val arbitraryJavaFloat =
+    Arbitrary(Arbitrary.arbitrary[Float].map(Injection[java.lang.Float, Float].invert))
+}
+

dataset/src/test/scala/frameless/functions/UdfTests.scala

@@ -83,7 +83,8 @@ class UdfTests extends TypedDatasetSuite {
       (dataset21 ?= d) && (dataset22 ?= d)
     }
 
-    check(forAll(prop[Int, Int, Int] _))
+    check(forAll(prop[Vector[X4[Long, X1[Int], Vector[X2[X1[Long], Vector[X1[Int]]]], Long]], Vector[Int],
+      X3[X1[Long], Vector[Char], String]] _))
     check(forAll(prop[String, Int, Int] _))
   }