diff --git a/build.sbt b/build.sbt index d89c428..28ccac8 100644 --- a/build.sbt +++ b/build.sbt @@ -1,12 +1,12 @@ name := "spark-tests" -version := "2.3.0-SNAPSHOT" +version := "2.3.0" deps ++= Seq( paths % "1.3.1", scalatest, spark, - spark_util % "2.0.0-SNAPSHOT", + spark_util % "2.0.0", testUtils ) diff --git a/project/plugins.sbt b/project/plugins.sbt index 686674b..48597a1 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1 +1 @@ -addSbtPlugin("org.hammerlab" % "sbt-parent" % "3.1.1") +addSbtPlugin("org.hammerlab" % "sbt-parent" % "3.3.0") diff --git a/src/main/scala/org/hammerlab/spark/test/suite/KryoSparkSuite.scala b/src/main/scala/org/hammerlab/spark/test/suite/KryoSparkSuite.scala index bfba002..f97417b 100644 --- a/src/main/scala/org/hammerlab/spark/test/suite/KryoSparkSuite.scala +++ b/src/main/scala/org/hammerlab/spark/test/suite/KryoSparkSuite.scala @@ -1,8 +1,6 @@ package org.hammerlab.spark.test.suite -import org.apache.spark.serializer.KryoRegistrator -import org.hammerlab.kryo.spark.Registrator -import org.hammerlab.spark.confs +import org.hammerlab.spark.SelfRegistrar /** * Base for test-suites that rely on Kryo serialization, including registering classes for serialization in a @@ -11,7 +9,4 @@ import org.hammerlab.spark.confs class KryoSparkSuite(override val registrationRequired: Boolean = true, override val referenceTracking: Boolean = false) extends SparkSuite - with Registrator - with confs.Kryo { - override def registrar: Class[_ <: KryoRegistrator] = getClass -} + with SelfRegistrar diff --git a/src/main/scala/org/hammerlab/spark/test/suite/MainSuite.scala b/src/main/scala/org/hammerlab/spark/test/suite/MainSuite.scala index a065d66..bb1d2c1 100644 --- a/src/main/scala/org/hammerlab/spark/test/suite/MainSuite.scala +++ b/src/main/scala/org/hammerlab/spark/test/suite/MainSuite.scala @@ -2,7 +2,6 @@ package org.hammerlab.spark.test.suite import java.lang.System.{ clearProperty, getProperty, setProperty } -import org.apache.spark.serializer.KryoRegistrator import org.hammerlab.spark.{ SparkConfBase, confs } import org.hammerlab.test.Suite @@ -14,7 +13,7 @@ import scala.collection.mutable * * Sets Spark configuration settings, including Kryo-serde with required registration, through system properties. */ -class MainSuite(override val registrar: Class[_ <: KryoRegistrator] = null) +class MainSuite extends Suite with SparkConfBase with TestConfs diff --git a/src/main/scala/org/hammerlab/spark/test/suite/SparkSuite.scala b/src/main/scala/org/hammerlab/spark/test/suite/SparkSuite.scala index 7b4847d..65b2ca2 100644 --- a/src/main/scala/org/hammerlab/spark/test/suite/SparkSuite.scala +++ b/src/main/scala/org/hammerlab/spark/test/suite/SparkSuite.scala @@ -5,7 +5,7 @@ import org.hammerlab.test.Suite /** * Base for test suites that shar one [[org.apache.spark.SparkContext]] across all test cases. */ -trait SparkSuite +abstract class SparkSuite extends Suite with SparkSuiteBase { diff --git a/src/test/scala/org/hammerlab/spark/test/suite/KryoSparkSuiteTest.scala b/src/test/scala/org/hammerlab/spark/test/suite/KryoSparkSuiteTest.scala index 88c1ded..19adf81 100644 --- a/src/test/scala/org/hammerlab/spark/test/suite/KryoSparkSuiteTest.scala +++ b/src/test/scala/org/hammerlab/spark/test/suite/KryoSparkSuiteTest.scala @@ -3,6 +3,7 @@ package org.hammerlab.spark.test.suite import com.esotericsoftware.kryo.io.{ Input, Output } import com.esotericsoftware.kryo.{ Kryo, Serializer } import org.apache.spark.SparkException +import org.hammerlab.kryo._ import scala.collection.mutable @@ -11,10 +12,9 @@ class KryoSparkSuiteTest with SparkSerialization { register( - classOf[mutable.WrappedArray.ofRef[_]], - classOf[Foo] → new FooSerializer, - classOf[Array[Foo]], - classOf[Bar] + cls[mutable.WrappedArray.ofRef[_]], + arr[Foo], + cls[Bar] ) test("spark job custom serializer") { @@ -35,11 +35,14 @@ class KryoSparkSuiteTest case class Foo(n: Int, s: String) -case class FooException() extends Exception - -class FooSerializer extends Serializer[Foo] { - override def write(kryo: Kryo, output: Output, `object`: Foo): Unit = throw FooException() - override def read(kryo: Kryo, input: Input, `type`: Class[Foo]): Foo = ??? +object Foo { + implicit val serializer: Serializer[Foo] = + new Serializer[Foo] { + override def write(kryo: Kryo, out: Output, foo: Foo): Unit = throw FooException() + override def read(kryo: Kryo, in: Input, cls: Class[Foo]): Foo = ??? + } } +case class FooException() extends Exception + case class Bar(n: Int, s: String)