Fix serialization fuzzing error

microsoft · Jul 5, 2019 · 07316a8 · 07316a8
1 parent f6df907
commit 07316a8
Show file tree

Hide file tree

Showing 5 changed files with 11 additions and 3 deletions.
diff --git a/build.sbt b/build.sbt
@@ -19,7 +19,7 @@ def getPythonVersion(baseVersion: String): String = {
 
 val baseVersion = "0.17.1"
 val condaEnvName = "mmlspark"
-name := "mmlspark-build"
+name := "mmlspark"
 organization := "com.microsoft.ml.spark"
 version := getVersion(baseVersion)
 scalaVersion := "2.11.12"

diff --git a/src/it/scala/com/microsoft/ml/spark/codegen/CodegenConfig.scala b/src/it/scala/com/microsoft/ml/spark/codegen/CodegenConfig.scala
@@ -12,6 +12,7 @@ object Config {
 
   val topDir = BuildInfo.baseDirectory
   val version = BuildInfo.version
+  val packageName = BuildInfo.name
   val targetDir = new File(topDir, "target/scala-2.11")
   val scalaSrcDir = "src/main/scala"
 

diff --git a/src/it/scala/com/microsoft/ml/spark/codegen/PySparkWrapperTest.scala b/src/it/scala/com/microsoft/ml/spark/codegen/PySparkWrapperTest.scala
@@ -53,7 +53,7 @@ abstract class PySparkWrapperParamsTest(entryPoint: Params,
         |spark = SparkSession.builder \\
         |    .master("local[*]") \\
         |    .appName("$entryPointName") \\
-        |    .config("spark.jars.packages", "com.microsoft.ml.spark:mmlspark-build_2.11:$version") \\
+        |    .config("spark.jars.packages", "com.microsoft.ml.spark:${packageName}_2.11:$version") \\
         |    .config("spark.executor.heartbeatInterval", "60s") \\
         |    .getOrCreate()
         |

diff --git a/src/test/python/mmlspark/recommendation/test_ranking.py b/src/test/python/mmlspark/recommendation/test_ranking.py
@@ -18,7 +18,7 @@
 spark = SparkSession.builder \
     .master("local[*]") \
     .appName("_FindBestModel") \
-    .config("spark.jars.packages", "com.microsoft.ml.spark:mmlspark-build_2.11:" + os.environ["MML_VERSION"]) \
+    .config("spark.jars.packages", "com.microsoft.ml.spark:mmlspark_2.11:" + os.environ["MML_VERSION"]) \
     .config("spark.executor.heartbeatInterval", "60s") \
     .getOrCreate()
 

diff --git a/src/test/scala/com/microsoft/ml/spark/cognitive/SpeechSuite.scala b/src/test/scala/com/microsoft/ml/spark/cognitive/SpeechSuite.scala
@@ -10,6 +10,8 @@ import com.microsoft.ml.spark.core.test.fuzzing.{TestObject, TransformerFuzzing}
 import org.apache.commons.compress.utils.IOUtils
 import org.apache.spark.ml.util.MLReadable
 import org.apache.spark.sql.DataFrame
+import org.scalactic.Equality
+import org.scalatest.Assertion
 
 trait SpeechKey {
   lazy val speechKey = sys.env.getOrElse("SPEECH_API_KEY", Secrets.speechApiKey)
@@ -35,6 +37,11 @@ class SpeechToTextSuite extends TransformerFuzzing[SpeechToText]
     Tuple1(audioBytes)
   ).toDF("audio")
 
+  override lazy val dfEq = new Equality[DataFrame] {
+    override def areEqual(a: DataFrame, b: Any): Boolean =
+      baseDfEq.areEqual(a.drop("audio"), b.asInstanceOf[DataFrame].drop("audio"))
+  }
+
   test("Basic Usage") {
     val toObj = SpeechResponse.makeFromRowConverter
     val result = toObj(stt.setFormat("simple")