Merge pull request #77 from holdenk/build-spark-mllib-with-spark-version

Build spark mllib tests with spark version
databricks · Aug 5, 2015 · 03f2ce3 · 03f2ce3
2 parents 9a48aaf + 1cb45d3
commit 03f2ce3
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 5 deletions.
diff --git a/lib/sparkperf/main.py b/lib/sparkperf/main.py
@@ -121,7 +121,7 @@
         "tests, but %s was not already present") % StreamingTests.test_jar_path
 
 if should_prep_mllib_tests:
-    MLlibTests.build()
+    MLlibTests.build(config.MLLIB_SPARK_VERSION)
 elif run_mllib_tests:
     assert MLlibTests.is_built(), ("You tried to skip packaging the MLlib perf " +
         "tests, but %s was not already present") % MLlibTests.test_jar_path

diff --git a/lib/sparkperf/testsuites.py b/lib/sparkperf/testsuites.py
@@ -251,8 +251,8 @@ class MLlibTests(JVMPerfTestSuite, MLlibTestHelper):
     test_jar_path = "%s/mllib-tests/target/mllib-perf-tests-assembly.jar" % PROJ_DIR
 
     @classmethod
-    def build(cls):
-        run_cmd("cd %s/mllib-tests; %s clean assembly" % (PROJ_DIR, SBT_CMD))
+    def build(cls, spark_version):
+        run_cmd("cd %s/mllib-tests; %s -Dspark.version=%s.0 clean assembly" % (PROJ_DIR, SBT_CMD, spark_version))
 
     @classmethod
     def process_output(cls, config, short_name, opt_list, stdout_filename, stderr_filename):

diff --git a/mllib-tests/project/MLlibTestsBuild.scala b/mllib-tests/project/MLlibTestsBuild.scala
@@ -17,7 +17,7 @@ object MLlibTestsBuild extends Build {
     organization := "org.spark-project",
     version := "0.1",
     scalaVersion := "2.10.4",
-    sparkVersion := sys.props.get("spark.version").getOrElse("1.3.1"),
+    sparkVersion := sys.props.getOrElse("spark.version", default="1.3.1"),
     libraryDependencies ++= Seq(
       "net.sf.jopt-simple" % "jopt-simple" % "4.6",
       "org.scalatest" %% "scalatest" % "2.2.1" % "test",
@@ -38,7 +38,7 @@ object MLlibTestsBuild extends Build {
           case v if v.startsWith("1.2.") => "v1p2"
           case v if v.startsWith("1.3.") => "v1p3"
           case v if v.startsWith("1.4.") => "v1p4"
-          case _ => throw new IllegalArgumentException(s"Do not support Spark $sparkVersion.")
+          case _ => throw new IllegalArgumentException(s"Do not support Spark ${sparkVersion.value}.")
         }
         baseDirectory.value / targetFolder / "src" / "main" / "scala"
       },