Skip MLLib tests for PyPy

dev/sparktestsupport/modules.py

@@ -30,7 +30,8 @@ class Module(object):
     """
 
     def __init__(self, name, dependencies, source_file_regexes, build_profile_flags=(),
-                 sbt_test_goals=(), python_test_goals=(), should_run_r_tests=False):
+                 sbt_test_goals=(), python_test_goals=(), blacklisted_python_implementations=(),
+                 should_run_r_tests=False):
         """
         Define a new module.
 
@@ -44,6 +45,9 @@ def __init__(self, name, dependencies, source_file_regexes, build_profile_flags=
             order to build and test this module (e.g. '-PprofileName').
         :param sbt_test_goals: A set of SBT test goals for testing this module.
         :param python_test_goals: A set of Python test goals for testing this module.
+        :param blacklisted_python_implementations: A set of Python implementations that are not
+            supported by this module's Python components. The values in this set should match
+            strings returned by Python's `platform.python_implementation()`.
         :param should_run_r_tests: If true, changes in this module will trigger all R tests.
         """
         self.name = name
@@ -52,6 +56,7 @@ def __init__(self, name, dependencies, source_file_regexes, build_profile_flags=
         self.sbt_test_goals = sbt_test_goals
         self.build_profile_flags = build_profile_flags
         self.python_test_goals = python_test_goals
+        self.blacklisted_python_implementations = blacklisted_python_implementations
         self.should_run_r_tests = should_run_r_tests
 
         self.dependent_modules = set()
@@ -308,6 +313,9 @@ def contains_file(self, filename):
         "pyspark.mllib.tree",
         "pyspark.mllib.util",
         "pyspark.mllib.tests",
+    ],
+    blacklisted_python_implementations=[
+        "PyPy"  # Skip these tests under PyPy since they require numpy and it isn't available there
     ]
 )
 
@@ -326,6 +334,9 @@ def contains_file(self, filename):
         "pyspark.ml.tuning",
         "pyspark.ml.tests",
         "pyspark.ml.evaluation",
+    ],
+    blacklisted_python_implementations=[
+        "PyPy"  # Skip these tests under PyPy since they require numpy and it isn't available there
     ]
 )
 

python/run-tests.py

@@ -46,7 +46,7 @@ def print_red(text):
 
 
 def run_individual_python_test(test_name, pyspark_python):
-    env = {'SPARK_TESTING': '1', 'PYSPARK_PYTHON': which(pyspark_python) }
+    env = {'SPARK_TESTING': '1', 'PYSPARK_PYTHON': which(pyspark_python)}
     print("    Running test: %s ..." % test_name, end='')
     start_time = time.time()
     with open(LOG_FILE, 'a') as log_file:
@@ -113,13 +113,16 @@ def main():
 
     start_time = time.time()
     for python_exec in python_execs:
+        python_implementation = subprocess.check_output(
+            [python_exec, "-c", "import platform; print platform.python_implementation()"]).strip()
         print("Testing with `%s`: " % python_exec, end='')
         subprocess.call([python_exec, "--version"])
 
         for module in modules_to_test:
-            print("Running %s tests ..." % module.name)
-            for test_goal in module.python_test_goals:
-                run_individual_python_test(test_goal, python_exec)
+            if python_implementation not in module.blacklisted_python_implementations:
+                print("Running %s tests ..." % module.name)
+                for test_goal in module.python_test_goals:
+                    run_individual_python_test(test_goal, python_exec)
     total_duration = time.time() - start_time
     print("Tests passed in %i seconds" % total_duration)