Increase testing parallelism

hail-is · May 17, 2018 · 65e2fa4 · 65e2fa4
1 parent 384dbed
commit 65e2fa4
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 22 deletions.
diff --git a/build.gradle b/build.gradle
@@ -183,6 +183,8 @@ task(checkSettings) << {
     System.setProperty("check.count", checkCount)
 }
 
+String parallelism = System.getProperty("test.parallelism", "2")
+
 test {
     useTestNG {}
 
@@ -199,22 +201,22 @@ test {
     beforeTest { descriptor ->
         logger.lifecycle("Running test: " + descriptor)
     }
+
+    maxParallelForks parallelism.toInteger()
 }
 
 test.dependsOn(checkSettings)
 
 task testPython(type: Exec, dependsOn: shadowJar) {
-     commandLine 'python3',
-             '-m',
-             'nose',
-             '-c',
-             'python/hail/setup.cfg',
-             'python/hail/tests'
-     environment SPARK_HOME: sparkHome
-     environment PYTHONPATH: '' + projectDir + '/python:' + sparkHome + '/python:' + sparkHome + '/python/lib/py4j-' + py4jVersion + '-src.zip'
-     environment PYSPARK_SUBMIT_ARGS: '--conf spark.driver.extraClassPath=' + projectDir + '/build/libs/hail-all-spark.jar --conf spark.executor.extraClassPath=' + projectDir + '/build/libs/hail-all-spark.jar pyspark-shell'
-     environment PYSPARK_PYTHON: 'python3'
-     environment NOSE_NOLOGCAPTURE: '1'
+    commandLine 'pytest',
+            '-n',
+            parallelism,
+            '--dist=loadscope',
+            'python/hail/tests'
+    environment SPARK_HOME: sparkHome
+    environment PYTHONPATH: '' + projectDir + '/python:' + sparkHome + '/python:' + sparkHome + '/python/lib/py4j-' + py4jVersion + '-src.zip'
+    environment PYSPARK_SUBMIT_ARGS: '--conf spark.driver.extraClassPath=' + projectDir + '/build/libs/hail-all-spark.jar --conf spark.executor.extraClassPath=' + projectDir + '/build/libs/hail-all-spark.jar pyspark-shell'
+    environment PYSPARK_PYTHON: 'python3'
 }
 
 task testAll(dependsOn: ['testPython', 'test'])

diff --git a/python/hail/dev-environment.yml b/python/hail/dev-environment.yml
@@ -17,3 +17,4 @@ dependencies:
 - pip:
     - parsimonious
     - ipykernel
+    - pytest-xdist
diff --git a/python/hail/environment.yml b/python/hail/environment.yml
@@ -12,4 +12,3 @@ dependencies:
 - pip:
     - parsimonious
     - ipykernel
-
diff --git a/python/hail/tests/test_methods.py b/python/hail/tests/test_methods.py
@@ -909,16 +909,6 @@ def test_pca(self):
 
         _, _, loadings = hl.pca(mt.GT.n_alt_alleles(), k=2, compute_loadings=False)
         self.assertEqual(loadings, None)
-
-    def test_pca_join(self):
-        mt = hl.balding_nichols_model(2, 10, 20)
-        eigenvalues, scores, loadings = hl.pca(mt.GT.n_alt_alleles(), k=1, compute_loadings=True)
-        eigenvalues2 , scores2, loadings2 = hl.pca(
-            mt.GT.n_alt_alleles() * hl.literal([1])[0], k=1, compute_loadings=True)
-
-        self.assertAlmostEqual(eigenvalues[0], eigenvalues2[0])
-        self.assertTrue(scores._same(scores2))
-        self.assertTrue(loadings._same(loadings2))
 
     def _R_pc_relate(self, mt, maf):
         plink_file = utils.uri_path(utils.new_temp_file())