Merge pull request #8059 from apache/revert-7675-py3-tox-2

Revert "[BEAM-6527] Use Gradle to parallel Python tox tests"

build.gradle

@@ -191,10 +191,7 @@ task goIntegrationTests() {
 }
 
 task pythonPreCommit() {
-  dependsOn ":beam-sdks-python:preCommitPy2"
-  dependsOn ":beam-sdks-python-test-suites-tox-py35:preCommitPy35"
-  dependsOn ":beam-sdks-python-test-suites-tox-py36:preCommitPy36"
-  dependsOn ":beam-sdks-python-test-suites-dataflow:preCommitIT"
+  dependsOn ":beam-sdks-python:preCommit"
 }
 
 task pythonPostCommit() {

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -1611,38 +1611,16 @@ class BeamModulePlugin implements Plugin<Project> {
         outputs.dirs(project.ext.envdir)
       }
 
-      def pythonSdkDeps = project.files(
-              project.fileTree(
-              dir: "${project.rootDir}",
-              include: ['model/**', 'sdks/python/**'],
-              // Exclude temporary directories used in build and test.
-              exclude: [
-                'sdks/python/build/**',
-                'sdks/python/dist/**',
-                'sdks/python/target/**',
-                'sdks/python/test-suites/**',
-              ])
-              )
-      def copiedSrcRoot = "${project.buildDir}/srcs"
-
       project.configurations { distConfig }
 
       project.task('sdist', dependsOn: 'setupVirtualenv') {
         doLast {
-          // Copy sdk sources to an isolated directory
-          project.copy {
-            from pythonSdkDeps
-            into copiedSrcRoot
-          }
-
-          // Build artifact
           project.exec {
             executable 'sh'
-            args '-c', ". ${project.ext.envdir}/bin/activate && cd ${copiedSrcRoot}/sdks/python && python setup.py sdist --formats zip,gztar --dist-dir ${project.buildDir}"
+            args '-c', ". ${project.ext.envdir}/bin/activate && cd ${pythonRootDir} && python setup.py sdist --keep-temp --formats zip,gztar --dist-dir ${project.buildDir}"
           }
           def collection = project.fileTree("${project.buildDir}"){ include '**/*.tar.gz' exclude '**/apache-beam.tar.gz'}
           println "sdist archive name: ${collection.singleFile}"
-
           // we need a fixed name for the artifact
           project.copy { from collection.singleFile; into "${project.buildDir}"; rename { 'apache-beam.tar.gz' } }
         }
@@ -1668,7 +1646,7 @@ class BeamModulePlugin implements Plugin<Project> {
           project.exec {
             executable 'sh'
             args '-c', "if [ -e ${activate} ]; then " +
-                    ". ${activate} && cd ${pythonRootDir} && python setup.py clean; " +
+                    ". ${activate} && python ${pythonRootDir}/setup.py clean; " +
                     "fi"
           }
           project.delete project.buildDir     // Gradle build directory
@@ -1695,21 +1673,6 @@ class BeamModulePlugin implements Plugin<Project> {
         }
         return argList.join(' ')
       }
-
-      project.ext.toxTask = { name, tox_env ->
-        project.tasks.create(name) {
-          dependsOn = ['sdist']
-          doLast {
-            def copiedPyRoot = "${copiedSrcRoot}/sdks/python"
-            project.exec {
-              executable 'sh'
-              args '-c', ". ${project.ext.envdir}/bin/activate && cd ${copiedPyRoot} && scripts/run_tox.sh $tox_env ${project.buildDir}/apache-beam.tar.gz"
-            }
-          }
-          inputs.files pythonSdkDeps
-          outputs.files project.fileTree(dir: "${pythonRootDir}/target/.tox/${tox_env}/log/")
-        }
-      }
     }
   }
 }

sdks/python/build.gradle

@@ -38,8 +38,34 @@ build.dependsOn buildPython
 
 
 /*************************************************************************************************/
-// Unit tests for Python 2
-// See Python 3 tests in test-suites/tox
+// Unit testing
+
+def pythonSdkDeps = files(
+    fileTree(dir: 'apache_beam', includes: ['**/*.py', '**/*.pyx', '**/*.pxd']),
+    fileTree(dir: 'apache_beam/testing/data'),
+    fileTree(dir: "${project.rootDir}/model"),
+    fileTree(dir: 'scripts'),
+    ".pylintrc",
+    "MANIFEST.in",
+    "gen_protos.py",
+    "setup.cfg",
+    "setup.py",
+    "test_config.py",
+    "tox.ini")
+
+def toxTask = {
+  name, tox_env -> tasks.create(name) {
+    dependsOn = ['setupVirtualenv']
+    doLast {
+      exec {
+        executable 'sh'
+        args '-c', ". ${project.ext.envdir}/bin/activate && ./scripts/run_tox.sh $tox_env"
+      }
+    }
+    inputs.files pythonSdkDeps
+    outputs.files fileTree(dir: "${project.rootDir}/sdks/python/target/.tox/${tox_env}/log/")
+  }
+}
 
 task lint {}
 check.dependsOn lint
@@ -50,30 +76,54 @@ lint.dependsOn lintPy27
 toxTask "lintPy27_3", "py27-lint3"
 lint.dependsOn lintPy27_3
 
+toxTask "lintPy35", "py35-lint"
+lint.dependsOn lintPy35
+
 toxTask "testPy2Gcp", "py27-gcp"
 test.dependsOn testPy2Gcp
 
+toxTask "testPy35Gcp", "py35-gcp"
+test.dependsOn testPy35Gcp
+
 toxTask "testPython2", "py27"
 test.dependsOn testPython2
 
+toxTask "testPython35", "py35"
+test.dependsOn testPython35
+
+toxTask "testPython36", "py36"
+test.dependsOn testPython36
+
 toxTask "testPy2Cython", "py27-cython"
 test.dependsOn testPy2Cython
 // Ensure that testPy2Cython runs exclusively to other tests. This line is not
 // actually required, since gradle doesn't do parallel execution within a
 // project.
 testPy2Cython.mustRunAfter testPython2, testPy2Gcp
 
+toxTask "testPy35Cython", "py35-cython"
+test.dependsOn testPy35Cython
+// Ensure that testPy35Cython runs exclusively to other tests. This line is not
+// actually required, since gradle doesn't do parallel execution within a
+// project.
+testPy35Cython.mustRunAfter testPython35, testPy35Gcp
+
 toxTask "docs", "docs"
 assemble.dependsOn docs
 
 toxTask "cover", "cover"
 
-task preCommitPy2() {
+task preCommit() {
   dependsOn "docs"
   dependsOn "testPy2Cython"
+  dependsOn "testPy35Cython"
   dependsOn "testPython2"
+  dependsOn "testPython35"
+  dependsOn "testPython36"
   dependsOn "testPy2Gcp"
+  dependsOn "testPy35Gcp"
   dependsOn "lint"
+  dependsOn ":beam-sdks-python-test-suites-dataflow:preCommitIT"
 }
 
 task portablePreCommit() {

sdks/python/scripts/generate_pydoc.sh

@@ -29,7 +29,7 @@ if [[ $PWD != *sdks/python* ]]; then
 fi
 
 # Go to the Apache Beam Python SDK root
-if [[ $PWD != *sdks/python ]]; then
+if [[ "*sdks/python" != $PWD ]]; then
   cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
 fi
 

sdks/python/scripts/run_integration_test.sh

@@ -163,7 +163,7 @@ if [[ -z $PIPELINE_OPTS ]]; then
   fi
 
   # Go to the Apache Beam Python SDK root
-  if [[ $PWD != *sdks/python ]]; then
+  if [[ "*sdks/python" != $PWD ]]; then
     cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
   fi
 

sdks/python/scripts/run_mini_py3lint.sh

@@ -29,7 +29,7 @@ if [[ $PWD != *sdks/python* ]]; then
 fi
 
 # Go to the Apache Beam Python SDK root
-if [[ $PWD != *sdks/python ]]; then
+if [[ "*sdks/python" != $PWD ]]; then
   cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
 fi
 

sdks/python/scripts/run_pylint.sh

@@ -30,7 +30,7 @@ if [[ $PWD != *sdks/python* ]]; then
 fi
 
 # Go to the Apache Beam Python SDK root
-if [[ $PWD != *sdks/python ]]; then
+if [[ "*sdks/python" != $PWD ]]; then
   cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
 fi
 

sdks/python/scripts/run_pylint_2to3.sh

@@ -29,7 +29,7 @@ if [[ $PWD != *sdks/python* ]]; then
 fi
 
 # Go to the Apache Beam Python SDK root
-if [[ $PWD != *sdks/python ]]; then
+if [[ "*sdks/python" != $PWD ]]; then
   cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
 fi
 

sdks/python/scripts/run_tox.sh

@@ -24,10 +24,9 @@
 
 ###########################################################################
 # Usage check.
-if [[ $# < 1 || $# > 2 ]]; then
-  printf "Usage: \n$> ./scripts/run_tox.sh <tox_environment> [<sdk_location>]"
+if [[ $# != 1 ]]; then
+  printf "Usage: \n$> ./scripts/run_tox.sh <tox_environment>"
   printf "\n\ttox_environment: [required] Tox environment to run the test in.\n"
-  printf "\n\tsdk_location: [optional] SDK tarball artifact location.\n"
   exit 1
 fi
 
@@ -38,19 +37,11 @@ if [[ $PWD != *sdks/python* ]]; then
 fi
 
 # Go to the Apache Beam Python SDK root
-if [[ $PWD != *sdks/python ]]; then
+if [[ "*sdks/python" != $PWD ]]; then
   cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
 fi
 
-# Used in tox.ini to isolate toxworkdir of each environment.
-export ENV_NAME=.tox-$1
-
-if [[ ! -z $2 ]]; then
-  tox -c tox.ini --recreate -e $1 --installpkg $2
-else
-  tox -c tox.ini --recreate -e $1
-fi
-
+tox -c tox.ini --recreate -e $1
 exit_code=$?
 # Retry once for the specific exit code 245.
 if [[ $exit_code == 245 ]]; then

sdks/python/scripts/run_tox_cleanup.sh

@@ -26,7 +26,7 @@ if [[ $PWD != *sdks/python* ]]; then
 fi
 
 # Go to the Apache Beam Python SDK root
-if [[ $PWD != *sdks/python ]]; then
+if [[ "*sdks/python" != $PWD ]]; then
   cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
 fi
 

sdks/python/tox.ini

@@ -18,7 +18,7 @@
 [tox]
 # new environments will be excluded by default unless explicitly added to envlist.
 envlist = py27,py35,py36,py27-{gcp,cython,lint,lint3},py35-{gcp,cython,lint},docs
-toxworkdir = {toxinidir}/target/{env:ENV_NAME:.tox}
+toxworkdir = {toxinidir}/target/.tox
 
 [pycodestyle]
 # Disable all errors and warnings except for the ones related to blank lines.

settings.gradle

@@ -207,10 +207,6 @@ include "beam-sdks-python-test-suites-dataflow-py3"
 project(":beam-sdks-python-test-suites-dataflow-py3").dir = file("sdks/python/test-suites/dataflow/py3")
 include "beam-sdks-python-test-suites-direct-py3"
 project(":beam-sdks-python-test-suites-direct-py3").dir = file("sdks/python/test-suites/direct/py3")
-include "beam-sdks-python-test-suites-tox-py35"
-project(":beam-sdks-python-test-suites-tox-py35").dir = file("sdks/python/test-suites/tox/py35")
-include "beam-sdks-python-test-suites-tox-py36"
-project(":beam-sdks-python-test-suites-tox-py36").dir = file("sdks/python/test-suites/tox/py36")
 include "beam-sdks-python-load-tests"
 project(":beam-sdks-python-load-tests").dir = file("sdks/python/apache_beam/testing/load_tests")
 include "beam-vendor-grpc-1_13_1"