Assorted tox updates (#6971)

* update tox, virtualenv versions, toxfile tweaks
dagster-io · Apr 22, 2022 · 697e701 · 697e701 · vercel · Apr 22, 2022
1 parent 3f1f0fd
commit 697e701
Show file tree

Hide file tree

Showing 88 changed files with 386 additions and 513 deletions.
diff --git a/.buildkite/dagster-buildkite/dagster_buildkite/steps/dagster.py b/.buildkite/dagster-buildkite/dagster_buildkite/steps/dagster.py
@@ -607,6 +607,25 @@ def pylint_steps():
     ]
 
 
+def isort_steps():
+    return [
+        StepBuilder(":isort:")
+        .run("pip install -e python_modules/dagster[isort]", "make check_isort")
+        .on_integration_image(SupportedPython.V3_7)
+        .build(),
+    ]
+
+
+def black_steps():
+    return [
+        StepBuilder(":python-black:")
+        # See: https://github.com/dagster-io/dagster/issues/1999
+        .run("pip install -e python_modules/dagster[black]", "make check_black")
+        .on_integration_image(SupportedPython.V3_7)
+        .build(),
+    ]
+
+
 def schema_checks(version=SupportedPython.V3_8):
     return [
         StepBuilder("SQL schema checks")
@@ -649,18 +668,11 @@ def dagster_steps():
     steps = []
     steps += publish_test_images()
 
+    # NOTE: these `pylint_steps` only cover misc python code, there are also package-specific pylint
+    # steps
     steps += pylint_steps()
-    steps += [
-        StepBuilder(":isort:")
-        .run("pip install -e python_modules/dagster[isort]", "make check_isort")
-        .on_integration_image(SupportedPython.V3_7)
-        .build(),
-        StepBuilder(":python-black:")
-        # See: https://github.com/dagster-io/dagster/issues/1999
-        .run("pip install -e python_modules/dagster[black]", "make check_black")
-        .on_integration_image(SupportedPython.V3_7)
-        .build(),
-    ]
+    steps += isort_steps()
+    steps += black_steps()
 
     for m in DAGSTER_PACKAGES_WITH_CUSTOM_TESTS:
         steps += m.get_tox_build_steps()

diff --git a/docs/tox.ini b/docs/tox.ini
@@ -3,11 +3,12 @@ envlist = py38-sphinx
 skipsdist = True
 
 [testenv]
-pip_version = pip==21.3.1
+setenv =
+  VIRTUALENV_PIP=22.0.4
 passenv = CI_* COVERALLS_REPO_TOKEN AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID BUILDKITE
 
 usedevelop = False
-whitelist_externals =
+allowlist_externals =
   /bin/bash
   make
 

diff --git a/examples/airflow_ingest/tox.ini b/examples/airflow_ingest/tox.ini
@@ -3,12 +3,13 @@ envlist = py{39,38,37,36}-{unix,windows},pylint
 skipsdist = True
 
 [testenv]
-pip_version = pip==21.3.1
+setenv =
+  VIRTUALENV_PIP=22.0.4
 passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE
 deps =
   -e ../../python_modules/dagster[test]
   -e ../../python_modules/libraries/dagster-airflow[test]
-whitelist_externals =
+allowlist_externals =
   /bin/bash
   echo
 commands =
@@ -17,6 +18,5 @@ commands =
   pytest -vv {posargs}
 
 [testenv:pylint]
-basepython = python
 commands =
-  /bin/bash -c 'cd .. && pylint -j 0 --rcfile=../pyproject.toml airflow_ingest/'
+  pylint -j0 --rcfile=../../pyproject.toml airflow_ingest
diff --git a/examples/airline_demo/tox.ini b/examples/airline_demo/tox.ini
@@ -2,7 +2,12 @@
 envlist = py{39,38,37,36}-{unix,windows},pylint
 
 [testenv]
-pip_version = pip==21.3.1
+usedevelop = true
+extras =
+  full
+  airflow
+setenv =
+  VIRTUALENV_PIP=22.0.4
 deps =
   -e ../../python_modules/dagster[test]
   -e ../../python_modules/dagster-graphql
@@ -17,9 +22,8 @@ deps =
   -e ../../python_modules/libraries/dagster-slack
   -e ../../python_modules/libraries/dagster-airflow
   -e ../../python_modules/libraries/dagster-databricks
-  -e .[full,airflow]
 passenv = CI_* COVERALLS_REPO_TOKEN TOXENV SLUGIFY_USES_TEXT_UNIDECODE AWS_* JAVA_HOME POSTGRES_TEST_DB_HOST
-whitelist_externals =
+allowlist_externals =
   /bin/bash
   echo
 commands =
@@ -32,10 +36,9 @@ commands =
   coverage report --omit='.tox/*,**/test_*.py' --skip-covered
   coverage html --omit='.tox/*,**/test_*.py'
   coverage xml --omit='.tox/*,**/test_*.py'
-usedevelop = true
 
 [testenv:py37]
-whitelist_externals =
+allowlist_externals =
   /bin/bash
   echo
 commands =
@@ -49,8 +52,5 @@ commands =
   coverage xml --omit='.tox/*,**/test_*.py'
 
 [testenv:pylint]
-whitelist_externals =
-  pylint
-basepython = python
 commands =
-  pylint -j 0 --rcfile=../../pyproject.toml airline_demo airline_demo_tests
+  pylint -j0 --rcfile=../../pyproject.toml airline_demo airline_demo_tests
diff --git a/examples/basic_pyspark/tox.ini b/examples/basic_pyspark/tox.ini
@@ -3,13 +3,14 @@ envlist = py{39,38,37,36}-{unix,windows},pylint
 skipsdist = True
 
 [testenv]
-pip_version = pip==21.3.1
+setenv =
+  VIRTUALENV_PIP=22.0.4
 passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE
 deps =
   -e ../../python_modules/dagster[test]
   -e ../../python_modules/libraries/dagster-spark
   -e ../../python_modules/libraries/dagster-pyspark
-whitelist_externals =
+allowlist_externals =
   /bin/bash
   echo
 commands =
@@ -18,6 +19,5 @@ commands =
   pytest -vv {posargs}
 
 [testenv:pylint]
-basepython = python
 commands =
-  /bin/bash -c 'cd .. && pylint -j 0 --rcfile=../pyproject.toml basic_pyspark/'
+  pylint -j0 --rcfile=../../pyproject.toml .
diff --git a/examples/bollinger/tox.ini b/examples/bollinger/tox.ini
@@ -2,27 +2,20 @@
 envlist = py{39, 38, 37, 36},pylint,mypy
 
 [testenv]
-pip_version = pip==21.3.1
+usedevelop = true
+setenv =
+  VIRTUALENV_PIP=22.0.4
 passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE SNOWFLAKE_ACCOUNT SNOWFLAKE_USER SNOWFLAKE_PASSWORD
 deps =
   -e ../../python_modules/dagster[test]
   -e ../../python_modules/libraries/dagster-pandera/
-  -e .
-whitelist_externals =
+allowlist_externals =
   /bin/bash
   echo
 commands =
   !windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster -e dagit'
   pytest -vv
 
 [testenv:pylint]
-whitelist_externals =
-  pylint
 commands =
-  pylint -j 0 --rcfile=../../pyproject.toml bollinger bollinger_tests
-
-[testenv:mypy]
-whitelist_externals =
-  mypy
-commands =
-  mypy -p bollinger -p bollinger_tests
+  pylint -j0 --rcfile=../../pyproject.toml bollinger bollinger_tests
diff --git a/examples/dbt_example/tox.ini b/examples/dbt_example/tox.ini
@@ -3,7 +3,9 @@ envlist = py{39,38,37,36}-{unix,windows},pylint
 skipsdist = True
 
 [testenv]
-pip_version = pip==21.3.1
+usedevelop = true
+setenv =
+  VIRTUALENV_PIP=22.0.4
 passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE DAGSTER_DBT_EXAMPLE_PGHOST
 deps =
   -e ../../python_modules/dagster[test]
@@ -14,8 +16,7 @@ deps =
   -e ../../python_modules/libraries/dagster-slack
   -e ../../python_modules/libraries/dagstermill
   -e ../../python_modules/libraries/dagster-dbt[test]
-  -e .
-whitelist_externals =
+allowlist_externals =
   /bin/bash
   echo
 commands =
@@ -24,6 +25,5 @@ commands =
   pytest -vv {posargs}
 
 [testenv:pylint]
-basepython = python
 commands =
-  pylint -j 0 --rcfile=../../pyproject.toml dbt_example/
+  pylint -j0 --rcfile=../../pyproject.toml dbt_example/
diff --git a/examples/deploy_docker/tox.ini b/examples/deploy_docker/tox.ini
@@ -3,11 +3,12 @@ envlist = py{39,38,37,36}-{unix,windows},pylint
 skipsdist = True
 
 [testenv]
-pip_version = pip==21.3.1
+setenv =
+  VIRTUALENV_PIP=22.0.4
 passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE DEPLOY_DOCKER_DAGIT_HOST
 deps =
   -e ../../python_modules/dagster[test]
-whitelist_externals =
+allowlist_externals =
   /bin/bash
   echo
 commands =
@@ -16,6 +17,5 @@ commands =
   pytest -vv {posargs}
 
 [testenv:pylint]
-basepython = python
 commands =
-  /bin/bash -c 'cd .. && pylint -j 0 --rcfile=../pyproject.toml deploy_docker/'
+  pylint -j0 --rcfile=../../pyproject.toml .
diff --git a/examples/deploy_ecs/tox.ini b/examples/deploy_ecs/tox.ini
@@ -3,12 +3,13 @@ envlist = py{39,38,37,36}-{unix,windows},pylint
 skipsdist = True
 
 [testenv]
-pip_version = pip==21.3.1
+setenv =
+  VIRTUALENV_PIP=22.0.4
 passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE DEPLOY_DOCKER_DAGIT_HOST
 deps =
   -e ../../python_modules/dagster[test]
   -e ../../python_modules/dagster-test
-whitelist_externals =
+allowlist_externals =
   /bin/bash
   echo
 commands =
@@ -17,6 +18,5 @@ commands =
   pytest -s -vv {posargs}
 
 [testenv:pylint]
-basepython = python
 commands =
-  /bin/bash -c 'cd .. && pylint -j 0 --rcfile=../pyproject.toml deploy_ecs/'
+  pylint -j0 --rcfile=../../pyproject.toml .
diff --git a/examples/deploy_k8s/tox.ini b/examples/deploy_k8s/tox.ini
@@ -3,7 +3,8 @@ envlist = py{39,38,37,36}-{unix,windows},pylint
 skipsdist = True
 
 [testenv]
-pip_version = pip==21.3.1
+setenv =
+  VIRTUALENV_PIP=22.0.4
 passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE
 deps =
   -e ../../python_modules/dagster[test]
@@ -12,7 +13,7 @@ deps =
   -e ../../python_modules/libraries/dagster-celery
   -e ../../python_modules/libraries/dagster-k8s
   -e ../../python_modules/libraries/dagster-celery-k8s
-whitelist_externals =
+allowlist_externals =
   /bin/bash
   echo
 commands =
@@ -21,6 +22,5 @@ commands =
   pytest -vv {posargs}
 
 [testenv:pylint]
-basepython = python
 commands =
-  /bin/bash -c 'cd .. && pylint -j 0 --rcfile=../pyproject.toml deploy_k8s/example_project'
+  pylint -j0 --rcfile=../../pyproject.toml example_project
diff --git a/examples/docs_snippets/tox.ini b/examples/docs_snippets/tox.ini
@@ -2,7 +2,11 @@
 envlist = py{39,38,37,36}-{unix,windows},pylint
 
 [testenv]
-pip_version = pip==21.3.1
+usedevelop = true
+extras =
+  full
+setenv =
+  VIRTUALENV_PIP=22.0.4
 passenv = CI_* COVERALLS_REPO_TOKEN POSTGRES_TEST_DB_HOST BUILDKITE
 deps =
   -e ../../python_modules/dagster[test]
@@ -18,9 +22,7 @@ deps =
   -e ../../python_modules/libraries/dagster-dbt
   -e ../../python_modules/libraries/dagster-k8s
   -e ../../python_modules/dagit
-  -e .[full]
-usedevelop = true
-whitelist_externals =
+allowlist_externals =
   /bin/bash
   echo
 commands =
@@ -29,8 +31,5 @@ commands =
   pytest -vv {posargs}
 
 [testenv:pylint]
-whitelist_externals =
-  pylint
-basepython = python
 commands =
-  pylint -j 0 --rcfile=../../pyproject.toml docs_snippets docs_snippets_tests
+  pylint -j0 --rcfile=../../pyproject.toml docs_snippets docs_snippets_tests
diff --git a/examples/emr_pyspark/tox.ini b/examples/emr_pyspark/tox.ini
@@ -3,14 +3,15 @@ envlist = py{39,38,37,36}-{unix,windows},pylint
 skipsdist = True
 
 [testenv]
-pip_version = pip==21.3.1
+setenv =
+  VIRTUALENV_PIP=22.0.4
 passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE
 deps =
   -e ../../python_modules/dagster[test]
   -e ../../python_modules/libraries/dagster-aws[test]
   -e ../../python_modules/libraries/dagster-spark
   -e ../../python_modules/libraries/dagster-pyspark
-whitelist_externals =
+allowlist_externals =
   /bin/bash
   echo
 commands =
@@ -19,6 +20,5 @@ commands =
   pytest -vv {posargs}
 
 [testenv:pylint]
-basepython = python
 commands =
-  /bin/bash -c 'cd .. && pylint -j 0 --rcfile=../pyproject.toml emr_pyspark/'
+  pylint -j0 --rcfile=../../pyproject.toml .
diff --git a/examples/ge_example/tox.ini b/examples/ge_example/tox.ini
@@ -3,16 +3,17 @@ envlist = py{39,38,37,36}-{unix,windows},pylint
 skipsdist = True
 
 [testenv]
-pip_version = pip==21.3.1
+usedevelop = true
+setenv =
+  VIRTUALENV_PIP=22.0.4
 passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE
 deps =
   -e ../../python_modules/dagster[test]
   -e ../../python_modules/dagster-graphql
   -e ../../python_modules/dagit
   -e ../../python_modules/libraries/dagster-pandas
   -e ../../python_modules/libraries/dagster-ge
-  -e .
-whitelist_externals =
+allowlist_externals =
   /bin/bash
   echo
 commands =
@@ -21,6 +22,5 @@ commands =
   pytest -vv {posargs}
 
 [testenv:pylint]
-basepython = python
 commands =
-  /bin/bash -c 'pylint -j 0 --rcfile=../../pyproject.toml ge_example/'
+  pylint -j0 --rcfile=../../pyproject.toml ge_example