test(ingest): update tox test configurations and test airflow 2.x by default

metadata-ingestion/setup.cfg

@@ -37,20 +37,28 @@ ignore_missing_imports = no
 ignore_missing_imports = no
 
 [tool:pytest]
-addopts = --cov src --cov-report term --cov-config setup.cfg --strict-markers
+addopts = --cov --cov-report term-missing --cov-config setup.cfg --strict-markers
 markers =
     integration: marks tests to only run in integration (deselect with '-m "not integration"')
 testpaths = 
-    tests/unit
-    tests/integration
+    tests
+
+[coverage:paths]
+# This is necessary for tox-based coverage combinations to be counted properly.
+source =
+   src
+   */site-packages
 
 [coverage:report]
-fail_under = 70
+# The fail_under value ensures that at least some coverage data is collected.
+# We override its value in the tox config.
+fail_under = 0.01
 show_missing = true
+precision = 2
 exclude_lines =
-  pragma: no cover
-  @abstract
-  if TYPE_CHECKING:
+    pragma: no cover
+    @abstract
+    if TYPE_CHECKING:
 omit =
     # omit codegen
     src/datahub/metadata/*

metadata-ingestion/setup.py

@@ -168,13 +168,12 @@ def get_long_description():
 
 dev_requirements = {
     *base_dev_requirements,
-    "apache-airflow==1.10.15",
-    "apache-airflow-backport-providers-snowflake",  # Used in the example DAGs.
+    "apache-airflow[snowflake]>=2.0.2",  # snowflake is used in example dags
 }
-dev_requirements_airflow_2 = {
+dev_requirements_airflow_1 = {
     *base_dev_requirements,
-    "apache-airflow>=2.0.2",
-    "apache-airflow-providers-snowflake",
+    "apache-airflow==1.10.15",
+    "apache-airflow-backport-providers-snowflake",
 }
 
 full_test_dev_requirements = {
@@ -242,7 +241,6 @@ def get_long_description():
         "Source": "https://github.com/linkedin/datahub",
         "Changelog": "https://github.com/linkedin/datahub/releases",
     },
-    author="DataHub Committers",
     license="Apache License 2.0",
     description="A CLI to work with DataHub metadata",
     long_description=get_long_description(),
@@ -296,7 +294,7 @@ def get_long_description():
             )
         ),
         "dev": list(dev_requirements),
-        "dev-airflow2": list(dev_requirements_airflow_2),
+        "dev-airflow1": list(dev_requirements_airflow_1),
         "integration-tests": list(full_test_dev_requirements),
     },
 )

metadata-ingestion/tox.ini

@@ -4,16 +4,20 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py3-airflow{2,1}
+envlist = py36-full,py36-airflow1,py39
 
 [testenv]
-setenv =
-    AIRFLOW_HOME = /tmp/airflow/thisshouldnotexist-{envname}
 extras = dev
 commands =
-    # TODO: look at https://github.com/pytest-dev/pytest-cov/blob/master/examples/src-layout/tox.ini
-    # For now, coverage doesn't work within tox.
-    pytest -m 'not integration' --cov-fail-under=0
+    pytest -vv -m 'not integration'
 
-[testenv:py3-airflow2]
-extras = dev-airflow2
+setenv =
+    AIRFLOW_HOME = /tmp/airflow/thisshouldnotexist-{envname}
+
+[testenv:py36-airflow1]
+extras = dev-airflow1
+
+[testenv:py36-full]
+extras = dev,integration-tests
+commands =
+    pytest -vv --cov-fail-under 70