CI: Split tests in two groups by backend (#2194)

ibis-project · May 7, 2020 · 8910d40 · 8910d40
1 parent 596cff9
commit 8910d40
Show file tree

Hide file tree

Showing 4 changed files with 50 additions and 9 deletions.
diff --git a/ci/azure/linux.yml b/ci/azure/linux.yml
@@ -12,30 +12,63 @@ jobs:
     AZURECI: 1
     COMPOSE_FILE: ci/docker-compose.yml
     PYTHONHASHSEED: "random"
-    PYTEST_MARK_EXPRESSION: "not udf"
-    BACKENDS: "clickhouse impala kudu-master kudu-tserver mysql omniscidb parquet postgres sqlite"
+    # TODO: omniscidb should be in BACKENDS_SQL_PARQUET, but it's not compatible with Python 3.8, so it needs to be set individually in py36 and py37 builds.
+    # It is not being added, because the conda solver takes forever to resolve when pymapd is present, so it's not being tested for now
+    BACKENDS_SQL_PARQUET: "mysql postgres sqlite parquet"
+    BACKENDS_IMPALA_KUDU_CLICKHOUSE: "impala kudu-master kudu-tserver clickhouse"
+    # TODO: there are some issues with pyspark, so not testing for now. Same for udf I guess, which was already not being tested.
+    PYTEST_MARK_EXPRESSION_SQL_PARQUET: "not udf and not spark and not pyspark and not clickhouse and not impala and not kudu and not hdfs"
+    PYTEST_MARK_EXPRESSION_IMPALA_KUDU_CLICKHOUSE: "not udf and not spark and not pyspark and not mysql and not parquet and not postgresql and not postgis and not postgres_extensions and not sqlite"
   strategy:
     matrix:
-      py36:
+      py36_backends1:
         PYTHON_MAJOR_VERSION: "3"
         PYTHON_MINOR_VERSION: "6"
         PYTHON_VERSION: $(PYTHON_MAJOR_VERSION).$(PYTHON_MINOR_VERSION)
         PYTHON_NO_DOT_VERSION: $(PYTHON_MAJOR_VERSION)$(PYTHON_MINOR_VERSION)
-      py37:
+        PYTEST_MARK_EXPRESSION: $(PYTEST_MARK_EXPRESSION_SQL_PARQUET)
+        BACKENDS: $(BACKENDS_SQL_PARQUET)
+      py36_backends2:
+        PYTHON_MAJOR_VERSION: "3"
+        PYTHON_MINOR_VERSION: "6"
+        PYTHON_VERSION: $(PYTHON_MAJOR_VERSION).$(PYTHON_MINOR_VERSION)
+        PYTHON_NO_DOT_VERSION: $(PYTHON_MAJOR_VERSION)$(PYTHON_MINOR_VERSION)
+        PYTEST_MARK_EXPRESSION: $(PYTEST_MARK_EXPRESSION_IMPALA_KUDU_CLICKHOUSE)
+        BACKENDS: $(BACKENDS_IMPALA_KUDU_CLICKHOUSE)
+      py37_backends1:
+        PYTHON_MAJOR_VERSION: "3"
+        PYTHON_MINOR_VERSION: "7"
+        PYTHON_VERSION: $(PYTHON_MAJOR_VERSION).$(PYTHON_MINOR_VERSION)
+        PYTHON_NO_DOT_VERSION: $(PYTHON_MAJOR_VERSION)$(PYTHON_MINOR_VERSION)
+        PYTEST_MARK_EXPRESSION: $(PYTEST_MARK_EXPRESSION_SQL_PARQUET)
+        BACKENDS: $(BACKENDS_SQL_PARQUET)
+      py37_backends2:
         PYTHON_MAJOR_VERSION: "3"
         PYTHON_MINOR_VERSION: "7"
         PYTHON_VERSION: $(PYTHON_MAJOR_VERSION).$(PYTHON_MINOR_VERSION)
         PYTHON_NO_DOT_VERSION: $(PYTHON_MAJOR_VERSION)$(PYTHON_MINOR_VERSION)
-      py38:
+        PYTEST_MARK_EXPRESSION: $(PYTEST_MARK_EXPRESSION_IMPALA_KUDU_CLICKHOUSE)
+        BACKENDS: $(BACKENDS_IMPALA_KUDU_CLICKHOUSE)
+      py38_backends1:
+        PYTHON_MAJOR_VERSION: "3"
+        PYTHON_MINOR_VERSION: "8"
+        PYTHON_VERSION: $(PYTHON_MAJOR_VERSION).$(PYTHON_MINOR_VERSION)
+        PYTHON_NO_DOT_VERSION: $(PYTHON_MAJOR_VERSION)$(PYTHON_MINOR_VERSION)
+        # pymapd and pyspark are not working on Ibis with Python 3.8
+        # https://github.com/ibis-project/ibis/issues/2091
+        # https://github.com/ibis-project/ibis/issues/2090
+        PYTEST_MARK_EXPRESSION: "$(PYTEST_MARK_EXPRESSION_SQL_PARQUET) and not omniscidb and not spark and not pyspark"
+        BACKENDS: $(BACKENDS_SQL_PARQUET)
+      py38_backends2:
         PYTHON_MAJOR_VERSION: "3"
         PYTHON_MINOR_VERSION: "8"
         PYTHON_VERSION: $(PYTHON_MAJOR_VERSION).$(PYTHON_MINOR_VERSION)
         PYTHON_NO_DOT_VERSION: $(PYTHON_MAJOR_VERSION)$(PYTHON_MINOR_VERSION)
         # pymapd and pyspark are not working on Ibis with Python 3.8
         # https://github.com/ibis-project/ibis/issues/2091
         # https://github.com/ibis-project/ibis/issues/2090
-        PYTEST_MARK_EXPRESSION: "not udf and not omniscidb and not spark and not pyspark"
-        BACKENDS: "clickhouse impala kudu-master kudu-tserver mysql parquet postgres sqlite"
+        PYTEST_MARK_EXPRESSION: "$(PYTEST_MARK_EXPRESSION_IMPALA_KUDU_CLICKHOUSE) and not omniscidb and not spark and not pyspark"
+        BACKENDS: $(BACKENDS_IMPALA_KUDU_CLICKHOUSE)
 
   steps:
     - bash: |
@@ -88,6 +121,7 @@ jobs:
       displayName: 'Load test datasets'
 
     - bash: |
+        echo "PYTEST_MARK_EXPRESSION: ${PYTEST_MARK_EXPRESSION}"
         docker-compose run \
           -e PYTHONHASHSEED=$PYTHONHASHSEED \
           -e AZURECI=$AZURECI \
@@ -98,6 +132,7 @@ jobs:
                       --numprocesses auto \
                       --doctest-modules \
                       --doctest-ignore-import-errors \
+                      -k"-compile -connect" \
                       --junitxml=/tmp/test-reports/pytest/junit.xml \
                       --cov=ibis \
                       --cov-report=xml:/tmp/test-reports/pytest-cov/coverage.xml

diff --git a/ci/requirements-3.6-dev.yml b/ci/requirements-3.6-dev.yml
@@ -29,7 +29,7 @@ dependencies:
   - pydata-google-auth
   - pydocstyle=4.0.1
   - pygit2
-  - pymapd>=0.21.0
+  # - pymapd>=0.21.0
   - pymysql
   - pyspark>=2.4.3
   - pytables>=3.0.0

diff --git a/ci/requirements-3.7-dev.yml b/ci/requirements-3.7-dev.yml
@@ -29,7 +29,7 @@ dependencies:
   - pydata-google-auth
   - pydocstyle=4.0.1
   - pygit2
-  - pymapd>=0.21.0
+  # - pymapd>=0.21.0
   - pymysql
   - pyspark>=2.4.3
   - pytables>=3.0.0

diff --git a/ci/requirements-docs.yml b/ci/requirements-docs.yml
@@ -12,3 +12,9 @@ sphinx-releases
 sphinx_rtd_theme
 # https://github.com/ibis-project/ibis/issues/2027
 semantic_version<2.7
+
+# TODO: The conda solver takes forever when pymapd is installed, with the rest of
+# Ibis dependencies. It has been excluded from the tests for now, so we add it here.
+# Once the solver problem is fixed, this should be in ci/requirements-3.*-dev.yml
+# (the one used for the docs), and removed from here.
+pymapd>=0.21.0