Split distributed tests into separate test suite (#1126)

ludwig-ai · Mar 19, 2021 · 32313e3 · 32313e3
1 parent b0e12a4
commit 32313e3
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 6 deletions.
diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
@@ -17,6 +17,7 @@ jobs:
       matrix:
         os: [ ubuntu-latest ]
         python-version: [ 3.6, 3.7, 3.8 ]
+        test-markers: [ "not distributed", "distributed" ]
         include:
           - python-version: 3.6
             tensorflow-version: 2.3.1
@@ -26,6 +27,9 @@ jobs:
             tensorflow-version: nightly
     env:
       TENSORFLOW: ${{ matrix.tensorflow-version }}
+      MARKERS: ${{ matrix.test-markers }}
+
+    name: py${{ matrix.python-version  }}, tf${{ matrix.tensorflow-version }}, ${{ matrix.test-markers }}, ${{ matrix.os }}
 
     timeout-minutes: 70
     steps:
@@ -93,4 +97,4 @@ jobs:
 
     - name: Tests
       run: |
-        pytest -v --timeout 300 --durations 10 tests
+        pytest -v --timeout 360 --durations 10 -m "$MARKERS" tests
diff --git a/ludwig/data/preprocessing.py b/ludwig/data/preprocessing.py
@@ -17,14 +17,12 @@
 import logging
 import os
 from abc import ABC, abstractmethod
-from copy import copy
 
 import numpy as np
 import pandas as pd
 
 import ludwig
 from ludwig.backend import LOCAL_BACKEND
-from ludwig.backend import LOCAL_BACKEND
 from ludwig.constants import *
 from ludwig.constants import TEXT
 from ludwig.data.concatenate_datasets import concatenate_files, concatenate_df
@@ -1681,7 +1679,8 @@ def preprocess_for_prediction(
         :returns: Dataset, Train set metadata
         """
     # TODO dask: support distributed backend for prediction
-    backend = LOCAL_BACKEND
+    if backend.df_engine != LOCAL_BACKEND.df_engine:
+        backend = LOCAL_BACKEND
 
     # Sanity Check to make sure some data source is provided
     if dataset is None:

diff --git a/tests/integration_tests/test_experiment.py b/tests/integration_tests/test_experiment.py
@@ -430,7 +430,8 @@ def test_experiment_image_dataset(
         # create hdf5 data set
         _, test_set, _, training_set_metadata_for_test = preprocess_for_training(
             model.config,
-            dataset=test_data
+            dataset=test_data,
+            backend=backend,
         )
         test_dataset_to_use = test_set.data_hdf5_fp
     else:

diff --git a/tests/integration_tests/test_ray.py b/tests/integration_tests/test_ray.py
@@ -14,7 +14,6 @@
 # limitations under the License.
 # ==============================================================================
 import os
-import shutil
 import tempfile
 
 import pytest