Dev data reader heads

.github/workflows/development.yml

@@ -26,7 +26,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install pre-commit
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install .[dev]
 
   test:
     needs: dependency-install
@@ -42,10 +42,10 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install pytest pytest-mock
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install .[dev]
     - name: Test with pytest
       run: |
-        python -m pytest
+        python -m pytest -v -m "development"
 
   devops:
     needs: test
@@ -62,7 +62,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install pre-commit
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install .[dev]
     - name: Lint with pylint
       run: |
         pip install pylint
@@ -71,7 +71,7 @@ jobs:
       run: |
         mkdir -p ./coverage
         pip install pytest-cov pytest-mock
-        python -m pytest --cov --cov-report=xml:./coverage/coverage.xml
+        python -m pytest -m "not staging" --cov --cov-report=xml:./coverage/coverage.xml
 
     - name: Upload coverage
       uses: codecov/codecov-action@v3

.github/workflows/documentation.yml

@@ -34,15 +34,13 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install .[dev]
         pip install -U sphinx
         pip install furo
 
     - name: Build documentation
       run: |
         cd docs
-
-
         sphinx-apidoc -e -M --force -o . ../utilities/
         make html
     - name: Upload build data

.github/workflows/master.yml

@@ -26,7 +26,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install pre-commit
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install .[dev]
 
   test:
     needs: dependency-install
@@ -42,10 +42,16 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install pytest pytest-mock
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-    - name: Test with pytest
+        pip install .[dev]
+    - name: Test with pytest development
       run: |
-        python -m pytest
+        python -m pytest -v -m "development"
+    - name: Test with pytest staging
+      run: |
+        python -m pytest -v -m "staging"
+    - name: Test with pytest production
+      run: |
+        python -m pytest -v -m "production"
 
   devops:
     needs: test
@@ -62,7 +68,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install pre-commit
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install .[dev]
     - name: Lint with pylint
       run: |
         pip install pylint

.github/workflows/staging.yml

@@ -26,7 +26,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install pre-commit
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install .[dev]
 
   test:
     needs: dependency-install
@@ -42,10 +42,13 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install pytest pytest-mock
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-    - name: Test with pytest
+        pip install .[dev]
+    - name: Test with pytest development
       run: |
-        python -m pytest
+        python -m pytest -v -m "development"
+    - name: Test with pytest staging
+      run: |
+        python -m pytest -v -m "staging"
 
   devops:
     needs: test
@@ -62,7 +65,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install pre-commit
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install .[dev]
     - name: Lint with pylint
       run: |
         pip install pylint

pyproject.toml

@@ -21,11 +21,20 @@ dependencies = [
     "Pillow >= 9.4.0",
     "tensorflow >= 2.10",
     "toml >= 0.10.2",
+    "tqdm >= 4.64.1",
+    "pandas >= 1.5.1",
+    "opencv-python-headless >= 4.8.0.76"
 ]
 
 [tool.setuptools]
 packages = ["utilities"]
 
+[tool.pytest.ini_options]
+markers = [
+    "staging: Mark a test as part of the staging environment",
+    "production: Mark a test as part of the production environment",
+    "development: Mark a test as part of the development environment",
+]
 
 [project.optional-dependencies]
 dev = [

tests/segmentation_utils_tests.py/flow_reader_test.py

@@ -36,6 +36,7 @@ def flow_from_directory_mock(*args, **kwargs):
 
 
 # tests
+@pytest.mark.development
 def test_makes_flow_generator() -> None:
     patch = MonkeyPatch()
     # mock an imagedatagenerator from keras
@@ -48,7 +49,7 @@ def test_makes_flow_generator() -> None:
     # create a flow generator
     FlowGenerator(**generator_args)
 
-
+@pytest.mark.development
 def test_makes_flow_generator_with_queue() -> None:
     patch = MonkeyPatch()
     # mock an imagedatagenerator from keras
@@ -78,7 +79,7 @@ def test_makes_flow_generator_with_queue() -> None:
     generator = FlowGenerator(**new_generator_args)
     generator.set_preprocessing_pipeline(image_queue, mask_queue)
 
-
+@pytest.mark.development
 def test_makes_flow_generator_wrong_shape() -> None:
     try:
         patch = MonkeyPatch()
@@ -98,7 +99,7 @@ def test_makes_flow_generator_wrong_shape() -> None:
     except ValueError:
         assert True
 
-
+@pytest.mark.development
 def test_makes_flow_generator_wrong_dimension() -> None:
     try:
         patch = MonkeyPatch()
@@ -118,7 +119,7 @@ def test_makes_flow_generator_wrong_dimension() -> None:
     except ValueError:
         assert True
 
-
+@pytest.mark.development
 def test_flow_generator_with_preprocess() -> None:
     patch = MonkeyPatch()
     # mock an imagedatagenerator from keras
@@ -141,7 +142,7 @@ def test_flow_generator_with_preprocess() -> None:
     patch.undo()
     patch.undo()
 
-
+@pytest.mark.development
 def test_get_dataset_size() -> None:
     patch = MonkeyPatch()
     patch.setattr(os, "listdir", lambda x: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
@@ -166,7 +167,7 @@ def test_get_dataset_size() -> None:
     patch.undo()
     patch.undo()
 
-
+@pytest.mark.development
 def test_get_generator() -> None:
     patch = MonkeyPatch()
 

tests/segmentation_utils_tests.py/image_preprocessor_test.py

@@ -31,7 +31,7 @@ def test_image_onehot_encoder_column() -> None:
     )
     assert np.array_equal(one_hot_image, onehot_test)
 
-
+@pytest.mark.development
 def test_image_onehot_encoder_squarematrix() -> None:
     # predifining input variables
     n_classes = 2
@@ -58,7 +58,7 @@ def test_image_onehot_encoder_squarematrix() -> None:
     )
     assert np.array_equal(one_hot_image, onehot_test)
 
-
+@pytest.mark.development
 def test_image_augmentation_pipeline_squarematrix() -> None:
     # predifining input variables
     image = np.zeros((512, 512, 3))
@@ -89,7 +89,7 @@ def test_image_augmentation_pipeline_squarematrix() -> None:
     assert image_new.shape == (512, 512, 3)
     assert mask_new.shape == (256, 256, 1)
 
-
+@pytest.mark.development
 def test_processing_queue() -> None:
     # creating dummy queues
 
@@ -102,7 +102,7 @@ def test_processing_queue() -> None:
 
     assert image_queue.queue[0].kwargs["seed"] == new_seed
 
-
+@pytest.mark.development
 def test_generate_default_queue() -> None:
     # creating default queues
     image_queue, mask_queue = ImagePreprocessor.generate_default_queue()
@@ -111,7 +111,7 @@ def test_generate_default_queue() -> None:
     assert image_queue.get_queue_length() == 5
     assert mask_queue.get_queue_length() == 2
 
-
+@pytest.mark.development
 def test_flatten() -> None:
     image = np.zeros((512, 512, 3))
     image = tf.convert_to_tensor(image)