[MAINT] restructure the test in folders.

.flake8

@@ -4,14 +4,14 @@ exclude =
     __pycache__,
     build,
     dist,
-	env,
-	venv,
+    env,
+    venv,
 per-file-ignores =
     # - docstrings rules that should not be applied to tests
-	tests/test_*.py: D100, D101, D103
-    __init__.py: D104, F401
+    **/test_*.py: D100, D101, D103
+    **/__init__.py: D104, F401
     # allow "weird indentation"
-	tests/test_workflow_*.py: D100, D103, E127, E131, E501
+    tests/unit/workflows/test_*.py: D103, E127, E131, E501
     docs/source/conf.py: E501
 docstring-convention = numpy
 max_complexity = 12

docs/source/cli_reference/pipeline_upload.rst

@@ -2,7 +2,7 @@
 ===========================
 
 .. note::
-   This command calls the :py:class:`nipoppy.workflows.pipeline_store.zenodo.ZenodoUploadWorkflow` class from the Python :term:`API` internally.
+   This command calls the :py:class:`nipoppy.workflows.pipeline_store.upload.ZenodoUploadWorkflow` class from the Python :term:`API` internally.
 
 .. click:: nipoppy.cli:pipeline_upload
    :prog: nipoppy pipeline upload

nipoppy/cli.py

@@ -676,7 +676,7 @@ def pipeline_validate(**params):
 @global_options
 def pipeline_upload(**params):
     """Upload a pipeline config directory to Zenodo."""
-    from nipoppy.workflows.pipeline_store.zenodo import ZenodoUploadWorkflow
+    from nipoppy.workflows.pipeline_store.upload import ZenodoUploadWorkflow
 
     params["zenodo_api"] = ZenodoAPI(
         sandbox=params.pop("sandbox"),

nipoppy/zenodo_api.py

@@ -280,7 +280,7 @@ def upload_pipeline(
             if not metadata["metadata"].get("creators"):
                 self._update_creators(record_id, owner_id, metadata)
 
-            files = list(input_dir.iterdir())
+            files = sorted(input_dir.iterdir())
             self._upload_files(files, record_id)
             doi = self._publish(record_id)
             return doi

tests/test_zenodo_api_integration.py → tests/integration/test_zenodo_api.py

@@ -5,8 +5,8 @@
 
 from nipoppy.zenodo_api import ZenodoAPI, ZenodoAPIError
 
-from .conftest import datetime_fixture  # noqa F401
-from .conftest import TEST_PIPELINE
+from ..conftest import datetime_fixture  # noqa F401
+from ..conftest import TEST_PIPELINE
 
 ZENODO_SANDBOX = True
 

tests/test_config_main.py → tests/unit/config/test_main.py

@@ -12,8 +12,7 @@
 from nipoppy.config.pipeline import BasePipelineConfig
 from nipoppy.env import CURRENT_SCHEMA_VERSION, PipelineTypeEnum
 from nipoppy.utils import FPATH_SAMPLE_CONFIG
-
-from .conftest import DPATH_TEST_DATA
+from tests.conftest import DPATH_TEST_DATA
 
 FIELDS_PIPELINE_VARIABLES = ["BIDSIFICATION", "PROCESSING", "EXTRACTION"]
 REQUIRED_FIELDS_CONFIG = []

tests/test_tabular_base.py → tests/unit/tabular/test_base.py

@@ -9,8 +9,7 @@
 from pydantic import ValidationError
 
 from nipoppy.tabular.base import BaseTabular, BaseTabularModel
-
-from .conftest import DPATH_TEST_DATA
+from tests.conftest import DPATH_TEST_DATA
 
 
 class Tabular(BaseTabular):

tests/test_tabular_curation_status.py → tests/unit/tabular/test_curation_status.py

@@ -13,8 +13,7 @@
     update_curation_status_table,
 )
 from nipoppy.tabular.dicom_dir_map import DicomDirMap
-
-from .conftest import DPATH_TEST_DATA, check_curation_status_table, prepare_dataset
+from tests.conftest import DPATH_TEST_DATA, check_curation_status_table, prepare_dataset
 
 
 @pytest.fixture

tests/test_tabular_dicom_dir_map.py → tests/unit/tabular/test_dicom_dir_map.py

@@ -5,8 +5,7 @@
 from nipoppy.tabular.dicom_dir_map import DicomDirMap
 from nipoppy.tabular.manifest import Manifest
 from nipoppy.utils import FPATH_SAMPLE_DICOM_DIR_MAP
-
-from .conftest import DPATH_TEST_DATA
+from tests.conftest import DPATH_TEST_DATA
 
 
 @pytest.mark.parametrize(

tests/test_tabular_manifest.py → tests/unit/tabular/test_manifest.py

@@ -6,8 +6,7 @@
 import pytest
 
 from nipoppy.tabular.manifest import Manifest
-
-from .conftest import DPATH_TEST_DATA
+from tests.conftest import DPATH_TEST_DATA
 
 
 @pytest.mark.parametrize(

tests/test_tabular_processing_status.py → tests/unit/tabular/test_processing_status.py

@@ -9,8 +9,7 @@
     ProcessingStatusModel,
     ProcessingStatusTable,
 )
-
-from .conftest import DPATH_TEST_DATA
+from tests.conftest import DPATH_TEST_DATA
 
 
 @pytest.mark.parametrize(
@@ -429,8 +428,7 @@ def test_get_completed_participants_sessions(
     [
         DPATH_TEST_DATA / "processing_status1.tsv",
         DPATH_TEST_DATA / "processing_status2.tsv",
-        Path(__file__).parent
-        / ".."
+        Path(__file__).parents[3]
         / "docs"
         / "source"
         / "how_to_guides"

tests/test_cli.py → tests/unit/test_cli.py

@@ -8,8 +8,7 @@
 
 from nipoppy.cli import cli
 from nipoppy.env import ReturnCode
-
-from .conftest import ATTR_TO_DPATH_MAP, PASSWORD_FILE
+from tests.conftest import ATTR_TO_DPATH_MAP, PASSWORD_FILE
 
 runner = CliRunner()
 

tests/test_layout.py → tests/unit/test_layout.py

@@ -9,8 +9,7 @@
 from nipoppy.env import PipelineTypeEnum
 from nipoppy.layout import DatasetLayout, PathInfo
 from nipoppy.utils import DPATH_LAYOUTS, FPATH_DEFAULT_LAYOUT
-
-from .conftest import (
+from tests.conftest import (
     ATTR_TO_DPATH_MAP,
     ATTR_TO_REQUIRED_FPATH_MAP,
     DPATH_TEST_DATA,

tests/test_pipeline_validation.py → tests/unit/test_pipeline_validation.py

@@ -26,8 +26,7 @@
     _load_pipeline_config_file,
     check_pipeline_bundle,
 )
-
-from .conftest import DPATH_TEST_DATA
+from tests.conftest import DPATH_TEST_DATA
 
 
 @pytest.fixture()
@@ -274,7 +273,6 @@ def test_check_pybids_ignore_file_invalid(fpath, exception_class, exception_mess
 def test_check_config_files(
     pipeline_config_data, pipeline_class, n_files_expected, valid_config_data
 ):
-
     pipeline_config = pipeline_class(**pipeline_config_data, **valid_config_data)
     files = _check_pipeline_files(pipeline_config, DPATH_TEST_DATA)
 

tests/test_utils.py → tests/unit/test_utils.py

@@ -27,9 +27,8 @@
     save_json,
     session_id_to_bids_session_id,
 )
-
-from .conftest import datetime_fixture  # noqa F401
-from .conftest import (
+from tests.conftest import datetime_fixture  # noqa F401
+from tests.conftest import (
     DPATH_TEST_DATA,
     create_empty_dataset,
 )

tests/test_zenodo_api.py → tests/unit/test_zenodo_api.py

@@ -10,8 +10,7 @@
 import pytest_mock
 
 from nipoppy.zenodo_api import InvalidChecksumError, ZenodoAPI, ZenodoAPIError
-
-from .conftest import PASSWORD_FILE
+from tests.conftest import PASSWORD_FILE
 
 
 @pytest.fixture(scope="function")

tests/test_workflow_pipeline_store_create.py → tests/unit/workflows/pipeline_store/test_create.py

@@ -1,3 +1,5 @@
+"""Tests for PipelineCreateWorkflow class."""
+
 from pathlib import Path
 
 import pytest

tests/test_workflow_pipeline_store_install.py → tests/unit/workflows/pipeline_store/test_install.py

@@ -19,9 +19,8 @@
 from nipoppy.layout import DatasetLayout
 from nipoppy.workflows.pipeline_store.install import PipelineInstallWorkflow
 from nipoppy.zenodo_api import ZenodoAPI
-
-from .conftest import record_id  # noqa: F401
-from .conftest import TEST_PIPELINE, create_pipeline_config_files, get_config
+from tests.conftest import record_id  # noqa: F401
+from tests.conftest import TEST_PIPELINE, create_pipeline_config_files, get_config
 
 
 @pytest.fixture(scope="function")
@@ -371,7 +370,6 @@ def test_run_main_file_not_found(workflow: PipelineInstallWorkflow):
 
 
 def test_download(workflow_zenodo: PipelineInstallWorkflow):
-
     workflow_zenodo.run_main()
 
     # Check that the pipeline was downloaded and moved correctly

tests/test_workflow_pipeline_store_list.py → tests/unit/workflows/pipeline_store/test_list.py

@@ -8,8 +8,7 @@
 from nipoppy.config.pipeline import BasePipelineConfig
 from nipoppy.env import CURRENT_SCHEMA_VERSION, PipelineTypeEnum
 from nipoppy.workflows.pipeline_store.list import PipelineListWorkflow
-
-from .conftest import create_empty_dataset
+from tests.conftest import create_empty_dataset
 
 
 @pytest.fixture()

tests/test_workflow_zenodo.py → tests/unit/workflows/pipeline_store/test_upload.py

@@ -1,3 +1,5 @@
+"""Test for the ZenodoUploadWorkflow class."""
+
 from contextlib import nullcontext
 
 import pytest
@@ -6,13 +8,12 @@
 from nipoppy.config.pipeline import BasePipelineConfig
 from nipoppy.env import PipelineTypeEnum
 from nipoppy.pipeline_validation import _load_pipeline_config_file
-from nipoppy.workflows.pipeline_store.zenodo import (
+from nipoppy.workflows.pipeline_store.upload import (
     ZenodoUploadWorkflow,
     _is_same_pipeline,
 )
 from nipoppy.zenodo_api import ZenodoAPI, ZenodoAPIError
-
-from .conftest import TEST_PIPELINE
+from tests.conftest import TEST_PIPELINE
 
 DATASET_PATH = "my_dataset"
 
@@ -23,10 +24,10 @@ def zenodo_api_mocker(mocker: pytest_mock.MockerFixture):
         "nipoppy.zenodo_api.ZenodoAPI.upload_pipeline",
     )
     mocker.patch(
-        "nipoppy.workflows.pipeline_store.zenodo.ZenodoUploadWorkflow._get_pipeline_metadata",
+        "nipoppy.workflows.pipeline_store.upload.ZenodoUploadWorkflow._get_pipeline_metadata",  # noqa: E501
     )
     mocker.patch(
-        "nipoppy.workflows.pipeline_store.zenodo.check_pipeline_bundle",
+        "nipoppy.workflows.pipeline_store.upload.check_pipeline_bundle",
     )
     yield mocker
 
@@ -36,10 +37,10 @@ def test_upload(mocker: pytest_mock.MockerFixture):
         "nipoppy.zenodo_api.ZenodoAPI.upload_pipeline",
     )
     get_pipeline_metadata = mocker.patch(
-        "nipoppy.workflows.pipeline_store.zenodo.ZenodoUploadWorkflow._get_pipeline_metadata",
+        "nipoppy.workflows.pipeline_store.upload.ZenodoUploadWorkflow._get_pipeline_metadata",  # noqa: E501
     )
     validator = mocker.patch(
-        "nipoppy.workflows.pipeline_store.zenodo.check_pipeline_bundle",
+        "nipoppy.workflows.pipeline_store.upload.check_pipeline_bundle",
     )
 
     zenodo_api = ZenodoAPI(sandbox=True)
@@ -206,7 +207,7 @@ def test_confirm_upload_no(
     zenodo_api_mocker: pytest_mock.MockerFixture, caplog: pytest.LogCaptureFixture
 ):
     zenodo_api_mocker.patch(
-        "nipoppy.workflows.pipeline_store.zenodo.CONSOLE_STDOUT.confirm",
+        "nipoppy.workflows.pipeline_store.upload.CONSOLE_STDOUT.confirm",
         return_value=False,
     )
     zenodo_api = ZenodoAPI(sandbox=True)
@@ -231,7 +232,9 @@ def test_confirm_upload_no(
                 {"links": {"self_html": "https://zenodo.org/records/123456"}},
                 {"links": {"self_html": "https://zenodo.org/records/123456"}},
             ],
-            "https://zenodo.org/records/123456, https://zenodo.org/records/123456, https://zenodo.org/records/123456",
+            # TODO: This should be a list of URLs, not a single string
+            # We can handle the conversion in the test itself
+            "https://zenodo.org/records/123456, https://zenodo.org/records/123456, https://zenodo.org/records/123456",  # noqa: E501
         ]
     ],
 )

tests/test_workflow_bids_conversion.py → tests/unit/workflows/test_bids_conversion.py

@@ -8,8 +8,11 @@
 from nipoppy.config.pipeline import BidsPipelineConfig
 from nipoppy.tabular.curation_status import CurationStatusTable
 from nipoppy.workflows.bids_conversion import BidsConversionRunner
-
-from .conftest import create_empty_dataset, create_pipeline_config_files, get_config
+from tests.conftest import (
+    create_empty_dataset,
+    create_pipeline_config_files,
+    get_config,
+)
 
 
 @pytest.fixture

tests/test_workflow_dataset.py → tests/unit/workflows/test_dataset.py

@@ -12,8 +12,12 @@
 from nipoppy.tabular.manifest import Manifest
 from nipoppy.utils import FPATH_SAMPLE_CONFIG, FPATH_SAMPLE_MANIFEST
 from nipoppy.workflows.base import BaseDatasetWorkflow
-
-from .conftest import DPATH_TEST_DATA, create_empty_dataset, get_config, prepare_dataset
+from tests.conftest import (
+    DPATH_TEST_DATA,
+    create_empty_dataset,
+    get_config,
+    prepare_dataset,
+)
 
 
 @pytest.fixture()

tests/test_workflow_dicom_reorg.py → tests/unit/workflows/test_dicom_reorg.py

@@ -15,8 +15,12 @@
     session_id_to_bids_session_id,
 )
 from nipoppy.workflows.dicom_reorg import DicomReorgWorkflow, is_derived_dicom
-
-from .conftest import DPATH_TEST_DATA, create_empty_dataset, get_config, prepare_dataset
+from tests.conftest import (
+    DPATH_TEST_DATA,
+    create_empty_dataset,
+    get_config,
+    prepare_dataset,
+)
 
 
 @pytest.fixture()

tests/test_workflow_extractor.py → tests/unit/workflows/test_extractor.py

@@ -13,8 +13,11 @@
     session_id_to_bids_session_id,
 )
 from nipoppy.workflows.extractor import ExtractionRunner
-
-from .conftest import create_empty_dataset, create_pipeline_config_files, get_config
+from tests.conftest import (
+    create_empty_dataset,
+    create_pipeline_config_files,
+    get_config,
+)
 
 
 @pytest.fixture(scope="function")
@@ -252,7 +255,6 @@ def test_run_single(
     extractor: ExtractionRunner,
     mocker: pytest_mock.MockerFixture,
 ):
-
     mocked_process_container_config = mocker.patch(
         "nipoppy.workflows.runner.PipelineRunner.process_container_config",
         # usually returns string and config object

tests/test_workflow_init.py → tests/unit/workflows/test_init.py

@@ -10,8 +10,7 @@
 from nipoppy.tabular.manifest import Manifest
 from nipoppy.utils import DPATH_HPC, DPATH_LAYOUTS
 from nipoppy.workflows.dataset_init import InitWorkflow
-
-from .conftest import ATTR_TO_DPATH_MAP, FPATH_CONFIG, FPATH_MANIFEST
+from tests.conftest import ATTR_TO_DPATH_MAP, FPATH_CONFIG, FPATH_MANIFEST
 
 
 @pytest.fixture(params=["my_dataset", "dataset_dir"])

tests/test_workflow_pipeline.py → tests/unit/workflows/test_pipeline.py

@@ -34,9 +34,8 @@
     apply_analysis_level,
     get_pipeline_version,
 )
-
-from .conftest import datetime_fixture  # noqa F401
-from .conftest import (
+from tests.conftest import datetime_fixture  # noqa F401
+from tests.conftest import (
     create_empty_dataset,
     create_pipeline_config_files,
     get_config,

tests/test_workflow_runner.py → tests/unit/workflows/test_runner.py

@@ -18,8 +18,7 @@
 from nipoppy.tabular.manifest import Manifest
 from nipoppy.tabular.processing_status import ProcessingStatusTable
 from nipoppy.workflows.runner import PipelineRunner
-
-from .conftest import (
+from tests.conftest import (
     create_empty_dataset,
     create_pipeline_config_files,
     get_config,