Feature/pytest markers

.flake8

@@ -0,0 +1,5 @@
+[flake8]
+ignore =  E501, W503
+max-line-length = 79
+max-complexity = 18
+select = B,C,E,F,W,T4,B9,N

.github/workflows/pyDataPipeline.yaml

@@ -43,7 +43,7 @@ jobs:
           fair pull simpleModel/ext/SEIRSconfig.yaml
           fair run simpleModel/ext/SEIRSconfig.yaml
 
-          python3 -m poetry run pytest --cov=fairdatapipeline.api  --cov-report=xml --cov-report=term -s tests/
+          python3 -m poetry run pytest --cov=fairdatapipeline  --cov-report=xml --cov-report=term -s tests/
       - uses: codecov/codecov-action@v2
         with:
           token: ${{ secrets.CODECOV_TOKEN }}

.zenodo.json

@@ -19,7 +19,7 @@
       },
       {
         "affiliation": "United Kingdom Atomic Energy Authority",
-        "name": "Viola, Bruno",        
+        "name": "Viola, Bruno",
         "orcid": "0000-0001-5406-5860"
       },
       {

README.md

@@ -26,7 +26,7 @@ Assume FDP_CONFIG_DIR, storage_locations and objects have been set by CLI tool
 
 ```
 import os
-import org.fairdatapipeline.api as pipeline
+import fairdatapipeline as pipeline
 
 token = os.environ.get('FDP_LOCAL_TOKEN')
 config_path = os.environ.get('FDP_CONFIG_DIR') + '/config.yaml'

fairdatapipeline/fdp_utils.py

@@ -203,10 +203,9 @@ def post_storage_root(
     Returns:
         |   dict: repsonse from the local registy
     """
-    if "local" in data.keys():
-        if data["local"]:
-            data["root"] = "file://" + data["root"]
-    if not data["root"][-1] == "/":
+    if "local" in data and data["local"]:
+        data["root"] = "file://" + data["root"]
+    if data["root"][-1] != "/":
         data["root"] = data["root"] + "/"
     return post_entry(url, "storage_root", data, token, api_version)
 
@@ -340,19 +339,19 @@ def get_handle_index_from_path(handle: dict, path: str) -> Optional[Any]:
         |   path: path as generated by link_read or link_write
     """
     tmp = None
-    if "output" in handle.keys():
+    if "output" in handle:
         for output in handle["output"]:
             if handle["output"][output]["path"] == path:
                 tmp = output
-    if "input" in handle.keys():
+    if "input" in handle:
         for input in handle["input"]:
             if handle["input"][input]["path"] == path:
                 tmp = input
     return tmp
 
 
 # flake8: noqa C901
-def register_issues(token: str, handle: dict) -> dict:
+def register_issues(token: str, handle: dict) -> dict:  # sourcery no-metrics
     """
     Internal function, should only be called from finalise.
     """
@@ -381,11 +380,11 @@ def register_issues(token: str, handle: dict) -> dict:
                 object_id = None
                 if type == "config":
                     object_id = handle["model_config"]
-                elif type == "submission_script":
-                    object_id = handle["submission_script"]
                 elif type == "github_repo":
                     object_id = handle["code_repo"]
 
+                elif type == "submission_script":
+                    object_id = handle["submission_script"]
                 if object_id:
                     component_url = get_entry(
                         url=api_url,
@@ -398,7 +397,7 @@ def register_issues(token: str, handle: dict) -> dict:
                     )[0]["url"]
 
                 if index:
-                    if "output" in handle.keys():
+                    if "output" in handle:
                         for ii in handle["output"]:
                             if handle["output"][ii] == index:
                                 if (
@@ -410,7 +409,7 @@ def register_issues(token: str, handle: dict) -> dict:
                                     ]
                                 else:
                                     logging.warning("No Component Found")
-                    if "input" in handle.keys():
+                    if "input" in handle:
                         for ii in handle["input"]:
                             if handle["input"][ii] == index:
                                 if (

pyproject.toml

@@ -4,7 +4,7 @@ version = "0.4.0"
 description = "Python api to interact with the Fair Data Pipeline"
 authors = [
     "Ryan J Field <ryan.field@glasgow.ac.uk>",
-    "Dennis Reddyhoff <d.reddyhoff@sheffield.ac.uk>",  
+    "Dennis Reddyhoff <d.reddyhoff@sheffield.ac.uk>",
     "Robert D Turner <r.d.turner@shef.ac.uk>",
     "Bruno Viola <bruno.viola@ukaea.uk>",
     "Kristian Zarebski <kristian.zarebski@ukaea.uk>"
@@ -78,14 +78,14 @@ line-length = 79
 
 [tool.pytest.ini_options]
 addopts = '-s -v'
+markers = [
+    "pipeline: tests for 'pipeline' module ",
+    "issue: tests for raising issues ",
+    "utilities: tests for 'utilities' functions ",
+    "apiversion: tests for api versioning ",
+]
 
 [tool.mypy]
 ignore_missing_imports = true
 disallow_untyped_defs = true
 disallow_untyped_calls = true
-
-[tool.flake8]
-ignore =  'E501, W503'
-max-line-length = 79
-max-complexity = 18
-select = 'B,C,E,F,W,T4,B9,N'

tests/test_fdp_utils.py

@@ -171,6 +171,7 @@ def storage_root_test(token: str, url: str, scope: str = "module") -> dict:
     )
 
 
+@pytest.mark.utilities
 def test_post_entry(token: str, url: str) -> None:
     storage_root = fdp_utils.post_entry(
         token=token,
@@ -181,6 +182,7 @@ def test_post_entry(token: str, url: str) -> None:
     assert type(storage_root) == dict
 
 
+@pytest.mark.utilities
 def test_post_entry_409(token: str, url: str) -> None:
     storage_root = fdp_utils.post_entry(
         token=token,
@@ -191,6 +193,7 @@ def test_post_entry_409(token: str, url: str) -> None:
     assert type(storage_root) == dict
 
 
+@pytest.mark.utilities
 def test_post_entry_equal(token: str, url: str) -> None:
     storage_root = fdp_utils.post_entry(
         token=token,
@@ -207,6 +210,7 @@ def test_post_entry_equal(token: str, url: str) -> None:
     assert storage_root == storage_root_2
 
 
+@pytest.mark.utilities
 def test_post_entry_500(token: str, url: str) -> None:
     with pytest.raises(Exception):
         fdp_utils.post_entry(
@@ -217,6 +221,7 @@ def test_post_entry_500(token: str, url: str) -> None:
         )
 
 
+@pytest.mark.utilities
 def test_get_entry(url: str, token: str, storage_root_test: dict) -> None:
     entry = fdp_utils.get_entry(
         url=url,
@@ -227,6 +232,7 @@ def test_get_entry(url: str, token: str, storage_root_test: dict) -> None:
     assert entry[0] == storage_root_test
 
 
+@pytest.mark.utilities
 def test_get_entity(url: str, storage_root_test: dict) -> None:
     entity = fdp_utils.get_entity(
         url=url,
@@ -236,6 +242,7 @@ def test_get_entity(url: str, storage_root_test: dict) -> None:
     assert entity == storage_root_test
 
 
+@pytest.mark.apiversion
 def test_wrong_api_version(token: str, url: str) -> None:
     with pytest.raises(Exception):
         fdp_utils.post_entry(
@@ -247,6 +254,7 @@ def test_wrong_api_version(token: str, url: str) -> None:
         )
 
 
+@pytest.mark.apiversion
 def test_wrong_api_version_get(token: str, url: str) -> None:
     with pytest.raises(Exception):
         fdp_utils.get_entry(
@@ -258,6 +266,7 @@ def test_wrong_api_version_get(token: str, url: str) -> None:
         )
 
 
+@pytest.mark.utilities
 def test_get_entity_with_token(
     url: str, storage_root_test: dict, token: str
 ) -> None:
@@ -270,6 +279,7 @@ def test_get_entity_with_token(
     assert entity == storage_root_test
 
 
+@pytest.mark.utilities
 def test_get_entity_non_200(url: str, storage_root_test: dict) -> None:
     with pytest.raises(Exception):
         fdp_utils.get_entity(

tests/test_raise_issue.py

@@ -31,17 +31,20 @@ def config(test_dir: str) -> str:
     return os.path.join(test_dir, "write_csv.yaml")
 
 
+@pytest.mark.pipeline
 def test_initialise(token: str, config: str, script: str) -> None:
     handle = pipeline.initialise(token, config, script)
     assert type(handle) == dict
 
 
+@pytest.mark.pipeline
 def test_link_write(token: str, config: str, script: str) -> None:
     handle = pipeline.initialise(token, config, script)
     pipeline.link_write(handle, "test/csv")
     assert handle["output"]["output_0"]["data_product"] == "test/csv"
 
 
+@pytest.mark.issue
 def test_raise_issue_by_index(token: str, config: str, script: str) -> None:
     handle = pipeline.initialise(token, config, script)
     link_write = pipeline.link_write(handle, "test/csv")
@@ -50,12 +53,14 @@ def test_raise_issue_by_index(token: str, config: str, script: str) -> None:
     assert handle["issues"]["issue_0"]["use_data_product"] == "test/csv"
 
 
+@pytest.mark.issue
 def test_raise_issue_with_config(token: str, config: str, script: str) -> None:
     handle = pipeline.initialise(token, config, script)
     pipeline.raise_issue_with_config(handle, "Test Issue with config", 4)
     assert handle["issues"]["issue_0"]["type"] == "config"
 
 
+@pytest.mark.issue
 def test_raise_issue_with_github_repo(
     token: str, config: str, script: str
 ) -> None:
@@ -66,6 +71,7 @@ def test_raise_issue_with_github_repo(
     assert handle["issues"]["issue_0"]["type"] == "github_repo"
 
 
+@pytest.mark.issue
 def test_raise_issue_with_script(token: str, config: str, script: str) -> None:
     handle = pipeline.initialise(token, config, script)
     pipeline.raise_issue_with_submission_script(
@@ -74,6 +80,7 @@ def test_raise_issue_with_script(token: str, config: str, script: str) -> None:
     assert handle["issues"]["issue_0"]["type"] == "submission_script"
 
 
+@pytest.mark.pipeline
 def test_link_read(
     token: str, config: str, script: str, test_dir: str
 ) -> None:
@@ -90,6 +97,7 @@ def test_link_read(
     assert type(link_read_1) == str and type(link_read_2) == str
 
 
+@pytest.mark.pipeline
 def test_link_read_data_product_exists(
     token: str, config: str, script: str, test_dir: str
 ) -> None:
@@ -100,6 +108,7 @@ def test_link_read_data_product_exists(
     pipeline.finalise(token, handle)
 
 
+@pytest.mark.issue
 def test_raise_issue_existing_data_product(
     token: str, config: str, script: str
 ) -> None:
@@ -115,6 +124,7 @@ def test_raise_issue_existing_data_product(
     pipeline.finalise(token, handle)
 
 
+@pytest.mark.issue
 def test_raise_issue_data_product_from_reads(
     token: str, script: str, test_dir: str
 ) -> None: