diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..b3e4f6b1 --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +ignore = E501, W503 +max-line-length = 79 +max-complexity = 18 +select = B,C,E,F,W,T4,B9,N diff --git a/.github/workflows/pyDataPipeline.yaml b/.github/workflows/pyDataPipeline.yaml index d1081993..562a508e 100644 --- a/.github/workflows/pyDataPipeline.yaml +++ b/.github/workflows/pyDataPipeline.yaml @@ -43,7 +43,7 @@ jobs: fair pull simpleModel/ext/SEIRSconfig.yaml fair run simpleModel/ext/SEIRSconfig.yaml - python3 -m poetry run pytest --cov=fairdatapipeline.api --cov-report=xml --cov-report=term -s tests/ + python3 -m poetry run pytest --cov=fairdatapipeline --cov-report=xml --cov-report=term -s tests/ - uses: codecov/codecov-action@v2 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.zenodo.json b/.zenodo.json index a65d13c9..e79a817f 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -19,7 +19,7 @@ }, { "affiliation": "United Kingdom Atomic Energy Authority", - "name": "Viola, Bruno", + "name": "Viola, Bruno", "orcid": "0000-0001-5406-5860" }, { diff --git a/README.md b/README.md index 40d3cd8c..59e2b223 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Assume FDP_CONFIG_DIR, storage_locations and objects have been set by CLI tool ``` import os -import org.fairdatapipeline.api as pipeline +import fairdatapipeline as pipeline token = os.environ.get('FDP_LOCAL_TOKEN') config_path = os.environ.get('FDP_CONFIG_DIR') + '/config.yaml' diff --git a/fairdatapipeline/fdp_utils.py b/fairdatapipeline/fdp_utils.py index 614f6c2b..f05c21ad 100644 --- a/fairdatapipeline/fdp_utils.py +++ b/fairdatapipeline/fdp_utils.py @@ -203,10 +203,9 @@ def post_storage_root( Returns: | dict: repsonse from the local registy """ - if "local" in data.keys(): - if data["local"]: - data["root"] = "file://" + data["root"] - if not data["root"][-1] == "/": + if "local" in data and data["local"]: + data["root"] = "file://" + data["root"] + if data["root"][-1] != "/": data["root"] = data["root"] + "/" return post_entry(url, "storage_root", data, token, api_version) @@ -340,11 +339,11 @@ def get_handle_index_from_path(handle: dict, path: str) -> Optional[Any]: | path: path as generated by link_read or link_write """ tmp = None - if "output" in handle.keys(): + if "output" in handle: for output in handle["output"]: if handle["output"][output]["path"] == path: tmp = output - if "input" in handle.keys(): + if "input" in handle: for input in handle["input"]: if handle["input"][input]["path"] == path: tmp = input @@ -352,7 +351,7 @@ def get_handle_index_from_path(handle: dict, path: str) -> Optional[Any]: # flake8: noqa C901 -def register_issues(token: str, handle: dict) -> dict: +def register_issues(token: str, handle: dict) -> dict: # sourcery no-metrics """ Internal function, should only be called from finalise. """ @@ -381,11 +380,11 @@ def register_issues(token: str, handle: dict) -> dict: object_id = None if type == "config": object_id = handle["model_config"] - elif type == "submission_script": - object_id = handle["submission_script"] elif type == "github_repo": object_id = handle["code_repo"] + elif type == "submission_script": + object_id = handle["submission_script"] if object_id: component_url = get_entry( url=api_url, @@ -398,7 +397,7 @@ def register_issues(token: str, handle: dict) -> dict: )[0]["url"] if index: - if "output" in handle.keys(): + if "output" in handle: for ii in handle["output"]: if handle["output"][ii] == index: if ( @@ -410,7 +409,7 @@ def register_issues(token: str, handle: dict) -> dict: ] else: logging.warning("No Component Found") - if "input" in handle.keys(): + if "input" in handle: for ii in handle["input"]: if handle["input"][ii] == index: if ( diff --git a/pyproject.toml b/pyproject.toml index 0239ef87..ed212f78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ version = "0.4.0" description = "Python api to interact with the Fair Data Pipeline" authors = [ "Ryan J Field ", - "Dennis Reddyhoff ", + "Dennis Reddyhoff ", "Robert D Turner ", "Bruno Viola ", "Kristian Zarebski " @@ -78,14 +78,14 @@ line-length = 79 [tool.pytest.ini_options] addopts = '-s -v' +markers = [ + "pipeline: tests for 'pipeline' module ", + "issue: tests for raising issues ", + "utilities: tests for 'utilities' functions ", + "apiversion: tests for api versioning ", +] [tool.mypy] ignore_missing_imports = true disallow_untyped_defs = true disallow_untyped_calls = true - -[tool.flake8] -ignore = 'E501, W503' -max-line-length = 79 -max-complexity = 18 -select = 'B,C,E,F,W,T4,B9,N' diff --git a/tests/test_fdp_utils.py b/tests/test_fdp_utils.py index e6048e6f..77744ba4 100644 --- a/tests/test_fdp_utils.py +++ b/tests/test_fdp_utils.py @@ -171,6 +171,7 @@ def storage_root_test(token: str, url: str, scope: str = "module") -> dict: ) +@pytest.mark.utilities def test_post_entry(token: str, url: str) -> None: storage_root = fdp_utils.post_entry( token=token, @@ -181,6 +182,7 @@ def test_post_entry(token: str, url: str) -> None: assert type(storage_root) == dict +@pytest.mark.utilities def test_post_entry_409(token: str, url: str) -> None: storage_root = fdp_utils.post_entry( token=token, @@ -191,6 +193,7 @@ def test_post_entry_409(token: str, url: str) -> None: assert type(storage_root) == dict +@pytest.mark.utilities def test_post_entry_equal(token: str, url: str) -> None: storage_root = fdp_utils.post_entry( token=token, @@ -207,6 +210,7 @@ def test_post_entry_equal(token: str, url: str) -> None: assert storage_root == storage_root_2 +@pytest.mark.utilities def test_post_entry_500(token: str, url: str) -> None: with pytest.raises(Exception): fdp_utils.post_entry( @@ -217,6 +221,7 @@ def test_post_entry_500(token: str, url: str) -> None: ) +@pytest.mark.utilities def test_get_entry(url: str, token: str, storage_root_test: dict) -> None: entry = fdp_utils.get_entry( url=url, @@ -227,6 +232,7 @@ def test_get_entry(url: str, token: str, storage_root_test: dict) -> None: assert entry[0] == storage_root_test +@pytest.mark.utilities def test_get_entity(url: str, storage_root_test: dict) -> None: entity = fdp_utils.get_entity( url=url, @@ -236,6 +242,7 @@ def test_get_entity(url: str, storage_root_test: dict) -> None: assert entity == storage_root_test +@pytest.mark.apiversion def test_wrong_api_version(token: str, url: str) -> None: with pytest.raises(Exception): fdp_utils.post_entry( @@ -247,6 +254,7 @@ def test_wrong_api_version(token: str, url: str) -> None: ) +@pytest.mark.apiversion def test_wrong_api_version_get(token: str, url: str) -> None: with pytest.raises(Exception): fdp_utils.get_entry( @@ -258,6 +266,7 @@ def test_wrong_api_version_get(token: str, url: str) -> None: ) +@pytest.mark.utilities def test_get_entity_with_token( url: str, storage_root_test: dict, token: str ) -> None: @@ -270,6 +279,7 @@ def test_get_entity_with_token( assert entity == storage_root_test +@pytest.mark.utilities def test_get_entity_non_200(url: str, storage_root_test: dict) -> None: with pytest.raises(Exception): fdp_utils.get_entity( diff --git a/tests/test_raise_issue.py b/tests/test_raise_issue.py index 371ab068..f996da73 100644 --- a/tests/test_raise_issue.py +++ b/tests/test_raise_issue.py @@ -31,17 +31,20 @@ def config(test_dir: str) -> str: return os.path.join(test_dir, "write_csv.yaml") +@pytest.mark.pipeline def test_initialise(token: str, config: str, script: str) -> None: handle = pipeline.initialise(token, config, script) assert type(handle) == dict +@pytest.mark.pipeline def test_link_write(token: str, config: str, script: str) -> None: handle = pipeline.initialise(token, config, script) pipeline.link_write(handle, "test/csv") assert handle["output"]["output_0"]["data_product"] == "test/csv" +@pytest.mark.issue def test_raise_issue_by_index(token: str, config: str, script: str) -> None: handle = pipeline.initialise(token, config, script) link_write = pipeline.link_write(handle, "test/csv") @@ -50,12 +53,14 @@ def test_raise_issue_by_index(token: str, config: str, script: str) -> None: assert handle["issues"]["issue_0"]["use_data_product"] == "test/csv" +@pytest.mark.issue def test_raise_issue_with_config(token: str, config: str, script: str) -> None: handle = pipeline.initialise(token, config, script) pipeline.raise_issue_with_config(handle, "Test Issue with config", 4) assert handle["issues"]["issue_0"]["type"] == "config" +@pytest.mark.issue def test_raise_issue_with_github_repo( token: str, config: str, script: str ) -> None: @@ -66,6 +71,7 @@ def test_raise_issue_with_github_repo( assert handle["issues"]["issue_0"]["type"] == "github_repo" +@pytest.mark.issue def test_raise_issue_with_script(token: str, config: str, script: str) -> None: handle = pipeline.initialise(token, config, script) pipeline.raise_issue_with_submission_script( @@ -74,6 +80,7 @@ def test_raise_issue_with_script(token: str, config: str, script: str) -> None: assert handle["issues"]["issue_0"]["type"] == "submission_script" +@pytest.mark.pipeline def test_link_read( token: str, config: str, script: str, test_dir: str ) -> None: @@ -90,6 +97,7 @@ def test_link_read( assert type(link_read_1) == str and type(link_read_2) == str +@pytest.mark.pipeline def test_link_read_data_product_exists( token: str, config: str, script: str, test_dir: str ) -> None: @@ -100,6 +108,7 @@ def test_link_read_data_product_exists( pipeline.finalise(token, handle) +@pytest.mark.issue def test_raise_issue_existing_data_product( token: str, config: str, script: str ) -> None: @@ -115,6 +124,7 @@ def test_raise_issue_existing_data_product( pipeline.finalise(token, handle) +@pytest.mark.issue def test_raise_issue_data_product_from_reads( token: str, script: str, test_dir: str ) -> None: