From ff665703ed5458b9cae8fa7e4171684b922669c2 Mon Sep 17 00:00:00 2001 From: bruvio Date: Mon, 10 Jan 2022 12:50:17 +0000 Subject: [PATCH 1/3] added pytest markers --- .github/workflows/pyDataPipeline.yaml | 2 +- .zenodo.json | 2 +- fairdatapipeline/fdp_utils.py | 21 ++++++++++----------- pyproject.toml | 11 ++++++----- tests/test_fdp_utils.py | 20 ++++++++++---------- tests/test_raise_issue.py | 20 ++++++++++---------- 6 files changed, 38 insertions(+), 38 deletions(-) diff --git a/.github/workflows/pyDataPipeline.yaml b/.github/workflows/pyDataPipeline.yaml index d1081993..562a508e 100644 --- a/.github/workflows/pyDataPipeline.yaml +++ b/.github/workflows/pyDataPipeline.yaml @@ -43,7 +43,7 @@ jobs: fair pull simpleModel/ext/SEIRSconfig.yaml fair run simpleModel/ext/SEIRSconfig.yaml - python3 -m poetry run pytest --cov=fairdatapipeline.api --cov-report=xml --cov-report=term -s tests/ + python3 -m poetry run pytest --cov=fairdatapipeline --cov-report=xml --cov-report=term -s tests/ - uses: codecov/codecov-action@v2 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.zenodo.json b/.zenodo.json index a65d13c9..e79a817f 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -19,7 +19,7 @@ }, { "affiliation": "United Kingdom Atomic Energy Authority", - "name": "Viola, Bruno", + "name": "Viola, Bruno", "orcid": "0000-0001-5406-5860" }, { diff --git a/fairdatapipeline/fdp_utils.py b/fairdatapipeline/fdp_utils.py index 614f6c2b..f05c21ad 100644 --- a/fairdatapipeline/fdp_utils.py +++ b/fairdatapipeline/fdp_utils.py @@ -203,10 +203,9 @@ def post_storage_root( Returns: | dict: repsonse from the local registy """ - if "local" in data.keys(): - if data["local"]: - data["root"] = "file://" + data["root"] - if not data["root"][-1] == "/": + if "local" in data and data["local"]: + data["root"] = "file://" + data["root"] + if data["root"][-1] != "/": data["root"] = data["root"] + "/" return post_entry(url, "storage_root", data, token, api_version) @@ -340,11 +339,11 @@ def get_handle_index_from_path(handle: dict, path: str) -> Optional[Any]: | path: path as generated by link_read or link_write """ tmp = None - if "output" in handle.keys(): + if "output" in handle: for output in handle["output"]: if handle["output"][output]["path"] == path: tmp = output - if "input" in handle.keys(): + if "input" in handle: for input in handle["input"]: if handle["input"][input]["path"] == path: tmp = input @@ -352,7 +351,7 @@ def get_handle_index_from_path(handle: dict, path: str) -> Optional[Any]: # flake8: noqa C901 -def register_issues(token: str, handle: dict) -> dict: +def register_issues(token: str, handle: dict) -> dict: # sourcery no-metrics """ Internal function, should only be called from finalise. """ @@ -381,11 +380,11 @@ def register_issues(token: str, handle: dict) -> dict: object_id = None if type == "config": object_id = handle["model_config"] - elif type == "submission_script": - object_id = handle["submission_script"] elif type == "github_repo": object_id = handle["code_repo"] + elif type == "submission_script": + object_id = handle["submission_script"] if object_id: component_url = get_entry( url=api_url, @@ -398,7 +397,7 @@ def register_issues(token: str, handle: dict) -> dict: )[0]["url"] if index: - if "output" in handle.keys(): + if "output" in handle: for ii in handle["output"]: if handle["output"][ii] == index: if ( @@ -410,7 +409,7 @@ def register_issues(token: str, handle: dict) -> dict: ] else: logging.warning("No Component Found") - if "input" in handle.keys(): + if "input" in handle: for ii in handle["input"]: if handle["input"][ii] == index: if ( diff --git a/pyproject.toml b/pyproject.toml index 0239ef87..8d4e266b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,14 +78,15 @@ line-length = 79 [tool.pytest.ini_options] addopts = '-s -v' +markers = [ + "pipeline: tests for 'pipeline' module ", + "issue: tests for raising issues ", + "utilities: tests for 'utilities' functions ", + "apiversion: tests for api versioning ", +] [tool.mypy] ignore_missing_imports = true disallow_untyped_defs = true disallow_untyped_calls = true -[tool.flake8] -ignore = 'E501, W503' -max-line-length = 79 -max-complexity = 18 -select = 'B,C,E,F,W,T4,B9,N' diff --git a/tests/test_fdp_utils.py b/tests/test_fdp_utils.py index e6048e6f..10b6ee23 100644 --- a/tests/test_fdp_utils.py +++ b/tests/test_fdp_utils.py @@ -170,7 +170,7 @@ def storage_root_test(token: str, url: str, scope: str = "module") -> dict: endpoint="storage_root", ) - +@pytest.mark.utilities def test_post_entry(token: str, url: str) -> None: storage_root = fdp_utils.post_entry( token=token, @@ -180,7 +180,7 @@ def test_post_entry(token: str, url: str) -> None: ) assert type(storage_root) == dict - +@pytest.mark.utilities def test_post_entry_409(token: str, url: str) -> None: storage_root = fdp_utils.post_entry( token=token, @@ -190,7 +190,7 @@ def test_post_entry_409(token: str, url: str) -> None: ) assert type(storage_root) == dict - +@pytest.mark.utilities def test_post_entry_equal(token: str, url: str) -> None: storage_root = fdp_utils.post_entry( token=token, @@ -206,7 +206,7 @@ def test_post_entry_equal(token: str, url: str) -> None: ) assert storage_root == storage_root_2 - +@pytest.mark.utilities def test_post_entry_500(token: str, url: str) -> None: with pytest.raises(Exception): fdp_utils.post_entry( @@ -216,7 +216,7 @@ def test_post_entry_500(token: str, url: str) -> None: endpoint="non_existant", ) - +@pytest.mark.utilities def test_get_entry(url: str, token: str, storage_root_test: dict) -> None: entry = fdp_utils.get_entry( url=url, @@ -226,7 +226,7 @@ def test_get_entry(url: str, token: str, storage_root_test: dict) -> None: ) assert entry[0] == storage_root_test - +@pytest.mark.utilities def test_get_entity(url: str, storage_root_test: dict) -> None: entity = fdp_utils.get_entity( url=url, @@ -235,7 +235,7 @@ def test_get_entity(url: str, storage_root_test: dict) -> None: ) assert entity == storage_root_test - +@pytest.mark.apiversion def test_wrong_api_version(token: str, url: str) -> None: with pytest.raises(Exception): fdp_utils.post_entry( @@ -246,7 +246,7 @@ def test_wrong_api_version(token: str, url: str) -> None: api_version="2.2.2", ) - +@pytest.mark.apiversion def test_wrong_api_version_get(token: str, url: str) -> None: with pytest.raises(Exception): fdp_utils.get_entry( @@ -257,7 +257,7 @@ def test_wrong_api_version_get(token: str, url: str) -> None: api_version="3.0.0", ) - +@pytest.mark.utilities def test_get_entity_with_token( url: str, storage_root_test: dict, token: str ) -> None: @@ -269,7 +269,7 @@ def test_get_entity_with_token( ) assert entity == storage_root_test - +@pytest.mark.utilities def test_get_entity_non_200(url: str, storage_root_test: dict) -> None: with pytest.raises(Exception): fdp_utils.get_entity( diff --git a/tests/test_raise_issue.py b/tests/test_raise_issue.py index 371ab068..53f0001a 100644 --- a/tests/test_raise_issue.py +++ b/tests/test_raise_issue.py @@ -30,18 +30,18 @@ def script(test_dir: str) -> str: def config(test_dir: str) -> str: return os.path.join(test_dir, "write_csv.yaml") - +@pytest.mark.pipeline def test_initialise(token: str, config: str, script: str) -> None: handle = pipeline.initialise(token, config, script) assert type(handle) == dict - +@pytest.mark.pipeline def test_link_write(token: str, config: str, script: str) -> None: handle = pipeline.initialise(token, config, script) pipeline.link_write(handle, "test/csv") assert handle["output"]["output_0"]["data_product"] == "test/csv" - +@pytest.mark.issue def test_raise_issue_by_index(token: str, config: str, script: str) -> None: handle = pipeline.initialise(token, config, script) link_write = pipeline.link_write(handle, "test/csv") @@ -49,13 +49,13 @@ def test_raise_issue_by_index(token: str, config: str, script: str) -> None: pipeline.raise_issue_by_index(handle, index, "Test Issue", 7) assert handle["issues"]["issue_0"]["use_data_product"] == "test/csv" - +@pytest.mark.issue def test_raise_issue_with_config(token: str, config: str, script: str) -> None: handle = pipeline.initialise(token, config, script) pipeline.raise_issue_with_config(handle, "Test Issue with config", 4) assert handle["issues"]["issue_0"]["type"] == "config" - +@pytest.mark.issue def test_raise_issue_with_github_repo( token: str, config: str, script: str ) -> None: @@ -65,7 +65,7 @@ def test_raise_issue_with_github_repo( ) assert handle["issues"]["issue_0"]["type"] == "github_repo" - +@pytest.mark.issue def test_raise_issue_with_script(token: str, config: str, script: str) -> None: handle = pipeline.initialise(token, config, script) pipeline.raise_issue_with_submission_script( @@ -73,7 +73,7 @@ def test_raise_issue_with_script(token: str, config: str, script: str) -> None: ) assert handle["issues"]["issue_0"]["type"] == "submission_script" - +@pytest.mark.pipeline def test_link_read( token: str, config: str, script: str, test_dir: str ) -> None: @@ -89,7 +89,7 @@ def test_link_read( link_read_2 = pipeline.link_read(handle, "test/csv") assert type(link_read_1) == str and type(link_read_2) == str - +@pytest.mark.pipeline def test_link_read_data_product_exists( token: str, config: str, script: str, test_dir: str ) -> None: @@ -99,7 +99,7 @@ def test_link_read_data_product_exists( shutil.copy(tmp_csv, link_write) pipeline.finalise(token, handle) - +@pytest.mark.issue def test_raise_issue_existing_data_product( token: str, config: str, script: str ) -> None: @@ -114,7 +114,7 @@ def test_raise_issue_existing_data_product( ) pipeline.finalise(token, handle) - +@pytest.mark.issue def test_raise_issue_data_product_from_reads( token: str, script: str, test_dir: str ) -> None: From ea6106558db8527e0b43479efeb8f7a6bed79f65 Mon Sep 17 00:00:00 2001 From: bruvio Date: Mon, 10 Jan 2022 12:55:24 +0000 Subject: [PATCH 2/3] flake8 cfg added outside pyproject.toml as not supported by poetry yet --- .flake8 | 5 +++++ pyproject.toml | 5 ++--- tests/test_fdp_utils.py | 10 ++++++++++ tests/test_raise_issue.py | 10 ++++++++++ 4 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..b3e4f6b1 --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +ignore = E501, W503 +max-line-length = 79 +max-complexity = 18 +select = B,C,E,F,W,T4,B9,N diff --git a/pyproject.toml b/pyproject.toml index 8d4e266b..ed212f78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ version = "0.4.0" description = "Python api to interact with the Fair Data Pipeline" authors = [ "Ryan J Field ", - "Dennis Reddyhoff ", + "Dennis Reddyhoff ", "Robert D Turner ", "Bruno Viola ", "Kristian Zarebski " @@ -83,10 +83,9 @@ markers = [ "issue: tests for raising issues ", "utilities: tests for 'utilities' functions ", "apiversion: tests for api versioning ", -] +] [tool.mypy] ignore_missing_imports = true disallow_untyped_defs = true disallow_untyped_calls = true - diff --git a/tests/test_fdp_utils.py b/tests/test_fdp_utils.py index 10b6ee23..77744ba4 100644 --- a/tests/test_fdp_utils.py +++ b/tests/test_fdp_utils.py @@ -170,6 +170,7 @@ def storage_root_test(token: str, url: str, scope: str = "module") -> dict: endpoint="storage_root", ) + @pytest.mark.utilities def test_post_entry(token: str, url: str) -> None: storage_root = fdp_utils.post_entry( @@ -180,6 +181,7 @@ def test_post_entry(token: str, url: str) -> None: ) assert type(storage_root) == dict + @pytest.mark.utilities def test_post_entry_409(token: str, url: str) -> None: storage_root = fdp_utils.post_entry( @@ -190,6 +192,7 @@ def test_post_entry_409(token: str, url: str) -> None: ) assert type(storage_root) == dict + @pytest.mark.utilities def test_post_entry_equal(token: str, url: str) -> None: storage_root = fdp_utils.post_entry( @@ -206,6 +209,7 @@ def test_post_entry_equal(token: str, url: str) -> None: ) assert storage_root == storage_root_2 + @pytest.mark.utilities def test_post_entry_500(token: str, url: str) -> None: with pytest.raises(Exception): @@ -216,6 +220,7 @@ def test_post_entry_500(token: str, url: str) -> None: endpoint="non_existant", ) + @pytest.mark.utilities def test_get_entry(url: str, token: str, storage_root_test: dict) -> None: entry = fdp_utils.get_entry( @@ -226,6 +231,7 @@ def test_get_entry(url: str, token: str, storage_root_test: dict) -> None: ) assert entry[0] == storage_root_test + @pytest.mark.utilities def test_get_entity(url: str, storage_root_test: dict) -> None: entity = fdp_utils.get_entity( @@ -235,6 +241,7 @@ def test_get_entity(url: str, storage_root_test: dict) -> None: ) assert entity == storage_root_test + @pytest.mark.apiversion def test_wrong_api_version(token: str, url: str) -> None: with pytest.raises(Exception): @@ -246,6 +253,7 @@ def test_wrong_api_version(token: str, url: str) -> None: api_version="2.2.2", ) + @pytest.mark.apiversion def test_wrong_api_version_get(token: str, url: str) -> None: with pytest.raises(Exception): @@ -257,6 +265,7 @@ def test_wrong_api_version_get(token: str, url: str) -> None: api_version="3.0.0", ) + @pytest.mark.utilities def test_get_entity_with_token( url: str, storage_root_test: dict, token: str @@ -269,6 +278,7 @@ def test_get_entity_with_token( ) assert entity == storage_root_test + @pytest.mark.utilities def test_get_entity_non_200(url: str, storage_root_test: dict) -> None: with pytest.raises(Exception): diff --git a/tests/test_raise_issue.py b/tests/test_raise_issue.py index 53f0001a..f996da73 100644 --- a/tests/test_raise_issue.py +++ b/tests/test_raise_issue.py @@ -30,17 +30,20 @@ def script(test_dir: str) -> str: def config(test_dir: str) -> str: return os.path.join(test_dir, "write_csv.yaml") + @pytest.mark.pipeline def test_initialise(token: str, config: str, script: str) -> None: handle = pipeline.initialise(token, config, script) assert type(handle) == dict + @pytest.mark.pipeline def test_link_write(token: str, config: str, script: str) -> None: handle = pipeline.initialise(token, config, script) pipeline.link_write(handle, "test/csv") assert handle["output"]["output_0"]["data_product"] == "test/csv" + @pytest.mark.issue def test_raise_issue_by_index(token: str, config: str, script: str) -> None: handle = pipeline.initialise(token, config, script) @@ -49,12 +52,14 @@ def test_raise_issue_by_index(token: str, config: str, script: str) -> None: pipeline.raise_issue_by_index(handle, index, "Test Issue", 7) assert handle["issues"]["issue_0"]["use_data_product"] == "test/csv" + @pytest.mark.issue def test_raise_issue_with_config(token: str, config: str, script: str) -> None: handle = pipeline.initialise(token, config, script) pipeline.raise_issue_with_config(handle, "Test Issue with config", 4) assert handle["issues"]["issue_0"]["type"] == "config" + @pytest.mark.issue def test_raise_issue_with_github_repo( token: str, config: str, script: str @@ -65,6 +70,7 @@ def test_raise_issue_with_github_repo( ) assert handle["issues"]["issue_0"]["type"] == "github_repo" + @pytest.mark.issue def test_raise_issue_with_script(token: str, config: str, script: str) -> None: handle = pipeline.initialise(token, config, script) @@ -73,6 +79,7 @@ def test_raise_issue_with_script(token: str, config: str, script: str) -> None: ) assert handle["issues"]["issue_0"]["type"] == "submission_script" + @pytest.mark.pipeline def test_link_read( token: str, config: str, script: str, test_dir: str @@ -89,6 +96,7 @@ def test_link_read( link_read_2 = pipeline.link_read(handle, "test/csv") assert type(link_read_1) == str and type(link_read_2) == str + @pytest.mark.pipeline def test_link_read_data_product_exists( token: str, config: str, script: str, test_dir: str @@ -99,6 +107,7 @@ def test_link_read_data_product_exists( shutil.copy(tmp_csv, link_write) pipeline.finalise(token, handle) + @pytest.mark.issue def test_raise_issue_existing_data_product( token: str, config: str, script: str @@ -114,6 +123,7 @@ def test_raise_issue_existing_data_product( ) pipeline.finalise(token, handle) + @pytest.mark.issue def test_raise_issue_data_product_from_reads( token: str, script: str, test_dir: str From 026229073eac1e84057a89851b5df83939425463 Mon Sep 17 00:00:00 2001 From: bruvio Date: Mon, 10 Jan 2022 16:52:48 +0000 Subject: [PATCH 3/3] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 40d3cd8c..59e2b223 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Assume FDP_CONFIG_DIR, storage_locations and objects have been set by CLI tool ``` import os -import org.fairdatapipeline.api as pipeline +import fairdatapipeline as pipeline token = os.environ.get('FDP_LOCAL_TOKEN') config_path = os.environ.get('FDP_CONFIG_DIR') + '/config.yaml'