From d505480c04a49e4f3b4dd40604294d3f1d091ef1 Mon Sep 17 00:00:00 2001 From: bruvio Date: Mon, 10 Jan 2022 16:52:48 +0000 Subject: [PATCH 01/11] Update README.md handling get entry exception --- README.md | 2 +- fairdatapipeline/fdp_utils.py | 6 +++-- fairdatapipeline/pipeline.py | 5 ++--- tests/test_fdp_utils.py | 42 ++++++++++++++++++++++------------- 4 files changed, 33 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 40d3cd8c..59e2b223 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Assume FDP_CONFIG_DIR, storage_locations and objects have been set by CLI tool ``` import os -import org.fairdatapipeline.api as pipeline +import fairdatapipeline as pipeline token = os.environ.get('FDP_LOCAL_TOKEN') config_path = os.environ.get('FDP_CONFIG_DIR') + '/config.yaml' diff --git a/fairdatapipeline/fdp_utils.py b/fairdatapipeline/fdp_utils.py index f05c21ad..2561b1b7 100644 --- a/fairdatapipeline/fdp_utils.py +++ b/fairdatapipeline/fdp_utils.py @@ -60,8 +60,10 @@ def get_entry( + " Query = " + url ) - - return response.json()["results"] + results = response.json()["results"] + if len(results) == 0: + raise IndexError(f'list {results} empty') + return results def get_entity( diff --git a/fairdatapipeline/pipeline.py b/fairdatapipeline/pipeline.py index 22dc57c6..21469c09 100644 --- a/fairdatapipeline/pipeline.py +++ b/fairdatapipeline/pipeline.py @@ -109,7 +109,8 @@ def initialise(token: str, config: str, script: str) -> dict: user_url = user["url"] user_id = fdp_utils.extract_id(user_url) - + print(user_id) + print(registry_url) # Get author(s) author = fdp_utils.get_entry( url=registry_url, @@ -117,7 +118,6 @@ def initialise(token: str, config: str, script: str) -> dict: query={"user": user_id}, api_version=api_version, )[0] - # Check user author exists if not author: raise ValueError( @@ -416,7 +416,6 @@ def finalise(token: str, handle: dict) -> None: directory ) ) - pass directory = os.path.split(directory)[0] i += 1 if i > 4: diff --git a/tests/test_fdp_utils.py b/tests/test_fdp_utils.py index 77744ba4..f1fa8b21 100644 --- a/tests/test_fdp_utils.py +++ b/tests/test_fdp_utils.py @@ -26,11 +26,12 @@ def write_csv_path(test_dir: str) -> str: # Test is_file() +@pytest.mark.utilities def test_is_file_exists(test_dir: str) -> None: test_file = os.path.join(test_dir, "test.csv") assert fdp_utils.is_file(test_file) - +@pytest.mark.utilities @pytest.mark.parametrize( "file_path", [ @@ -42,13 +43,13 @@ def test_is_file_exists(test_dir: str) -> None: def test_is_file_not_exists(file_path: str) -> None: assert not fdp_utils.is_file(file_path) - +@pytest.mark.utilities @pytest.mark.parametrize("file_path", ["read_csv_path", "write_csv_path"]) def test_is_yaml(file_path: str, request: FixtureRequest) -> None: file_path = request.getfixturevalue(file_path) assert fdp_utils.is_yaml(file_path) - +@pytest.mark.utilities @pytest.mark.parametrize( "file_path", [ @@ -61,13 +62,13 @@ def test_is_yaml(file_path: str, request: FixtureRequest) -> None: def test_is_yaml_not(file_path: str) -> None: assert not fdp_utils.is_yaml(file_path) - +@pytest.mark.utilities @pytest.mark.parametrize("file_path", ["read_csv_path", "write_csv_path"]) def test_is_valid_yaml(file_path: str, request: FixtureRequest) -> None: file_path = request.getfixturevalue(file_path) assert fdp_utils.is_yaml(file_path) - +@pytest.mark.utilities @pytest.mark.parametrize( "file_path", [ @@ -80,7 +81,7 @@ def test_is_valid_yaml(file_path: str, request: FixtureRequest) -> None: def test_is_valid_yaml_not(file_path: str) -> None: assert not fdp_utils.is_valid_yaml(file_path) - +@pytest.mark.utilities def test_read_token(test_dir: str) -> None: token = os.path.join(test_dir, "test_token") assert ( @@ -88,7 +89,7 @@ def test_read_token(test_dir: str) -> None: == "1a2b3c4d5e6d7f8a9b8c7d6e5f1a2b3c4d5e6d7f" ) - +@pytest.mark.utilities def test_get_token(test_dir: str) -> None: token = os.path.join(test_dir, "test_token") assert ( @@ -96,7 +97,7 @@ def test_get_token(test_dir: str) -> None: == "1a2b3c4d5e6d7f8a9b8c7d6e5f1a2b3c4d5e6d7f" ) - +@pytest.mark.utilities def test_read_token_get_token(test_dir: str) -> None: token = os.path.join(test_dir, "test_token") assert fdp_utils.read_token(token) == fdp_utils.get_token(token) @@ -108,7 +109,7 @@ def token() -> str: os.path.join(os.path.expanduser("~"), ".fair/registry/token") ) - +@pytest.mark.utilities def test_get_file_hash(test_dir: str) -> None: file_path = os.path.join(test_dir, "test.csv") if platform.system() == "Windows": @@ -122,33 +123,33 @@ def test_get_file_hash(test_dir: str) -> None: == "51345410c236d375ccf47149196746bc7f4db29d" ) - +@pytest.mark.utilities def test_random_hash_is_string() -> None: assert type(fdp_utils.random_hash()) == str - +@pytest.mark.utilities def test_random_hash_length() -> None: assert len(fdp_utils.random_hash()) == 40 - +@pytest.mark.utilities def test_extract_id() -> None: assert fdp_utils.extract_id("http://localhost:8000/api/object/85") == "85" - +@pytest.mark.utilities def test_get_headers() -> None: assert type(fdp_utils.get_headers()) == dict - +@pytest.mark.utilities def test_get_headers_with_token(token: str) -> None: headers = fdp_utils.get_headers(token=token) assert headers["Authorization"] == "token " + token - +@pytest.mark.utilities def test_get_headers_post() -> None: headers = fdp_utils.get_headers(request_type="post") assert headers["Content-Type"] == "application/json" - +@pytest.mark.utilities def test_get_headers_api_version() -> None: headers = fdp_utils.get_headers(api_version="0.0.1") assert headers["Accept"] == "application/json; version=0.0.1" @@ -231,6 +232,15 @@ def test_get_entry(url: str, token: str, storage_root_test: dict) -> None: ) assert entry[0] == storage_root_test +@pytest.mark.utilities +def test_get_entry_author(url: str, token: str, storage_root_test: dict) -> None: + with pytest.raises(IndexError) as e_info: + entry = fdp_utils.get_entry( + url=url, + query={"root": "https://storage-root-test.com"}, + token=token, + endpoint="user_author", + ) @pytest.mark.utilities def test_get_entity(url: str, storage_root_test: dict) -> None: From 15e06cf4fe73198dac8e094e747ee3c125fee9e8 Mon Sep 17 00:00:00 2001 From: bruvio Date: Tue, 11 Jan 2022 13:38:17 +0000 Subject: [PATCH 02/11] handling get entry exception --- fairdatapipeline/fdp_utils.py | 5 +---- fairdatapipeline/pipeline.py | 21 +++++++++++++-------- tests/test_fdp_utils.py | 25 ++++++++++++++++--------- tests/test_raise_issue.py | 1 + 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/fairdatapipeline/fdp_utils.py b/fairdatapipeline/fdp_utils.py index 2561b1b7..50da71c9 100644 --- a/fairdatapipeline/fdp_utils.py +++ b/fairdatapipeline/fdp_utils.py @@ -60,10 +60,7 @@ def get_entry( + " Query = " + url ) - results = response.json()["results"] - if len(results) == 0: - raise IndexError(f'list {results} empty') - return results + return response.json()["results"] def get_entity( diff --git a/fairdatapipeline/pipeline.py b/fairdatapipeline/pipeline.py index 21469c09..1d6c531e 100644 --- a/fairdatapipeline/pipeline.py +++ b/fairdatapipeline/pipeline.py @@ -92,14 +92,17 @@ def initialise(token: str, config: str, script: str) -> dict: config_filetype_url = config_filetype_response["url"] # Get user for registry admin account - user = fdp_utils.get_entry( + results = fdp_utils.get_entry( url=registry_url, endpoint="users", query={"username": "admin"}, token=token, api_version=api_version, - )[0] - + ) + if len(results) == 0: + raise IndexError(f"list {results} empty") + else: + user = results[0] # Check users exists if not user: raise ValueError( @@ -109,15 +112,17 @@ def initialise(token: str, config: str, script: str) -> dict: user_url = user["url"] user_id = fdp_utils.extract_id(user_url) - print(user_id) - print(registry_url) # Get author(s) - author = fdp_utils.get_entry( + results = fdp_utils.get_entry( url=registry_url, endpoint="user_author", query={"user": user_id}, api_version=api_version, - )[0] + ) + if len(results) == 0: + raise IndexError(f"list {results} empty") + else: + author = results[0] # Check user author exists if not author: raise ValueError( @@ -334,7 +339,6 @@ def finalise(token: str, handle: dict) -> None: api_version = handle["yaml"]["run_metadata"]["api_version"] datastore = fdp_utils.remove_local_from_root(datastore) - datastore_root = fdp_utils.get_entry( url=registry_url, endpoint="storage_root", @@ -416,6 +420,7 @@ def finalise(token: str, handle: dict) -> None: directory ) ) + pass directory = os.path.split(directory)[0] i += 1 if i > 4: diff --git a/tests/test_fdp_utils.py b/tests/test_fdp_utils.py index f1fa8b21..3b5e92a9 100644 --- a/tests/test_fdp_utils.py +++ b/tests/test_fdp_utils.py @@ -31,6 +31,7 @@ def test_is_file_exists(test_dir: str) -> None: test_file = os.path.join(test_dir, "test.csv") assert fdp_utils.is_file(test_file) + @pytest.mark.utilities @pytest.mark.parametrize( "file_path", @@ -43,12 +44,14 @@ def test_is_file_exists(test_dir: str) -> None: def test_is_file_not_exists(file_path: str) -> None: assert not fdp_utils.is_file(file_path) + @pytest.mark.utilities @pytest.mark.parametrize("file_path", ["read_csv_path", "write_csv_path"]) def test_is_yaml(file_path: str, request: FixtureRequest) -> None: file_path = request.getfixturevalue(file_path) assert fdp_utils.is_yaml(file_path) + @pytest.mark.utilities @pytest.mark.parametrize( "file_path", @@ -62,12 +65,14 @@ def test_is_yaml(file_path: str, request: FixtureRequest) -> None: def test_is_yaml_not(file_path: str) -> None: assert not fdp_utils.is_yaml(file_path) + @pytest.mark.utilities @pytest.mark.parametrize("file_path", ["read_csv_path", "write_csv_path"]) def test_is_valid_yaml(file_path: str, request: FixtureRequest) -> None: file_path = request.getfixturevalue(file_path) assert fdp_utils.is_yaml(file_path) + @pytest.mark.utilities @pytest.mark.parametrize( "file_path", @@ -81,6 +86,7 @@ def test_is_valid_yaml(file_path: str, request: FixtureRequest) -> None: def test_is_valid_yaml_not(file_path: str) -> None: assert not fdp_utils.is_valid_yaml(file_path) + @pytest.mark.utilities def test_read_token(test_dir: str) -> None: token = os.path.join(test_dir, "test_token") @@ -89,6 +95,7 @@ def test_read_token(test_dir: str) -> None: == "1a2b3c4d5e6d7f8a9b8c7d6e5f1a2b3c4d5e6d7f" ) + @pytest.mark.utilities def test_get_token(test_dir: str) -> None: token = os.path.join(test_dir, "test_token") @@ -97,6 +104,7 @@ def test_get_token(test_dir: str) -> None: == "1a2b3c4d5e6d7f8a9b8c7d6e5f1a2b3c4d5e6d7f" ) + @pytest.mark.utilities def test_read_token_get_token(test_dir: str) -> None: token = os.path.join(test_dir, "test_token") @@ -109,6 +117,7 @@ def token() -> str: os.path.join(os.path.expanduser("~"), ".fair/registry/token") ) + @pytest.mark.utilities def test_get_file_hash(test_dir: str) -> None: file_path = os.path.join(test_dir, "test.csv") @@ -123,32 +132,39 @@ def test_get_file_hash(test_dir: str) -> None: == "51345410c236d375ccf47149196746bc7f4db29d" ) + @pytest.mark.utilities def test_random_hash_is_string() -> None: assert type(fdp_utils.random_hash()) == str + @pytest.mark.utilities def test_random_hash_length() -> None: assert len(fdp_utils.random_hash()) == 40 + @pytest.mark.utilities def test_extract_id() -> None: assert fdp_utils.extract_id("http://localhost:8000/api/object/85") == "85" + @pytest.mark.utilities def test_get_headers() -> None: assert type(fdp_utils.get_headers()) == dict + @pytest.mark.utilities def test_get_headers_with_token(token: str) -> None: headers = fdp_utils.get_headers(token=token) assert headers["Authorization"] == "token " + token + @pytest.mark.utilities def test_get_headers_post() -> None: headers = fdp_utils.get_headers(request_type="post") assert headers["Content-Type"] == "application/json" + @pytest.mark.utilities def test_get_headers_api_version() -> None: headers = fdp_utils.get_headers(api_version="0.0.1") @@ -232,15 +248,6 @@ def test_get_entry(url: str, token: str, storage_root_test: dict) -> None: ) assert entry[0] == storage_root_test -@pytest.mark.utilities -def test_get_entry_author(url: str, token: str, storage_root_test: dict) -> None: - with pytest.raises(IndexError) as e_info: - entry = fdp_utils.get_entry( - url=url, - query={"root": "https://storage-root-test.com"}, - token=token, - endpoint="user_author", - ) @pytest.mark.utilities def test_get_entity(url: str, storage_root_test: dict) -> None: diff --git a/tests/test_raise_issue.py b/tests/test_raise_issue.py index f996da73..ad579a21 100644 --- a/tests/test_raise_issue.py +++ b/tests/test_raise_issue.py @@ -92,6 +92,7 @@ def test_link_read( config = os.path.join(test_dir, "read_csv.yaml") handle = pipeline.initialise(token, config, script) + link_read_1 = pipeline.link_read(handle, "test/csv") link_read_2 = pipeline.link_read(handle, "test/csv") assert type(link_read_1) == str and type(link_read_2) == str From e52a6ff1fe3e4960d68b788d86445894d46a8e8f Mon Sep 17 00:00:00 2001 From: bruvio Date: Tue, 11 Jan 2022 17:15:50 +0000 Subject: [PATCH 03/11] testing user_author and users endpoint in get_entry if raise indexerror exceptio --- tests/test_fdp_utils.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_fdp_utils.py b/tests/test_fdp_utils.py index 3b5e92a9..f6638e65 100644 --- a/tests/test_fdp_utils.py +++ b/tests/test_fdp_utils.py @@ -249,6 +249,32 @@ def test_get_entry(url: str, token: str, storage_root_test: dict) -> None: assert entry[0] == storage_root_test +@pytest.mark.utilities +def test_get_entry_author(url: str, token: str) -> None: + + results = fdp_utils.get_entry( + url=url, + query={"user": 2}, + token=token, + endpoint="user_author", + ) + with pytest.raises(IndexError): + _ = results[0] + + +@pytest.mark.utilities +def test_get_entry_users(url: str, token: str) -> None: + + results = fdp_utils.get_entry( + url=url, + query={"username": "admin1"}, + token=token, + endpoint="users", + ) + with pytest.raises(IndexError): + _ = results[0] + + @pytest.mark.utilities def test_get_entity(url: str, storage_root_test: dict) -> None: entity = fdp_utils.get_entity( From 70b44651a322a2cde52f3313e41b2834462bde86 Mon Sep 17 00:00:00 2001 From: bruvio Date: Wed, 12 Jan 2022 06:54:12 +0000 Subject: [PATCH 04/11] improving get headers test --- tests/test_fdp_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_fdp_utils.py b/tests/test_fdp_utils.py index f6638e65..f2edd52b 100644 --- a/tests/test_fdp_utils.py +++ b/tests/test_fdp_utils.py @@ -151,6 +151,8 @@ def test_extract_id() -> None: @pytest.mark.utilities def test_get_headers() -> None: assert type(fdp_utils.get_headers()) == dict + headers = {"Accept": "application/json; version=" + "1.0.0"} + assert headers == fdp_utils.get_headers() @pytest.mark.utilities From 63f94e4261d4396a8b110d4ae29b21170e26d45a Mon Sep 17 00:00:00 2001 From: bruvio Date: Wed, 12 Jan 2022 09:14:44 +0000 Subject: [PATCH 05/11] adding coverage to default pytest options --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ed212f78..43c0f0ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,7 +77,7 @@ line_length = 79 line-length = 79 [tool.pytest.ini_options] -addopts = '-s -v' +addopts = '-s -v --cov=fairdatapipeline --cov-report=xml --cov-report=term' markers = [ "pipeline: tests for 'pipeline' module ", "issue: tests for raising issues ", From b08caf0dcb09806c0423f419c7eda1be91e7befa Mon Sep 17 00:00:00 2001 From: bruvio Date: Wed, 12 Jan 2022 13:19:48 +0000 Subject: [PATCH 06/11] pytest coverage output changed to html increase test coverage adding more tests removed use of reserved work 'type' changed to 'issue_type' handling of exception when API assumes that variables are in the correct state when accessing indexes --- fairdatapipeline/__init__.py | 2 + fairdatapipeline/fdp_utils.py | 7 ++-- fairdatapipeline/raise_issue.py | 6 +-- pyproject.toml | 2 +- tests/test_fdp_utils.py | 6 +++ tests/test_raise_issue.py | 68 ++++++++++++++++++++++++++++++++- 6 files changed, 83 insertions(+), 8 deletions(-) diff --git a/fairdatapipeline/__init__.py b/fairdatapipeline/__init__.py index 9a5e5d34..46217a6e 100644 --- a/fairdatapipeline/__init__.py +++ b/fairdatapipeline/__init__.py @@ -6,6 +6,7 @@ "raise_issue_by_data_product", "raise_issue_by_index", "raise_issue_with_config", + "raise_issue_by_type", "raise_issue_by_existing_data_product", "raise_issue_with_submission_script", "raise_issue_with_github_repo", @@ -19,6 +20,7 @@ raise_issue_by_data_product, raise_issue_by_existing_data_product, raise_issue_by_index, + raise_issue_by_type, raise_issue_with_config, raise_issue_with_github_repo, raise_issue_with_submission_script, diff --git a/fairdatapipeline/fdp_utils.py b/fairdatapipeline/fdp_utils.py index 50da71c9..ae6745fa 100644 --- a/fairdatapipeline/fdp_utils.py +++ b/fairdatapipeline/fdp_utils.py @@ -104,7 +104,10 @@ def extract_id(url: str) -> str: Returns: | str: id derrived from the url """ - return list(filter(None, urlsplit(url).path.split("/")))[-1] + try: + return list(filter(None, urlsplit(url).path.split("/")))[-1] + except: + raise IndexError(f"Unable to index input url: {url}") def post_entry( @@ -131,8 +134,6 @@ def post_entry( response = requests.post(_url, _data, headers=headers) - # print(response.request.headers) - if response.status_code == 409: logging.info("Entry Exists: Attempting to return Existing Entry") existing_entry = get_entry(url, endpoint, data) diff --git a/fairdatapipeline/raise_issue.py b/fairdatapipeline/raise_issue.py index 43281e01..58602d80 100644 --- a/fairdatapipeline/raise_issue.py +++ b/fairdatapipeline/raise_issue.py @@ -148,7 +148,7 @@ def raise_issue_by_type( # flake8: noqa C901 def raise_issue( handle: dict, - type: str, + issue_type: str, issue: str, severity: int, index: bool = None, @@ -159,7 +159,7 @@ def raise_issue( group: bool = True, ) -> None: current_group = issue + ":" + str(severity) - if type in [ + if issue_type in [ "config", "submission_script", "github_repo", @@ -233,7 +233,7 @@ def raise_issue( # Write to handle and return path issues_dict = { "index": index, - "type": type, + "type": issue_type, "use_data_product": data_product, "use_component": component, "version": version, diff --git a/pyproject.toml b/pyproject.toml index 43c0f0ff..47079cf2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,7 +77,7 @@ line_length = 79 line-length = 79 [tool.pytest.ini_options] -addopts = '-s -v --cov=fairdatapipeline --cov-report=xml --cov-report=term' +addopts = '-s -v --cov=fairdatapipeline --cov-report=html --cov-report=term' markers = [ "pipeline: tests for 'pipeline' module ", "issue: tests for raising issues ", diff --git a/tests/test_fdp_utils.py b/tests/test_fdp_utils.py index f2edd52b..c2079c54 100644 --- a/tests/test_fdp_utils.py +++ b/tests/test_fdp_utils.py @@ -148,6 +148,12 @@ def test_extract_id() -> None: assert fdp_utils.extract_id("http://localhost:8000/api/object/85") == "85" +@pytest.mark.utilities +def test_extract_id_should_fail() -> None: + with pytest.raises(IndexError): + fdp_utils.extract_id("") + + @pytest.mark.utilities def test_get_headers() -> None: assert type(fdp_utils.get_headers()) == dict diff --git a/tests/test_raise_issue.py b/tests/test_raise_issue.py index ad579a21..2e8e5061 100644 --- a/tests/test_raise_issue.py +++ b/tests/test_raise_issue.py @@ -53,6 +53,15 @@ def test_raise_issue_by_index(token: str, config: str, script: str) -> None: assert handle["issues"]["issue_0"]["use_data_product"] == "test/csv" +@pytest.mark.issue +def test_raise_issue_by_type(token: str, config: str, script: str) -> None: + handle = pipeline.initialise(token, config, script) + with pytest.raises(ValueError): + pipeline.raise_issue_by_type( + handle, "testing_type", "Test Issue by type", 1, group=True + ) + + @pytest.mark.issue def test_raise_issue_with_config(token: str, config: str, script: str) -> None: handle = pipeline.initialise(token, config, script) @@ -107,6 +116,13 @@ def test_link_read_data_product_exists( link_write = pipeline.link_write(handle, "test/csv") shutil.copy(tmp_csv, link_write) pipeline.finalise(token, handle) + assert len(handle["yaml"]["write"]) == 1 + assert handle["yaml"]["write"][0] == { + "data_product": "test/csv", + "description": "test csv file with simple data", + "file_type": "csv", + "use": {"version": "0.0.1"}, + } @pytest.mark.issue @@ -123,6 +139,17 @@ def test_raise_issue_existing_data_product( 5, ) pipeline.finalise(token, handle) + assert handle["issues"]["issue_0"] == { + "index": None, + "type": "existing_data_product", + "use_data_product": "test/csv", + "use_component": None, + "version": "0.0.1", + "use_namespace": "testing", + "issue": "Problem with csv File : Test Issue # 2", + "severity": 5, + "group": "Problem with csv File : Test Issue # 2:5", + } @pytest.mark.issue @@ -136,7 +163,46 @@ def test_raise_issue_data_product_from_reads( "test/csv", "0.0.1", "testing", - "Problem with csv File : Test Issue # 3", + "Problem with reading csv File : Test Issue # 3", + 5, + ) + pipeline.finalise(token, handle) + assert handle["issues"]["issue_0"] == { + "index": None, + "type": "data_product", + "use_data_product": "test/csv", + "use_component": None, + "version": "0.0.1", + "use_namespace": "testing", + "issue": "Problem with reading csv File : Test Issue # 3", + "severity": 5, + "group": "Problem with reading csv File : Test Issue # 3:5", + } + + +@pytest.mark.issue +def test_raise_issue_data_product_from_writes( + token: str, script: str, test_dir: str +) -> None: + config = os.path.join(test_dir, "write_csv.yaml") + handle = pipeline.initialise(token, config, script) + pipeline.raise_issue_by_data_product( + handle, + "test/csv", + "0.0.1", + "testing", + "Problem with writing csv File : Test Issue # 4", 5, ) pipeline.finalise(token, handle) + assert handle["issues"]["issue_0"] == { + "index": None, + "type": "data_product", + "use_data_product": "test/csv", + "use_component": None, + "version": "0.0.1", + "use_namespace": "testing", + "issue": "Problem with writing csv File : Test Issue # 4", + "severity": 5, + "group": "Problem with writing csv File : Test Issue # 4:5", + } From a0b5c1dd2cad0ad27b972b134e8e28f30a0ba56e Mon Sep 17 00:00:00 2001 From: bruvio Date: Thu, 13 Jan 2022 07:13:31 +0000 Subject: [PATCH 07/11] integrating codecov into PR --- codecov.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..aa84c676 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,2 @@ +ignore: + - "*/tests/*" From af812804f079cad43079088d76c8bd2f6b0f1ea8 Mon Sep 17 00:00:00 2001 From: bruvio Date: Thu, 13 Jan 2022 07:26:47 +0000 Subject: [PATCH 08/11] improving test intialise pipeline --- tests/test_raise_issue.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_raise_issue.py b/tests/test_raise_issue.py index 2e8e5061..2cc7ef12 100644 --- a/tests/test_raise_issue.py +++ b/tests/test_raise_issue.py @@ -35,6 +35,7 @@ def config(test_dir: str) -> str: def test_initialise(token: str, config: str, script: str) -> None: handle = pipeline.initialise(token, config, script) assert type(handle) == dict + assert handle["yaml"]["run_metadata"]["script"] == "python3 py.test" @pytest.mark.pipeline From eee356ce1648c0bb5b424b1324da1607168bdb6a Mon Sep 17 00:00:00 2001 From: bruvio Date: Thu, 13 Jan 2022 08:20:12 +0000 Subject: [PATCH 09/11] Specify an exception class to catch or reraise the exception in extract_id function catching config and script exceptions in initialise pipeline --- fairdatapipeline/fdp_utils.py | 2 +- fairdatapipeline/pipeline.py | 1 + tests/test_raise_issue.py | 18 ++++++++++++++++-- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/fairdatapipeline/fdp_utils.py b/fairdatapipeline/fdp_utils.py index ae6745fa..7ae14442 100644 --- a/fairdatapipeline/fdp_utils.py +++ b/fairdatapipeline/fdp_utils.py @@ -106,7 +106,7 @@ def extract_id(url: str) -> str: """ try: return list(filter(None, urlsplit(url).path.split("/")))[-1] - except: + except IndexError: raise IndexError(f"Unable to index input url: {url}") diff --git a/fairdatapipeline/pipeline.py b/fairdatapipeline/pipeline.py index 1d6c531e..88b37fd4 100644 --- a/fairdatapipeline/pipeline.py +++ b/fairdatapipeline/pipeline.py @@ -99,6 +99,7 @@ def initialise(token: str, config: str, script: str) -> dict: token=token, api_version=api_version, ) + if len(results) == 0: raise IndexError(f"list {results} empty") else: diff --git a/tests/test_raise_issue.py b/tests/test_raise_issue.py index 2cc7ef12..01fabdce 100644 --- a/tests/test_raise_issue.py +++ b/tests/test_raise_issue.py @@ -6,8 +6,6 @@ import fairdatapipeline as pipeline import fairdatapipeline.fdp_utils as fdp_utils -# from org.fairdatapipeline.api.common.link_read import link_read - @pytest.fixture def test_dir() -> str: @@ -38,6 +36,22 @@ def test_initialise(token: str, config: str, script: str) -> None: assert handle["yaml"]["run_metadata"]["script"] == "python3 py.test" +@pytest.mark.pipeline +def test_initialise_noconfig( + token: str, script: str, config: str = "file_that_does_not_exist" +) -> None: + with pytest.raises(ValueError): + _ = pipeline.initialise(token, config, script) + + +@pytest.mark.pipeline +def test_initialise_noscript( + token: str, config: str, script: str = "file_that_does_not_exist" +) -> None: + with pytest.raises(ValueError): + _ = pipeline.initialise(token, config, script) + + @pytest.mark.pipeline def test_link_write(token: str, config: str, script: str) -> None: handle = pipeline.initialise(token, config, script) From da633cf4ff6972be2634156a86e5304362203f6f Mon Sep 17 00:00:00 2001 From: bruvio Date: Thu, 13 Jan 2022 08:47:07 +0000 Subject: [PATCH 10/11] refactor of pipeline --- fairdatapipeline/pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fairdatapipeline/pipeline.py b/fairdatapipeline/pipeline.py index 88b37fd4..c41dc2c1 100644 --- a/fairdatapipeline/pipeline.py +++ b/fairdatapipeline/pipeline.py @@ -100,7 +100,7 @@ def initialise(token: str, config: str, script: str) -> dict: api_version=api_version, ) - if len(results) == 0: + if not results: raise IndexError(f"list {results} empty") else: user = results[0] @@ -120,7 +120,7 @@ def initialise(token: str, config: str, script: str) -> dict: query={"user": user_id}, api_version=api_version, ) - if len(results) == 0: + if not results: raise IndexError(f"list {results} empty") else: author = results[0] From 126960d2c322f2436ca028624892e22481073332 Mon Sep 17 00:00:00 2001 From: bruvio Date: Thu, 13 Jan 2022 09:45:37 +0000 Subject: [PATCH 11/11] refactor of extract_id function --- fairdatapipeline/fdp_utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fairdatapipeline/fdp_utils.py b/fairdatapipeline/fdp_utils.py index 7ae14442..f5fa17c7 100644 --- a/fairdatapipeline/fdp_utils.py +++ b/fairdatapipeline/fdp_utils.py @@ -104,10 +104,11 @@ def extract_id(url: str) -> str: Returns: | str: id derrived from the url """ - try: - return list(filter(None, urlsplit(url).path.split("/")))[-1] - except IndexError: - raise IndexError(f"Unable to index input url: {url}") + + split_url_path = urlsplit(url).path.split("/") + if not split_url_path: + raise IndexError(f"Unable to extract ID from registry URL: {url}") + return [s for s in split_url_path if s != ""][-1] def post_entry(