diff --git a/README.md b/README.md index 40d3cd8c..59e2b223 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Assume FDP_CONFIG_DIR, storage_locations and objects have been set by CLI tool ``` import os -import org.fairdatapipeline.api as pipeline +import fairdatapipeline as pipeline token = os.environ.get('FDP_LOCAL_TOKEN') config_path = os.environ.get('FDP_CONFIG_DIR') + '/config.yaml' diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..aa84c676 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,2 @@ +ignore: + - "*/tests/*" diff --git a/fairdatapipeline/__init__.py b/fairdatapipeline/__init__.py index 9a5e5d34..46217a6e 100644 --- a/fairdatapipeline/__init__.py +++ b/fairdatapipeline/__init__.py @@ -6,6 +6,7 @@ "raise_issue_by_data_product", "raise_issue_by_index", "raise_issue_with_config", + "raise_issue_by_type", "raise_issue_by_existing_data_product", "raise_issue_with_submission_script", "raise_issue_with_github_repo", @@ -19,6 +20,7 @@ raise_issue_by_data_product, raise_issue_by_existing_data_product, raise_issue_by_index, + raise_issue_by_type, raise_issue_with_config, raise_issue_with_github_repo, raise_issue_with_submission_script, diff --git a/fairdatapipeline/fdp_utils.py b/fairdatapipeline/fdp_utils.py index f05c21ad..f5fa17c7 100644 --- a/fairdatapipeline/fdp_utils.py +++ b/fairdatapipeline/fdp_utils.py @@ -60,7 +60,6 @@ def get_entry( + " Query = " + url ) - return response.json()["results"] @@ -105,7 +104,11 @@ def extract_id(url: str) -> str: Returns: | str: id derrived from the url """ - return list(filter(None, urlsplit(url).path.split("/")))[-1] + + split_url_path = urlsplit(url).path.split("/") + if not split_url_path: + raise IndexError(f"Unable to extract ID from registry URL: {url}") + return [s for s in split_url_path if s != ""][-1] def post_entry( @@ -132,8 +135,6 @@ def post_entry( response = requests.post(_url, _data, headers=headers) - # print(response.request.headers) - if response.status_code == 409: logging.info("Entry Exists: Attempting to return Existing Entry") existing_entry = get_entry(url, endpoint, data) diff --git a/fairdatapipeline/pipeline.py b/fairdatapipeline/pipeline.py index 22dc57c6..c41dc2c1 100644 --- a/fairdatapipeline/pipeline.py +++ b/fairdatapipeline/pipeline.py @@ -92,14 +92,18 @@ def initialise(token: str, config: str, script: str) -> dict: config_filetype_url = config_filetype_response["url"] # Get user for registry admin account - user = fdp_utils.get_entry( + results = fdp_utils.get_entry( url=registry_url, endpoint="users", query={"username": "admin"}, token=token, api_version=api_version, - )[0] + ) + if not results: + raise IndexError(f"list {results} empty") + else: + user = results[0] # Check users exists if not user: raise ValueError( @@ -109,15 +113,17 @@ def initialise(token: str, config: str, script: str) -> dict: user_url = user["url"] user_id = fdp_utils.extract_id(user_url) - # Get author(s) - author = fdp_utils.get_entry( + results = fdp_utils.get_entry( url=registry_url, endpoint="user_author", query={"user": user_id}, api_version=api_version, - )[0] - + ) + if not results: + raise IndexError(f"list {results} empty") + else: + author = results[0] # Check user author exists if not author: raise ValueError( @@ -334,7 +340,6 @@ def finalise(token: str, handle: dict) -> None: api_version = handle["yaml"]["run_metadata"]["api_version"] datastore = fdp_utils.remove_local_from_root(datastore) - datastore_root = fdp_utils.get_entry( url=registry_url, endpoint="storage_root", diff --git a/fairdatapipeline/raise_issue.py b/fairdatapipeline/raise_issue.py index 43281e01..58602d80 100644 --- a/fairdatapipeline/raise_issue.py +++ b/fairdatapipeline/raise_issue.py @@ -148,7 +148,7 @@ def raise_issue_by_type( # flake8: noqa C901 def raise_issue( handle: dict, - type: str, + issue_type: str, issue: str, severity: int, index: bool = None, @@ -159,7 +159,7 @@ def raise_issue( group: bool = True, ) -> None: current_group = issue + ":" + str(severity) - if type in [ + if issue_type in [ "config", "submission_script", "github_repo", @@ -233,7 +233,7 @@ def raise_issue( # Write to handle and return path issues_dict = { "index": index, - "type": type, + "type": issue_type, "use_data_product": data_product, "use_component": component, "version": version, diff --git a/pyproject.toml b/pyproject.toml index ed212f78..47079cf2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,7 +77,7 @@ line_length = 79 line-length = 79 [tool.pytest.ini_options] -addopts = '-s -v' +addopts = '-s -v --cov=fairdatapipeline --cov-report=html --cov-report=term' markers = [ "pipeline: tests for 'pipeline' module ", "issue: tests for raising issues ", diff --git a/tests/test_fdp_utils.py b/tests/test_fdp_utils.py index 77744ba4..c2079c54 100644 --- a/tests/test_fdp_utils.py +++ b/tests/test_fdp_utils.py @@ -26,11 +26,13 @@ def write_csv_path(test_dir: str) -> str: # Test is_file() +@pytest.mark.utilities def test_is_file_exists(test_dir: str) -> None: test_file = os.path.join(test_dir, "test.csv") assert fdp_utils.is_file(test_file) +@pytest.mark.utilities @pytest.mark.parametrize( "file_path", [ @@ -43,12 +45,14 @@ def test_is_file_not_exists(file_path: str) -> None: assert not fdp_utils.is_file(file_path) +@pytest.mark.utilities @pytest.mark.parametrize("file_path", ["read_csv_path", "write_csv_path"]) def test_is_yaml(file_path: str, request: FixtureRequest) -> None: file_path = request.getfixturevalue(file_path) assert fdp_utils.is_yaml(file_path) +@pytest.mark.utilities @pytest.mark.parametrize( "file_path", [ @@ -62,12 +66,14 @@ def test_is_yaml_not(file_path: str) -> None: assert not fdp_utils.is_yaml(file_path) +@pytest.mark.utilities @pytest.mark.parametrize("file_path", ["read_csv_path", "write_csv_path"]) def test_is_valid_yaml(file_path: str, request: FixtureRequest) -> None: file_path = request.getfixturevalue(file_path) assert fdp_utils.is_yaml(file_path) +@pytest.mark.utilities @pytest.mark.parametrize( "file_path", [ @@ -81,6 +87,7 @@ def test_is_valid_yaml_not(file_path: str) -> None: assert not fdp_utils.is_valid_yaml(file_path) +@pytest.mark.utilities def test_read_token(test_dir: str) -> None: token = os.path.join(test_dir, "test_token") assert ( @@ -89,6 +96,7 @@ def test_read_token(test_dir: str) -> None: ) +@pytest.mark.utilities def test_get_token(test_dir: str) -> None: token = os.path.join(test_dir, "test_token") assert ( @@ -97,6 +105,7 @@ def test_get_token(test_dir: str) -> None: ) +@pytest.mark.utilities def test_read_token_get_token(test_dir: str) -> None: token = os.path.join(test_dir, "test_token") assert fdp_utils.read_token(token) == fdp_utils.get_token(token) @@ -109,6 +118,7 @@ def token() -> str: ) +@pytest.mark.utilities def test_get_file_hash(test_dir: str) -> None: file_path = os.path.join(test_dir, "test.csv") if platform.system() == "Windows": @@ -123,32 +133,47 @@ def test_get_file_hash(test_dir: str) -> None: ) +@pytest.mark.utilities def test_random_hash_is_string() -> None: assert type(fdp_utils.random_hash()) == str +@pytest.mark.utilities def test_random_hash_length() -> None: assert len(fdp_utils.random_hash()) == 40 +@pytest.mark.utilities def test_extract_id() -> None: assert fdp_utils.extract_id("http://localhost:8000/api/object/85") == "85" +@pytest.mark.utilities +def test_extract_id_should_fail() -> None: + with pytest.raises(IndexError): + fdp_utils.extract_id("") + + +@pytest.mark.utilities def test_get_headers() -> None: assert type(fdp_utils.get_headers()) == dict + headers = {"Accept": "application/json; version=" + "1.0.0"} + assert headers == fdp_utils.get_headers() +@pytest.mark.utilities def test_get_headers_with_token(token: str) -> None: headers = fdp_utils.get_headers(token=token) assert headers["Authorization"] == "token " + token +@pytest.mark.utilities def test_get_headers_post() -> None: headers = fdp_utils.get_headers(request_type="post") assert headers["Content-Type"] == "application/json" +@pytest.mark.utilities def test_get_headers_api_version() -> None: headers = fdp_utils.get_headers(api_version="0.0.1") assert headers["Accept"] == "application/json; version=0.0.1" @@ -232,6 +257,32 @@ def test_get_entry(url: str, token: str, storage_root_test: dict) -> None: assert entry[0] == storage_root_test +@pytest.mark.utilities +def test_get_entry_author(url: str, token: str) -> None: + + results = fdp_utils.get_entry( + url=url, + query={"user": 2}, + token=token, + endpoint="user_author", + ) + with pytest.raises(IndexError): + _ = results[0] + + +@pytest.mark.utilities +def test_get_entry_users(url: str, token: str) -> None: + + results = fdp_utils.get_entry( + url=url, + query={"username": "admin1"}, + token=token, + endpoint="users", + ) + with pytest.raises(IndexError): + _ = results[0] + + @pytest.mark.utilities def test_get_entity(url: str, storage_root_test: dict) -> None: entity = fdp_utils.get_entity( diff --git a/tests/test_raise_issue.py b/tests/test_raise_issue.py index f996da73..01fabdce 100644 --- a/tests/test_raise_issue.py +++ b/tests/test_raise_issue.py @@ -6,8 +6,6 @@ import fairdatapipeline as pipeline import fairdatapipeline.fdp_utils as fdp_utils -# from org.fairdatapipeline.api.common.link_read import link_read - @pytest.fixture def test_dir() -> str: @@ -35,6 +33,23 @@ def config(test_dir: str) -> str: def test_initialise(token: str, config: str, script: str) -> None: handle = pipeline.initialise(token, config, script) assert type(handle) == dict + assert handle["yaml"]["run_metadata"]["script"] == "python3 py.test" + + +@pytest.mark.pipeline +def test_initialise_noconfig( + token: str, script: str, config: str = "file_that_does_not_exist" +) -> None: + with pytest.raises(ValueError): + _ = pipeline.initialise(token, config, script) + + +@pytest.mark.pipeline +def test_initialise_noscript( + token: str, config: str, script: str = "file_that_does_not_exist" +) -> None: + with pytest.raises(ValueError): + _ = pipeline.initialise(token, config, script) @pytest.mark.pipeline @@ -53,6 +68,15 @@ def test_raise_issue_by_index(token: str, config: str, script: str) -> None: assert handle["issues"]["issue_0"]["use_data_product"] == "test/csv" +@pytest.mark.issue +def test_raise_issue_by_type(token: str, config: str, script: str) -> None: + handle = pipeline.initialise(token, config, script) + with pytest.raises(ValueError): + pipeline.raise_issue_by_type( + handle, "testing_type", "Test Issue by type", 1, group=True + ) + + @pytest.mark.issue def test_raise_issue_with_config(token: str, config: str, script: str) -> None: handle = pipeline.initialise(token, config, script) @@ -92,6 +116,7 @@ def test_link_read( config = os.path.join(test_dir, "read_csv.yaml") handle = pipeline.initialise(token, config, script) + link_read_1 = pipeline.link_read(handle, "test/csv") link_read_2 = pipeline.link_read(handle, "test/csv") assert type(link_read_1) == str and type(link_read_2) == str @@ -106,6 +131,13 @@ def test_link_read_data_product_exists( link_write = pipeline.link_write(handle, "test/csv") shutil.copy(tmp_csv, link_write) pipeline.finalise(token, handle) + assert len(handle["yaml"]["write"]) == 1 + assert handle["yaml"]["write"][0] == { + "data_product": "test/csv", + "description": "test csv file with simple data", + "file_type": "csv", + "use": {"version": "0.0.1"}, + } @pytest.mark.issue @@ -122,6 +154,17 @@ def test_raise_issue_existing_data_product( 5, ) pipeline.finalise(token, handle) + assert handle["issues"]["issue_0"] == { + "index": None, + "type": "existing_data_product", + "use_data_product": "test/csv", + "use_component": None, + "version": "0.0.1", + "use_namespace": "testing", + "issue": "Problem with csv File : Test Issue # 2", + "severity": 5, + "group": "Problem with csv File : Test Issue # 2:5", + } @pytest.mark.issue @@ -135,7 +178,46 @@ def test_raise_issue_data_product_from_reads( "test/csv", "0.0.1", "testing", - "Problem with csv File : Test Issue # 3", + "Problem with reading csv File : Test Issue # 3", + 5, + ) + pipeline.finalise(token, handle) + assert handle["issues"]["issue_0"] == { + "index": None, + "type": "data_product", + "use_data_product": "test/csv", + "use_component": None, + "version": "0.0.1", + "use_namespace": "testing", + "issue": "Problem with reading csv File : Test Issue # 3", + "severity": 5, + "group": "Problem with reading csv File : Test Issue # 3:5", + } + + +@pytest.mark.issue +def test_raise_issue_data_product_from_writes( + token: str, script: str, test_dir: str +) -> None: + config = os.path.join(test_dir, "write_csv.yaml") + handle = pipeline.initialise(token, config, script) + pipeline.raise_issue_by_data_product( + handle, + "test/csv", + "0.0.1", + "testing", + "Problem with writing csv File : Test Issue # 4", 5, ) pipeline.finalise(token, handle) + assert handle["issues"]["issue_0"] == { + "index": None, + "type": "data_product", + "use_data_product": "test/csv", + "use_component": None, + "version": "0.0.1", + "use_namespace": "testing", + "issue": "Problem with writing csv File : Test Issue # 4", + "severity": 5, + "group": "Problem with writing csv File : Test Issue # 4:5", + }