diff --git a/.github/workflows/fair-cli.yaml b/.github/workflows/fair-cli.yaml index 52ba755b..bdf08371 100644 --- a/.github/workflows/fair-cli.yaml +++ b/.github/workflows/fair-cli.yaml @@ -30,8 +30,17 @@ jobs: run: python -m poetry install - name: Install Python API for API Tests run: python -m poetry run pip install git+https://github.com/FAIRDataPipeline/pyDataPipeline.git@dev - - name: Run Tests - run: python -m poetry run pytest --cov=fair --cov-report=xml --cov-report=term -s tests/ + - name: Run Tests for Each Marker + run: | + for marker in $(poetry run pytest --markers | grep -oE "faircli_[a-zA-Z|_|0-9]+") + do + echo "Running tests for marker '$marker'" + python -m poetry run pytest -m $marker --cov=fair --cov-report=xml --cov-report=term --cov-append -s tests/ + exit_code=$? + if [ "$exit_code" != "0" ]; then + echo "ERROR: Tests for marker '$marker' failed" + fi + done - uses: codecov/codecov-action@v2 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/implementations.yml b/.github/workflows/implementations.yml index 2e657ca1..b6f56b69 100644 --- a/.github/workflows/implementations.yml +++ b/.github/workflows/implementations.yml @@ -18,35 +18,34 @@ jobs: python-version: "3.9" architecture: "x64" + - name: Install memcached + run: sudo apt install -y libmemcached-dev + - name: Install and initialise FAIR CLI run: | pip install poetry poetry install - - name: Setup Python API + - name: Build Python API + run: | + poetry install + poetry build + working-directory: python_example + + - name: run Python Model with fair cli run: | - rm pyproject.toml poetry.lock # Remove conflicting poetry setup in Python API + rm pyproject.toml poetry.lock + poetry run pip install --find-links=dist/ pydatapipeline poetry run fair registry install poetry run fair registry install --directory ${GITHUB_WORKSPACE}/registry-rem poetry run fair init --ci poetry run fair registry start poetry run ${GITHUB_WORKSPACE}/registry-rem/scripts/start_fair_registry -p 8001 cp ${GITHUB_WORKSPACE}/registry-rem/token $PWD/token - poetry run pip install git+https://github.com/FAIRDataPipeline/pyDataPipeline.git@dev # Install the Python API via Poetry itself - working-directory: python_example - - - name: Test Pull - run: poetry run fair pull --debug simpleModel/ext/SEIRSconfig.yaml - working-directory: python_example - - - name: Test Run - run: poetry run fair run --dirty --debug simpleModel/ext/SEIRSconfig.yaml - working-directory: python_example - - - name: Test Push - run: | - poetry run fair add testing:SEIRS_model/parameters@v1.0.0 - poetry run fair push + poetry run fair pull --debug simpleModel/ext/SEIRSconfig.yaml + poetry run fair run --dirty --debug simpleModel/ext/SEIRSconfig.yaml + poetry run fair add PSU:SEIRS_model/parameters@v1.0.0 + poetry run fair push --dirty working-directory: python_example Java: @@ -78,18 +77,16 @@ jobs: pip install poetry poetry install - - name: Setup javaSimpleModel + - name: run javaSimpleModel with fair cli run: | poetry run fair registry install + poetry run fair registry install --directory ${GITHUB_WORKSPACE}/registry-rem + poetry run fair registry start + poetry run ${GITHUB_WORKSPACE}/registry-rem/scripts/start_fair_registry -p 8001 + cp ${GITHUB_WORKSPACE}/registry-rem/token $PWD/token poetry run fair init --ci - working-directory: java_example - - - name: Test Pull - run: poetry run fair pull --debug src/main/resources/seirs-config.yaml - working-directory: java_example - - - name: Test Run - run: poetry run fair run --dirty --debug src/main/resources/seirs-config.yaml + poetry run fair pull --debug src/main/resources/seirs-config.yaml + poetry run fair run --dirty --debug src/main/resources/seirs-config.yaml working-directory: java_example R: @@ -156,18 +153,16 @@ jobs: pip install poetry poetry install - - name: Setup rSimpleModel + - name: run rSimpleModel with fair cli run: | poetry run fair registry install + poetry run fair registry install --directory ${GITHUB_WORKSPACE}/registry-rem + poetry run fair registry start + poetry run ${GITHUB_WORKSPACE}/registry-rem/scripts/start_fair_registry -p 8001 + cp ${GITHUB_WORKSPACE}/registry-rem/token $PWD/token poetry run fair init --ci - working-directory: r_example - - - name: Test Pull - run: poetry run fair pull --debug inst/extdata/SEIRSconfig.yaml - working-directory: r_example - - - name: Test Run - run: poetry run fair run --dirty --debug inst/extdata/SEIRSconfig.yaml + poetry run fair pull --debug inst/extdata/SEIRSconfig.yaml + poetry run fair run --dirty --debug inst/extdata/SEIRSconfig.yaml working-directory: r_example Julia: @@ -196,16 +191,57 @@ jobs: pip install poetry poetry install - - name: Setup SEIRS Model + - name: Run SEIRS Model with fair cli run: | poetry run fair registry install + poetry run fair registry install --directory ${GITHUB_WORKSPACE}/registry-rem + poetry run fair registry start + poetry run ${GITHUB_WORKSPACE}/registry-rem/scripts/start_fair_registry -p 8001 + cp ${GITHUB_WORKSPACE}/registry-rem/token $PWD/token poetry run fair init --ci + poetry run fair pull --debug examples/fdp/SEIRSconfig.yaml + poetry run fair run --dirty --debug examples/fdp/SEIRSconfig.yaml working-directory: julia_example - - name: Test Pull - run: poetry run fair pull --debug examples/fdp/SEIRSconfig.yaml - working-directory: julia_example - - - name: Test Run - run: poetry run fair run --dirty --debug examples/fdp/SEIRSconfig.yaml - working-directory: julia_example + CPlusPlus: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Clone CPlusPlus Model + uses: actions/checkout@v2 + with: + repository: FAIRDataPipeline/cppSimpleModel + path: cpp_example + ref: 'main' + + - name: Install requirements + run: | + sudo apt install -y libmemcached-dev graphviz gnuplot + sudo apt install -y libjsoncpp-dev curl libcurl4-openssl-dev + sudo apt install -y libyaml-cpp-dev libhdf5-dev + + - name: Install Model + run: | + cmake -Bbuild + cmake --build build + working-directory: cpp_example + + - name: install fair-cli + run: | + pip install poetry + poetry install + + - name: Run Example model with fair-cli + run: | + poetry run fair registry install + poetry run fair registry install --directory ${GITHUB_WORKSPACE}/registry-rem + poetry run fair registry start + poetry run ${GITHUB_WORKSPACE}/registry-rem/scripts/start_fair_registry -p 8001 + cp ${GITHUB_WORKSPACE}/registry-rem/token $PWD/token + poetry run fair init --ci + poetry run fair pull --debug data/seirs_config.yaml + poetry run fair run --dirty --debug data/seirs_config.yaml + working-directory: cpp_example + + diff --git a/CHANGELOG.md b/CHANGELOG.md index 09ee9e63..f824ca0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,6 @@ # Unreleased +- Wildcard '*' parsing introduced for data products. +- Ability to `push` to a registry added. - Added `--dirty` option to `fair run` to allow running with uncommitted changes. - Added `config.yaml` file validation. - Added initialisation from existing registry. diff --git a/fair/cli.py b/fair/cli.py index faa1ec75..dd3d882a 100644 --- a/fair/cli.py +++ b/fair/cli.py @@ -271,7 +271,8 @@ def install(debug: bool, force: bool, directory: str): try: if debug: logging.getLogger("FAIRDataPipeline").setLevel(logging.DEBUG) - fdp_svr.install_registry(install_dir=directory, force=force) + _version = fdp_svr.install_registry(install_dir=directory, force=force) + click.echo(f"Installed registry version '{_version}'") except fdp_exc.FAIRCLIException as e: if debug: raise e @@ -424,7 +425,6 @@ def run(config: str, script: str, debug: bool, ci: bool, dirty: bool): """Initialises a job with the option to specify a bash command""" # Allow no config to be specified, if that is the case use default local config = config[0] if config else fdp_com.local_user_config(os.getcwd()) - _run_mode = fdp_run.CMD_MODE.RUN if not ci else fdp_run.CMD_MODE.PASS try: with fdp_session.FAIR( os.getcwd(), @@ -433,7 +433,7 @@ def run(config: str, script: str, debug: bool, ci: bool, dirty: bool): server_mode=fdp_svr.SwitchMode.CLI, allow_dirty=dirty ) as fair_session: - _hash = fair_session.run_job(script, mode=_run_mode) + _hash = fair_session.run(script, passive=ci, allow_dirty=dirty) if ci: click.echo(fdp_run.get_job_dir(_hash)) except fdp_exc.FAIRCLIException as e: @@ -533,12 +533,18 @@ def modify(ctx, label: str, url: str, debug: bool) -> None: @cli.command() @click.argument("remote", nargs=-1) @click.option("--debug/--no-debug", help="Run in debug mode", default=False) -def push(remote: str, debug: bool): +@click.option( + "--dirty/--clean", help="Allow running with uncommitted changes", default=False +) +def push(remote: str, debug: bool, dirty: bool): """Push data between the local and remote registry""" remote = "origin" if not remote else remote[0] try: with fdp_session.FAIR( - os.getcwd(), debug=debug, server_mode=fdp_svr.SwitchMode.CLI + os.getcwd(), + debug=debug, + server_mode=fdp_svr.SwitchMode.CLI, + allow_dirty=dirty ) as fair_session: fair_session.push(remote) except fdp_exc.FAIRCLIException as e: @@ -572,15 +578,16 @@ def config_email(user_email: str) -> None: @click.option("--debug/--no-debug") def pull(config: str, debug: bool): """Update local registry from remotes and sources""" - config = config[0] if config != "" else fdp_com.local_user_config(os.getcwd()) + config = config[0] if config else fdp_com.local_user_config(os.getcwd()) try: with fdp_session.FAIR( os.getcwd(), config, server_mode=fdp_svr.SwitchMode.CLI, debug=debug, + allow_dirty=True ) as fair: - fair.run_job(mode=fdp_run.CMD_MODE.PULL) + fair.pull() except fdp_exc.FAIRCLIException as e: if debug: raise e diff --git a/fair/exceptions.py b/fair/exceptions.py index 15bc3b29..39f28d42 100644 --- a/fair/exceptions.py +++ b/fair/exceptions.py @@ -30,6 +30,9 @@ import click import typing +import json +import logging + class FAIRCLIException(Exception): @@ -152,10 +155,11 @@ class ValidationError(FAIRCLIException): def __init__(self, info: typing.List[typing.Dict]) -> None: _invalid_data: typing.List[typing.Dict] = [] - for data in info: - _location = ":".join(data["loc"]) - _type = data["type"] - _msg = data["msg"] + for data in json.loads(info): + _location = map(str, data["loc"]) + _location = ":".join(_location) + _type = str(data["type"]) + _msg = str(data["msg"]) _invalid_data.append(f"{_location:<50} {_type:<20} {_msg:<20}") _msg = "User 'config.yaml' file validation failed with:\n" diff --git a/fair/identifiers.py b/fair/identifiers.py index a39a0c13..31f75881 100644 --- a/fair/identifiers.py +++ b/fair/identifiers.py @@ -149,6 +149,8 @@ def check_id_permitted(identifier: str, retries: int = 5) -> bool: ---------- identifier : str identifier URL candidate + retries: int + number of attempts Returns ------- diff --git a/fair/parsing/__init__.py b/fair/parsing/__init__.py deleted file mode 100644 index 05f8599c..00000000 --- a/fair/parsing/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/python3 -# -*- coding: utf-8 -*- -""" - -Parse User Config -================= - -Perform parsing of the user updated `config.yaml` file. - - -Contents -======== - -Modules -------- - - globbing - swap glob expressions for registry entries - variables - substitute recognised FAIR CLI variables - -""" - -__date__ = "2021-08-04" diff --git a/fair/parsing/globbing.py b/fair/parsing/globbing.py deleted file mode 100644 index 97e46069..00000000 --- a/fair/parsing/globbing.py +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/python3 -# -*- coding: utf-8 -*- -""" - -Glob User Config -================= - -Substitute globular expressions in `config.yaml` for entries -within the local registry - - -Contents -======== - -Functions -------- - - glob_read_write - glob expressions in 'read' and 'write' blocks - -""" - -__date__ = "2021-08-16" - -import copy -import typing - -import fair.exceptions as fdp_exc -import fair.registry.requests as fdp_req -import fair.registry.versioning as fdp_ver -import fair.utilities as fdp_util -import fair.register as fdp_reg - - -def glob_read_write( - user_config: typing.Dict, - blocktype: str, - version: str, - registry_url: str, - search_key: typing.Optional[str] = None, - remove_wildcard: bool = False, -) -> typing.List: - """Substitute glob expressions in the 'read' or 'write' part of a user config - - Parameters - ---------- - user_config : typing.Dict - config yaml - blocktype : str - block type to process - version : str - version string - registry_url : str - URL of the registry to process - search_key : str, optional - key to search under, default is taken from SEARCH_KEYS - remove_wildcard: bool, optional - whether to delete wildcard from yaml file, default is False - """ - - _block_cfg = user_config[blocktype] - _parsed: typing.List[typing.Dict] = [] - - # Iterate through all entries in the section looking for any - # key-value pairs that contain glob statements. - for entry in _block_cfg: - # We still want to keep the wildcard version in case the - # user wants to write to this namespace. - # Wipe version info for this object to start from beginning - _orig_entry = copy.deepcopy(entry) - - _orig_entry["use"]["version"] = str( - fdp_ver.get_correct_version(version, free_write=blocktype != "read") - ) - - _glob_vals = [ - (k, v) for k, v in entry.items() if isinstance(v, str) and "*" in v - ] - if len(_glob_vals) > 1: - # For now only allow one value within the dictionary to have them - raise fdp_exc.NotImplementedError( - "Only one key-value pair in a 'read' list entry may contain a" - " globbable value" - ) - elif len(_glob_vals) == 0: - # If no globbables keep existing statement - _parsed.append(entry) - continue - elif not remove_wildcard: - # If we're going ahead, add in the wildcard too if wanted - _parsed.append(_orig_entry) - - _key_glob, _globbable = _glob_vals[0] - - if not search_key: - search_key = fdp_reg.SEARCH_KEYS[_key_glob] - - _search_dict = {search_key: _globbable} - - # Update search from 'use' block - _search_dict.update(entry["use"]) - - try: - fdp_ver.parse_incrementer(_search_dict["version"]) - # Is an incrementer, so get rid of it - _search_dict.pop("version", None) - except fdp_exc.UserConfigError: # Should be an exact version, so keep - None - - # Send a request to the relevant registry using the search string - # and the selected search key - _results = fdp_req.get(registry_url, _key_glob, params=_search_dict) - - # Iterate through all results, make a copy of the entry and swap - # the globbable statement for the result statement appending this - # to the output list - for result in _results: - _entry_dict = copy.deepcopy(entry) - if _key_glob in _entry_dict["use"]: - _entry_dict["use"][_key_glob] = result[search_key] - if _key_glob in _entry_dict: - _entry_dict[_key_glob] = result[search_key] - _parsed.append(_entry_dict) - - # Before returning the list of dictionaries remove any duplicates - user_config[blocktype] = fdp_util.remove_dictlist_dupes(_parsed) diff --git a/fair/register.py b/fair/register.py index f4ecb756..0c350d87 100644 --- a/fair/register.py +++ b/fair/register.py @@ -22,28 +22,22 @@ import copy import logging import os +import urllib.parse import shutil import typing -import urllib.parse - -import requests +import platform +from fair.registry import SEARCH_KEYS import fair.exceptions as fdp_exc import fair.registry.requests as fdp_req import fair.registry.storage as fdp_store import fair.registry.versioning as fdp_ver +import fair.registry.sync as fdp_sync +logger = logging.getLogger("FAIRDataPipeline.Register") -SEARCH_KEYS = { - "data_product": "name", - "namespace": "name", - "file_type": "extension", - "storage_root": "root", - "storage_location": "hash", -} - -def convert_key_value_to_id(uri: str, obj_type: str, value: str, check: bool = False) -> int: +def convert_key_value_to_id(uri: str, obj_type: str, value: str, token: str) -> int: """Converts a config key value to the relevant URL on the local registry Parameters @@ -54,6 +48,8 @@ def convert_key_value_to_id(uri: str, obj_type: str, value: str, check: bool = F object type value : str search term to use + token: str + registry access token Returns ------- @@ -61,7 +57,7 @@ def convert_key_value_to_id(uri: str, obj_type: str, value: str, check: bool = F ID on the local registry matching the entry """ _params = {SEARCH_KEYS[obj_type]: value} - _result = fdp_req.get(uri, obj_type, params=_params) + _result = fdp_req.get(uri, obj_type, token, params=_params) if not _result: raise fdp_exc.RegistryError( f"Failed to obtain result for '{obj_type}' with parameters '{_params}'" @@ -98,7 +94,6 @@ def fetch_registrations( "version", "public", ] - _logger = logging.getLogger("FAIRDataPipeline.Run") _stored_objects: typing.List[str] = [] @@ -149,7 +144,11 @@ def fetch_registrations( f"'alternate_identifier_type' in external object '{_name}'" ) try: - _data_product_id = convert_key_value_to_id(local_uri, "data_product", entry["use"]["data_product"]) + _data_product_id = convert_key_value_to_id( + local_uri, "data_product", + entry["use"]["data_product"], + fdp_req.local_token() + ) _search_data["data_product"] = _data_product_id except fdp_exc.RegistryError: _is_present = "absent" @@ -172,23 +171,19 @@ def fetch_registrations( "Only one unique identifier may be provided (doi/unique_name)" ) - if "cache" in entry: # Do we have a local cache already? + if "cache" in entry: _temp_data_file = entry["cache"] - else: # Need to download it - _root, _path = entry["root"], entry["path"] - - # Encode the path first - _path = urllib.parse.quote_plus(_path) - _url = f"{_root}{_path}" - try: - _temp_data_file = fdp_req.download_file(_url) - except requests.HTTPError as r_in: - raise fdp_exc.UserConfigError( - f"Failed to fetch item '{_url}' with exit code {r_in.response}" - ) + else: + _local_parsed = urllib.parse.urlparse(local_uri) + _local_url = f"{_local_parsed.scheme}://{_local_parsed.netloc}" + _temp_data_file = fdp_sync.download_from_registry( + _local_url, + root=entry["root"], + path=entry["path"] + ) # Need to fix the path for Windows - if os.path.sep != "/": + if platform.system() == "Windows": _name = _name.replace("/", os.path.sep) _local_dir = os.path.join(write_data_store, _namespace, _name) @@ -197,15 +192,17 @@ def fetch_registrations( _is_present = fdp_store.check_if_object_exists( local_uri=local_uri, file_loc=_temp_data_file, + token=fdp_req.local_token(), obj_type=_obj_type, search_data=_search_data, ) # Hash matched version already present if _is_present == "hash_match": - _logger.debug( - f"Skipping item '{_name}' as a hash matched entry is already" - " present with this name" + logger.debug( + "Skipping item '%s' as a hash matched entry is already" + " present with this name, deleting temporary data file", + _name ) os.remove(_temp_data_file) continue @@ -218,14 +215,14 @@ def fetch_registrations( free_write=True, version=entry["use"]["version"], ) - _logger.debug("Found results for %s", str(_results)) + logger.debug("Found existing results for %s", _results) else: _user_version = fdp_ver.get_correct_version( results_list=None, free_write=True, version=entry["use"]["version"], ) - _logger.debug("Found nothing for %s", str(_search_data)) + logger.debug("No existing results found for %s", _search_data) # Create object location directory, ignoring if already present # as multiple version files can exist @@ -234,10 +231,10 @@ def fetch_registrations( _local_file = os.path.join(_local_dir, f"{_user_version}.{entry['file_type']}") # Copy the temporary file into the data store # then remove temporary file to save space + logger.debug("Saving data file to '%s'", _local_file) shutil.copy(_temp_data_file, _local_file) - if "cache" not in entry: - os.remove(_temp_data_file) + os.remove(_temp_data_file) if "public" in entry: _public = entry["public"] @@ -249,6 +246,7 @@ def fetch_registrations( _file_url = fdp_store.store_data_file( uri=local_uri, repo_dir=repo_dir, + token=fdp_req.local_token(), data=data, local_file=_local_file, write_data_store=write_data_store, diff --git a/fair/registry/__init__.py b/fair/registry/__init__.py index e69de29b..a2e36e8f 100644 --- a/fair/registry/__init__.py +++ b/fair/registry/__init__.py @@ -0,0 +1,7 @@ +SEARCH_KEYS = { + "data_product": "name", + "namespace": "name", + "file_type": "extension", + "storage_root": "root", + "storage_location": "hash", +} \ No newline at end of file diff --git a/fair/registry/requests.py b/fair/registry/requests.py index 9d11cc60..80834284 100644 --- a/fair/registry/requests.py +++ b/fair/registry/requests.py @@ -23,7 +23,6 @@ import json import logging import os -import re import tempfile import typing import urllib.parse @@ -81,14 +80,12 @@ def local_token(registry_dir: str = None) -> str: def _access( uri: str, method: str, + token: str, obj_path: str = None, response_codes: typing.List[int] = [201, 200], - token: str = None, headers: typing.Dict[str, typing.Any] = None, params: typing.Dict = None, - data: typing.Dict = None, - *args, - **kwargs, + data: typing.Dict = None ): if not headers: headers: typing.Dict[str, str] = {} @@ -99,9 +96,6 @@ def _access( if not data: data: typing.Dict[str, str] = {} - if not token: - token = local_token() - # Make sure we have the right number of '/' in the components _uri = uri _uri = fdp_util.check_trailing_slash(_uri) @@ -119,14 +113,14 @@ def _access( if method == "get": logger.debug("Query parameters: %s", params) _request = requests.get( - _url, headers=_headers, params=params, *args, **kwargs + _url, headers=_headers, params=params ) elif method == "post": logger.debug("Post data: %s", data) - _request = requests.post(_url, headers=_headers, data=data, *args, **kwargs) + _request = requests.post(_url, headers=_headers, data=data) else: _request = getattr(requests, method)( - _url, headers=_headers, *args, **kwargs + _url, headers=_headers ) except requests.exceptions.ConnectionError: raise fdp_exc.UnexpectedRegistryServerState( @@ -135,8 +129,8 @@ def _access( ) _info = f"url = {_url}, " - _info += f' parameters = {kwargs["params"]},' if "params" in kwargs else "" - _info += f' data = {kwargs["data"]}' if "data" in kwargs else "" + _info += f' parameters = {params},' if params else "" + _info += f' data = {data}' if data else "" # Case of unrecognised object if _request.status_code == 404: @@ -154,7 +148,7 @@ def _access( error_code=403, ) elif _request.status_code == 409: - _searchable = uri if not obj_path else "/".join(obj_path) + _searchable = obj_path or uri raise fdp_exc.RegistryAPICallError( f"Cannot post object of type '{_searchable}' " f"using method '{method}' as it already exists." @@ -187,9 +181,9 @@ def _access( def post( uri: str, obj_path: str, + token: str, data: typing.Dict[str, typing.Any], headers: typing.Dict[str, typing.Any] = None, - token: str = None, ) -> typing.Dict: """Post an object to the registry @@ -199,12 +193,12 @@ def post( endpoint of the registry obj_path : str type of object to post + token : str + token for accessing the registry data : typing.Dict[str, typing.Any] data for the object headers : typing.Dict[str, typing.Any], optional any additional headers for the request, by default None - token : str, optional - token for accessing the registry, by default None Returns ------- @@ -214,21 +208,24 @@ def post( if headers is None: headers = {} - if not token: - token = local_token() - headers.update({"Content-Type": "application/json"}) + + for param, value in data.copy().items(): + if not value: + logger.warning(f"Key in post data '{param}' has no value so will be ignored") + del data[param] + return _access( uri, "post", + token, obj_path, headers=headers, data=json.dumps(data, cls=fdp_util.JSONDateTimeEncoder), - token=token, ) -def url_get(url: str, *args, **kwargs) -> typing.Dict: +def url_get(url: str, token: str) -> typing.Dict: """Send a URL only request and retrieve results Unlike 'get' this method is 'raw' in that there is no validation of @@ -238,20 +235,22 @@ def url_get(url: str, *args, **kwargs) -> typing.Dict: ---------- url : str URL to send request to + token: str + url access token Returns ------- typing.Dict results dictionary """ - return _access(url, "get", *args, **kwargs) + return _access(url, "get", token) def get( uri: str, obj_path: str, + token: str, headers: typing.Dict[str, typing.Any] = None, params: typing.Dict[str, typing.Any] = None, - token: str = None, ) -> typing.Dict: """Retrieve an object from the given registry @@ -261,12 +260,12 @@ def get( endpoint of the registry obj_path : str type of the object to fetch + token : str + token for accessing the registry headers : typing.Dict[str, typing.Any], optional any additional headers for the request, by default None params : typing.Dict[str, typing.Any], optional search parameters for the object, by default None - token : str, optional - token for accessing the registry, by default None Returns ------- @@ -285,18 +284,20 @@ def get( params = {} if not params else copy.deepcopy(params) - if not token: - token = local_token() + for param, value in params.copy().items(): + if not value: + logger.warning(f"Key in get parameters '{param}' has no value so will be ignored") + del params[param] - return _access(uri, "get", obj_path, headers=headers, params=params, token=token) + return _access(uri, "get", token, obj_path=obj_path, headers=headers, params=params) def post_else_get( uri: str, obj_path: str, + token: str, data: typing.Dict[str, typing.Any], params: typing.Dict[str, typing.Any] = None, - token: str = None, ) -> str: """Post to the registry if an object does not exist else retrieve URL @@ -306,12 +307,12 @@ def post_else_get( endpoint of the registry obj_path : str object type to post + token : str + token to access registry data : typing.Dict[str, typing.Any] data for the object to be posted params : typing.Dict[str, typing.Any], optional parameters for searching if object exists, by default None - token : str, optional - token to access registry, by default None Returns ------- @@ -321,23 +322,23 @@ def post_else_get( if not params: params = {} - if not token: - token = local_token() - try: logger.debug("Attempting to post an instance of '%s' to '%s'", obj_path, uri) _loc = post(uri, obj_path, data=data, token=token) except fdp_exc.RegistryAPICallError as e: - # If the item is already in the registry then ignore the - # conflict error and continue, else raise exception - if e.error_code == 409: - logger.debug("Object already exists, retrieving entry") - _loc = get(uri, obj_path, params=params, token=token) - else: + if e.error_code != 409: raise e - + logger.debug(e.msg) + logger.debug('Object already exists, retrieving entry') + _loc = get(uri, obj_path, token, params=params) if isinstance(_loc, list): if not _loc: + logger.error(f"Results of URL query empty: {_loc}") + try: + _full_listing = get(uri, obj_path, token) + logger.debug(f"Available {obj_path}s: {_full_listing}") + except fdp_exc.RegistryError: + logger.debug("No entries of type '{obj_path}' exist") raise fdp_exc.RegistryError( "Expected to receive a URL location from registry post" ) @@ -348,7 +349,7 @@ def post_else_get( def filter_object_dependencies( - uri: str, obj_path: str, filter: typing.Dict[str, typing.Any] + uri: str, obj_path: str, token: str, filter: typing.Dict[str, typing.Any] ) -> typing.List[str]: """Filter dependencies of an API object based on a set of conditions @@ -358,6 +359,8 @@ def filter_object_dependencies( endpoint of the registry object : str path of object type, e.g. 'code_run' + token : str + registry access token filter : typing.Dict[str, typing.Any] list of filters to apply to listing @@ -366,27 +369,28 @@ def filter_object_dependencies( typing.List[str] list of object type paths """ + logger.debug("Filtering dependencies for object '%s' and filter '%s'", obj_path, filter) try: - _actions = _access(uri, "options", obj_path)["actions"]["POST"] + _actions = _access(uri, "options", token, obj_path)["actions"]["POST"] except KeyError: # No 'actions' key means no dependencies return [] _fields: typing.List[str] = [] for name, info in _actions.items(): - _filter_result: typing.List[bool] = [] - for filt, value in filter.items(): - # Some objects may not have the key - if filt not in info: - continue - _filter_result.append(info[filt] == value) + _filter_result: typing.List[bool] = [ + info[filt] == value + for filt, value in filter.items() + if filt in info + ] + if all(_filter_result): _fields.append(name) return _fields -def get_filter_variables(uri: str, obj_path: str) -> typing.List[str]: +def get_filter_variables(uri: str, obj_path: str, token: str) -> typing.List[str]: """Retrieves a list of variables you can filter by for a given object Parameters @@ -395,6 +399,8 @@ def get_filter_variables(uri: str, obj_path: str) -> typing.List[str]: endpoint of registry obj_path : str type of object + token : str + registry access token Returns ------- @@ -402,14 +408,14 @@ def get_filter_variables(uri: str, obj_path: str) -> typing.List[str]: list of filterable fields """ try: - _filters = _access(uri, "options", obj_path)["filter_fields"] + _filters = _access(uri, "options", token, obj_path)["filter_fields"] except KeyError: # No 'filter_fields' key means no filters return [] return [*_filters] -def get_writable_fields(uri: str, obj_path: str) -> typing.List[str]: +def get_writable_fields(uri: str, obj_path: str, token: str) -> typing.List[str]: """Retrieve a list of writable fields for the given RestAPI object Parameters @@ -418,13 +424,15 @@ def get_writable_fields(uri: str, obj_path: str) -> typing.List[str]: endpoint of the registry object : str path of object type, e.g. 'code_run' + token: str + registry access token Returns ------- typing.List[str] list of object type paths """ - return filter_object_dependencies(uri, obj_path, {"read_only": False}) + return filter_object_dependencies(uri, obj_path, token, {"read_only": False}) def download_file(url: str, chunk_size: int = 8192) -> str: @@ -445,40 +453,48 @@ def download_file(url: str, chunk_size: int = 8192) -> str: # Save the data to a temporary file so we can calculate the hash _file, _fname = tempfile.mkstemp() - with requests.get(url, stream=True) as r_in: - try: - r_in.raise_for_status() - except requests.HTTPError: - raise fdp_exc.FileNotFoundError( - f"Failed to download file from '{url}'" - f" with status code {r_in.status_code}" - ) - with os.fdopen(_file, "wb") as in_f: - for chunk in r_in.iter_content(chunk_size=chunk_size): - in_f.write(chunk) + try: + with requests.get(url, stream=True) as r_in: + try: + r_in.raise_for_status() + except requests.HTTPError: + raise fdp_exc.FileNotFoundError( + f"Failed to download file from '{url}'" + f" with status code {r_in.status_code}" + ) + + with os.fdopen(_file, "wb") as in_f: + for chunk in r_in.iter_content(chunk_size=chunk_size): + in_f.write(chunk) + except requests.exceptions.ConnectionError: + raise fdp_exc.FAIRCLIException( + f"Failed to download file '{url}'" + f" due to connection error" + ) return _fname -def get_dependency_listing(uri: str) -> typing.Dict: +def get_dependency_listing(uri: str, token: str) -> typing.Dict: """Get complete listing of all objects and their registry based dependencies Parameters ---------- uri : str endpoint of the registry + token : str + registry access token Returns ------- typing.Dict dictionary of object types and their registry based dependencies """ - - _registry_objs = url_get(uri) + _registry_objs = url_get(uri, token) return { obj: filter_object_dependencies( - uri, obj, {"read_only": False, "type": "field", "local": True} + uri, obj, token, {"read_only": False, "type": "field", "local": True} ) for obj in _registry_objs } @@ -501,13 +517,15 @@ def get_obj_id_from_url(object_url: str) -> int: return [i for i in _url.path.split("/") if i.strip()][-1] -def get_obj_type_from_url(request_url: str) -> str: +def get_obj_type_from_url(request_url: str, token: str) -> str: """Retrieves the type of object from the given URL Parameters ---------- request_url : str url to type check + token: str + token for accessing specified registry Returns ------- @@ -515,7 +533,7 @@ def get_obj_type_from_url(request_url: str) -> str: object type if recognised else empty string """ _uri, _ = split_api_url(request_url) - for obj_type in sorted([*url_get(_uri)], key=len, reverse=True): + for obj_type in sorted([*url_get(_uri, token=token)], key=len, reverse=True): if obj_type in request_url: return obj_type return "" diff --git a/fair/registry/server.py b/fair/registry/server.py index d654779d..43493a0e 100644 --- a/fair/registry/server.py +++ b/fair/registry/server.py @@ -291,7 +291,7 @@ def install_registry( silent: bool = False, force: bool = False, venv_dir: str = None, -) -> None: +) -> str: logger = logging.getLogger("FAIRDataPipeline.Server") @@ -392,6 +392,8 @@ def install_registry( ) rebuild_local(_venv_python, install_dir, silent) + + return reference def uninstall_registry() -> None: @@ -437,15 +439,15 @@ def update_registry_post_setup(repo_dir: str, global_setup: bool = False) -> Non if global_setup: logger.debug("Populating file types") - fdp_store.populate_file_type(fdp_conf.get_local_uri()) + fdp_store.populate_file_type(fdp_conf.get_local_uri(), fdp_req.local_token()) logger.debug("Adding 'author' and 'UserAuthor' entries if not present") # Add author and UserAuthor - _author_url = fdp_store.store_user(repo_dir, fdp_conf.get_local_uri()) + _author_url = fdp_store.store_user(repo_dir, fdp_conf.get_local_uri(), fdp_req.local_token()) try: _admin_url = fdp_req.get( - fdp_conf.get_local_uri(), "users", params={"username": "admin"} + fdp_conf.get_local_uri(), "users", fdp_req.local_token(), params={"username": "admin"} )[0]["url"] except (KeyError, IndexError): raise fdp_exc.RegistryError( @@ -455,6 +457,7 @@ def update_registry_post_setup(repo_dir: str, global_setup: bool = False) -> Non fdp_req.post_else_get( fdp_conf.get_local_uri(), "user_author", + fdp_req.local_token(), data={"user": _admin_url, "author": _author_url}, ) diff --git a/fair/registry/storage.py b/fair/registry/storage.py index 7b5d594d..de009bb9 100644 --- a/fair/registry/storage.py +++ b/fair/registry/storage.py @@ -24,7 +24,9 @@ import hashlib import os +from threading import local import typing +import logging import yaml @@ -35,8 +37,10 @@ import fair.registry.requests as fdp_req import fair.registry.versioning as fdp_ver +logger = logging.getLogger("FAIRDataPipeline.Storage") -def get_write_storage(uri: str, write_data_store: str) -> str: + +def get_write_storage(uri: str, write_data_store: str, token: str) -> str: """Construct storage root if it does not exist Parameters @@ -45,6 +49,8 @@ def get_write_storage(uri: str, write_data_store: str) -> str: end point of the RestAPI write_data_store : str path of the write data store + token: str + registry access token Returns ------- @@ -56,6 +62,7 @@ def get_write_storage(uri: str, write_data_store: str) -> str: fdp_exc.UserConfigError If 'write_data_store' not present in the working config or global config """ + logger.debug("Constructing a storage root for '%s'", write_data_store) # Convert local file path to a valid data store path _write_store_root = f"file://{write_data_store}" @@ -63,7 +70,7 @@ def get_write_storage(uri: str, write_data_store: str) -> str: _write_store_root += os.path.sep # Check if the data store already exists by querying for it - _search_root = fdp_req.get(uri, "storage_root", params={"root": _write_store_root}) + _search_root = fdp_req.get(uri, "storage_root", token, params={"root": _write_store_root}) # If the data store already exists just return the URI else create it # and then do the same @@ -71,15 +78,54 @@ def get_write_storage(uri: str, write_data_store: str) -> str: return _search_root[0]["url"] _post_data = {"root": _write_store_root, "local": True} - _storage_root = fdp_req.post(uri, "storage_root", data=_post_data) + _storage_root = fdp_req.post(uri, "storage_root", token, data=_post_data) return _storage_root["url"] -def store_user(repo_dir: str, uri: str) -> str: +def store_author( + uri: str, + token: str, + name: str, + identifier: str = None, + uuid: str = None) -> str: + """Creates an Author entry if one does not exist + + Parameters + ---------- + uri : str + registry RestAPI endpoint + token: str + registry access token + data: typing.Dict + author data to post + params: typing.Dict, optional + parameters to search if exists already + + Returns + ------- + str + URI for created author + """ + _data = { + "name": name, + "identifier": identifier, + "uuid": uuid + } + + return fdp_req.post_else_get( + uri, "author", token, _data, {"name": name} + ) + + +def store_user(repo_dir: str, uri: str, token: str) -> str: """Creates an Author entry for the user if one does not exist Parameters ---------- + repo_dir: str + repository directory + token: str + registry access token uri : str registry RestAPI endpoint @@ -88,39 +134,44 @@ def store_user(repo_dir: str, uri: str) -> str: str URI for created author """ + _user = fdp_conf.get_current_user_name(repo_dir) - _data = {"name": " ".join(_user) if _user[1] else _user[0]} + name = " ".join(_user) if _user[1] else _user[0] + _id = None + _uuid = None + + logger.debug("Storing user '%s'", name) try: _id = fdp_conf.get_current_user_uri(repo_dir) - _data["identifier"] = _id - return fdp_req.post_else_get( - uri, "author", data=_data, params={"identifier": _id} - ) except fdp_exc.CLIConfigurationError: _uuid = fdp_conf.get_current_user_uuid(repo_dir) - _data["uuid"] = _uuid - return fdp_req.post_else_get(uri, "author", data=_data, params={"uuid": _uuid}) + + return store_author(uri, token, name, _id, _uuid) -def populate_file_type(uri: str) -> typing.List[typing.Dict]: +def populate_file_type(uri: str, token: str) -> typing.List[typing.Dict]: """Populates file_type table with common file file_types Parameters ---------- uri: str registry RestAPI end point + token: str + registry access token """ + logger.debug("Adding file types to storage") + _type_objs = [] for _extension in fdp_file.FILE_TYPES: # Use post_else_get in case some file types exist already - _result = create_file_type(uri, _extension) + _result = create_file_type(uri, _extension, token) _type_objs.append(_result) return _type_objs -def create_file_type(uri: str, extension: str) -> str: +def create_file_type(uri: str, extension: str, token: str) -> str: """Creates a new file type on the registry Parameters @@ -129,6 +180,8 @@ def create_file_type(uri: str, extension: str) -> str: registry RestAPI end point extension : str file extension + token: str + registry access string Returns ------- @@ -136,15 +189,19 @@ def create_file_type(uri: str, extension: str) -> str: URI for created file type """ _name = fdp_file.FILE_TYPES[extension] + + logger.debug("Adding file type '%s' with extension '%s'", _name, extension) + return fdp_req.post_else_get( uri, "file_type", + token, data={"name": _name, "extension": extension.lower()}, params={"extension": extension.lower()}, ) -def store_working_config(repo_dir: str, uri: str, work_cfg_yml: str) -> str: +def store_working_config(repo_dir: str, uri: str, work_cfg_yml: str, token: str) -> str: """Construct a storage location and object for the working config Parameters @@ -155,6 +212,8 @@ def store_working_config(repo_dir: str, uri: str, work_cfg_yml: str) -> str: RestAPI end point work_cfg_yml : str location of working config yaml + token: str + registry access token Returns ------- @@ -166,7 +225,9 @@ def store_working_config(repo_dir: str, uri: str, work_cfg_yml: str) -> str: fair.exceptions.RegistryAPICallError if bad status code returned from the registry """ - _root_store = get_write_storage(uri, work_cfg_yml) + logger.debug("Storing working config on registry") + + _root_store = get_write_storage(uri, work_cfg_yml, token) _work_cfg = yaml.safe_load(open(work_cfg_yml)) _work_cfg_data_store = _work_cfg["run_metadata"]["write_data_store"] @@ -188,21 +249,19 @@ def store_working_config(repo_dir: str, uri: str, work_cfg_yml: str) -> str: } try: - _post_store_loc = fdp_req.post(uri, "storage_location", data=_storage_loc_data) + _post_store_loc = fdp_req.post(uri, "storage_location", token, data=_storage_loc_data) except fdp_exc.RegistryAPICallError as e: - if not e.error_code == 409: + if e.error_code != 409: raise e else: raise fdp_exc.RegistryAPICallError( - f"Cannot post storage_location " - f"'{_rel_path}' with hash" - f" '{_hash}', object already exists", + f"Cannot post storage_location '{_rel_path}' with hash '{_hash}', object already exists", error_code=409, ) - _user = store_user(repo_dir, uri) + _user = store_user(repo_dir, uri, token) - _yaml_type = create_file_type(uri, "yaml") + _yaml_type = create_file_type(uri, "yaml", token) _desc = f"Working configuration file for timestamp {_time_stamp_dir}" _object_data = { @@ -213,12 +272,12 @@ def store_working_config(repo_dir: str, uri: str, work_cfg_yml: str) -> str: } return fdp_req.post_else_get( - uri, "object", data=_object_data, params={"description": _desc} + uri, "object", token, data=_object_data, params={"description": _desc} ) def store_working_script( - repo_dir: str, uri: str, script_path: str, working_config: str + repo_dir: str, uri: str, script_path: str, working_config: str, token: str ) -> str: """Construct a storage location and object for the CLI run script @@ -232,6 +291,8 @@ def store_working_script( location of working CLI run script data_store : str data store path + token: str + registry access token Returns ------- @@ -243,8 +304,10 @@ def store_working_script( fair.exceptions.RegistryAPICallError if bad status code returned from the registry """ + logger.debug("Storing working script on registry") + _work_cfg = yaml.safe_load(open(working_config)) - _root_store = get_write_storage(uri, working_config) + _root_store = get_write_storage(uri, working_config, token) _data_store = _work_cfg["run_metadata"]["write_data_store"] _rel_path = os.path.relpath(script_path, _data_store) @@ -266,9 +329,9 @@ def store_working_script( } try: - _post_store_loc = fdp_req.post(uri, "storage_location", data=_storage_loc_data) + _post_store_loc = fdp_req.post(uri, "storage_location", token, data=_storage_loc_data) except fdp_exc.RegistryAPICallError as e: - if not e.error_code == 409: + if e.error_code != 409: raise e else: raise fdp_exc.RegistryAPICallError( @@ -278,9 +341,9 @@ def store_working_script( error_code=409, ) - _user = store_user(repo_dir, uri) + _user = store_user(repo_dir, uri, token) - _shell_script_type = create_file_type(uri, "sh") + _shell_script_type = create_file_type(uri, "sh", token) _time_stamp_dir = os.path.basename(os.path.dirname(script_path)) _desc = f"Run script for timestamp {_time_stamp_dir}" @@ -292,12 +355,16 @@ def store_working_script( } return fdp_req.post_else_get( - uri, "object", data=_object_data, params={"description": _desc} + uri, "object", token, data=_object_data, params={"description": _desc} ) def store_namespace( - uri: str, namespace_label: str, full_name: str = None, website: str = None + uri: str, + token: str, + name: str, + full_name: str = None, + website: str = None ) -> str: """Create a namespace on the registry @@ -307,6 +374,8 @@ def store_namespace( endpoint of the registry namespace_label : str name of the Namespace + token : str + registry access token full_name : str, optional full title of the namespace, by default None website : str, optional @@ -317,27 +386,32 @@ def store_namespace( str URL of the created namespace """ + logger.debug("Storing namespace '%s' on registry", name) + _data = { - "name": namespace_label, + "name": name, "full_name": full_name, "website": website, } + return fdp_req.post_else_get( - uri, "namespace", data=_data, params={"name": namespace_label} + uri, "namespace", token, data=_data, params={"name": name} ) -# flake8: noqa: C901 def store_data_file( uri: str, repo_dir: str, + token: str, data: typing.Dict, local_file: str, write_data_store: str, public: bool, ) -> None: - _root_store = get_write_storage(uri, write_data_store) + logger.debug("Storing data file '%s' on registry", local_file) + + _root_store = get_write_storage(uri, write_data_store, token) _rel_path = os.path.relpath(local_file, write_data_store) @@ -347,60 +421,189 @@ def store_data_file( "registry submission but none found" ) + _post_store_loc = _get_url_from_storage_loc( + local_file=local_file, + registry_uri=uri, + registry_token=token, + relative_path=_rel_path, + root_store_url=_root_store, + is_public=public + ) + + _user = store_user(repo_dir, uri, token) + + _file_type = _get_url_from_file_type( + data=data, + local_file=local_file, + registry_uri=uri, + registry_token=token + ) + + _namespace_url = _get_url_from_namespace( + data=data, + label=local_file, + registry_uri=uri, + registry_token=token + ) + + _obj_url = _get_url_from_object( + data=data, + registry_uri=uri, + registry_token=token, + user=_user, + storage_loc_url=_post_store_loc, + file_type_url=_file_type + ) + + _data_prod_url = _get_url_from_data_product( + data=data, + label=local_file, + registry_uri=uri, + registry_token=token, + namespace_url=_namespace_url, + object_url=_obj_url + ) + + # If 'data_product' key present finish here and return URL + # else this is an external object + if "data_product" in data: + return _data_prod_url + + return _get_url_from_external_obj( + data=data, + local_file=local_file, + registry_uri=uri, + registry_token=token, + data_product_url=_data_prod_url + ) + + +def _get_url_from_storage_loc( + local_file: str, + registry_uri: str, + registry_token: str, + relative_path: str, + root_store_url: str, + is_public: bool +) -> str: _hash = calculate_file_hash(local_file) _storage_loc_data = { - "path": _rel_path, - "storage_root": _root_store, - "public": public, + "path": relative_path, + "storage_root": root_store_url, + "public": is_public, "hash": _hash, } _search_data = {"hash": _hash} - _post_store_loc = fdp_req.post_else_get( - uri, "storage_location", data=_storage_loc_data, params=_search_data + return fdp_req.post_else_get( + registry_uri, + "storage_location", + registry_token, + data=_storage_loc_data, + params=_search_data ) - _user = store_user(repo_dir, uri) +def _get_url_from_file_type( + data: typing.Dict, + local_file: str, + registry_uri: str, + registry_token: str +) -> str: if "file_type" in data: _file_type = data["file_type"] else: _file_type = os.path.splitext(local_file)[1] - _file_type = create_file_type(uri, _file_type) + return create_file_type(registry_uri, _file_type, registry_token) + +def _get_url_from_namespace( + data: typing.Dict, + label: str, + registry_uri: str, + registry_token: str +) -> str: # Namespace is read from the source information if "namespace_name" not in data: raise fdp_exc.UserConfigError( - f"Expected 'namespace_name' for item '{local_file}'" + f"Expected 'namespace_name' for item '{label}'" ) _namespace_args = { - "uri": uri, - "namespace_label": data["namespace_name"], + "name": data["namespace_name"], "full_name": data["namespace_full_name"] if "namespace_full_name" in data else None, - "website": data["namespace_website"] if "namespace_website" in data else None, + "website": data.get("namespace_website", None), + } + + return store_namespace( + registry_uri, + registry_token, + **_namespace_args + ) + + +def _get_url_from_external_obj( + data: typing.Dict, + local_file: str, + registry_uri: str, + registry_token: str, + data_product_url: str +) -> typing.Dict: + _expected_ext_obj_keys = ("release_date", "primary", "title") + + + for key in _expected_ext_obj_keys: + if key not in data: + raise fdp_exc.UserConfigError( + f"Expected key '{key}' for item '{local_file}'" + ) + + _external_obj_data = { + "data_product": data_product_url, + "title": data["title"], + "primary_not_supplement": data["primary"], + "release_date": data["release_date"], } + _external_obj_data.update(_get_identifier_from_data(data, local_file)) + + return fdp_req.post( + registry_uri, + "external_object", + registry_token, + data=_external_obj_data + ) - _namespace_url = store_namespace(**_namespace_args) - _desc = data["description"] if "description" in data else None +def _get_url_from_object( + data: typing.Dict, + registry_uri: str, + registry_token: str, + user: str, + storage_loc_url: str, + file_type_url: str) -> str: + _desc = data.get("description", None) _object_data = { "description": _desc, - "file_type": _file_type, - "storage_location": _post_store_loc, - "authors": [_user], + "file_type": file_type_url, + "storage_location": storage_loc_url, + "authors": [user], } try: - _obj_url = fdp_req.post(uri, "object", data=_object_data)["url"] + return fdp_req.post( + registry_uri, + "object", + registry_token, + data=_object_data + )["url"] except fdp_exc.RegistryAPICallError as e: - if not e.error_code == 409: + if e.error_code != 409: raise e else: raise fdp_exc.RegistryAPICallError( @@ -413,6 +616,15 @@ def store_data_file( f" for post object '{_desc}'" ) + +def _get_url_from_data_product( + data: typing.Dict, + label: str, + registry_uri: str, + registry_token: str, + namespace_url: str, + object_url: str) -> str: + """Retrieve the URL for a given config data product""" # Get the name of the entry if "external_object" in data: _name = data["external_object"] @@ -420,7 +632,7 @@ def store_data_file( _name = data["data_product"] else: raise fdp_exc.UserConfigError( - f"Failed to determine type while storing item '{local_file}'" + f"Failed to determine type while storing item '{label}'" "into registry" ) @@ -428,16 +640,21 @@ def store_data_file( _name = data["use"]["data_product"] _data_prod_data = { - "namespace": _namespace_url, - "object": _obj_url, + "namespace": namespace_url, + "object": object_url, "version": str(data["use"]["version"]), "name": _name, } try: - _data_prod_url = fdp_req.post(uri, "data_product", data=_data_prod_data)["url"] + return fdp_req.post( + registry_uri, + "data_product", + registry_token, + data=_data_prod_data + )["url"] except fdp_exc.RegistryAPICallError as e: - if not e.error_code == 409: + if e.error_code != 409: raise e else: raise fdp_exc.RegistryAPICallError( @@ -450,55 +667,33 @@ def store_data_file( f" for post object '{_name}'" ) - # If 'data_product' key present finish here and return URL - # else this is an external object - if "data_product" in data: - return _data_prod_url - - _expected_ext_obj_keys = ["release_date", "primary", "title"] - _identifier = None - _alternate_identifier = None - _alternate_identifier_type = None +def _get_identifier_from_data(data: typing.Dict, label: str) -> typing.Dict[str, str]: + """Retrieve the identifier metadata from the data entry""" + _identifier: typing.Dict[str, str] = {} - if "identifier" in data: - _identifier = data["identifier"] - if not fdp_id.check_id_permitted(_identifier): + if data.get("identifier", None): + if not fdp_id.check_id_permitted(data["identifier"]): raise fdp_exc.UserConfigError( - f"Identifier '{_identifier}' is not a valid identifier" + "Identifier '"+data["identifier"]+"' is not a valid identifier" ) - - if not _identifier: - if "unique_name" not in data: + _identifier["identifier"] = data["identifier"] + else: + try: + _identifier["alternate_identifier"] = data["unique_name"] + except KeyError: raise fdp_exc.UserConfigError( - "No identifier/alternate_identifier given for " f"item '{local_file}'", + "No identifier/alternate_identifier given for " + f"item '{label}'", hint="You must provide either a URL 'identifier', or " "'unique_name' and 'source_name' keys", ) - else: - _alternate_identifier = data["unique_name"] - if "alternate_identifier_type" in data: - _alternate_identifier_type = data["alternate_identifier_type"] - else: - _alternate_identifier_type = "local source descriptor" - - for key in _expected_ext_obj_keys: - if key not in data: - raise fdp_exc.UserConfigError( - f"Expected key '{key}' for item '{local_file}'" - ) - - _external_obj_data = { - "data_product": _data_prod_url, - "title": data["title"], - "primary_not_supplement": data["primary"], - "release_date": data["release_date"], - "identifier": _identifier, - "alternate_identifier": _alternate_identifier, - "alternate_identifier_type": _alternate_identifier_type, - } - - return fdp_req.post(uri, "external_object", data=_external_obj_data) + _identifier["alternate_identifier"] = data.get( + "alternate_identifier_type", + "local source descriptor" + ) + + return _identifier def calculate_file_hash(file_name: str, buffer_size: int = 64 * 1024) -> str: @@ -584,7 +779,7 @@ def check_if_object_exists( file_loc: str, obj_type: str, search_data: typing.Dict, - token: str = None, + token: str, ) -> str: """Checks if a data product is already present in the registry @@ -629,7 +824,7 @@ def check_if_object_exists( if obj_type == "external_object": _results = [res["data_product"] for res in _results] - _results = [fdp_req.url_get(r) for r in _results] + _results = [fdp_req.url_get(r, token=token) for r in _results] _object_urls = [res["object"] for res in _results] diff --git a/fair/registry/sync.py b/fair/registry/sync.py index 895d5b64..359e4d4f 100644 --- a/fair/registry/sync.py +++ b/fair/registry/sync.py @@ -19,16 +19,22 @@ import typing import collections import logging +import requests import re +import os +import shutil +import urllib.parse +import click import fair.exceptions as fdp_exc import fair.registry.requests as fdp_req import fair.utilities as fdp_util -from fair.register import SEARCH_KEYS + +from fair.registry import SEARCH_KEYS logger = logging.getLogger("FAIRDataPipeline.Sync") -def get_dependency_chain(object_url: str) -> collections.deque: +def get_dependency_chain(object_url: str, token: str) -> collections.deque: """Get all objects relating to an object in order of dependency For a given URL this function fetches all component URLs ordering them @@ -39,24 +45,25 @@ def get_dependency_chain(object_url: str) -> collections.deque: ---------- object_url : str Full URL of an object within a registry + token: str + registry access token Returns ------- collections.deque ordered iterable of component object URLs """ - _logger = logging.getLogger("FAIRDataPipeline.Sync") - _logger.debug(f"Retrieving dependency chain for '{object_url}'") + logger.debug(f"Retrieving dependency chain for '{object_url}'") _local_uri, _ = fdp_req.split_api_url(object_url) - _dependency_list = fdp_req.get_dependency_listing(_local_uri) + _dependency_list = fdp_req.get_dependency_listing(_local_uri, token) def _dependency_of(url_list: collections.deque, item: str): if item in url_list: return url_list.appendleft(item) - _results = fdp_req.url_get(item) - _type = fdp_req.get_obj_type_from_url(item) + _results = fdp_req.url_get(item, token) + _type = fdp_req.get_obj_type_from_url(item, token) for req, val in _results.items(): if req in _dependency_list[_type] and val: if isinstance(val, list): @@ -72,11 +79,64 @@ def _dependency_of(url_list: collections.deque, item: str): return _urls -def push_dependency_chain( +def pull_all_namespaces( + local_uri: str, + remote_uri: str, + local_token: str, + remote_token: str +) -> typing.List[str]: + """Pull all namespaces from a remote registry + + This ensures a user does not try to register locally a namespace that + already exists on the remote and so lowers the risk of conflicting + metadata when running a pull + + Parameters + ---------- + local_uri : str + endpoint of the local registry + remote_uri : str + endpoint of the remote registry + local_token : str + access token for the local registry + remote_token : str + access token for the remote registry + + Returns + ------- + typing.List[str] + list of identified namespaces + """ + logger.debug("Pulling all namespaces to local registry") + + _remote_namespaces = fdp_req.get(remote_uri, "namespace", remote_token) + + logger.debug( + "Found %s namespace%s on remote", len(_remote_namespaces), + "s" if len(_remote_namespaces) != 1 else "" + ) + + if not _remote_namespaces: + return + + _writable_fields = fdp_req.get_writable_fields(local_uri, "namespace", local_token) + + for namespace in _remote_namespaces: + _writable_data = { + k: v + for k, v in namespace.items() + if k in _writable_fields + } + logger.debug("Writable local object data: %s", _writable_data) + fdp_req.post_else_get(local_uri, "namespace", local_token, _writable_data) + + +def sync_dependency_chain( object_url: str, dest_uri: str, - local_uri: str, + origin_uri: str, dest_token: str, + origin_token: str ) -> typing.Dict[str, str]: """Push an object and all of its dependencies to the remote registry @@ -89,88 +149,66 @@ def push_dependency_chain( object to push dest_uri : str endpoint of the destination registry - local_uri : str - endpoint of the local registry + origin_uri : str + endpoint of the origin registry dest_token : str access token for the destination registry + origin_token : str + access token for the origin registry Returns ------- typing.Dict[str, str] dictionary showing conversion from source registry URL to destination """ - _logger = logging.getLogger("FAIRDataPipeline.Sync") - _logger.debug(f"Attempting to push object '{object_url}' to '{dest_uri}'") + logger.debug(f"Attempting to push object '{object_url}' to '{dest_uri}'") + + if not origin_token: + raise fdp_exc.InternalError("Expected an origin token to be provided") + + _dependency_chain: collections.deque = get_dependency_chain( + object_url, + origin_token + ) - _dependency_chain: collections.deque = get_dependency_chain(object_url) _new_urls: typing.Dict[str, str] = {k: "" for k in _dependency_chain} + _writable_fields: typing.Dict[str, str] ={} + # For every object (and the order) in the dependency chain # post the object then store the URL so it can be used to assemble those # further down the chain for object_url in _dependency_chain: - _logger.debug("Preparing object '%s'", object_url) + logger.debug("Preparing object '%s'", object_url) # Retrieve the data for the object from the registry - _obj_data = fdp_req.url_get(object_url) - # Get the URI from the URL - _uri, _ = fdp_req.split_api_url(object_url) + _obj_data = fdp_req.url_get(object_url, token=origin_token) # Deduce the object type from its URL - _obj_type = fdp_req.get_obj_type_from_url(object_url) + _obj_type = fdp_req.get_obj_type_from_url(object_url, token=origin_token) + + if _obj_type not in _writable_fields: + _writable_fields[_obj_type] = fdp_req.get_writable_fields( + origin_uri, + _obj_type, + origin_token + ) # Filter object data to only the writable fields _writable_data = { k: v for k, v in _obj_data.items() - if k in fdp_req.get_writable_fields(_uri, _obj_type) + if k in _writable_fields[_obj_type] } - _logger.debug("Writable local object data: %s", _writable_data) - _new_obj_data: typing.Dict[str, typing.Any] = {} - _url_fields: typing.List[str] = [] - - # Iterate through the object data, for any values which are URLs - # substitute the local URL for the created remote ones. - # For the first object there should be no URL values at all. - for key, value in _writable_data.items(): - # Check if value is URL - _not_str = not isinstance(value, str) - _not_url = isinstance(value, str) and not fdp_util.is_api_url( - local_uri, value - ) - if _not_str or _not_url: - _new_obj_data[key] = value - continue - # Store which fields have URLs to use later - _url_fields.append(key) - # Make sure that a URL for the component does exist - if value not in _new_urls: - raise fdp_exc.RegistryError( - f"Expected URL from remote '{dest_uri}' for component " - f"'{key}' of local object '{value}' during push." - ) - - # Retrieve from the new URLs the correct value and substitute - _new_obj_data[key] = _new_urls[value] - - # Filters are all variables returned by 'filter_fields' request for a - # given object minus any variables which have a URL value - # (as remote URL will never match local) - - _filters = { - k: v - for k, v in _new_obj_data.items() - if k in fdp_req.get_filter_variables(_uri, _obj_type) - and isinstance(v, str) - and k not in _url_fields - } - - _logger.debug(f"Pushing member '{object_url}' to '{dest_uri}'") + logger.debug("Writable local object data: %s", _writable_data) - if dest_uri == local_uri: - raise fdp_exc.InternalError("Cannot push object to its source address") - - _new_url = fdp_req.post_else_get( - dest_uri, _obj_type, data=_new_obj_data, token=dest_token, params=_filters + _new_url = _get_new_url( + origin_uri=origin_uri, + origin_token=origin_token, + dest_uri=dest_uri, + dest_token=dest_token, + object_url=object_url, + new_urls=_new_urls, + writable_data=_writable_data ) if not fdp_util.is_api_url(dest_uri, _new_url): @@ -184,19 +222,136 @@ def push_dependency_chain( return _new_urls -def push_data_products( - local_uri: str, dest_uri: str, dest_token: str, data_products: typing.List[str] +def _get_new_url( + origin_uri: str, + origin_token: str, + dest_uri: str, + dest_token: str, + object_url: str, + new_urls: typing.Dict, + writable_data: typing.Dict +) -> typing.Tuple[typing.Dict, typing.List]: + _new_obj_data: typing.Dict[str, typing.Any] = {} + _url_fields: typing.List[str] = [] + + # Iterate through the object data, for any values which are URLs + # substitute the local URL for the created remote ones. + # For the first object there should be no URL values at all. + for key, value in writable_data.items(): + # Check if value is URL + _not_str = not isinstance(value, str) + _not_url = isinstance(value, str) and not fdp_util.is_api_url( + origin_uri, value + ) + if _not_str or _not_url: + _new_obj_data[key] = value + continue + # Store which fields have URLs to use later + _url_fields.append(key) + # Make sure that a URL for the component does exist + if value not in new_urls: + raise fdp_exc.RegistryError( + f"Expected URL from remote '{dest_uri}' for component " + f"'{key}' of local object '{value}' during push." + ) + + # Retrieve from the new URLs the correct value and substitute + _new_obj_data[key] = new_urls[value] + + # Filters are all variables returned by 'filter_fields' request for a + # given object minus any variables which have a URL value + # (as remote URL will never match local) + + _obj_type = fdp_req.get_obj_type_from_url(object_url, token=origin_token) + + _filters = { + k: v + for k, v in _new_obj_data.items() + if k in fdp_req.get_filter_variables(dest_uri, _obj_type, dest_token) + and isinstance(v, str) + and k not in _url_fields + } + + logger.debug(f"Pushing member '{object_url}' to '{dest_uri}'") + + if dest_uri == origin_uri: + raise fdp_exc.InternalError("Cannot push object to its source address") + + return fdp_req.post_else_get( + dest_uri, _obj_type, data=_new_obj_data, token=dest_token, params=_filters + ) + + +def sync_data_products( + origin_uri: str, + dest_uri: str, + dest_token: str, + origin_token: str, + remote_label: str, + data_products: typing.List[str], + local_data_store: str = None ) -> None: + """Transfer data products from one registry to another + + Parameters + ---------- + origin_uri : str + origin data registry URL + dest_uri : str + destination data registry URL + dest_token : str + path to token for destination data registry + origin_token : str + path to token for origin data registry + remote_label : str + name of remote in listing + data_products : typing.List[str] + list of data products to push + local_data_store : optional, str + specified when pulling from remote registry to local + """ for data_product in data_products: namespace, name, version = re.split("[:@]", data_product) + _existing_namespace = fdp_req.get( + dest_uri, + "namespace", + params={SEARCH_KEYS["namespace"]: namespace}, + token=dest_token + ) + + if _existing_namespace: + _namespace_id = fdp_req.get_obj_id_from_url(_existing_namespace[0]["url"]) + _existing = fdp_req.get( + dest_uri, + "data_product", + dest_token, + params={ + "namespace": _namespace_id, + "name": name, + "version": version.replace("v", "") + } + ) + if _existing: + click.echo( + f"Data product '{data_product}' already present " + f"on remote '{remote_label}', ignoring.", + ) + continue + # Convert namespace name to an ID for retrieval _namespaces = fdp_req.get( - local_uri, + origin_uri, "namespace", - params={SEARCH_KEYS["namespace"]: namespace} + params={SEARCH_KEYS["namespace"]: namespace}, + token=origin_token ) + if not _namespaces: + raise fdp_exc.RegistryError( + f"Failed to find namespace '{namespace}' on registry {origin_uri}" + ) + _namespace_id = fdp_req.get_obj_id_from_url(_namespaces[0]["url"]) query_params = { @@ -205,11 +360,150 @@ def push_data_products( "version": version.replace("v", "") } - result = fdp_req.get(local_uri, "data_product", params=query_params) + result = fdp_req.get( + origin_uri, + "data_product", + params=query_params, + token=origin_token + ) if not result: raise fdp_exc.RegistryError( f"Failed to find data product matching descriptor '{data_product}'" ) - push_dependency_chain(result[0]["url"], dest_uri, local_uri, dest_token) + sync_dependency_chain( + object_url=result[0]["url"], + dest_uri=dest_uri, + origin_uri=origin_uri, + dest_token=dest_token, + origin_token=origin_token + ) + + if local_data_store: + logger.debug("Retrieving files from remote registry data storage") + fetch_data_product(origin_token, local_data_store, result[0]) + + +def fetch_data_product( + remote_token: str, + local_data_store: str, + data_product: typing.Dict +) -> None: + """ + Retrieve a file using the given user configuration metadata + + Parameters + ---------- + + remote_uri : str + remote registry URI + remote_token : str + remote registry access token + config_metadata : typing.Dict + user configuration file block describing an object + """ + _object = fdp_req.url_get(data_product["object"], remote_token) + + _endpoint = data_product["object"].split("data_product")[0] + + if not _object.get("storage_location", None): + logger.debug( + "Skipping item '%s' for download " + "as there is no physical storage location", + data_product + ) + + _storage_loc = fdp_req.url_get(_object["storage_location"], remote_token) + + _path = _storage_loc["path"] + _path = urllib.parse.quote(_path) + _root = fdp_req.url_get(_storage_loc["storage_root"], remote_token) + + _reg_parse = urllib.parse.urlparse(_endpoint) + _reg_url = f"{_reg_parse.scheme}://{_reg_parse.netloc}" + + _downloaded_file = download_from_registry(_reg_url, _root["root"], _path) + + _namespace = fdp_req.url_get(data_product["namespace"], remote_token) + + _file_type_url = _object.get("file_type", None) + + if _file_type_url: + _file_type = f'.{fdp_req.url_get(_file_type_url, remote_token)["extension"]}' + else: + _file_type = "" + + _local_dir = os.path.join( + local_data_store, + _namespace["name"], + data_product["data_product"] + ) + + os.makedirs(_local_dir, exist_ok=True) + + _out_file = os.path.join( + _local_dir, + f'{data_product["version"]}{_file_type}' + ) + + if os.path.exists(_out_file): + logger.debug("File '%s' already exists skipping download", _out_file) + return + + shutil.copy(_downloaded_file, _out_file) + + +def download_from_registry( + registry_url: str, + root: str, + path: str +) -> str: + """ + Download a file from the registry given the storage root and path. + + If the root starts with '/' assume the file exists on the same location as + the registry itself and try to download from that. + + Parameters + ---------- + registry_url : str + net location of the registry (not the endpoint of the API) + root : str + storage root + path : str + path of file on storage location + + Returns + ------- + str + path of downloaded temporary file + + Raises + ------ + fdp_exc.UserConfigError + if download failed + """ + + if root.startswith("/"): + logger.warning( + "Root of data storage location is '/' assuming data exists" + " on registry server" + ) + + if not registry_url.endswith("/"): + registry_url = registry_url[:-1] + + root = f"{registry_url}{root}" + + _download_url = f"{root}{path}" + + try: + _temp_data_file = fdp_req.download_file(_download_url) + logger.debug("Downloaded file from '%s' to temporary file", _download_url) + except requests.HTTPError as r_in: + raise fdp_exc.UserConfigError( + f"Failed to fetch item '{_download_url}' with exit code {r_in.response}" + ) + + return _temp_data_file diff --git a/fair/registry/versioning.py b/fair/registry/versioning.py index c0f33fd8..b0b48fe6 100644 --- a/fair/registry/versioning.py +++ b/fair/registry/versioning.py @@ -115,6 +115,16 @@ def get_correct_version( version: str, results_list: typing.List = None, free_write: bool = True ) -> semver.VersionInfo: + # Version is already specified + if isinstance(version, semver.VersionInfo): + return version + + try: + return semver.VersionInfo.parse(version) + except ValueError: + pass + + _zero = semver.VersionInfo.parse("0.0.0") if results_list: diff --git a/fair/run.py b/fair/run.py index 5b5dce04..ad487ce3 100644 --- a/fair/run.py +++ b/fair/run.py @@ -12,24 +12,14 @@ __date__ = "2021-06-30" -import datetime import glob import hashlib import logging import os -import platform -import subprocess -import sys import typing -import click - import fair.common as fdp_com -import fair.configuration as fdp_conf import fair.exceptions as fdp_exc -import fair.history as fdp_hist -import fair.user_config as fdp_user -from fair.common import CMD_MODE logger = logging.getLogger("FAIRDataPipeline.Run") @@ -55,227 +45,6 @@ } -def run_command( - repo_dir: str, - config_yaml: str = None, - mode: CMD_MODE = CMD_MODE.RUN, - bash_cmd: str = "", - allow_dirty: bool = False, -) -> str: - """Execute a process as part of job - - Executes a command from the given job config file, if a command is - given this is job instead and overwrites that in the job config file. - - Parameters - ---------- - local_uri : str - local registry endpoint - repo_dir : str - directory of repository to run from - config_yaml : str, optional - run from a given config.yaml file - bash_cmd : str, optional - override execution command with a bash command - allow_dirty : bool, optional - allow runs with uncommitted changes, default is False - """ - - if not config_yaml: - config_yaml = os.path.join(fdp_com.find_fair_root(), fdp_com.USER_CONFIG_FILE) - - logger.debug("Using user configuration file: %s", config_yaml) - click.echo(f"Updating registry from {config_yaml}", err=True) - - # Record the time the job was commenced, create a log and both - # print output and write it to the log file - _now = datetime.datetime.now() - _timestamp = _now.strftime("%Y-%m-%d_%H_%M_%S_%f") - _logs_dir = fdp_hist.history_directory(repo_dir) - - if not os.path.exists(_logs_dir): - os.mkdir(_logs_dir) - - _log_file = os.path.join(_logs_dir, f"job_{_timestamp}.log") - - # Check that the specified user config file for a job actually exists - if not os.path.exists(config_yaml): - raise fdp_exc.FileNotFoundError( - "Failed to read user configuration, " - f"file '{config_yaml}' does not exist." - ) - - logger.debug(f"Creating user configuration job object from {config_yaml}") - - _job_cfg = fdp_user.JobConfiguration(config_yaml) - - _job_cfg.update_from_fair(repo_dir) - - if bash_cmd: - _job_cfg.set_command(bash_cmd) - - _job_dir = os.path.join(fdp_com.default_jobs_dir(), _timestamp) - logger.debug("Using job directory: %s", _job_dir) - os.makedirs(_job_dir, exist_ok=True) - - _job_cfg.prepare(_job_dir, _timestamp, mode, allow_dirty=allow_dirty) - - _run_executable = ( - "script" in _job_cfg["run_metadata"] - or "script_path" in _job_cfg["run_metadata"] - ) - _run_executable = _run_executable and mode in [CMD_MODE.RUN, CMD_MODE.PASS] - - if mode == CMD_MODE.PASS: - logger.debug("Run called in passive mode, no command will be executed") - - # Set location of working config.yaml to the job directory - _work_cfg_yml = os.path.join(_job_dir, fdp_com.USER_CONFIG_FILE) - - # Fetch the CLI configurations for logging information - _user = fdp_conf.get_current_user_name(repo_dir) - _email = fdp_conf.get_current_user_email(repo_dir) - - if mode in [CMD_MODE.PULL]: - # If not a fair run then the log file will have less metadata - # all commands should produce a log so that the 'fair log' history - # can be displayed - with open(_log_file, "a") as f: - _out_str = _now.strftime("%a %b %d %H:%M:%S %Y %Z") - f.writelines( - [ - "--------------------------------\n", - f" Commenced = {_out_str}\n", - f" Author = {' '.join(_user)} <{_email}>\n", - " Command = fair pull\n", - "--------------------------------\n", - ] - ) - - _job_cfg.write(_work_cfg_yml) - - logger.debug("Creating working configuration storage location") - - if _run_executable: - - # Create a run script if 'script' is specified instead of 'script_path' - # else use the script - _cmd_setup = setup_job_script( - _job_cfg.content, _job_cfg.env["FDP_CONFIG_DIR"], _job_dir - ) - - _job_cfg.set_script(_cmd_setup["script"]) - _job_cfg.write(_work_cfg_yml) - - if _job_cfg.shell not in SHELLS: - raise fdp_exc.UserConfigError( - f"Unrecognised shell '{_job_cfg.shell}' specified." - ) - - _exec = SHELLS[_job_cfg.shell]["exec"] - _cmd_list = _exec.format(_cmd_setup["script"]).split() - - if not _job_cfg.command: - click.echo("Nothing to run.") - sys.exit(0) - - # Generate a local job log for the CLI, this is NOT - # related to metadata sent to the registry - # this log is viewable via the `fair view ` - with open(_log_file, "a") as f: - _out_str = _now.strftime("%a %b %d %H:%M:%S %Y %Z") - _user = _user[0] if not _user[1] else " ".join(_user) - f.writelines( - [ - "--------------------------------\n", - f" Commenced = {_out_str}\n", - f" Author = {_user} <{_email}>\n", - f" Namespace = {_job_cfg.default_output_namespace}\n", - f" Command = {' '.join(_cmd_list)}\n", - "--------------------------------\n", - ] - ) - - if mode == CMD_MODE.RUN: - execute_run(_cmd_list, _job_cfg, _log_file, _now) - else: # CMD_MODE.PASS - _end_time = datetime.datetime.now() - with open(_log_file, "a") as f: - _duration = _end_time - _now - f.writelines( - [ - "Operating in ci mode without running script\n", - f"------- time taken {_duration} -------\n", - ] - ) - else: - _end_time = datetime.datetime.now() - with open(_log_file, "a") as f: - _duration = _end_time - _now - f.writelines([f"------- time taken {_duration} -------\n"]) - - return get_job_hash(_job_dir) - - -def execute_run( - command: typing.List[str], - job_config: fdp_user.JobConfiguration, - log_file: str, - timestamp: datetime.datetime, -) -> None: - """Execute a run initialised by a CLI run of mode RUN - - Parameters - ---------- - command : typing.List[str] - command to execute - job_config : str - job configuration - log_file : str - log file to write stdout - timestamp : datetime.datetime - job execution start time - - Raises - ------ - fdp_exc.CommandExecutionError - [description] - """ - logger.debug("Executing command: %s", " ".join(command)) - - # Run the submission script - _process = subprocess.Popen( - command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True, - bufsize=1, - text=True, - shell=False, - env=job_config.environment, - cwd=job_config.local_repository, - ) - - # Write any stdout to the job log - for line in iter(_process.stdout.readline, ""): - with open(log_file, "a") as f: - f.writelines([line]) - click.echo(line, nl=False) - sys.stdout.flush() - _process.wait() - _end_time = datetime.datetime.now() - with open(log_file, "a") as f: - _duration = _end_time - timestamp - f.writelines([f"------- time taken {_duration} -------\n"]) - - # Exit the session if the job failed - if _process.returncode != 0: - raise fdp_exc.CommandExecutionError( - f"Run failed with exit code '{_process.returncode}'", - exit_code=_process.returncode, - ) - - def get_job_hash(job_dir: str) -> str: """Retrieve the hash for a given job @@ -322,85 +91,3 @@ def get_job_dir(job_hash: str) -> str: return job return "" - - -def setup_job_script( - user_config: typing.Dict, config_dir: str, output_dir: str -) -> typing.Dict[str, typing.Any]: - """Setup a job script from the given configuration. - - Checks the user configuration file for the required 'script' or 'script_path' - keys and determines the process to be executed. Also sets up an environment - usable when executing the submission script. - - Parameters - ---------- - local_repo : str - local FAIR repository - script : str - script to write to file - config_dir : str - final location of output config.yaml - output_dir : str - location to store submission/job script - - Returns - ------- - Dict[str, Any] - a dictionary containing information on the command to execute, - which shell to run it in and the environment to use - """ - logger.debug("Setting up job script for execution") - _cmd = None - - if config_dir[-1] != os.path.sep: - config_dir += os.path.sep - - # Check if a specific shell has been defined for the script - _shell = None - _out_file = None - - if "shell" in user_config["run_metadata"]: - _shell = user_config["run_metadata"]["shell"] - else: - _shell = "batch" if platform.system() == "Windows" else "sh" - - logger.debug("Will use shell: %s", _shell) - - if "script" in user_config["run_metadata"]: - _cmd = user_config["run_metadata"]["script"] - - if "extension" not in SHELLS[_shell]: - raise fdp_exc.InternalError( - f"Failed to retrieve an extension for shell '{_shell}'" - ) - _ext = SHELLS[_shell]["extension"] - _out_file = os.path.join(output_dir, f"script.{_ext}") - if _cmd: - with open(_out_file, "w") as f: - f.write(_cmd) - - elif "script_path" in user_config["run_metadata"]: - _path = user_config["run_metadata"]["script_path"] - if not os.path.exists(_path): - raise fdp_exc.CommandExecutionError( - f"Failed to execute run, script '{_path}' was not found, or" - " failed to be created.", - exit_code=1, - ) - _cmd = open(_path).read() - _out_file = os.path.join(output_dir, os.path.basename(_path)) - if _cmd: - with open(_out_file, "w") as f: - f.write(_cmd) - - logger.debug("Script command: %s", _cmd) - logger.debug("Script written to: %s", _out_file) - - if not _cmd or not _out_file: - raise fdp_exc.UserConfigError( - "Configuration file must contain either a valid " - "'script' or 'script_path' entry under 'run_metadata'" - ) - - return {"shell": _shell, "script": _out_file} diff --git a/fair/session.py b/fair/session.py index c7eaf652..692b8ef4 100644 --- a/fair/session.py +++ b/fair/session.py @@ -53,10 +53,11 @@ import fair.history as fdp_hist import fair.registry.server as fdp_serv import fair.registry.sync as fdp_sync -import fair.run as fdp_run +import fair.registry.requests as fdp_req import fair.staging as fdp_stage import fair.templates as fdp_tpl import fair.testing as fdp_test +import fair.user_config as fdp_user import fair.configuration.validation as fdp_clivalid @@ -110,6 +111,8 @@ def __init__( """ if debug: logging.getLogger("FAIRDataPipeline").setLevel(logging.DEBUG) + else: + logging.getLogger("FAIRDataPipeline").setLevel(logging.CRITICAL) self._logger.debug("Starting new session.") self._testing = testing self._session_loc = repo_loc @@ -120,6 +123,7 @@ def __init__( self._session_id = ( uuid.uuid4() if server_mode == fdp_serv.SwitchMode.CLI else None ) + self._session_config = None if user_config and not os.path.exists(user_config): raise fdp_exc.FileNotFoundError( @@ -127,9 +131,8 @@ def __init__( "file not found." ) - self._session_config = user_config or fdp_com.local_user_config( - self._session_loc - ) + self._session_config = fdp_user.JobConfiguration(user_config) + if server_mode != fdp_serv.SwitchMode.NO_SERVER and not os.path.exists( fdp_com.registry_home() @@ -150,33 +153,24 @@ def __init__( self._logger.debug( "Initialising session with:\n" + "\tlocation = %s\n" "\tsession_config = %s\n" "\ttesting = %s\n" "\trun_mode = %s\n" "\tstaging_file = %s\n" - "\tsession_id = %s\n" - "\tallow_dirty = %s\n", - self._session_config, + "\tsession_id = %s\n", + self._session_loc, + user_config, self._testing, self._run_mode, self._stager._staging_file, self._session_id, - self._allow_dirty ) self._load_configurations() self._setup_server(server_port) - def push(self, remote: str = "origin"): - _staged_data_products = self._stager.get_item_list(True, "data_product") - fdp_sync.push_data_products( - fdp_conf.get_local_uri(), - fdp_conf.get_remote_uri(self._session_loc, remote), - fdp_conf.get_remote_token(self._session_loc, remote), - _staged_data_products, - ) - def purge( self, verbose: bool = True, @@ -308,46 +302,159 @@ def _setup_server_user_start(self, port: int) -> None: pathlib.Path(_cache_addr).touch() fdp_serv.launch_server(port=port, verbose=True) - def run_job( + def _pre_job_setup(self, remote: str = None) -> None: + self._logger.debug("Running pre-job setup") + self.check_is_repo() + self._session_config.update_from_fair( + fdp_com.find_fair_root(self._session_loc), remote + ) + + def _post_job_breakdown(self, add_run: bool = False) -> None: + if add_run: + self._logger.debug(f"Tracking job hash {self._session_config.hash}") + + self._logger.debug("Updating staging post-run") + + self._stager.update_data_product_staging() + + if add_run: + # Automatically add the run to tracking but unstaged + self._stager.add_to_staging(self._session_config.hash, "job") + + self._session_config.close_log() + + def push(self, remote: str = "origin"): + self._pre_job_setup(remote) + self._session_config.prepare( + fdp_com.CMD_MODE.PUSH, + allow_dirty=self._allow_dirty + ) + _staged_data_products = self._stager.get_item_list(True, "data_product") + + if not _staged_data_products: + click.echo("Nothing to push.") + + fdp_sync.sync_data_products( + origin_uri=fdp_conf.get_local_uri(), + dest_uri=fdp_conf.get_remote_uri(self._session_loc, remote), + dest_token=fdp_conf.get_remote_token(self._session_loc, remote), + origin_token=fdp_req.local_token(), + remote_label=remote, + data_products=_staged_data_products, + ) + + self._session_config.write_log_lines( + [f"Pushing data products to remote '{remote}':"] + + [f'\t- {data_product}' for data_product in _staged_data_products] + ) + + self._post_job_breakdown() + + # When push successful unstage data products again + for data_product in _staged_data_products: + self._stager.change_stage_status(data_product, "data_product", False) + + def pull(self, remote: str = "origin"): + self._logger.debug("Performing pull on remote '%s'", remote) + + _remote_addr = fdp_conf.get_remote_uri(self._session_loc, remote) + + if not fdp_serv.check_server_running(_remote_addr): + raise fdp_exc.UnexpectedRegistryServerState( + f"Cannot perform pull from registry '{remote}' as the" + f" server does not exist. Expected response from '{_remote_addr}'.", + hint="Is your FAIR repository configured correctly?" + ) + + self._logger.debug("Retrieving namespaces from remote") + + fdp_sync.pull_all_namespaces( + fdp_conf.get_local_uri(), + fdp_conf.get_remote_uri(self._session_loc, remote), + fdp_req.local_token(), + fdp_conf.get_remote_token(self._session_loc, remote) + ) + + self._logger.debug("Performing pre-job setup") + + self._pre_job_setup(remote) + + self._session_config.prepare( + fdp_com.CMD_MODE.PULL, + allow_dirty=self._allow_dirty, + remote_uri=fdp_conf.get_remote_uri(self._session_loc, remote), + remote_token=fdp_conf.get_remote_token(self._session_loc, remote) + ) + + _readables = self._session_config.get_readables() + + self._session_config.write() + + self._logger.debug("Preparing to retrieve %s items", len(_readables)) + + self._logger.debug("Pulling data products locally") + + # Only push data products if there are any to do so, this covers the + # case whereby no remote has been setup and we just want to register + # items on the local registry + if _readables: + fdp_sync.sync_data_products( + origin_uri=fdp_conf.get_remote_uri(self._session_loc, remote), + dest_uri=fdp_conf.get_local_uri(), + dest_token=fdp_req.local_token(), + origin_token=fdp_conf.get_remote_token(self._session_loc, remote), + remote_label=remote, + data_products=_readables, + local_data_store=self._session_config.default_data_store + ) + + self._session_config.write_log_lines( + [f"Pulled data products from remote '{remote}':"] + + [f'\t- {data_product}' for data_product in _readables] + ) + else: + click.echo(f"No items to retrieve from remote '{remote}'.") + + self._logger.debug("Performing post-job breakdown") + + self._post_job_breakdown() + + def run( self, bash_cmd: str = "", - mode: fdp_run.CMD_MODE = fdp_run.CMD_MODE.RUN, + passive: bool = False, + allow_dirty: bool = False, ) -> str: """Execute a run using the given user configuration file""" - self.check_is_repo() - if not os.path.exists(self._session_config): - self.make_starter_config() + self._pre_job_setup() + + self._session_config.prepare( + fdp_com.CMD_MODE.PASS if passive else fdp_com.CMD_MODE.RUN, + allow_dirty=self._allow_dirty + ) self._logger.debug("Setting up command execution") + if bash_cmd: + self._session_config.set_command(bash_cmd) + + self._session_config.setup_job_script() - if self._allow_dirty: + if allow_dirty: self._logger.debug("Allowing uncommitted changes during run.") # Only apply constraint for clean repository when executing a run - if mode != fdp_com.CMD_MODE.RUN: - self._allow_dirty = True - - self.check_git_repo_state() - - _hash = fdp_run.run_command( - repo_dir=self._session_loc, - config_yaml=self._session_config, - bash_cmd=bash_cmd, - mode=mode, - allow_dirty=self._allow_dirty, - ) + if passive: + allow_dirty = True - self._logger.debug(f"Tracking job hash {_hash}") + self.check_git_repo_state(allow_dirty=allow_dirty) - self._logger.debug("Updating staging post-run") + self._session_config.write() - if mode in [fdp_com.CMD_MODE.RUN, fdp_com.CMD_MODE.PULL]: - self._stager.update_data_product_staging() + self._session_config.execute() - # Automatically add the run to tracking but unstaged - self._stager.add_to_staging(_hash, "job") + self._post_job_breakdown(add_run=True) - return _hash + return self._session_config.hash def check_is_repo(self, location: str = None) -> None: """Check that the current location is a FAIR repository""" @@ -360,7 +467,7 @@ def check_is_repo(self, location: str = None) -> None: ) def check_git_repo_state( - self, remote_label: str = "origin" + self, remote_label: str = "origin", allow_dirty: bool = False ) -> bool: """Checks the git repository is clean and that local matches remote""" _repo_root = fdp_com.find_git_root(self._session_loc) @@ -375,7 +482,7 @@ def check_git_repo_state( # Get the latest commit on the current branch locally _loc_commit = _repo.refs[_current_branch].commit.hexsha except (TypeError, IndexError) as e: - if self._allow_dirty: + if allow_dirty: click.echo(f"Warning: {' '.join(e.args)}") else: raise fdp_exc.FDPRepositoryError(" ".join(e.args)) @@ -398,7 +505,7 @@ def check_git_repo_state( ) except IndexError: _msg = f"Failed to find branch '{_current_branch}' on remote repository" - if self._allow_dirty: + if allow_dirty: click.echo(f"Warning: {_msg}") else: raise fdp_exc.FDPRepositoryError(_msg) @@ -407,7 +514,7 @@ def check_git_repo_state( _com_match = _loc_commit == _rem_commit if not _com_match: - if self._allow_dirty: + if allow_dirty: click.echo("Warning: local git repository is ahead/behind remote") else: raise fdp_exc.FDPRepositoryError( @@ -415,7 +522,7 @@ def check_git_repo_state( f"remote '{remote_label}'" ) if _repo.is_dirty(): - if self._allow_dirty: + if allow_dirty: click.echo("Warning: running with uncommitted changes") else: raise fdp_exc.FDPRepositoryError( @@ -701,7 +808,8 @@ def initialise( if self._testing: using = fdp_test.create_configurations( - registry, fdp_com.find_git_root(os.getcwd()), os.getcwd() + registry, fdp_com.find_git_root(os.getcwd()), os.getcwd(), + os.path.join(os.getcwd(), "data_store") ) if os.path.exists(_fair_dir): @@ -771,6 +879,8 @@ def initialise( "Initialisation failed, validation of global CLI config file did not pass" ) + os.makedirs(fdp_hist.history_directory(self._session_loc), exist_ok=True) + click.echo(f"Initialised empty fair repository in {_fair_dir}") def _clean_reset(self, _fair_dir, e: Exception = None, local_only: bool = False): diff --git a/fair/staging.py b/fair/staging.py index 92c523af..5785c7fd 100644 --- a/fair/staging.py +++ b/fair/staging.py @@ -183,7 +183,7 @@ def find_registry_entry_for_file(self, local_uri: str, file_path: str) -> str: # parent_directory/file_name _obj_type = "storage_location" - _results = fdp_req.get(local_uri, _obj_type, params={"path": file_path}) + _results = fdp_req.get(local_uri, _obj_type, fdp_req.local_token(), {"path": file_path}) if not _results: raise fdp_exc.StagingError( @@ -232,7 +232,12 @@ def _get_code_run_entries(self, local_uri: str, job_dir: str) -> typing.List[str _runs = [i.strip() for i in open(_code_run_file).readlines()] for run in _runs: - _results = fdp_req.get(local_uri, "code_run", params={"uuid": run}) + _results = fdp_req.get( + local_uri, + "code_run", + fdp_req.local_token(), + params={"uuid": run} + ) if not _results: raise fdp_exc.ImplementationError( @@ -253,7 +258,7 @@ def _get_written_obj_entries(self, local_uri: str, config_dict: typing.Dict): for write_obj in config_dict["write"]: _data_product = write_obj["data_product"] _results = fdp_req.get( - local_uri, "data_product", params={"name": _data_product} + local_uri, "data_product", fdp_req.local_token(), params={"name": _data_product} ) if not _results: @@ -390,10 +395,16 @@ def update_data_product_staging(self) -> None: with open(self._staging_file) as f: _staging_dict = yaml.safe_load(f) - result = fdp_req.url_get(f"{fdp_com.DEFAULT_LOCAL_REGISTRY_URL}data_product") + result = fdp_req.url_get( + f"{fdp_com.DEFAULT_LOCAL_REGISTRY_URL}data_product", + fdp_req.local_token() + ) for data_product in result: - namespace = fdp_req.url_get(data_product["namespace"])["name"] + namespace = fdp_req.url_get( + data_product["namespace"], + fdp_req.local_token() + )["name"] name = data_product["name"] version = data_product["version"] key = f"{namespace}:{name}@v{version}" diff --git a/fair/user_config/__init__.py b/fair/user_config/__init__.py index 6be14002..08216b4e 100644 --- a/fair/user_config/__init__.py +++ b/fair/user_config/__init__.py @@ -1,3 +1,31 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +""" +User Config Management +====================== + +Contains classes for the parsing and preparation of the user's 'config.yaml' +prior to the execution of a run or synchronisation + +Contents +======== + +Constants +--------- + + - JOB2CLI_MAPPINGS: mappings from CLI configuration to config.yaml keys + - SHELLS: commands for executing scripts depending on specified shell + + +Classes +------- + + - JobConfiguration: handles the setup of the configuration file + +""" + +__date__ = "2021-09-10" + import copy import datetime import logging @@ -5,24 +33,33 @@ import os.path import platform import re +import sys +import json import typing +import subprocess from collections.abc import MutableMapping import git +import click import yaml import pydantic import fair.common as fdp_com import fair.configuration as fdp_conf import fair.exceptions as fdp_exc -import fair.parsing.globbing as fdp_glob import fair.register as fdp_reg import fair.registry.requests as fdp_req import fair.registry.storage as fdp_store import fair.registry.versioning as fdp_ver import fair.utilities as fdp_util +import fair.run as fdp_run +import fair.history as fdp_hist + +from fair.registry import SEARCH_KEYS from fair.common import CMD_MODE -from fair.user_config.validation import UserConfigModel + +import fair.user_config.validation as fdp_valid +import fair.user_config.globbing as fdp_glob JOB2CLI_MAPPINGS = { "run_metadata.local_repo": "git.local_repo", @@ -33,32 +70,81 @@ "run_metadata.write_data_store": "registries.local.data_store", } +SHELLS: typing.Dict[str, str] = { + "pwsh": {"exec": "pwsh -command \". '{0}'\"", "extension": "ps1"}, + "batch": {"exec": "{0}", "extension": "bat"}, + "powershell": { + "exec": "powershell -command \". '{0}'\"", + "extension": "ps1", + }, + "python2": {"exec": "python2 {0}", "extension": "py"}, + "python3": {"exec": "python3 {0}", "extension": "py"}, + "python": {"exec": "python {0}", "extension": "py"}, + "R": {"exec": "R -f {0}", "extension": "R"}, + "julia": {"exec": "julia {0}", "extension": "jl"}, + "bash": { + "exec": "bash -eo pipefail {0}", + "extension": "sh", + }, + "java": {"exec": "java {0}", "extension": "java"}, + "sh": {"exec": "sh -e {0}", "extension": "sh"}, +} + + class JobConfiguration(MutableMapping): _logger = logging.getLogger("FAIRDataPipeline.ConfigYAML") _block_types = ("register", "write", "read") + _final_permitted = { + "all": ("data_product", + "public", + "use", + "description" + ), + "write": ("file_type",), + } + _status_tags = ("registered",) + + def __init__(self, config_yaml: str = None) -> None: + self._config = {"run_metadata": {}} + self._input_file = config_yaml + if config_yaml: + if not os.path.exists(config_yaml): + raise fdp_exc.FileNotFoundError( + f"Cannot load job configuration from file '{config_yaml}', " + "file does not exist" + ) - def __init__(self, config_yaml: str) -> None: - if not os.path.exists(config_yaml): - raise fdp_exc.FileNotFoundError( - f"Cannot load job configuration from file '{config_yaml}', " - "file does not exist" - ) - - self._logger.debug("Loading file '%s'", config_yaml) - - self._config: typing.Dict = yaml.safe_load(open(config_yaml)) + self._logger.debug("Loading file '%s'", config_yaml) + self._config: typing.Dict = yaml.safe_load(open(config_yaml)) self._fill_missing() + self._now = datetime.datetime.now() + self._parsed = {"namespace": [], "author": []} self.env = None + self._job_dir = None + self._log_file = None + + def _get_local_namespaces(self) -> typing.List[str]: + _namespaces = fdp_req.get(self.local_uri, "namespace", fdp_req.local_token()) + return _namespaces if not _namespaces else [n["name"] for n in _namespaces] + + def _get_local_authors(self) -> typing.List[str]: + _authors = fdp_req.get(self.local_uri, "author", fdp_req.local_token()) + return _authors if not _authors else [a["name"] for a in _authors] def __contains__(self, key_addr: str) -> bool: - return key_addr in fdp_util.flatten_dict(self._config) + return any( + [ + key_addr in self._config, + key_addr in fdp_util.flatten_dict(self._config) + ] + ) def __setitem__( self, key_addr: str, value: typing.Any, separator: str = "." ) -> None: - if key_addr in self._config: + if key_addr in self: self._config[key_addr] = value _flat_cfg = fdp_util.flatten_dict(self._config, separator) _flat_cfg[key_addr] = value @@ -70,7 +156,7 @@ def __delitem__(self, key_addr: str, separator: str = ".") -> None: del self._config[key_addr] _flat_cfg = fdp_util.flatten_dict(self._config, separator) if key_addr not in _flat_cfg: - raise fdp_exc.KeyPathError(key_addr, fdp_com.USER_CONFIG_FILE) + raise fdp_exc.KeyPathError(key_addr, f'UserConfig[{self._input_file}]') self._logger.debug(f"Removing '{key_addr}'") del _flat_cfg[key_addr] self._config = fdp_util.expand_dict(_flat_cfg) @@ -80,7 +166,7 @@ def __getitem__(self, key_addr: str, separator: str = ".") -> None: return self._config[key_addr] _flat_cfg = fdp_util.flatten_dict(self._config, separator) if key_addr not in _flat_cfg: - raise fdp_exc.KeyPathError(key_addr, fdp_com.USER_CONFIG_FILE) + raise fdp_exc.KeyPathError(key_addr, f'UserConfig[{self._input_file}]') return _flat_cfg[key_addr] def __len__(self) -> int: @@ -95,7 +181,7 @@ def __iter__(self) -> typing.Any: def _fill_missing(self) -> None: self._logger.debug("Filling missing metadata") - if "run_metadata" not in self._config: + if "run_metadata" not in self: self._logger.debug( "Failed to find 'run_metadata' in configuration, creating" ) @@ -120,48 +206,43 @@ def _fill_missing(self) -> None: self[item[0]] = item[1] def _handle_register_namespaces(self) -> typing.Dict: - self._logger.debug("Handling 'register' namespaces") + self._logger.debug("Handling 'register:namespace'") _new_register_block: typing.List[typing.Dict] = [] + + # Register explicit namespace objects and remove from config for item in self["register"]: - if any(i in item for i in ("external_object", "data_product")): - if "namespace_name" in item: - _namespace_store_args = { - "namespace_label": item["namespace_name"], - "full_name": item.get("namespace_full_name", None), - "website": item.get("namespace_website", None), - } - elif "namespace" in item and isinstance(item["namespace"], dict): - _namespace_store_args = item["namespace"] - _new_register_block.append(item) - elif "namespace" in item: - _namespace_store_args = { - "namespace_label": item["namespace"], - "full_name": item.get("full_name", None), - "website": item.get("website", None), - } - else: + # Not a namespace + if 'namespace' not in item: _new_register_block.append(item) - fdp_store.store_namespace(self.local_uri, **_namespace_store_args) - return _new_register_block - - def _unpack_register_namespaces(self) -> None: - self._logger.debug("Unpacking 'register' namespaces") - for i, item in enumerate(self._config["register"]): - if all(it not in item for it in ["external_object", "data_product"]): continue - if "namespace" not in item: + if any(n in item for n in ["data_product", "external_object"]): + raise fdp_exc.UserConfigError( + "Invalid use of tag 'namespace' in non-namespace registration", + "Did you mean 'namespace_name'?" + ) + + _namespace_metadata = { + "name": item["namespace"], + "full_name": item.get("full_name", None), + "website": item.get("website", None) + } + + if item["namespace"] in self._parsed["namespace"]: + self._logger.warning( + "Ignoring registration of namespace '%s' as it already exists", + item["namespace"] + ) continue - if isinstance(item["namespace"], str): - self._config["register"][i]["namespace_name"] = item["namespace"] - elif isinstance(item["namespace"], dict): - self._config["register"][i]["namespace_name"] = item["namespace"][ - "name" - ] - self._config["register"][i]["namespace_full_name"] = None - self._config["register"][i]["namespace_website"] = None - del self._config["register"][i]["namespace"] + fdp_store.store_namespace( + self.local_uri, fdp_req.local_token(), **_namespace_metadata + ) + + self._parsed["namespace"].append(_namespace_metadata["name"]) + + return _new_register_block + def _fill_namespaces(self, block_type: str) -> typing.List[typing.Dict]: self._logger.debug("Filling all namespaces") @@ -179,16 +260,6 @@ def _fill_namespaces(self, block_type: str) -> typing.List[typing.Dict]: _new_item = copy.deepcopy(item) _new_item["use"] = item.get("use", {}) - # --------------------------------------------------------------- # - # FIXME: Schema needs to be finalised, allowing item:namespace and - # also item:use:namespace will cause confusion. - - if "namespace" in item and "namespace" not in _new_item["use"]: - _new_item["use"]["namespace"] = _new_item["namespace"] - _new_item.pop("namespace") - - # --------------------------------------------------------------- # - if "namespace" not in _new_item["use"]: _new_item["use"]["namespace"] = _default_ns @@ -196,10 +267,41 @@ def _fill_namespaces(self, block_type: str) -> typing.List[typing.Dict]: return _entries + def _switch_namespace_name_to_use(self, register_block: typing.List): + """ + Checks namespace listed in 'namespace_name' if given, if there is a + match the entry is removed and replaced with a 'use:namespace' entry + """ + _new_register_block: typing.List[typing.Dict] = [] + for register_entry in register_block: + _new_entry = register_entry.copy() + if "namespace_name" not in register_entry: + continue + if register_entry["namespace_name"] not in self._parsed["namespace"]: + self._logger.error( + "'%s' not in available namespaces:\n\t-%s", + register_entry['namespace_name'], + '\n\t-'.join(self._parsed["namespace"]) + ) + raise fdp_exc.UserConfigError( + "Attempt to register object with unknown namespace " + "'"+register_entry["namespace_name"]+"'", + "Add new 'namespace' as separate 'register' entry" + ) + if "use" not in _new_entry: + _new_entry["use"] = {} + _new_entry["use"]["namespace"] = register_entry["namespace_name"] + _new_register_block.append(_new_entry) + + return _new_register_block + def _update_namespaces(self) -> None: self._logger.debug("Updating namespace list") - if "register" in self._config: + + # Only allow namespaces to be registered directly + if "register" in self: self["register"] = self._handle_register_namespaces() + self["register"] = self._switch_namespace_name_to_use(self["register"]) if not self.default_input_namespace: raise fdp_exc.UserConfigError("Input namespace cannot be None") @@ -207,38 +309,110 @@ def _update_namespaces(self) -> None: if not self.default_output_namespace: raise fdp_exc.UserConfigError("Output namespace cannot be None") - fdp_store.store_namespace(self.local_uri, self.default_input_namespace) - fdp_store.store_namespace(self.local_uri, self.default_output_namespace) - for block_type in self._block_types: if block_type not in self: continue self[block_type] = self._fill_namespaces(block_type) - if "register" in self._config: - self._unpack_register_namespaces() + def _globular_registry_search( + self, + registry_uri: str, + registry_token: str, + block_entry: typing.Dict[str, typing.Any], + block_type: str) -> typing.List[typing.Dict]: + """Performs globular search in the specified registry + + Any '*' wildcards are used to perform + """ + _vals_to_check = (i for i in block_entry.values() if isinstance(i, str)) + if all('*' not in v for v in _vals_to_check): + return [block_entry] + + _new_entries: typing.List[typing.Dict] = [] + _obj_type = None + for obj in fdp_valid.VALID_OBJECTS: + # Identify object type + if obj in block_entry: + _obj_type = obj + break + + if not _obj_type: + raise fdp_exc.UserConfigError( + f"Unrecognised object type for wildcard search in: {block_entry}" + ) - def _expand_wildcards_from_local_reg(self, block_type: str) -> None: - self._logger.debug("Expanding wildcards using local registry") - _version = ( - self.default_read_version - if block_type == "read" - else self.default_write_version - ) - fdp_glob.glob_read_write( - user_config=self._config, - blocktype=block_type, - search_key=None, - registry_url=self.local_uri, - version=_version, - remove_wildcard=block_type == "read", - ) + _search_key = SEARCH_KEYS[_obj_type] + + try: + _results_local = fdp_req.get( + registry_uri, _obj_type, registry_token, + params={_search_key: block_entry[_obj_type]} + ) + except fdp_exc.RegistryAPICallError: + raise fdp_exc.UserConfigError( + f"Failed to retrieve entries on local registry for {_obj_type}" + f" wildcard '{block_entry[_obj_type]}'" + ) + + if _obj_type in ("namespace", "author"): + # If the object is a namespace or an author then there is no + # additional info in the registry so we can just add the entries + # as they are + _new_entries = fdp_glob.get_single_layer_objects(_results_local, _obj_type) + + elif _obj_type == "external_object": + # If the object is an external_object we firstly need to get the + # name of the data product, version and the namespace of this object + # as well as the identifier + _version = block_entry.get("version", None) + + _new_entries = fdp_glob.get_external_objects( + registry_token, _results_local, block_type, _version + ) + + elif _obj_type == "data_product": + _version = block_entry.get("version", None) + + _new_entries = fdp_glob.get_data_product_objects( + registry_token, _results_local, block_type, _version + ) + + if block_type == "write": + _new_entries.append(block_entry) + + return _new_entries + + + def _expand_wildcards( + self, + registry_uri: str, + registry_token: str) -> None: + self._logger.debug(f"Expanding wildcards using registry '{registry_uri}") + for block in self._block_types: + if block not in self: + continue + _new_block: typing.List[typing.Dict] = [] + for block_entry in self[block]: + _new_block_entries = self._globular_registry_search( + registry_uri, + registry_token, + block_entry, + block + ) + _new_block += _new_block_entries + + self[block] = _new_block def _fetch_latest_commit(self, allow_dirty: bool = False) -> None: self._logger.debug( f"Retrieving latest commit SHA with allow_dirty={allow_dirty}" ) - _repository = git.Repo(fdp_com.find_git_root(self.local_repository)) + try: + _repository = git.Repo(fdp_com.find_git_root(self.local_repository)) + except git.InvalidGitRepositoryError: + raise fdp_exc.FDPRepositoryError( + f"Location '{self._local_repository}' is not a valid git repository" + ) try: _latest = _repository.head.commit.hexsha @@ -255,6 +429,12 @@ def _fetch_latest_commit(self, allow_dirty: bool = False) -> None: if _repository.is_dirty(): if not allow_dirty: + _changes = [i.a_path for i in _repository.index.diff(None)] + self._logger.error( + "Cannot retrieve latest commit for repository with allow_dirty=False, " + "the follow files have uncommitted changes:\n\t- %s", + '\n\t- '.join(_changes) + ) raise fdp_exc.FDPRepositoryError( "Cannot retrieve latest commit, " "repository contains uncommitted changes" @@ -263,6 +443,92 @@ def _fetch_latest_commit(self, allow_dirty: bool = False) -> None: return _latest + def setup_job_script(self) -> typing.Dict[str, typing.Any]: + """Setup a job script from the given configuration. + + Checks the user configuration file for the required 'script' or 'script_path' + keys and determines the process to be executed. Also sets up an environment + usable when executing the submission script. + + Parameters + ---------- + local_repo : str + local FAIR repository + script : str + script to write to file + config_dir : str + final location of output config.yaml + output_dir : str + location to store submission/job script + + Returns + ------- + Dict[str, Any] + a dictionary containing information on the command to execute, + which shell to run it in and the environment to use + """ + self._logger.debug("Setting up job script for execution") + _cmd = None + + if not self._job_dir: + raise fdp_exc.InternalError("Job directory initialisation failed") + + config_dir = self._job_dir + + if config_dir[-1] != os.path.sep: + config_dir += os.path.sep + + # Check if a specific shell has been defined for the script + _shell = None + _out_file = None + + if "shell" in self["run_metadata"]: + _shell = self["run_metadata.shell"] + else: + _shell = "batch" if platform.system() == "Windows" else "sh" + + self._logger.debug("Will use shell: %s", _shell) + + if "script" in self["run_metadata"]: + _cmd = self["run_metadata.script"] + + if "extension" not in SHELLS[_shell]: + raise fdp_exc.InternalError( + f"Failed to retrieve an extension for shell '{_shell}'" + ) + _ext = SHELLS[_shell]["extension"] + _out_file = os.path.join(self._job_dir, f"script.{_ext}") + if _cmd: + with open(_out_file, "w") as f: + f.write(_cmd) + + elif "script_path" in self["run_metadata"]: + _path = self["run_metadata.script_path"] + if not os.path.exists(_path): + raise fdp_exc.CommandExecutionError( + f"Failed to execute run, script '{_path}' was not found, or" + " failed to be created.", + exit_code=1, + ) + _cmd = open(_path).read() + _out_file = os.path.join(self._job_dir, os.path.basename(_path)) + if _cmd: + with open(_out_file, "w") as f: + f.write(_cmd) + + self._logger.debug("Script command: %s", _cmd) + self._logger.debug("Script written to: %s", _out_file) + + if not _cmd or not _out_file: + raise fdp_exc.UserConfigError( + "Configuration file must contain either a valid " + "'script' or 'script_path' entry under 'run_metadata'" + ) + + self.set_script(_out_file) + + return {"shell": _shell, "script": _out_file} + def update_from_fair( self, fair_repo_dir: str = None, remote_label: str = None ) -> None: @@ -304,7 +570,24 @@ def update_from_fair( if fair_repo_dir and "run_metadata.remote_repo" not in self: _remote = _fdpconfig["git.remote"] - _git_repo = git.Repo(fair_repo_dir) + + # If local repository stated in loaded config use that, else if + # already defined use existing location, else use specified directory + try: + _local_repo = _fdpconfig["git.local_repo"] + except KeyError: + if "run_metadata.local_repo" in self: + _local_repo = self["run_metadata.local_repo"] + else: + _local_repo = fair_repo_dir + + try: + _git_repo = git.Repo(_local_repo) + except git.InvalidGitRepositoryError: + raise fdp_exc.FDPRepositoryError( + f"Failed to update job configuration from location '{fair_repo_dir}', " + "not a valid git repository." + ) _url = _git_repo.remotes[_remote].url self["run_metadata.remote_repo"] = _url @@ -333,11 +616,17 @@ def set_command(self, cmd: str, shell: typing.Optional[str] = None) -> None: self["run_metadata.shell"] = shell self.pop("run_metadata.script_path") - def _create_environment(self, output_dir: str) -> None: + def _create_environment(self) -> None: """Create the environment for running a job""" _environment = os.environ.copy() _environment["FDP_LOCAL_REPO"] = self.local_repository - _environment["FDP_CONFIG_DIR"] = output_dir + if "PYTHONPATH" in _environment: + _new_py_path = _environment["PYTHONPATH"] + _new_py_path += os.pathsep + self.local_repository + else: + _new_py_path = self.local_repository + _environment["PYTHONPATH"] = _new_py_path + _environment["FDP_CONFIG_DIR"] = self._job_dir _environment["FDP_LOCAL_TOKEN"] = fdp_req.local_token() return _environment @@ -347,51 +636,71 @@ def set_script(self, command_script: str) -> None: self["run_metadata.script_path"] = command_script self.pop("run_metadata.script") - def _substitute_variables( - self, job_dir: str, time_stamp: datetime.datetime - ) -> None: - self._logger.debug("Performing variable substitution") - _config_str = self._subst_cli_vars(job_dir, time_stamp) - self._config = yaml.safe_load(_config_str) + def _create_log(self, command: str = None) -> None: + _logs_dir = fdp_hist.history_directory(self.local_repository) + + if not os.path.exists(_logs_dir): + os.makedirs(_logs_dir) + + _time_stamp = self._now.strftime("%Y-%m-%d_%H_%M_%S_%f") + self._log_file_path = os.path.join(_logs_dir, f"job_{_time_stamp}.log") + self._logger.debug(f"Will write session log to '{self._log_file_path}'") + command = command or self.command + self._log_file = open(self._log_file_path, "w") def prepare( self, - job_dir: str, - time_stamp: datetime.datetime, job_mode: CMD_MODE, allow_dirty: bool = False, - ) -> None: + remote_uri: str = None, + remote_token: str = None + ) -> str: """Initiate a job execution""" + _time_stamp = self._now.strftime("%Y-%m-%d_%H_%M_%S_%f") + self._job_dir = os.path.join(fdp_com.default_jobs_dir(), _time_stamp) + + # For push we do not need to do anything to the config as information + # is taken from staging + if job_mode == CMD_MODE.PUSH: + self._create_log() + return os.path.join(self._job_dir, fdp_com.USER_CONFIG_FILE) + self._logger.debug("Preparing configuration") self._update_namespaces() - self._substitute_variables(job_dir, time_stamp) - self._fill_all_block_types() + os.makedirs(self._job_dir) + self._create_log() + self._subst_cli_vars(self._now) - if job_mode in [CMD_MODE.PULL, CMD_MODE.PUSH]: - self._pull_metadata() + self._fill_all_block_types() - for block_type in ("read", "write"): - if block_type not in self: - continue - try: - self._expand_wildcards_from_local_reg(block_type) - except fdp_exc.InternalError: - continue + if job_mode == CMD_MODE.PULL: + _cmd = f'pull {self._input_file}' + self._pull_push_log_header(_cmd) + elif job_mode == CMD_MODE.PUSH: + _cmd = 'push' + self._pull_push_log_header(_cmd) + + # If pulling glob from the remote, else glob from local + if job_mode == CMD_MODE.PULL: + if not remote_uri: + raise fdp_exc.InternalError( + "Expected URI during wildcard unpacking for 'pull'" + ) + if not remote_token: + raise fdp_exc.InternalError( + "Expected token during wildcard unpacking for 'pull'" + ) + self._expand_wildcards(remote_uri, remote_token) + else: + self._expand_wildcards(self.local_uri, fdp_req.local_token()) for block_type in self._block_types: if block_type in self: self[block_type] = self._fill_versions(block_type) - if "register" in self: - if "read" not in self: - self["read"] = [] - self["read"] += self._register_to_read() - - if job_mode in [CMD_MODE.PULL, CMD_MODE.PUSH]: - self._pull_data() - if job_mode == CMD_MODE.PULL and "register" in self: + self._logger.debug("Fetching registrations") _objs = fdp_reg.fetch_registrations( local_uri=self.local_uri, repo_dir=self.local_repository, @@ -400,22 +709,42 @@ def prepare( ) self._logger.debug("Fetched objects:\n %s", _objs) - self._config = self._clean() + self._parsed["namespace"] = self._get_local_namespaces() + self._parsed["author"] = self._get_local_authors() + + if "register" in self: + if "read" not in self: + self["read"] = [] + self["read"] += self._register_to_read(self["register"]) - _unparsed = self._check_for_unparsed() + if "read" in self: + self["read"] = self._update_use_sections(self["read"]) - if _unparsed: - raise fdp_exc.InternalError(f"Failed to parse variables '{_unparsed}'") + self._config = self._clean() + + self._check_for_unparsed() self["run_metadata.latest_commit"] = self._fetch_latest_commit(allow_dirty) # Perform config validation self._logger.debug("Running configuration validation") - try: - UserConfigModel(**self._config) - except pydantic.ValidationError as e: - raise fdp_exc.ValidationError(e.json()) + return os.path.join(self._job_dir, fdp_com.USER_CONFIG_FILE) + + def _pull_push_log_header(self, _cmd): + _cmd = f'fair {_cmd}' + _out_str = self._now.strftime("%a %b %d %H:%M:%S %Y %Z") + _user = fdp_conf.get_current_user_name(self.local_repository) + _email = fdp_conf.get_current_user_email(self.local_repository) + self._log_file.writelines( + [ + "--------------------------------\n", + f" Commenced = {_out_str}\n", + f" Author = {' '.join(_user)} <{_email}>\n", + f" Command = {_cmd}\n", + "--------------------------------\n", + ] + ) def _check_for_unparsed(self) -> typing.List[str]: self._logger.debug("Checking for unparsed variables") @@ -424,18 +753,21 @@ def _check_for_unparsed(self) -> typing.List[str]: # Additional parser for formatted datetime _regex_fmt = re.compile(r"\$\{\{\s*([^}${\s]+)\s*\}\}") - return _regex_fmt.findall(_conf_str) + _unparsed = _regex_fmt.findall(_conf_str) + + if _unparsed: + raise fdp_exc.InternalError(f"Failed to parse variables '{_unparsed}'") - def _subst_cli_vars(self, job_dir: str, job_time: datetime.datetime) -> str: + def _subst_cli_vars(self, job_time: datetime.datetime) -> str: self._logger.debug("Searching for CLI variables") def _get_id(): try: - return fdp_conf.get_current_user_uri(job_dir) + return fdp_conf.get_current_user_uri(self._job_dir) except fdp_exc.CLIConfigurationError: - return fdp_conf.get_current_user_uuid(job_dir) + return fdp_conf.get_current_user_uuid(self._job_dir) - def _tag_check(*args, **kwargs): + def _tag_check(): _repo = git.Repo(fdp_conf.local_git_repo(self.local_repository)) if len(_repo.tags) < 1: fdp_exc.UserConfigError( @@ -449,7 +781,7 @@ def _tag_check(*args, **kwargs): "USER": lambda: fdp_conf.get_current_user_name(self.local_repository), "USER_ID": lambda: _get_id(), "REPO_DIR": lambda: self.local_repository, - "CONFIG_DIR": lambda: job_dir + os.path.sep, + "CONFIG_DIR": lambda: self._job_dir + os.path.sep, "LOCAL_TOKEN": lambda: fdp_req.local_token(), "GIT_BRANCH": lambda: self.git_branch, "GIT_REMOTE": lambda: self.git_remote_uri, @@ -498,10 +830,9 @@ def _tag_check(*args, **kwargs): _config_str = re.sub(subst, str(_value), _config_str) self._logger.debug("Substituting %s: %s", var, str(_value)) - # Load the YAML (this also verifies the write was successful) and return it - return _config_str + self._config = yaml.safe_load(_config_str) - def _register_to_read(self) -> typing.Dict: + def _register_to_read(self, register_block: typing.List[typing.Dict]) -> typing.List[typing.Dict]: """Construct 'read' block entries from 'register' block entries Parameters @@ -511,66 +842,129 @@ def _register_to_read(self) -> typing.Dict: Returns ------- - typing.Dict + typing.List[typing.Dict] new read entries extract from register statements """ _read_block: typing.List[typing.Dict] = [] - for item in self._config["register"]: - _readable = {} - if "use" in item: - _readable["use"] = copy.deepcopy(item["use"]) + for item in register_block: + _readable = item.copy() if "external_object" in item: _readable["data_product"] = item["external_object"] - elif "data_product" in item: - _readable["data_product"] = item["data_product"] + _readable.pop("external_object") elif "namespace" in item: - fdp_store.store_namespace(**item) + try: + fdp_valid.Namespace(**_readable) + except pydantic.ValidationError as e: + raise fdp_exc.ValidationError(e.json()) + + if item["namespace"] in self._parsed["namespace"]: + self._logger.warning( + "Namespace '%s' already added, ignoring duplicate", + item["namespace"] + ) + else: + _readable["name"] = item["namespace"] + _readable.pop("namespace") + + fdp_store.store_namespace( + self.local_uri, + fdp_req.local_token(), + **_readable + ) + + # We do not want to register a namespace twice so + # keep track of which we have + self._parsed["namespace"].append(_readable["name"]) + elif "author" in item: + try: + fdp_valid.Author(**_readable) + except pydantic.ValidationError as e: + raise fdp_exc.ValidationError(e.json()) + + if item["author"] in self._parsed["author"]: + self._logger.warning( + "Author '%s' already added, ignoring duplicate", + item["author"] + ) + else: + _readable["name"] = item["author"] + _readable.pop("author") + + fdp_store.store_author( + self.local_uri, + fdp_req.local_token(), + **_readable + ) + + # We do not want to register a namespace twice so + # keep track of which we have + self._parsed["author"].append(_readable["name"]) + + else: # unknown raise fdp_exc.UserConfigError( f"Found registration for unknown item with keys {[*item]}" ) - _readable["use"]["version"] = fdp_ver.undo_incrementer( - _readable["use"]["version"] - ) - _read_block.append(_readable) + # 'public' only valid for writables + if "public" in _readable: + _readable.pop("public") + + # Add extra tag for tracking objects which have been registered + # as opposed to pulled from a remote + _readable["registered"] = True + _read_block.append(_readable) return _read_block + def _update_use_sections(self, read_block: typing.List[typing.Dict]) -> typing.List[typing.Dict]: + _new_read_block: typing.List[typing.Dict] = [] + + for entry in read_block: + _new_entry = entry.copy() + _new_entry["use"]["version"] = fdp_ver.undo_incrementer( + entry["use"]["version"] + ) + if "namespace" not in entry["use"]: + if "namespace_name" in entry: + _new_entry["use"]["namespace"] = entry["namespace_name"] + else: + _new_entry["use"]["namespace"] = self.default_input_namespace + _new_read_block.append(_new_entry) + return _new_read_block + def _clean(self) -> typing.Dict: self._logger.debug("Cleaning configuration") - _new_config: typing.Dict = {} - _new_config["run_metadata"] = copy.deepcopy(self["run_metadata"]) + _new_config: typing.Dict = { + 'run_metadata': copy.deepcopy(self["run_metadata"]) + } for action in ("read", "write"): if f"default_{action}_version" in _new_config["run_metadata"]: del _new_config["run_metadata"][f"default_{action}_version"] - _namespaces = ( - self.default_input_namespace, - self.default_output_namespace, - ) - - for namespace, block_type in zip(_namespaces, ["read", "write"]): - if block_type not in self._config: + for block_type in ("read", "write"): + if block_type not in self: continue _new_config[block_type] = [] for item in self[block_type]: - for use_item in [*item["use"]]: - # Get rid of duplicates - if use_item in item and item["use"][use_item] == item[use_item]: - item["use"].pop(use_item) - - if block_type == "write" and item["public"] == self.is_public_global: - item.pop("public") + _new_item = item.copy() + # Keep only the final permitted keys, this may vary depending + # on block type, also allow internal status check tags to + # pass at this stage + _allowed = list(self._final_permitted["all"]) + if block_type in self._final_permitted: + _allowed += list(self._final_permitted[block_type]) + _allowed += list(self._status_tags) - if item["use"]["namespace"] == namespace: - item["use"].pop("namespace") + for key in item.keys(): + if key not in _allowed: + _new_item.pop(key) - _new_config[block_type].append(item) + _new_config[block_type].append(_new_item) for block in self._block_types: if block in _new_config and not _new_config[block]: @@ -586,11 +980,12 @@ def _fill_versions(self, block_type: str) -> typing.List[typing.Dict]: _default_ver = self.default_read_version else: _default_ver = self.default_write_version - - for item in self._config[block_type]: + + for item in self[block_type]: if all(i not in item for i in ("data_product", "external_object")): _entries.append(item) continue + _new_item = copy.deepcopy(item) _new_item["use"] = item.get("use", {}) @@ -620,28 +1015,39 @@ def _fill_versions(self, block_type: str) -> typing.List[typing.Dict]: _name = item["use"]["data_product"] _namespace = item["use"]["namespace"] - _id_namespace = fdp_reg.convert_key_value_to_id( - self.local_uri, - "namespace", - _namespace - ) - if "${{" in _version: - _results = fdp_req.get( - self.local_uri, - "data_product", - params={"name": _name, "namespace": _id_namespace}, - ) - else: - _results = fdp_req.get( + # If no ID exists for the namespace then this object has not yet + # been written to the target registry and so a version number + # cannot be deduced this way + try: + _id_namespace = fdp_reg.convert_key_value_to_id( self.local_uri, - "data_product", - params={ - "name": _name, - "namespace": _id_namespace, - "version": _version, - }, + "namespace", + _namespace, + fdp_req.local_token() ) + if "${{" in _version: + _results = fdp_req.get( + self.local_uri, + "data_product", + fdp_req.local_token(), + params={"name": _name, "namespace": _id_namespace}, + ) + if "LATEST" in _version: + _version = fdp_ver.get_latest_version(_results) + else: + _results = fdp_req.get( + self.local_uri, + "data_product", + fdp_req.local_token(), + params={ + "name": _name, + "namespace": _id_namespace, + "version": _version, + }, + ) + except fdp_exc.RegistryError: + _results = [] try: _version = fdp_ver.get_correct_version( @@ -655,27 +1061,19 @@ def _fill_versions(self, block_type: str) -> typing.List[typing.Dict]: f"Already found this version ({e}), but may be identical" ) else: - self._logger.error(str(item)) + self._logger.error(f"Failed to find version match for {item}") raise e if "${{" in _version: self._logger.error(f"Found a version ({_version}) that needs resolving") if str(_version) != item["use"]["version"]: - item["use"]["version"] = str(_version) + _new_item["use"]["version"] = str(_version) - _entries.append(item) + _entries.append(_new_item) return _entries - def _pull_metadata(self) -> None: - self._logger.debug("Pulling metadata from remote registry") - self._logger.warning("Remote registry pulls are not yet implemented") - - def _pull_data(self) -> None: - self._logger.debug("Pulling data from remote registry") - self._logger.warning("Remote registry pulls are not yet implemented") - def _fill_block_data_product( self, block_type: str, item: typing.Dict ) -> typing.Dict: @@ -739,6 +1137,7 @@ def _fill_all_block_types(self) -> bool: for block_type in self._block_types: self._logger.debug(f"Filling '{block_type}' block") _new_block: typing.List[typing.Dict] = [] + if block_type not in self: continue @@ -746,7 +1145,24 @@ def _fill_all_block_types(self) -> bool: _new_item = self._fill_block_item(block_type, item) _new_block.append(_new_item) - self._config[block_type] = _new_block + self[block_type] = _new_block + + def _remove_status_tags(self) -> bool: + """ + Removes any internal tags added by the config class for + tracking status of objects + """ + for block in self._block_types: + if block not in self: + continue + for i, _ in enumerate(self[block]): + for key in self._status_tags: + self[block][i].pop(key, None) + + @property + def script(self) -> str: + """Retrieve path of session executable script""" + return self["run_metadata.script_path"] @property def content(self) -> typing.Dict: @@ -839,11 +1255,171 @@ def environment(self) -> typing.Dict: """Returns the job execution environment""" return self.env - def write(self, output_file: str) -> None: + def execute(self) -> int: + """Execute script/command if specified + + Returns + ------- + int + exit code of the executed process + """ + if not self.command: + raise fdp_exc.UserConfigError( + "No command specified to execute" + ) + _out_str = self._now.strftime("%a %b %d %H:%M:%S %Y %Z") + _user = fdp_conf.get_current_user_name(self.local_repository) + _email = fdp_conf.get_current_user_email(self.local_repository) + + self._log_file.writelines( + [ + "--------------------------------\n", + f" Commenced = {_out_str}\n", + f" Author = {' '.join(_user)} <{_email}>\n", + f" Command = {self.command}\n", + "--------------------------------\n", + ] + ) + + if not self.env: + raise fdp_exc.InternalError("Command execution environment setup failed") + + _exec = SHELLS[self.shell]["exec"].format( + self.script + ) + + self._logger.debug("Executing command: %s", _exec) + + _log_tail: typing.List[str] = [] + + _process = subprocess.Popen( + _exec.split(), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + bufsize=1, + text=True, + shell=False, + env=self.env, + cwd=self.local_repository, + ) + + # Write any stdout to the job log + for line in iter(_process.stdout.readline, ""): + self._log_file.writelines([line]) + _log_tail.append(line) + click.echo(line, nl=False) + sys.stdout.flush() + + _process.wait() + + if _process.returncode != 0: + self.close_log() + self._logger.error( + "Command '%s' failed with exit code %s, log tail:\n\t%s", + _exec, _process.returncode, '\n\t'.join(_log_tail) + ) + raise fdp_exc.CommandExecutionError( + f"Executed 'run' command failed with exit code {_process.returncode}", + _process.returncode, + ) + + def close_log(self) -> None: + _time_finished = datetime.datetime.now() + _duration = _time_finished - self._now + self._log_file.writelines( + [ + f"------- time taken {_duration} -------\n", + ] + ) + self._log_file.close() + + def get_readables(self) -> typing.List[str]: + """Returns list form of items to retrieve + + Returns + ------- + typing.List[str] + list of data products to retrieve + """ + self._logger.debug("Retrieving list of 'read' items") + _readables: typing.List[str] = [] + if "read" not in self: + return _readables + + #TODO: For now only supports data products + for readable in self["read"]: + # In this context readables are items to be read from a remote + # registry, not items registered locally + if "registered" in readable: + continue + + if "data_product" not in readable: + continue + if "use" not in readable: + self._logger.error( + "Incomplete read block, expected key 'use' in:\n" + f'\t{readable}' + ) + raise fdp_exc.UserConfigError( + "Attempt to access 'read' listings before parsing complete" + ) + if any(v not in readable["use"] for v in ("version", "namespace")): + self._logger.error( + "Incomplete read block, expected keys 'namespace' and 'version' in:\n" + f'\t{readable["use"]}' + ) + raise fdp_exc.UserConfigError( + "Attempt to access 'read' listings before parsing complete" + ) + _version = readable["use"]["version"] + _namespace = readable["use"]["namespace"] + _name = readable["data_product"] + + # If the user has requested to use a cached version, do not + # add to the list of items to read externally + if "cache" not in readable["use"]: + _readables.append(f"{_namespace}:{_name}@v{_version}") + + return _readables + + @property + def hash(self) -> str: + """Get job hash""" + return fdp_run.get_job_hash(self._job_dir) + + def write_log_lines(self, log_file_lines: typing.List[str]) -> None: + """Add lines to the current session log file""" + self._log_file.writelines(log_file_lines) + + def write(self, output_file: str = None) -> str: """Write job configuration to file""" + self._remove_status_tags() + + # Validate the file before writing + try: + fdp_valid.UserConfigModel(**self._config) + except pydantic.ValidationError as e: + self._logger.error( + "Validation of generated user configuration file failed:" + "\nCONFIG:\n%s\n\nRESULT:\n%s", + self._config, + json.loads(e.json()) + ) + raise fdp_exc.ValidationError(e.json()) + + if not output_file: + if not self._job_dir: + raise fdp_exc.UserConfigError( + "Cannot write new user configuration file, " + "no job directory created and no alternative filename provided" + ) + output_file = os.path.join(self._job_dir, fdp_com.USER_CONFIG_FILE) with open(output_file, "w") as out_f: yaml.dump(self._config, out_f) - self.env = self._create_environment(os.path.dirname(output_file)) + self.env = self._create_environment() self._logger.debug(f"Configuration written to '{output_file}'") + + return output_file diff --git a/fair/user_config/globbing.py b/fair/user_config/globbing.py new file mode 100644 index 00000000..d58c7334 --- /dev/null +++ b/fair/user_config/globbing.py @@ -0,0 +1,201 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +""" +User Config Remote Globbing +=========================== + +Handles the inclusion of wildcards in configuration statements by speaking to +the local registry and extracting items. + + +Constants +--------- + + - DISPOSABLES: tuple of keys to be removed before adding to config.yaml + +""" +import typing + +import fair.registry.requests as fdp_req +import fair.exceptions as fdp_exc + +from fair.registry import SEARCH_KEYS + +__date__ = "2022-01-11" + + +DISPOSABLES = ( + "name", + "object", + "last_updated", + "namespace", + "release_date", + "updated_by", + "original_store", + "prov_report", + "external_object", + "internal_format", + "url" +) + + +def get_single_layer_objects( + results_list: typing.List[typing.Dict], + object_type: str) -> typing.List[typing.Dict]: + """ + Retrieve results for a wildcard search for the given object + + This object should not have any requirements (other registry URLs) + + Parameters + ---------- + results_list : typing.List[typing.Dict] + results of registry search for the wildcard + + Returns + ------- + typing.List[typing.Dict] + entries for the config.yaml file + """ + _new_entries: typing.List[typing.Dict] = [] + + for result in results_list: + _data = result.copy() + for key, value in result.items(): + if not value: + _data.pop(key, None) + _data[object_type] = _data["name"] + + for key in DISPOSABLES: + _data.pop(key, None) + + _new_entries.append(_data) + return _new_entries + + +def get_data_product_objects( + registry_token: str, + results_list: typing.List[typing.Dict], + block_type: str, + version: str = None + ) -> typing.List[typing.Dict]: + """ + Retrieve results for a wildcard search of a data_product + + Parameters + ---------- + results_list : typing.List[typing.Dict] + results of registry search for the wildcard + + Returns + ------- + typing.List[typing.Dict] + entries for the config.yaml file + """ + _new_entries: typing.List[typing.Dict] = [] + + # If a data product need to retrieve the namespace name + for entry in results_list: + _data = entry.copy() + + for key, value in entry.items(): + if not value: + _data.pop(key, None) + + _namespace = fdp_req.url_get( + entry["namespace"], + registry_token + ) + + if not _namespace: + raise fdp_exc.InternalError( + "Failed to retrieve namespace for external_object " + f"{entry[SEARCH_KEYS['data_product']]}" + ) + + _version = entry["version"] + + if block_type == "write" and version: + _version = version + + _data["use"] = {} + _data["use"]["namespace"] = _namespace["name"] + _data["data_product"] = _data["name"] + _data["use"]["data_product"] = _data["name"] + _data["use"]["version"] = _version + + for key in DISPOSABLES: + _data.pop(key, None) + + _new_entries.append(_data) + + return _new_entries + + +def get_external_objects( + registry_token: str, + results_list: typing.List[typing.Dict], + block_type: str, + version: str = None + ) -> typing.List[typing.Dict]: + """ + Retrieve results for a wildcard search of a external_object + + Parameters + ---------- + results_list : typing.List[typing.Dict] + results of registry search for the wildcard + + Returns + ------- + typing.List[typing.Dict] + entries for the config.yaml file + """ + _new_entries: typing.List[typing.Dict] = [] + + for result in results_list: + _data = result.copy() + + for key, value in result.items(): + if not value: + _data.pop(key, None) + + _data_product = fdp_req.url_get( + result["data_product"], + registry_token + ) + + if not _data_product: + raise fdp_exc.InternalError( + "Failed to retrieve data_product for external_object " + f"{result[SEARCH_KEYS['data_product']]}" + ) + + _namespace = fdp_req.url_get( + _data_product["namespace"], + fdp_req.local_token() + ) + + if not _namespace: + raise fdp_exc.InternalError( + "Failed to retrieve namespace for external_object " + f"{result[SEARCH_KEYS['data_product']]}" + ) + + _version = result["version"] + + if block_type == "write" and version: + _version = version + + _data["use"] = {} + _data["use"]["namespace"] = _namespace["name"], + _data["use"]["version"] = _version + _data.pop("name", None) + _data.pop("last_updated", None) + + for key in DISPOSABLES: + _data.pop(key, None) + + _new_entries.append(_data) + + return _new_entries diff --git a/fair/user_config/validation.py b/fair/user_config/validation.py index bedb450a..0b65e3f4 100644 --- a/fair/user_config/validation.py +++ b/fair/user_config/validation.py @@ -35,7 +35,6 @@ import uuid import datetime - class SupportedShells(enum.Enum): POWERSHELL = "powershell" PWSH = "pwsh" @@ -49,7 +48,6 @@ class SupportedShells(enum.Enum): JAVA = "java" SH = "sh" - class RunMetadata(pydantic.BaseModel): local_repo: pathlib.Path = pydantic.Field( ..., @@ -192,6 +190,9 @@ class Use(pydantic.BaseModel): namespace: typing.Optional[str] = pydantic.Field( None, title="namespace", description="namespace to read/write object using" ) + cache: typing.Optional[str] = pydantic.Field( + None, title="cache", description="local copy of requested file to use" + ) class Config: extra = "forbid" @@ -219,6 +220,11 @@ class DataProductWrite(DataProduct): title="file type", description="extension of type of file the data product is", ) + public: typing.Optional[bool] = pydantic.Field( + True, + title="public", + description="whether items are/should be publically accessible", + ) class Config: extra = "forbid" @@ -229,7 +235,7 @@ class Namespace(pydantic.BaseModel): ..., title="namespace label", description="label for the namespace" ) full_name: str = pydantic.Field( - ..., title="namespace full name", description="longer name for the namespace" + None, title="namespace full name", description="longer name for the namespace" ) website: typing.Optional[pydantic.AnyHttpUrl] = pydantic.Field( None, @@ -261,6 +267,15 @@ class Config: extra = "forbid" +# Permitted objects which are recognised by the schema and registry +VALID_OBJECTS = { + "author": Author, + "data_product": DataProduct, + "namespace": Namespace, + "external_object": ExternalObject +} + + class UserConfigModel(pydantic.BaseModel): run_metadata: RunMetadata read: typing.Optional[typing.List[DataProduct]] diff --git a/fair/utilities.py b/fair/utilities.py index 8f1e5fc1..76ebcefe 100644 --- a/fair/utilities.py +++ b/fair/utilities.py @@ -128,7 +128,7 @@ def remove_dictlist_dupes( new list without duplicates """ # Convert single layer dictionary to a list of key-value tuples - _tupleify = [[(k, v) for k, v in d.items()] for d in dicts] + _tupleify = [list(d.items()) for d in dicts] # Only append unique tuple lists _set_tupleify = [] diff --git a/poetry.lock b/poetry.lock index 7841d6d6..d0ff0db9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -8,32 +8,17 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "attrs" -version = "21.2.0" +version = "21.4.0" description = "Classes Without Boilerplate" category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit"] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins"] - -[[package]] -name = "backports.entry-points-selectable" -version = "1.1.1" -description = "Compatibility shim providing selectable entry points for older implementations" -category = "main" -optional = false -python-versions = ">=2.7" - -[package.dependencies] -importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} - -[package.extras] -docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -testing = ["pytest", "pytest-flake8", "pytest-cov", "pytest-black (>=0.3.7)", "pytest-mypy", "pytest-checkdocs (>=2.4)", "pytest-enabler (>=1.0.1)"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] [[package]] name = "bandit" @@ -135,7 +120,7 @@ python-versions = ">=3.6.1" [[package]] name = "charset-normalizer" -version = "2.0.9" +version = "2.0.10" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." category = "main" optional = false @@ -231,7 +216,7 @@ python-versions = ">=3.6,<4.0" [[package]] name = "cryptography" -version = "36.0.0" +version = "36.0.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "dev" optional = false @@ -250,7 +235,7 @@ test = ["pytest (>=6.2.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pr [[package]] name = "decorator" -version = "5.1.0" +version = "5.1.1" description = "Decorators for Humans" category = "main" optional = false @@ -272,7 +257,7 @@ cli = ["click (==8.0.3)", "pyyaml (==5.4.1)", "toml (==0.10.2)", "clevercsv (==0 [[package]] name = "distlib" -version = "0.3.3" +version = "0.3.4" description = "Distribution utilities" category = "main" optional = false @@ -318,11 +303,11 @@ testing = ["pre-commit"] [[package]] name = "filelock" -version = "3.4.0" +version = "3.4.2" description = "A platform independent file lock." category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.extras] docs = ["furo (>=2021.8.17b43)", "sphinx (>=4.1)", "sphinx-autodoc-typehints (>=1.12)"] @@ -355,7 +340,7 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.24" +version = "3.1.26" description = "GitPython is a python library used to interact with Git repositories" category = "main" optional = false @@ -363,7 +348,7 @@ python-versions = ">=3.7" [package.dependencies] gitdb = ">=4.0.1,<5" -typing-extensions = {version = ">=3.7.4.3", markers = "python_version < \"3.10\""} +typing-extensions = {version = ">=3.7.4.3", markers = "python_version < \"3.8\""} [[package]] name = "html5lib" @@ -385,7 +370,7 @@ lxml = ["lxml"] [[package]] name = "identify" -version = "2.4.0" +version = "2.4.2" description = "File identification library for Python" category = "main" optional = false @@ -538,7 +523,7 @@ python-versions = "*" [[package]] name = "mypy" -version = "0.930" +version = "0.931" description = "Optional static typing for Python" category = "dev" optional = false @@ -599,15 +584,15 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "path" -version = "16.2.0" +version = "16.3.0" description = "A module wrapper for os.path" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.extras] docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)", "jaraco.tidelift (>=1.4)"] -testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "appdirs", "packaging", "pygments", "pytest-black (>=0.3.7)", "pytest-mypy", "pywin32"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "appdirs", "packaging", "pygments", "pytest-black (>=0.3.7)", "pytest-mypy", "pywin32"] [[package]] name = "path.py" @@ -664,11 +649,11 @@ testing = ["coverage", "nose"] [[package]] name = "platformdirs" -version = "2.4.0" +version = "2.4.1" description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.extras] docs = ["Sphinx (>=4)", "furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)"] @@ -816,7 +801,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pygments" -version = "2.10.0" +version = "2.11.2" description = "Pygments is a syntax highlighting package written in Python." category = "main" optional = false @@ -986,7 +971,7 @@ python-versions = ">=3.6" [[package]] name = "requests" -version = "2.27.0" +version = "2.27.1" description = "Python HTTP for Humans." category = "main" optional = false @@ -1031,7 +1016,7 @@ requests = ">=2.0.1,<3.0.0" [[package]] name = "rich" -version = "10.16.2" +version = "11.0.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" category = "main" optional = false @@ -1136,7 +1121,7 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "tomli" -version = "1.2.2" +version = "1.2.3" description = "A lil' TOML parser" category = "dev" optional = false @@ -1144,19 +1129,19 @@ python-versions = ">=3.6" [[package]] name = "tomlkit" -version = "0.7.2" +version = "0.8.0" description = "Style preserving TOML library" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.6,<4.0" [[package]] name = "typed-ast" -version = "1.4.3" +version = "1.5.1" description = "a fork of Python 2 and 3 ast modules with type comment support" category = "dev" optional = false -python-versions = "*" +python-versions = ">=3.6" [[package]] name = "typing-extensions" @@ -1168,7 +1153,7 @@ python-versions = ">=3.6" [[package]] name = "urllib3" -version = "1.26.7" +version = "1.26.8" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false @@ -1196,14 +1181,13 @@ test = ["pytest (>=2.2.3)", "flake8 (>=2.4.0)", "isort (>=4.2.2)"] [[package]] name = "virtualenv" -version = "20.10.0" +version = "20.13.0" description = "Virtual Python Environment builder" category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" [package.dependencies] -"backports.entry-points-selectable" = ">=1.0.4" distlib = ">=0.3.1,<1" filelock = ">=3.2,<4" importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} @@ -1224,20 +1208,20 @@ python-versions = "*" [[package]] name = "zipp" -version = "3.6.0" +version = "3.7.0" description = "Backport of pathlib-compatible object wrapper for zip files" category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.extras] docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] [metadata] lock-version = "1.1" python-versions = "^3.7.1,<4.0" -content-hash = "07df483f8476cbb131c342e31a4ea247b6b80163cc77e8981884150d48b3495d" +content-hash = "efab86aa65d3c3da1897100eed9f4701a837896ef77d9b6d25b3270ed2e679b4" [metadata.files] atomicwrites = [ @@ -1245,12 +1229,8 @@ atomicwrites = [ {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, ] attrs = [ - {file = "attrs-21.2.0-py2.py3-none-any.whl", hash = "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1"}, - {file = "attrs-21.2.0.tar.gz", hash = "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"}, -] -"backports.entry-points-selectable" = [ - {file = "backports.entry_points_selectable-1.1.1-py2.py3-none-any.whl", hash = "sha256:7fceed9532a7aa2bd888654a7314f864a3c16a4e710b34a58cfc0f08114c663b"}, - {file = "backports.entry_points_selectable-1.1.1.tar.gz", hash = "sha256:914b21a479fde881635f7af5adc7f6e38d6b274be32269070c53b698c60d5386"}, + {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, + {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, ] bandit = [ {file = "bandit-1.7.1-py3-none-any.whl", hash = "sha256:f5acd838e59c038a159b5c621cf0f8270b279e884eadd7b782d7491c02add0d4"}, @@ -1329,8 +1309,8 @@ cfgv = [ {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, ] charset-normalizer = [ - {file = "charset-normalizer-2.0.9.tar.gz", hash = "sha256:b0b883e8e874edfdece9c28f314e3dd5badf067342e42fb162203335ae61aa2c"}, - {file = "charset_normalizer-2.0.9-py3-none-any.whl", hash = "sha256:1eecaa09422db5be9e29d7fc65664e6c33bd06f9ced7838578ba40d58bdf3721"}, + {file = "charset-normalizer-2.0.10.tar.gz", hash = "sha256:876d180e9d7432c5d1dfd4c5d26b72f099d503e8fcc0feb7532c9289be60fcbd"}, + {file = "charset_normalizer-2.0.10-py3-none-any.whl", hash = "sha256:cb957888737fc0bbcd78e3df769addb41fd1ff8cf950dc9e7ad7793f1bf44455"}, ] cleo = [ {file = "cleo-0.8.1-py2.py3-none-any.whl", hash = "sha256:141cda6dc94a92343be626bb87a0b6c86ae291dfc732a57bf04310d4b4201753"}, @@ -1410,39 +1390,38 @@ crashtest = [ {file = "crashtest-0.3.1.tar.gz", hash = "sha256:42ca7b6ce88b6c7433e2ce47ea884e91ec93104a4b754998be498a8e6c3d37dd"}, ] cryptography = [ - {file = "cryptography-36.0.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:9511416e85e449fe1de73f7f99b21b3aa04fba4c4d335d30c486ba3756e3a2a6"}, - {file = "cryptography-36.0.0-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:97199a13b772e74cdcdb03760c32109c808aff7cd49c29e9cf4b7754bb725d1d"}, - {file = "cryptography-36.0.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:494106e9cd945c2cadfce5374fa44c94cfadf01d4566a3b13bb487d2e6c7959e"}, - {file = "cryptography-36.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6fbbbb8aab4053fa018984bb0e95a16faeb051dd8cca15add2a27e267ba02b58"}, - {file = "cryptography-36.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:684993ff6f67000a56454b41bdc7e015429732d65a52d06385b6e9de6181c71e"}, - {file = "cryptography-36.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c702855cd3174666ef0d2d13dcc879090aa9c6c38f5578896407a7028f75b9f"}, - {file = "cryptography-36.0.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d91bc9f535599bed58f6d2e21a2724cb0c3895bf41c6403fe881391d29096f1d"}, - {file = "cryptography-36.0.0-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:b17d83b3d1610e571fedac21b2eb36b816654d6f7496004d6a0d32f99d1d8120"}, - {file = "cryptography-36.0.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:8982c19bb90a4fa2aad3d635c6d71814e38b643649b4000a8419f8691f20ac44"}, - {file = "cryptography-36.0.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:24469d9d33217ffd0ce4582dfcf2a76671af115663a95328f63c99ec7ece61a4"}, - {file = "cryptography-36.0.0-cp36-abi3-win32.whl", hash = "sha256:f6a5a85beb33e57998dc605b9dbe7deaa806385fdf5c4810fb849fcd04640c81"}, - {file = "cryptography-36.0.0-cp36-abi3-win_amd64.whl", hash = "sha256:2deab5ec05d83ddcf9b0916319674d3dae88b0e7ee18f8962642d3cde0496568"}, - {file = "cryptography-36.0.0-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2049f8b87f449fc6190350de443ee0c1dd631f2ce4fa99efad2984de81031681"}, - {file = "cryptography-36.0.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a776bae1629c8d7198396fd93ec0265f8dd2341c553dc32b976168aaf0e6a636"}, - {file = "cryptography-36.0.0-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:aa94d617a4cd4cdf4af9b5af65100c036bce22280ebb15d8b5262e8273ebc6ba"}, - {file = "cryptography-36.0.0-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:5c49c9e8fb26a567a2b3fa0343c89f5d325447956cc2fc7231c943b29a973712"}, - {file = "cryptography-36.0.0-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ef216d13ac8d24d9cd851776662f75f8d29c9f2d05cdcc2d34a18d32463a9b0b"}, - {file = "cryptography-36.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:231c4a69b11f6af79c1495a0e5a85909686ea8db946935224b7825cfb53827ed"}, - {file = "cryptography-36.0.0-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:f92556f94e476c1b616e6daec5f7ddded2c082efa7cee7f31c7aeda615906ed8"}, - {file = "cryptography-36.0.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:d73e3a96c38173e0aa5646c31bf8473bc3564837977dd480f5cbeacf1d7ef3a3"}, - {file = "cryptography-36.0.0.tar.gz", hash = "sha256:52f769ecb4ef39865719aedc67b4b7eae167bafa48dbc2a26dd36fa56460507f"}, + {file = "cryptography-36.0.1-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:73bc2d3f2444bcfeac67dd130ff2ea598ea5f20b40e36d19821b4df8c9c5037b"}, + {file = "cryptography-36.0.1-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:2d87cdcb378d3cfed944dac30596da1968f88fb96d7fc34fdae30a99054b2e31"}, + {file = "cryptography-36.0.1-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:74d6c7e80609c0f4c2434b97b80c7f8fdfaa072ca4baab7e239a15d6d70ed73a"}, + {file = "cryptography-36.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:6c0c021f35b421ebf5976abf2daacc47e235f8b6082d3396a2fe3ccd537ab173"}, + {file = "cryptography-36.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d59a9d55027a8b88fd9fd2826c4392bd487d74bf628bb9d39beecc62a644c12"}, + {file = "cryptography-36.0.1-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a817b961b46894c5ca8a66b599c745b9a3d9f822725221f0e0fe49dc043a3a3"}, + {file = "cryptography-36.0.1-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:94ae132f0e40fe48f310bba63f477f14a43116f05ddb69d6fa31e93f05848ae2"}, + {file = "cryptography-36.0.1-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:7be0eec337359c155df191d6ae00a5e8bbb63933883f4f5dffc439dac5348c3f"}, + {file = "cryptography-36.0.1-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:e0344c14c9cb89e76eb6a060e67980c9e35b3f36691e15e1b7a9e58a0a6c6dc3"}, + {file = "cryptography-36.0.1-cp36-abi3-win32.whl", hash = "sha256:4caa4b893d8fad33cf1964d3e51842cd78ba87401ab1d2e44556826df849a8ca"}, + {file = "cryptography-36.0.1-cp36-abi3-win_amd64.whl", hash = "sha256:391432971a66cfaf94b21c24ab465a4cc3e8bf4a939c1ca5c3e3a6e0abebdbcf"}, + {file = "cryptography-36.0.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:bb5829d027ff82aa872d76158919045a7c1e91fbf241aec32cb07956e9ebd3c9"}, + {file = "cryptography-36.0.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ebc15b1c22e55c4d5566e3ca4db8689470a0ca2babef8e3a9ee057a8b82ce4b1"}, + {file = "cryptography-36.0.1-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:596f3cd67e1b950bc372c33f1a28a0692080625592ea6392987dba7f09f17a94"}, + {file = "cryptography-36.0.1-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:30ee1eb3ebe1644d1c3f183d115a8c04e4e603ed6ce8e394ed39eea4a98469ac"}, + {file = "cryptography-36.0.1-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ec63da4e7e4a5f924b90af42eddf20b698a70e58d86a72d943857c4c6045b3ee"}, + {file = "cryptography-36.0.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca238ceb7ba0bdf6ce88c1b74a87bffcee5afbfa1e41e173b1ceb095b39add46"}, + {file = "cryptography-36.0.1-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:ca28641954f767f9822c24e927ad894d45d5a1e501767599647259cbf030b903"}, + {file = "cryptography-36.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:39bdf8e70eee6b1c7b289ec6e5d84d49a6bfa11f8b8646b5b3dfe41219153316"}, + {file = "cryptography-36.0.1.tar.gz", hash = "sha256:53e5c1dc3d7a953de055d77bef2ff607ceef7a2aac0353b5d630ab67f7423638"}, ] decorator = [ - {file = "decorator-5.1.0-py3-none-any.whl", hash = "sha256:7b12e7c3c6ab203a29e157335e9122cb03de9ab7264b137594103fd4a683b374"}, - {file = "decorator-5.1.0.tar.gz", hash = "sha256:e59913af105b9860aa2c8d3272d9de5a56a4e608db9a2f167a8480b323d529a7"}, + {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"}, + {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"}, ] deepdiff = [ {file = "deepdiff-5.7.0-py3-none-any.whl", hash = "sha256:1ffb38c3b5d9174eb2df95850c93aee55ec00e19396925036a2e680f725079e0"}, {file = "deepdiff-5.7.0.tar.gz", hash = "sha256:838766484e323dcd9dec6955926a893a83767dc3f3f94542773e6aa096efe5d4"}, ] distlib = [ - {file = "distlib-0.3.3-py2.py3-none-any.whl", hash = "sha256:c8b54e8454e5bf6237cc84c20e8264c3e991e824ef27e8f1e81049867d861e31"}, - {file = "distlib-0.3.3.zip", hash = "sha256:d982d0751ff6eaaab5e2ec8e691d949ee80eddf01a62eaa96ddb11531fe16b05"}, + {file = "distlib-0.3.4-py2.py3-none-any.whl", hash = "sha256:6564fe0a8f51e734df6333d08b8b94d4ea8ee6b99b5ed50613f731fd4089f34b"}, + {file = "distlib-0.3.4.zip", hash = "sha256:e4b58818180336dc9c529bfb9a0b58728ffc09ad92027a3f30b7cd91e3458579"}, ] dnspython = [ {file = "dnspython-2.1.0-py3-none-any.whl", hash = "sha256:95d12f6ef0317118d2a1a6fc49aac65ffec7eb8087474158f42f26a639135216"}, @@ -1457,8 +1436,8 @@ execnet = [ {file = "execnet-1.9.0.tar.gz", hash = "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5"}, ] filelock = [ - {file = "filelock-3.4.0-py3-none-any.whl", hash = "sha256:2e139a228bcf56dd8b2274a65174d005c4a6b68540ee0bdbb92c76f43f29f7e8"}, - {file = "filelock-3.4.0.tar.gz", hash = "sha256:93d512b32a23baf4cac44ffd72ccf70732aeff7b8050fcaf6d3ec406d954baf4"}, + {file = "filelock-3.4.2-py3-none-any.whl", hash = "sha256:cf0fc6a2f8d26bd900f19bf33915ca70ba4dd8c56903eeb14e1e7a2fd7590146"}, + {file = "filelock-3.4.2.tar.gz", hash = "sha256:38b4f4c989f9d06d44524df1b24bd19e167d851f19b50bf3e3559952dddc5b80"}, ] flake8 = [ {file = "flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"}, @@ -1469,16 +1448,16 @@ gitdb = [ {file = "gitdb-4.0.9.tar.gz", hash = "sha256:bac2fd45c0a1c9cf619e63a90d62bdc63892ef92387424b855792a6cabe789aa"}, ] gitpython = [ - {file = "GitPython-3.1.24-py3-none-any.whl", hash = "sha256:dc0a7f2f697657acc8d7f89033e8b1ea94dd90356b2983bca89dc8d2ab3cc647"}, - {file = "GitPython-3.1.24.tar.gz", hash = "sha256:df83fdf5e684fef7c6ee2c02fc68a5ceb7e7e759d08b694088d0cacb4eba59e5"}, + {file = "GitPython-3.1.26-py3-none-any.whl", hash = "sha256:26ac35c212d1f7b16036361ca5cff3ec66e11753a0d677fb6c48fa4e1a9dd8d6"}, + {file = "GitPython-3.1.26.tar.gz", hash = "sha256:fc8868f63a2e6d268fb25f481995ba185a85a66fcad126f039323ff6635669ee"}, ] html5lib = [ {file = "html5lib-1.1-py2.py3-none-any.whl", hash = "sha256:0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d"}, {file = "html5lib-1.1.tar.gz", hash = "sha256:b2e5b40261e20f354d198eae92afc10d750afb487ed5e50f9c4eaf07c184146f"}, ] identify = [ - {file = "identify-2.4.0-py2.py3-none-any.whl", hash = "sha256:eba31ca80258de6bb51453084bff4a923187cd2193b9c13710f2516ab30732cc"}, - {file = "identify-2.4.0.tar.gz", hash = "sha256:a33ae873287e81651c7800ca309dc1f84679b763c9c8b30680e16fbfa82f0107"}, + {file = "identify-2.4.2-py2.py3-none-any.whl", hash = "sha256:67c1e66225870dce721228176637a8ef965e8dd58450bcc7592249d0dfc4da6c"}, + {file = "identify-2.4.2.tar.gz", hash = "sha256:93e8ec965e888f2212aa5c24b2b662f4832c39acb1d7196a70ea45acb626a05e"}, ] idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, @@ -1633,26 +1612,26 @@ msgpack = [ {file = "msgpack-1.0.3.tar.gz", hash = "sha256:51fdc7fb93615286428ee7758cecc2f374d5ff363bdd884c7ea622a7a327a81e"}, ] mypy = [ - {file = "mypy-0.930-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:221cc94dc6a801ccc2be7c0c9fd791c5e08d1fa2c5e1c12dec4eab15b2469871"}, - {file = "mypy-0.930-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:db3a87376a1380f396d465bed462e76ea89f838f4c5e967d68ff6ee34b785c31"}, - {file = "mypy-0.930-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1d2296f35aae9802eeb1327058b550371ee382d71374b3e7d2804035ef0b830b"}, - {file = "mypy-0.930-cp310-cp310-win_amd64.whl", hash = "sha256:959319b9a3cafc33a8185f440a433ba520239c72e733bf91f9efd67b0a8e9b30"}, - {file = "mypy-0.930-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:45a4dc21c789cfd09b8ccafe114d6de66f0b341ad761338de717192f19397a8c"}, - {file = "mypy-0.930-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1e689e92cdebd87607a041585f1dc7339aa2e8a9f9bad9ba7e6ece619431b20c"}, - {file = "mypy-0.930-cp36-cp36m-win_amd64.whl", hash = "sha256:ed4e0ea066bb12f56b2812a15ff223c57c0a44eca817ceb96b214bb055c7051f"}, - {file = "mypy-0.930-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a9d8dffefba634b27d650e0de2564379a1a367e2e08d6617d8f89261a3bf63b2"}, - {file = "mypy-0.930-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b419e9721260161e70d054a15abbd50603c16f159860cfd0daeab647d828fc29"}, - {file = "mypy-0.930-cp37-cp37m-win_amd64.whl", hash = "sha256:601f46593f627f8a9b944f74fd387c9b5f4266b39abad77471947069c2fc7651"}, - {file = "mypy-0.930-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1ea7199780c1d7940b82dbc0a4e37722b4e3851264dbba81e01abecc9052d8a7"}, - {file = "mypy-0.930-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:70b197dd8c78fc5d2daf84bd093e8466a2b2e007eedaa85e792e513a820adbf7"}, - {file = "mypy-0.930-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5feb56f8bb280468fe5fc8e6f56f48f99aa0df9eed3c507a11505ee4657b5380"}, - {file = "mypy-0.930-cp38-cp38-win_amd64.whl", hash = "sha256:2e9c5409e9cb81049bb03fa1009b573dea87976713e3898561567a86c4eaee01"}, - {file = "mypy-0.930-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:554873e45c1ca20f31ddf873deb67fa5d2e87b76b97db50669f0468ccded8fae"}, - {file = "mypy-0.930-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0feb82e9fa849affca7edd24713dbe809dce780ced9f3feca5ed3d80e40b777f"}, - {file = "mypy-0.930-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:bc1a0607ea03c30225347334af66b0af12eefba018a89a88c209e02b7065ea95"}, - {file = "mypy-0.930-cp39-cp39-win_amd64.whl", hash = "sha256:f9f665d69034b1fcfdbcd4197480d26298bbfb5d2dfe206245b6498addb34999"}, - {file = "mypy-0.930-py3-none-any.whl", hash = "sha256:bf4a44e03040206f7c058d1f5ba02ef2d1820720c88bc4285c7d9a4269f54173"}, - {file = "mypy-0.930.tar.gz", hash = "sha256:51426262ae4714cc7dd5439814676e0992b55bcc0f6514eccb4cf8e0678962c2"}, + {file = "mypy-0.931-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3c5b42d0815e15518b1f0990cff7a705805961613e701db60387e6fb663fe78a"}, + {file = "mypy-0.931-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c89702cac5b302f0c5d33b172d2b55b5df2bede3344a2fbed99ff96bddb2cf00"}, + {file = "mypy-0.931-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:300717a07ad09525401a508ef5d105e6b56646f7942eb92715a1c8d610149714"}, + {file = "mypy-0.931-cp310-cp310-win_amd64.whl", hash = "sha256:7b3f6f557ba4afc7f2ce6d3215d5db279bcf120b3cfd0add20a5d4f4abdae5bc"}, + {file = "mypy-0.931-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:1bf752559797c897cdd2c65f7b60c2b6969ffe458417b8d947b8340cc9cec08d"}, + {file = "mypy-0.931-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4365c60266b95a3f216a3047f1d8e3f895da6c7402e9e1ddfab96393122cc58d"}, + {file = "mypy-0.931-cp36-cp36m-win_amd64.whl", hash = "sha256:1b65714dc296a7991000b6ee59a35b3f550e0073411ac9d3202f6516621ba66c"}, + {file = "mypy-0.931-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e839191b8da5b4e5d805f940537efcaa13ea5dd98418f06dc585d2891d228cf0"}, + {file = "mypy-0.931-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:50c7346a46dc76a4ed88f3277d4959de8a2bd0a0fa47fa87a4cde36fe247ac05"}, + {file = "mypy-0.931-cp37-cp37m-win_amd64.whl", hash = "sha256:d8f1ff62f7a879c9fe5917b3f9eb93a79b78aad47b533911b853a757223f72e7"}, + {file = "mypy-0.931-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f9fe20d0872b26c4bba1c1be02c5340de1019530302cf2dcc85c7f9fc3252ae0"}, + {file = "mypy-0.931-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1b06268df7eb53a8feea99cbfff77a6e2b205e70bf31743e786678ef87ee8069"}, + {file = "mypy-0.931-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8c11003aaeaf7cc2d0f1bc101c1cc9454ec4cc9cb825aef3cafff8a5fdf4c799"}, + {file = "mypy-0.931-cp38-cp38-win_amd64.whl", hash = "sha256:d9d2b84b2007cea426e327d2483238f040c49405a6bf4074f605f0156c91a47a"}, + {file = "mypy-0.931-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ff3bf387c14c805ab1388185dd22d6b210824e164d4bb324b195ff34e322d166"}, + {file = "mypy-0.931-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5b56154f8c09427bae082b32275a21f500b24d93c88d69a5e82f3978018a0266"}, + {file = "mypy-0.931-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8ca7f8c4b1584d63c9a0f827c37ba7a47226c19a23a753d52e5b5eddb201afcd"}, + {file = "mypy-0.931-cp39-cp39-win_amd64.whl", hash = "sha256:74f7eccbfd436abe9c352ad9fb65872cc0f1f0a868e9d9c44db0893440f0c697"}, + {file = "mypy-0.931-py3-none-any.whl", hash = "sha256:1171f2e0859cfff2d366da2c7092b06130f232c636a3f7301e3feb8b41f6377d"}, + {file = "mypy-0.931.tar.gz", hash = "sha256:0038b21890867793581e4cb0d810829f5fd4441aa75796b53033af3aa30430ce"}, ] mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, @@ -1674,8 +1653,8 @@ pastel = [ {file = "pastel-0.2.1.tar.gz", hash = "sha256:e6581ac04e973cac858828c6202c1e1e81fee1dc7de7683f3e1ffe0bfd8a573d"}, ] path = [ - {file = "path-16.2.0-py3-none-any.whl", hash = "sha256:340054c5bb459fc9fd40e7eb6768c5989f3e599d18224238465b5333bc8faa7d"}, - {file = "path-16.2.0.tar.gz", hash = "sha256:2de925e8d421f93bcea80d511b81accfb6a7e6b249afa4a5559557b0cf817097"}, + {file = "path-16.3.0-py3-none-any.whl", hash = "sha256:19429b3aa39982d372bd9c0e109fa8e991539ae3ae4151490da18b00586324a7"}, + {file = "path-16.3.0.tar.gz", hash = "sha256:5d8c2063ffade0766e0aa04255f8c0a0fd0eda5fdfb190375871b2f2b2e4afd7"}, ] "path.py" = [ {file = "path.py-12.5.0-py3-none-any.whl", hash = "sha256:a43e82eb2c344c3fd0b9d6352f6b856f40b8b7d3d65cc05978b42c3715668496"}, @@ -1698,8 +1677,8 @@ pkginfo = [ {file = "pkginfo-1.8.2.tar.gz", hash = "sha256:542e0d0b6750e2e21c20179803e40ab50598d8066d51097a0e382cba9eb02bff"}, ] platformdirs = [ - {file = "platformdirs-2.4.0-py3-none-any.whl", hash = "sha256:8868bbe3c3c80d42f20156f22e7131d2fb321f5bc86a2a345375c6481a67021d"}, - {file = "platformdirs-2.4.0.tar.gz", hash = "sha256:367a5e80b3d04d2428ffa76d33f124cf11e8fff2acdaa9b43d545f5c7d661ef2"}, + {file = "platformdirs-2.4.1-py3-none-any.whl", hash = "sha256:1d7385c7db91728b83efd0ca99a5afb296cab9d0ed8313a45ed8ba17967ecfca"}, + {file = "platformdirs-2.4.1.tar.gz", hash = "sha256:440633ddfebcc36264232365d7840a970e75e1018d15b4327d11f91909045fda"}, ] pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, @@ -1779,8 +1758,8 @@ pyflakes = [ {file = "pyflakes-2.3.1.tar.gz", hash = "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"}, ] pygments = [ - {file = "Pygments-2.10.0-py3-none-any.whl", hash = "sha256:b8e67fe6af78f492b3c4b3e2970c0624cbf08beb1e493b2c99b9fa1b67a20380"}, - {file = "Pygments-2.10.0.tar.gz", hash = "sha256:f398865f7eb6874156579fdf36bc840a03cab64d1cde9e93d68f46a425ec52c6"}, + {file = "Pygments-2.11.2-py3-none-any.whl", hash = "sha256:44238f1b60a76d78fc8ca0528ee429702aae011c265fe6a8dd8b63049ae41c65"}, + {file = "Pygments-2.11.2.tar.gz", hash = "sha256:4e426f72023d88d03b2fa258de560726ce890ff3b630f88c21cbb8b2503b8c6a"}, ] pylama = [ {file = "pylama-7.7.1-py2.py3-none-any.whl", hash = "sha256:fd61c11872d6256b019ef1235be37b77c922ef37ac9797df6bd489996dddeb15"}, @@ -1867,8 +1846,8 @@ pyyaml = [ {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] requests = [ - {file = "requests-2.27.0-py2.py3-none-any.whl", hash = "sha256:f71a09d7feba4a6b64ffd8e9d9bc60f9bf7d7e19fd0e04362acb1cfc2e3d98df"}, - {file = "requests-2.27.0.tar.gz", hash = "sha256:8e5643905bf20a308e25e4c1dd379117c09000bf8a82ebccc462cfb1b34a16b5"}, + {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"}, + {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"}, ] requests-mock = [ {file = "requests-mock-1.9.3.tar.gz", hash = "sha256:8d72abe54546c1fc9696fa1516672f1031d72a55a1d66c85184f972a24ba0eba"}, @@ -1879,8 +1858,8 @@ requests-toolbelt = [ {file = "requests_toolbelt-0.9.1-py2.py3-none-any.whl", hash = "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f"}, ] rich = [ - {file = "rich-10.16.2-py3-none-any.whl", hash = "sha256:c59d73bd804c90f747c8d7b1d023b88f2a9ac2454224a4aeaf959b21eeb42d03"}, - {file = "rich-10.16.2.tar.gz", hash = "sha256:720974689960e06c2efdb54327f8bf0cdbdf4eae4ad73b6c94213cad405c371b"}, + {file = "rich-11.0.0-py3-none-any.whl", hash = "sha256:d7a8086aa1fa7e817e3bba544eee4fd82047ef59036313147759c11475f0dafd"}, + {file = "rich-11.0.0.tar.gz", hash = "sha256:c32a8340b21c75931f157466fefe81ae10b92c36a5ea34524dff3767238774a4"}, ] secretstorage = [ {file = "SecretStorage-3.3.1-py3-none-any.whl", hash = "sha256:422d82c36172d88d6a0ed5afdec956514b189ddbfb72fefab0c8a1cee4eaf71f"}, @@ -1981,66 +1960,55 @@ toml = [ {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] tomli = [ - {file = "tomli-1.2.2-py3-none-any.whl", hash = "sha256:f04066f68f5554911363063a30b108d2b5a5b1a010aa8b6132af78489fe3aade"}, - {file = "tomli-1.2.2.tar.gz", hash = "sha256:c6ce0015eb38820eaf32b5db832dbc26deb3dd427bd5f6556cf0acac2c214fee"}, + {file = "tomli-1.2.3-py3-none-any.whl", hash = "sha256:e3069e4be3ead9668e21cb9b074cd948f7b3113fd9c8bba083f48247aab8b11c"}, + {file = "tomli-1.2.3.tar.gz", hash = "sha256:05b6166bff487dc068d322585c7ea4ef78deed501cc124060e0f238e89a9231f"}, ] tomlkit = [ - {file = "tomlkit-0.7.2-py2.py3-none-any.whl", hash = "sha256:173ad840fa5d2aac140528ca1933c29791b79a374a0861a80347f42ec9328117"}, - {file = "tomlkit-0.7.2.tar.gz", hash = "sha256:d7a454f319a7e9bd2e249f239168729327e4dd2d27b17dc68be264ad1ce36754"}, + {file = "tomlkit-0.8.0-py3-none-any.whl", hash = "sha256:b824e3466f1d475b2b5f1c392954c6cb7ea04d64354ff7300dc7c14257dc85db"}, + {file = "tomlkit-0.8.0.tar.gz", hash = "sha256:29e84a855712dfe0e88a48f6d05c21118dbafb283bb2eed614d46f80deb8e9a1"}, ] typed-ast = [ - {file = "typed_ast-1.4.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:2068531575a125b87a41802130fa7e29f26c09a2833fea68d9a40cf33902eba6"}, - {file = "typed_ast-1.4.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:c907f561b1e83e93fad565bac5ba9c22d96a54e7ea0267c708bffe863cbe4075"}, - {file = "typed_ast-1.4.3-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:1b3ead4a96c9101bef08f9f7d1217c096f31667617b58de957f690c92378b528"}, - {file = "typed_ast-1.4.3-cp35-cp35m-win32.whl", hash = "sha256:dde816ca9dac1d9c01dd504ea5967821606f02e510438120091b84e852367428"}, - {file = "typed_ast-1.4.3-cp35-cp35m-win_amd64.whl", hash = "sha256:777a26c84bea6cd934422ac2e3b78863a37017618b6e5c08f92ef69853e765d3"}, - {file = "typed_ast-1.4.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f8afcf15cc511ada719a88e013cec87c11aff7b91f019295eb4530f96fe5ef2f"}, - {file = "typed_ast-1.4.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:52b1eb8c83f178ab787f3a4283f68258525f8d70f778a2f6dd54d3b5e5fb4341"}, - {file = "typed_ast-1.4.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:01ae5f73431d21eead5015997ab41afa53aa1fbe252f9da060be5dad2c730ace"}, - {file = "typed_ast-1.4.3-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:c190f0899e9f9f8b6b7863debfb739abcb21a5c054f911ca3596d12b8a4c4c7f"}, - {file = "typed_ast-1.4.3-cp36-cp36m-win32.whl", hash = "sha256:398e44cd480f4d2b7ee8d98385ca104e35c81525dd98c519acff1b79bdaac363"}, - {file = "typed_ast-1.4.3-cp36-cp36m-win_amd64.whl", hash = "sha256:bff6ad71c81b3bba8fa35f0f1921fb24ff4476235a6e94a26ada2e54370e6da7"}, - {file = "typed_ast-1.4.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0fb71b8c643187d7492c1f8352f2c15b4c4af3f6338f21681d3681b3dc31a266"}, - {file = "typed_ast-1.4.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:760ad187b1041a154f0e4d0f6aae3e40fdb51d6de16e5c99aedadd9246450e9e"}, - {file = "typed_ast-1.4.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5feca99c17af94057417d744607b82dd0a664fd5e4ca98061480fd8b14b18d04"}, - {file = "typed_ast-1.4.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:95431a26309a21874005845c21118c83991c63ea800dd44843e42a916aec5899"}, - {file = "typed_ast-1.4.3-cp37-cp37m-win32.whl", hash = "sha256:aee0c1256be6c07bd3e1263ff920c325b59849dc95392a05f258bb9b259cf39c"}, - {file = "typed_ast-1.4.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9ad2c92ec681e02baf81fdfa056fe0d818645efa9af1f1cd5fd6f1bd2bdfd805"}, - {file = "typed_ast-1.4.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b36b4f3920103a25e1d5d024d155c504080959582b928e91cb608a65c3a49e1a"}, - {file = "typed_ast-1.4.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:067a74454df670dcaa4e59349a2e5c81e567d8d65458d480a5b3dfecec08c5ff"}, - {file = "typed_ast-1.4.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7538e495704e2ccda9b234b82423a4038f324f3a10c43bc088a1636180f11a41"}, - {file = "typed_ast-1.4.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:af3d4a73793725138d6b334d9d247ce7e5f084d96284ed23f22ee626a7b88e39"}, - {file = "typed_ast-1.4.3-cp38-cp38-win32.whl", hash = "sha256:f2362f3cb0f3172c42938946dbc5b7843c2a28aec307c49100c8b38764eb6927"}, - {file = "typed_ast-1.4.3-cp38-cp38-win_amd64.whl", hash = "sha256:dd4a21253f42b8d2b48410cb31fe501d32f8b9fbeb1f55063ad102fe9c425e40"}, - {file = "typed_ast-1.4.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f328adcfebed9f11301eaedfa48e15bdece9b519fb27e6a8c01aa52a17ec31b3"}, - {file = "typed_ast-1.4.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:2c726c276d09fc5c414693a2de063f521052d9ea7c240ce553316f70656c84d4"}, - {file = "typed_ast-1.4.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:cae53c389825d3b46fb37538441f75d6aecc4174f615d048321b716df2757fb0"}, - {file = "typed_ast-1.4.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:b9574c6f03f685070d859e75c7f9eeca02d6933273b5e69572e5ff9d5e3931c3"}, - {file = "typed_ast-1.4.3-cp39-cp39-win32.whl", hash = "sha256:209596a4ec71d990d71d5e0d312ac935d86930e6eecff6ccc7007fe54d703808"}, - {file = "typed_ast-1.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:9c6d1a54552b5330bc657b7ef0eae25d00ba7ffe85d9ea8ae6540d2197a3788c"}, - {file = "typed_ast-1.4.3.tar.gz", hash = "sha256:fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65"}, + {file = "typed_ast-1.5.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d8314c92414ce7481eee7ad42b353943679cf6f30237b5ecbf7d835519e1212"}, + {file = "typed_ast-1.5.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b53ae5de5500529c76225d18eeb060efbcec90ad5e030713fe8dab0fb4531631"}, + {file = "typed_ast-1.5.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:24058827d8f5d633f97223f5148a7d22628099a3d2efe06654ce872f46f07cdb"}, + {file = "typed_ast-1.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:a6d495c1ef572519a7bac9534dbf6d94c40e5b6a608ef41136133377bba4aa08"}, + {file = "typed_ast-1.5.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:de4ecae89c7d8b56169473e08f6bfd2df7f95015591f43126e4ea7865928677e"}, + {file = "typed_ast-1.5.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:256115a5bc7ea9e665c6314ed6671ee2c08ca380f9d5f130bd4d2c1f5848d695"}, + {file = "typed_ast-1.5.1-cp36-cp36m-win_amd64.whl", hash = "sha256:7c42707ab981b6cf4b73490c16e9d17fcd5227039720ca14abe415d39a173a30"}, + {file = "typed_ast-1.5.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:71dcda943a471d826ea930dd449ac7e76db7be778fcd722deb63642bab32ea3f"}, + {file = "typed_ast-1.5.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4f30a2bcd8e68adbb791ce1567fdb897357506f7ea6716f6bbdd3053ac4d9471"}, + {file = "typed_ast-1.5.1-cp37-cp37m-win_amd64.whl", hash = "sha256:ca9e8300d8ba0b66d140820cf463438c8e7b4cdc6fd710c059bfcfb1531d03fb"}, + {file = "typed_ast-1.5.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9caaf2b440efb39ecbc45e2fabde809cbe56272719131a6318fd9bf08b58e2cb"}, + {file = "typed_ast-1.5.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c9bcad65d66d594bffab8575f39420fe0ee96f66e23c4d927ebb4e24354ec1af"}, + {file = "typed_ast-1.5.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:591bc04e507595887160ed7aa8d6785867fb86c5793911be79ccede61ae96f4d"}, + {file = "typed_ast-1.5.1-cp38-cp38-win_amd64.whl", hash = "sha256:a80d84f535642420dd17e16ae25bb46c7f4c16ee231105e7f3eb43976a89670a"}, + {file = "typed_ast-1.5.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:38cf5c642fa808300bae1281460d4f9b7617cf864d4e383054a5ef336e344d32"}, + {file = "typed_ast-1.5.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5b6ab14c56bc9c7e3c30228a0a0b54b915b1579613f6e463ba6f4eb1382e7fd4"}, + {file = "typed_ast-1.5.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a2b8d7007f6280e36fa42652df47087ac7b0a7d7f09f9468f07792ba646aac2d"}, + {file = "typed_ast-1.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:b6d17f37f6edd879141e64a5db17b67488cfeffeedad8c5cec0392305e9bc775"}, + {file = "typed_ast-1.5.1.tar.gz", hash = "sha256:484137cab8ecf47e137260daa20bafbba5f4e3ec7fda1c1e69ab299b75fa81c5"}, ] typing-extensions = [ {file = "typing_extensions-4.0.1-py3-none-any.whl", hash = "sha256:7f001e5ac290a0c0401508864c7ec868be4e701886d5b573a9528ed3973d9d3b"}, {file = "typing_extensions-4.0.1.tar.gz", hash = "sha256:4ca091dea149f945ec56afb48dae714f21e8692ef22a395223bcd328961b6a0e"}, ] urllib3 = [ - {file = "urllib3-1.26.7-py2.py3-none-any.whl", hash = "sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844"}, - {file = "urllib3-1.26.7.tar.gz", hash = "sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece"}, + {file = "urllib3-1.26.8-py2.py3-none-any.whl", hash = "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed"}, + {file = "urllib3-1.26.8.tar.gz", hash = "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c"}, ] validators = [ {file = "validators-0.18.2-py3-none-any.whl", hash = "sha256:0143dcca8a386498edaf5780cbd5960da1a4c85e0719f3ee5c9b41249c4fefbd"}, {file = "validators-0.18.2.tar.gz", hash = "sha256:37cd9a9213278538ad09b5b9f9134266e7c226ab1fede1d500e29e0a8fbb9ea6"}, ] virtualenv = [ - {file = "virtualenv-20.10.0-py2.py3-none-any.whl", hash = "sha256:4b02e52a624336eece99c96e3ab7111f469c24ba226a53ec474e8e787b365814"}, - {file = "virtualenv-20.10.0.tar.gz", hash = "sha256:576d05b46eace16a9c348085f7d0dc8ef28713a2cabaa1cf0aea41e8f12c9218"}, + {file = "virtualenv-20.13.0-py2.py3-none-any.whl", hash = "sha256:339f16c4a86b44240ba7223d0f93a7887c3ca04b5f9c8129da7958447d079b09"}, + {file = "virtualenv-20.13.0.tar.gz", hash = "sha256:d8458cf8d59d0ea495ad9b34c2599487f8a7772d796f9910858376d1600dd2dd"}, ] webencodings = [ {file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"}, {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"}, ] zipp = [ - {file = "zipp-3.6.0-py3-none-any.whl", hash = "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc"}, - {file = "zipp-3.6.0.tar.gz", hash = "sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832"}, + {file = "zipp-3.7.0-py3-none-any.whl", hash = "sha256:b47250dd24f92b7dd6a0a8fc5244da14608f3ca90a5efcd37a3b1642fac9a375"}, + {file = "zipp-3.7.0.tar.gz", hash = "sha256:9f50f446828eb9d45b267433fd3e9da8d801f614129124863f9c51ebceafb87d"}, ] diff --git a/pyproject.toml b/pyproject.toml index 0a861928..227bb89d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ python = "^3.7.1,<4.0" click = "^8.0.0" requests = "^2.23.0" toml = "^0.10.2" -rich = "^10.2.3" +rich = ">=10.2.3,<12.0.0" GitPython = "^3.1.18" semver = "^2.13.0" Jinja2 = "^3.0.1" @@ -65,7 +65,7 @@ email-validator = "^1.1.3" [tool.poetry.dev-dependencies] pytest = "^6.2.4" black = "^21.12b0" -mypy = "^0.930" +mypy = "^0.931" bandit = "^1.7.1" pycodestyle = "^2.7.0" flake8 = "^3.9.2" @@ -76,11 +76,12 @@ pytest-mock = "^3.6.1" pytest-dependency = "^0.5.1" pytest-cov = "^3.0.0" requests-mock = "^1.9.3" -deepdiff = "^5.7.0" +deepdiff = "^5.5.0" pytest-virtualenv = "^1.7.0" loremipsum = "^1.0.5" pre-commit = "^2.16.0" isort = "^5.10.1" +coverage = "^6.2" [tool.poetry.scripts] fair = 'fair.cli:cli' diff --git a/pytest.ini b/pytest.ini index 09032fd0..c39cbecf 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,17 +1,21 @@ [pytest] addopts = -s -v +testpaths= + tests markers= - history: tests for 'history' submodule - ids: tests for 'identifiers' submodule - common: tests for 'common' submodule - utilities: tests for 'utilities' submodule - configuration: tests for the 'configuration' submodule - versioning: tests for the 'registry.versioning' submodule - requests: tests for the 'registry.requests' submodule - storage: tests for the 'registry.storage' submodule - server: tests for the 'registry.server' submodule - user_config: tests for the 'user_config' submodule - staging: tests for the 'staging' submodule - cli: tests for the CLI itself - variables: tests for 'parsing.variables' submodule - with_api: tests using the Python API \ No newline at end of file + faircli_history: tests for 'history' submodule + faircli_ids: tests for 'identifiers' submodule + faircli_common: tests for 'common' submodule + faircli_utilities: tests for 'utilities' submodule + faircli_configuration: tests for the 'configuration' submodule + faircli_versioning: tests for the 'registry.versioning' submodule + faircli_requests: tests for the 'registry.requests' submodule + faircli_storage: tests for the 'registry.storage' submodule + faircli_server: tests for the 'registry.server' submodule + faircli_user_config: tests for the 'user_config' submodule + faircli_staging: tests for the 'staging' submodule + faircli_cli: tests for the CLI itself + faircli_run: 'fair run' tests + faircli_pull: 'fair pull' tests + faircli_push: 'fair push' tests + faircli_sync: sync tests diff --git a/tests/conftest.py b/tests/conftest.py index 208126e4..d476547a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,8 @@ import os import signal import tempfile +import git +import typing import pytest import pytest_fixture_config @@ -17,10 +19,56 @@ from . import registry_install as test_reg TEST_JOB_FILE_TIMESTAMP = "2021-10-11_10_0_0_100000" +PYTHON_API_GIT = "https://github.com/FAIRDataPipeline/pyDataPipeline.git" + +TEST_OUT_DIR = os.path.join(os.getcwd(), "test_outputs") +os.makedirs(TEST_OUT_DIR, exist_ok=True) logging.getLogger("FAIRDataPipeline").setLevel(logging.DEBUG) +def get_example_entries(registry_dir: str): + """ + With the registry examples regularly changing this function parses the + relevant file in the reg repository to obtain all example object metadata + """ + SEARCH_STR = "StorageLocation.objects.get_or_create" + _example_file = os.path.join( + registry_dir, + "data_management", + "management", + "commands", + "_example_data.py" + ) + + _objects: typing.List[typing.Tuple[str, str, str]] = [] + + with open(_example_file) as in_f: + _lines = in_f.readlines() + for i, line in enumerate(_lines): + if SEARCH_STR in line: + _path_line_offset = 0 + while "path" not in _lines[i+_path_line_offset]: + _path_line_offset += 1 + _candidate = _lines[i+_path_line_offset] + _candidate = _candidate.replace('"', "") + _candidate = _candidate.replace("path=", "") + _metadata, _file = _candidate.rsplit("/", 1) + _metadata = _metadata.replace("path=", "") + _version = ".".join(_file.split(".")[:3]) + _objects.append((*_metadata.split("/", 1), _version)) + + return _objects + + +@pytest.fixture() +def pyDataPipeline(): + with tempfile.TemporaryDirectory() as temp_d: + _repo_path = os.path.join(temp_d, 'repo') + _repo = git.Repo.clone_from(PYTHON_API_GIT, _repo_path) + _repo.git.checkout("dev") + yield _repo_path + @pytest.fixture(scope="session") @pytest_fixture_config.yield_requires_config( @@ -119,18 +167,20 @@ def job_log(mocker: pytest_mock.MockerFixture) -> str: class RegistryTest: - def __init__(self, install_loc: str, venv_dir: str, port: int = 8000): + def __init__(self, install_loc: str, venv: pytest_virtualenv.VirtualEnv, port: int = 8000): self._install = install_loc - self._venv = os.path.join(venv_dir, ".env") + self._venv = venv + self._venv_dir = os.path.join(venv.workspace, ".env") self._process = None self._port = port + self._url = f"http://127.0.0.1:{port}/api/" if not os.path.exists(os.path.join(install_loc, "manage.py")): test_reg.install_registry( - install_dir=install_loc, silent=True, venv_dir=self._venv + install_dir=install_loc, silent=True, venv_dir=self._venv_dir ) # Start then stop to generate key _process = test_reg.launch( - self._install, silent=True, venv_dir=self._venv, port=self._port + self._install, silent=True, venv_dir=self._venv_dir, port=self._port ) while not os.path.exists(os.path.join(self._install, "token")): time.sleep(5) @@ -140,13 +190,13 @@ def __init__(self, install_loc: str, venv_dir: str, port: int = 8000): def rebuild(self): test_reg.rebuild_local( - os.path.join(self._venv, "bin", "python"), self._install + os.path.join(self._venv_dir, "bin", "python"), self._install ) def __enter__(self): try: self._process = test_reg.launch( - self._install, silent=True, venv_dir=self._venv, port=self._port + self._install, silent=True, venv_dir=self._venv_dir, port=self._port ) except KeyboardInterrupt as e: os.kill(self._process.pid, signal.SIGTERM) @@ -157,19 +207,19 @@ def __exit__(self, type, value, tb): self._process = None -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def local_registry(session_virtualenv: pytest_virtualenv.VirtualEnv): if fdp_serv.check_server_running('http://127.0.0.1:8000'): pytest.skip("Cannot run registry tests, a server is already running on port 8000") with tempfile.TemporaryDirectory() as tempd: session_virtualenv.env = test_reg.django_environ(session_virtualenv.env) - yield RegistryTest(tempd, session_virtualenv.workspace, port=8000) + yield RegistryTest(tempd, session_virtualenv, port=8000) -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def remote_registry(session_virtualenv: pytest_virtualenv.VirtualEnv): if fdp_serv.check_server_running('http://127.0.0.1:8001'): pytest.skip("Cannot run registry tests, a server is already running on port 8001") with tempfile.TemporaryDirectory() as tempd: session_virtualenv.env = test_reg.django_environ(session_virtualenv.env) - yield RegistryTest(tempd, session_virtualenv.workspace, port=8001) + yield RegistryTest(tempd, session_virtualenv, port=8001) diff --git a/tests/data/test_config.yaml b/tests/data/test_config.yaml deleted file mode 100644 index 9a37319b..00000000 --- a/tests/data/test_config.yaml +++ /dev/null @@ -1,34 +0,0 @@ -run_metadata: - default_input_namespace: unit_testing - description: SEIRS Model R - script: R -f inst/extdata/SEIRS.R - -register: -- namespace: PSU - full_name: Pennsylvania State University - website: https://ror.org/04p491231 - -- external_object: SEIRS_model/parameters - namespace: PSU - root: https://raw.githubusercontent.com/ - path: FAIRDataPipeline/rSimpleModel/main/inst/extdata/static_params_SEIRS.csv - title: Parameters of SEIRS model - description: Static parameters of SEIRS model from Figure 1 - identifier: https://doi.org/10.1038/s41592-020-0856-2 - file_type: csv - release_date: 2020-06-01T12:00 - version: "1.0.0" - primary: False - -write: -- data_product: model_output - description: SEIRS model results - file_type: csv - use: - data_product: SEIRS_model/results/model_output/R - -- data_product: figure - description: SEIRS output plot - file_type: pdf - use: - data_product: SEIRS_model/results/figure/R diff --git a/tests/data/test_pull_config.yaml b/tests/data/test_pull_config.yaml new file mode 100644 index 00000000..60d3df9a --- /dev/null +++ b/tests/data/test_pull_config.yaml @@ -0,0 +1,8 @@ +run_metadata: + default_input_namespace: + description: pull testing +read: + - data_product: + use: + namespace: + version: \ No newline at end of file diff --git a/tests/data/test_wildcards_config.yaml b/tests/data/test_wildcards_config.yaml new file mode 100644 index 00000000..98d8187e --- /dev/null +++ b/tests/data/test_wildcards_config.yaml @@ -0,0 +1,5 @@ +run_metadata: + default_input_namespace: + description: wildcards testing +read: +- data_product: diff --git a/tests/registry_install.py b/tests/registry_install.py index b056b34a..1fc9e0aa 100644 --- a/tests/registry_install.py +++ b/tests/registry_install.py @@ -88,7 +88,7 @@ def rebuild_local(python: str, install_dir: str = None, silent: bool = False): def install_registry( repository: str = FAIR_REGISTRY_REPO, - head: str = "main", + reference: str = "", install_dir: str = None, silent: bool = False, force: bool = False, @@ -107,12 +107,11 @@ def install_registry( _repo = git.Repo.clone_from(repository, install_dir) - if head not in _repo.heads: - raise FileNotFoundError( - f"No such HEAD '{head}' in registry repository" - ) - else: - _repo.heads[head].checkout() + # If no reference is specified, use the latest tag for the registry + if not reference: + reference = _repo.tags[-1].name + + _repo.git.checkout(reference) if not venv_dir: venv_dir = os.path.join(install_dir, "venv") @@ -153,6 +152,10 @@ def install_registry( rebuild_local(_venv_python, install_dir, silent) + print(f"[REGISTRY] Installed registry version '{reference}'") + + return reference + def refresh( install_dir: str = None, silent: bool = False, venv_dir: str = None diff --git a/tests/test_cli.py b/tests/test_cli.py index 293b071a..966d2528 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -40,7 +40,7 @@ def click_test(): yield click_test -@pytest.mark.cli +@pytest.mark.faircli_cli def test_status( local_config: typing.Tuple[str, str], local_registry: conf.RegistryTest, @@ -73,7 +73,7 @@ def test_status( 'data_product': {}, } - _urls_list = {i: 'http://dummyurl.com' for i in _dummy_job_staging['job']} + _urls_list = {i: 'https://dummyurl.com' for i in _dummy_job_staging['job']} mocker.patch.object(fair.staging.Stager, 'get_job_data', lambda *args: _urls_list) mocker.patch('fair.registry.server.stop_server', lambda *args: None) @@ -89,7 +89,7 @@ def test_status( assert _result.exit_code == 0 -@pytest.mark.cli +@pytest.mark.faircli_cli def test_create( local_registry: conf.RegistryTest, click_test: click.testing.CliRunner, @@ -113,7 +113,7 @@ def test_create( assert os.path.exists(_out_config) -@pytest.mark.cli +@pytest.mark.faircli_cli def test_init_from_existing( local_registry: conf.RegistryTest, click_test: click.testing.CliRunner, @@ -158,7 +158,7 @@ def test_init_from_existing( assert os.path.exists(os.path.join(os.getcwd(), fdp_com.FAIR_FOLDER)) -@pytest.mark.cli +@pytest.mark.faircli_cli def test_init_from_env( local_registry: conf.RegistryTest, click_test: click.testing.CliRunner, @@ -205,7 +205,7 @@ def test_init_from_env( assert os.path.exists(os.path.join(os.getcwd(), fdp_com.FAIR_FOLDER)) -@pytest.mark.cli +@pytest.mark.faircli_cli def test_init_full( local_registry: conf.RegistryTest, click_test: click.testing.CliRunner, @@ -276,7 +276,7 @@ def test_init_full( assert _cli_cfg["user"]["uuid"] -@pytest.mark.cli +@pytest.mark.faircli_cli def test_purge( local_config: typing.Tuple[str, str], click_test: click.testing.CliRunner, @@ -309,7 +309,7 @@ def test_purge( ) -@pytest.mark.cli +@pytest.mark.faircli_cli def test_registry_cli( local_config: typing.Tuple[str, str], click_test: click.testing.CliRunner, diff --git a/tests/test_common.py b/tests/test_common.py index 84d86fc2..2f0523e3 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -10,7 +10,7 @@ import fair.exceptions as fdp_exc -@pytest.mark.common +@pytest.mark.faircli_common def test_find_git_root(): with tempfile.TemporaryDirectory() as tempd: with pytest.raises(fdp_exc.UserConfigError): @@ -22,7 +22,7 @@ def test_find_git_root(): assert fdp_com.find_git_root(_proj_dir) == tempd -@pytest.mark.common +@pytest.mark.faircli_common def test_find_fair_root(): with tempfile.TemporaryDirectory() as tempd: assert not fdp_com.find_fair_root(tempd) @@ -33,7 +33,7 @@ def test_find_fair_root(): assert fdp_com.find_fair_root(_proj_dir) == tempd -@pytest.mark.common +@pytest.mark.faircli_common def test_staging_cache(): with tempfile.TemporaryDirectory() as tempd: _fair_dir = os.path.join(tempd, fdp_com.FAIR_FOLDER) @@ -43,7 +43,7 @@ def test_staging_cache(): ) -@pytest.mark.common +@pytest.mark.faircli_common def test_default_data(mocker: pytest_mock.MockerFixture): with tempfile.TemporaryDirectory() as tempd: _glob_conf = os.path.join(tempd, "cli-config.yaml") @@ -65,7 +65,7 @@ def test_default_data(mocker: pytest_mock.MockerFixture): assert fdp_com.default_data_dir() == "data_store_1" -@pytest.mark.common +@pytest.mark.faircli_common def test_registry_home(mocker: pytest_mock.MockerFixture): with tempfile.TemporaryDirectory() as tempd: _glob_conf = os.path.join(tempd, "cli-config.yaml") diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 1a34b88a..41e38db7 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -11,7 +11,7 @@ import fair.common as fdp_com -@pytest.mark.configuration +@pytest.mark.faircli_configuration def test_local_cli_config_read(local_config: typing.Tuple[str, str]): _read = fdp_conf.read_local_fdpconfig(local_config[1]) assert _read["git"]["local_repo"] == os.path.join( @@ -20,7 +20,7 @@ def test_local_cli_config_read(local_config: typing.Tuple[str, str]): assert _read["namespaces"]["input"] == "testing" -@pytest.mark.configuration +@pytest.mark.faircli_configuration def test_global_cli_config_read(local_config: typing.Tuple[str, str]): _read = fdp_conf.read_global_fdpconfig() assert _read["git"]["local_repo"] == os.path.join( @@ -29,7 +29,7 @@ def test_global_cli_config_read(local_config: typing.Tuple[str, str]): assert _read["namespaces"]["input"] == "testing" -@pytest.mark.configuration +@pytest.mark.faircli_configuration def test_email_set(local_config: typing.Tuple[str, str]): TEST_EMAIL = "testemail@nowhere" TEST_EMAIL2 = "otheremail@nowhere" @@ -47,7 +47,7 @@ def test_email_set(local_config: typing.Tuple[str, str]): assert fdp_conf.read_global_fdpconfig()["user"]["email"] == TEST_EMAIL2 -@pytest.mark.configuration +@pytest.mark.faircli_configuration def test_user_set(local_config: typing.Tuple[str, str]): TEST_USER = "john smith" TEST_USER2 = "victor Chester bloggs" @@ -78,7 +78,7 @@ def test_user_set(local_config: typing.Tuple[str, str]): assert fdp_conf.read_global_fdpconfig()["user"]["family_name"] == "Bloggs" -@pytest.mark.configuration +@pytest.mark.faircli_configuration def test_get_user(local_config: typing.Tuple[str, str]): assert fdp_conf.get_current_user_name(local_config[1]) == ( "Interface", @@ -86,7 +86,7 @@ def test_get_user(local_config: typing.Tuple[str, str]): ) -@pytest.mark.configuration +@pytest.mark.faircli_configuration def test_get_remote_uri(local_config: typing.Tuple[str, str]): assert ( fdp_conf.get_remote_uri(local_config[1]) @@ -94,7 +94,7 @@ def test_get_remote_uri(local_config: typing.Tuple[str, str]): ) -@pytest.mark.configuration +@pytest.mark.faircli_configuration def test_get_remote_token(mocker: pytest_mock.MockerFixture): with tempfile.TemporaryDirectory() as tempd: _token = "t35tt0k3n" @@ -107,7 +107,7 @@ def test_get_remote_token(mocker: pytest_mock.MockerFixture): assert fdp_conf.get_remote_token("") == _token -@pytest.mark.configuration +@pytest.mark.faircli_configuration def test_get_git_remote(local_config: typing.Tuple[str, str]): _proj_dir = os.path.join(local_config[0], "project") assert fdp_conf.get_session_git_remote(_proj_dir) == "origin" @@ -117,7 +117,7 @@ def test_get_git_remote(local_config: typing.Tuple[str, str]): ) -@pytest.mark.configuration +@pytest.mark.faircli_configuration def test_get_orcid(local_config: typing.Tuple[str, str]): assert ( fdp_conf.get_current_user_uri(local_config[0]) @@ -125,7 +125,7 @@ def test_get_orcid(local_config: typing.Tuple[str, str]): ) -@pytest.mark.configuration +@pytest.mark.faircli_configuration def test_get_uuid(local_config: typing.Tuple[str, str]): assert ( fdp_conf.get_current_user_uuid(local_config[0]) @@ -133,7 +133,7 @@ def test_get_uuid(local_config: typing.Tuple[str, str]): ) -@pytest.mark.configuration +@pytest.mark.faircli_configuration def test_registry_exists( mocker: pytest_mock.MockerFixture, local_config: typing.Tuple[str, str] ): @@ -142,17 +142,17 @@ def test_registry_exists( assert fdp_conf.check_registry_exists(local_config[0]) -@pytest.mark.configuration +@pytest.mark.faircli_configuration def test_local_uri(local_config: typing.Tuple[str, str]): assert fdp_conf.get_local_uri() == "http://127.0.0.1:8000/api/" -@pytest.mark.configuration +@pytest.mark.faircli_configuration def test_local_port(local_config: typing.Tuple[str, str]): assert fdp_conf.get_local_port() == 8000 -@pytest.mark.configuration +@pytest.mark.faircli_configuration def test_user_info(mocker: pytest_mock.MockerFixture): _namepaces = {"input": "ispace", "output": "jbloggs"} _override = { @@ -195,7 +195,7 @@ def test_user_info(mocker: pytest_mock.MockerFixture): assert not deepdiff.DeepDiff(_orc, _expect_orcid) -@pytest.mark.configuration +@pytest.mark.faircli_configuration def test_global_config_query( mocker: pytest_mock.MockerFixture, local_config: typing.Tuple[str, str] ): @@ -256,7 +256,7 @@ def test_global_config_query( ) -@pytest.mark.configuration +@pytest.mark.faircli_configuration def test_local_config_query( local_config: typing.Tuple[str, str], mocker: pytest_mock.MockerFixture ): @@ -319,7 +319,7 @@ def test_local_config_query( assert not deepdiff.DeepDiff(_glob_conf, _usr_config) -@pytest.mark.configuration +@pytest.mark.faircli_configuration def test_update_port(local_config: typing.Tuple[str, str]): assert fdp_conf.get_local_uri() == fdp_com.DEFAULT_LOCAL_REGISTRY_URL fdp_conf.update_local_port() diff --git a/tests/test_history.py b/tests/test_history.py index bcc012a4..f77b765a 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -7,7 +7,7 @@ from fair.common import FAIR_FOLDER -@pytest.mark.history +@pytest.mark.faircli_history def test_history_directory(job_directory: str): os.makedirs(os.path.join(os.path.dirname(job_directory), FAIR_FOLDER)) _expected = os.path.join( @@ -16,7 +16,7 @@ def test_history_directory(job_directory: str): assert fdp_hist.history_directory(job_directory) == _expected -@pytest.mark.history +@pytest.mark.faircli_history def test_show_history( capsys: pytest.CaptureFixture, job_directory: str, job_log: str ): @@ -34,7 +34,7 @@ def test_show_history( ) -@pytest.mark.history +@pytest.mark.faircli_history def test_job_log_show( capsys: pytest.CaptureFixture, job_directory: str, job_log: str ): diff --git a/tests/test_identifiers.py b/tests/test_identifiers.py index 9c1c681d..9dd57a14 100644 --- a/tests/test_identifiers.py +++ b/tests/test_identifiers.py @@ -3,7 +3,7 @@ import fair.identifiers as fdp_id -@pytest.mark.ids +@pytest.mark.faircli_ids def test_check_orcid(): _data = fdp_id.check_orcid("0000-0002-6773-1049") assert _data["name"] == "Kristian Zarębski" @@ -13,7 +13,7 @@ def test_check_orcid(): assert not fdp_id.check_orcid("notanid!") -@pytest.mark.ids +@pytest.mark.faircli_ids def test_check_ror(): _data = fdp_id.check_ror("049s0ch10") assert _data["name"] == "Rakon (France)" == _data["family_name"] @@ -21,7 +21,7 @@ def test_check_ror(): assert not fdp_id.check_grid("notanid!") -@pytest.mark.ids +@pytest.mark.faircli_ids def test_check_grid(): _data = fdp_id.check_grid("grid.438622.9") assert _data["name"] == "Rakon (France)" == _data["family_name"] @@ -29,7 +29,7 @@ def test_check_grid(): assert not fdp_id.check_grid("notanid!") -@pytest.mark.ids +@pytest.mark.faircli_ids def test_check_permitted(): assert fdp_id.check_id_permitted("https://orcid.org/0000-0002-6773-1049") assert not fdp_id.check_id_permitted("notanid!") diff --git a/tests/test_requests.py b/tests/test_requests.py index c1217c41..2a7c31e8 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -12,7 +12,7 @@ LOCAL_URL = "http://127.0.0.1:8000/api" -@pytest.mark.requests +@pytest.mark.faircli_requests def test_split_url(): _test_url = "https://not_a_site.com/api/object?something=other" assert fdp_req.split_api_url(_test_url) == ( @@ -25,7 +25,7 @@ def test_split_url(): ) -@pytest.mark.requests +@pytest.mark.faircli_requests def test_local_token(mocker: pytest_mock.MockerFixture): _dummy_key = "sdfd234ersdf45234" with tempfile.TemporaryDirectory() as tempd: @@ -37,7 +37,7 @@ def test_local_token(mocker: pytest_mock.MockerFixture): assert fdp_req.local_token() == _dummy_key -@pytest.mark.requests +@pytest.mark.faircli_requests @pytest.mark.dependency(name="post") def test_post(local_registry: conf.RegistryTest, mocker: pytest_mock.MockerFixture): mocker.patch("fair.common.registry_home", lambda: local_registry._install) @@ -45,20 +45,21 @@ def test_post(local_registry: conf.RegistryTest, mocker: pytest_mock.MockerFixtu _orcid = "https://orcid.org/0000-0000-0000-0000" with local_registry: _result = fdp_req.post( - LOCAL_URL, "author", data={"name": _name, "identifier": _orcid} + LOCAL_URL, "author", local_registry._token, + data={"name": _name, "identifier": _orcid} ) assert _result["url"] -@pytest.mark.requests +@pytest.mark.faircli_requests @pytest.mark.dependency(name="get", depends=["post"]) def test_get(local_registry: conf.RegistryTest, mocker: pytest_mock.MockerFixture): mocker.patch("fair.common.registry_home", lambda: local_registry._install) with local_registry: - assert fdp_req.get(LOCAL_URL, "author") + assert fdp_req.get(LOCAL_URL, "author", local_registry._token) -@pytest.mark.requests +@pytest.mark.faircli_requests def test_post_else_get( local_registry: conf.RegistryTest, mocker: pytest_mock.MockerFixture ): @@ -73,7 +74,7 @@ def test_post_else_get( mock_get = mocker.patch("fair.registry.requests.get") # Perform method twice, first should post, second retrieve assert fdp_req.post_else_get( - LOCAL_URL, _obj_path, data=_data, params=_params + LOCAL_URL, _obj_path, local_registry._token, data=_data, params=_params ) mock_post.assert_called_once() @@ -93,6 +94,7 @@ def raise_it(*kwargs, **args): assert fdp_req.post_else_get( LOCAL_URL, "file_type", + local_registry._token, data={"name": "Comma Separated Values", "extension": "csv"}, params={"extension": "csv"}, ) @@ -100,27 +102,27 @@ def raise_it(*kwargs, **args): mock_get.assert_called_once() -@pytest.mark.requests +@pytest.mark.faircli_requests def test_filter_variables( local_registry: conf.RegistryTest, mocker: pytest_mock.MockerFixture ): mocker.patch("fair.common.registry_home", lambda: local_registry._install) with local_registry: - assert fdp_req.get_filter_variables(LOCAL_URL, "data_product") + assert fdp_req.get_filter_variables(LOCAL_URL, "data_product", local_registry._token) -@pytest.mark.requests +@pytest.mark.faircli_requests def test_writable_fields( local_registry: conf.RegistryTest, mocker: pytest_mock.MockerFixture ): mocker.patch("fair.common.registry_home", lambda: local_registry._install) with local_registry: fdp_req.filter_object_dependencies( - LOCAL_URL, "data_product", {"read_only": True} + LOCAL_URL, "data_product", local_registry._token, {"read_only": True} ) -@pytest.mark.requests +@pytest.mark.faircli_requests def test_download(local_registry: conf.RegistryTest, mocker: pytest_mock.MockerFixture): mocker.patch("fair.common.registry_home", lambda: local_registry._install) with local_registry: @@ -129,21 +131,21 @@ def test_download(local_registry: conf.RegistryTest, mocker: pytest_mock.MockerF assert os.path.exists(_out_file) -@pytest.mark.requests +@pytest.mark.faircli_requests def test_dependency_list( local_registry: conf.RegistryTest, mocker: pytest_mock.MockerFixture ): mocker.patch("fair.common.registry_home", lambda: local_registry._install) with local_registry: - _reqs = fdp_req.get_dependency_listing(LOCAL_URL) + _reqs = fdp_req.get_dependency_listing(LOCAL_URL, local_registry._token) assert _reqs["data_product"] == ["object", "namespace"] -@pytest.mark.requests +@pytest.mark.faircli_requests def test_object_type_fetch( local_registry: conf.RegistryTest, mocker: pytest_mock.MockerFixture ): mocker.patch("fair.common.registry_home", lambda: local_registry._install) with local_registry: for obj in ["object", "data_product", "author", "file_type"]: - assert fdp_req.get_obj_type_from_url(f"{LOCAL_URL}/{obj}") == obj + assert fdp_req.get_obj_type_from_url(f"{LOCAL_URL}/{obj}", local_registry._token) == obj diff --git a/tests/test_server.py b/tests/test_server.py index dbb5e1ae..abeeac6d 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -13,7 +13,7 @@ LOCAL_REGISTRY_URL = "http://127.0.0.1:8000/api" -@pytest.mark.server +@pytest.mark.faircli_server def test_check_server_running( local_registry: conf.RegistryTest, mocker: pytest_mock.MockerFixture ): @@ -23,7 +23,7 @@ def test_check_server_running( assert fdp_serv.check_server_running(LOCAL_REGISTRY_URL) -@pytest.mark.server +@pytest.mark.faircli_server def test_launch_stop_server( local_config: typing.Tuple[str, str], local_registry: conf.RegistryTest, @@ -36,7 +36,7 @@ def test_launch_stop_server( fdp_serv.stop_server(force=True) -@pytest.mark.server +@pytest.mark.faircli_server def test_registry_install_uninstall(mocker: pytest_mock.MockerFixture): with tempfile.TemporaryDirectory() as tempd: reg_dir = os.path.join(tempd, "registry") diff --git a/tests/test_staging.py b/tests/test_staging.py index bd66ace8..29fc12f4 100644 --- a/tests/test_staging.py +++ b/tests/test_staging.py @@ -3,6 +3,7 @@ import tempfile import typing import uuid +import git import pytest import pytest_mock @@ -24,7 +25,7 @@ def stager(local_config: typing.Tuple[str, str]): return _stager -@pytest.mark.staging +@pytest.mark.faircli_staging def test_job_status_change( stager: fdp_stage.Stager, mocker: pytest_mock.MockerFixture ): @@ -49,13 +50,13 @@ def test_job_status_change( assert not any(_dict["job"].values()) -@pytest.mark.staging +@pytest.mark.faircli_staging def test_registry_entry_for_file( stager: fdp_stage.Stager, mocker: pytest_mock.MockerFixture ): _url = "http://127.0.0.1:8000/api/storage_location/1" - def dummy_get(uri, obj_path, params): + def dummy_get(uri, obj_path, token, params): if uri != LOCAL_REGISTRY_URL: raise fdp_exc.RegistryError("No such registry") if obj_path != "storage_location": @@ -68,18 +69,20 @@ def dummy_get(uri, obj_path, params): "fair.registry.requests.get", lambda *args, **kwargs: dummy_get(*args, **kwargs), ) + mocker.patch("fair.registry.requests.local_token", lambda: "") assert ( stager.find_registry_entry_for_file(LOCAL_REGISTRY_URL, "/not/a/path") == _url ) -@pytest.mark.staging +@pytest.mark.faircli_staging def test_get_job_data( local_registry, stager: fdp_stage.Stager, local_config: typing.Tuple[str, str], mocker: pytest_mock.MockerFixture, + pyDataPipeline: str ): with local_registry: mocker.patch( @@ -110,8 +113,15 @@ def test_get_job_data( lambda *args, **kwargs: [{"url": _dummy_url}], ) + _cfg_path = os.path.join( + pyDataPipeline, + "simpleModel", + "ext", + "SEIRSconfig.yaml" + ) + shutil.copy( - os.path.join(TEST_DATA, "test_config.yaml"), + _cfg_path, os.path.join(_job_dir, fdp_com.USER_CONFIG_FILE), ) diff --git a/tests/test_storage.py b/tests/test_storage.py index c64e54e4..71f076c0 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -15,7 +15,7 @@ LOCAL_REGISTRY_URL = "http://127.0.0.1:8000/api" -@pytest.mark.storage +@pytest.mark.faircli_storage @pytest.mark.dependency(name="store_author") def test_store_user( local_config: typing.Tuple[str, str], @@ -24,10 +24,10 @@ def test_store_user( ): mocker.patch("fair.common.registry_home", lambda: local_registry._install) with local_registry: - assert fdp_store.store_user(local_config[1], LOCAL_URL) + assert fdp_store.store_user(local_config[1], LOCAL_URL, local_registry._token) -@pytest.mark.storage +@pytest.mark.faircli_storage def test_populate_file_type( local_config: typing.Tuple[str, str], local_registry: conf.RegistryTest, @@ -35,12 +35,12 @@ def test_populate_file_type( ): mocker.patch("fair.common.registry_home", lambda: local_registry._install) with local_registry: - assert len(fdp_store.populate_file_type(LOCAL_URL)) == len( + assert len(fdp_store.populate_file_type(LOCAL_URL, local_registry._token)) == len( fdp_file.FILE_TYPES ) -@pytest.mark.storage +@pytest.mark.faircli_storage def test_store_working_config( local_config: typing.Tuple[str, str], local_registry: conf.RegistryTest, @@ -56,11 +56,11 @@ def test_store_working_config( ) assert fdp_store.store_working_config( - local_config[1], LOCAL_URL, tempf.name + local_config[1], LOCAL_URL, tempf.name, local_registry._token ) -@pytest.mark.storage +@pytest.mark.faircli_storage def test_store_working_script( local_config: typing.Tuple[str, str], local_registry: conf.RegistryTest, @@ -78,11 +78,11 @@ def test_store_working_script( _temp_script = tempfile.NamedTemporaryFile(suffix=".sh", delete=False) assert fdp_store.store_working_script( - local_config[1], LOCAL_URL, _temp_script.name, tempf.name + local_config[1], LOCAL_URL, _temp_script.name, tempf.name, local_registry._token ) -@pytest.mark.storage +@pytest.mark.faircli_storage def test_store_namespace( local_registry: conf.RegistryTest, mocker: pytest_mock.MockerFixture ): @@ -90,13 +90,14 @@ def test_store_namespace( with local_registry: assert fdp_store.store_namespace( LOCAL_URL, + local_registry._token, "test_namespace", "Testing Namespace", "https://www.notarealsite.com", ) -@pytest.mark.storage +@pytest.mark.faircli_storage def test_calc_file_hash(): with tempfile.NamedTemporaryFile( mode="w+", suffix=".txt", delete=False diff --git a/tests/test_sync.py b/tests/test_sync.py new file mode 100644 index 00000000..0d1e62c1 --- /dev/null +++ b/tests/test_sync.py @@ -0,0 +1,93 @@ +import pytest +import tempfile +import os.path +import pathlib +import glob +import pytest_mock + +import fair.registry.sync as fdp_sync +import fair.registry.requests as fdp_req + +from .conftest import RegistryTest + + +@pytest.mark.faircli_sync +def test_pull_download(): + + _root = "https://github.com/" + _path = "FAIRDataPipeline/FAIR-CLI/blob/main/README.md" + + _file = fdp_sync.download_from_registry("http://127.0.0.1:8000", _root, _path) + + assert open(_file).read() + + +@pytest.mark.faircli_sync +def test_fetch_data_product( + mocker: pytest_mock.MockerFixture +): + + with tempfile.TemporaryDirectory() as tempd: + _dummy_data_product_name = "test" + _dummy_data_product_version = "2.3.0" + _dummy_data_product_namespace = "testing" + + def mock_get(url, obj, *args, **kwargs): + if obj == "storage_location": + return [{ + "path": "/this/is/a/dummy/path", + "storage_root": "https://dummyurl/" + }] + elif obj == "storage_root": + return [{ + "root": "https://fake/root/" + }] + elif obj == "namespace": + return [{ + "name": _dummy_data_product_namespace, + "url": "namespace" + }] + elif obj == "data_product": + return [{ + "data_product": _dummy_data_product_name, + "version": _dummy_data_product_version, + "namespace": "namespace" + }] + + def mock_url_get(url, *args, **kwargs): + if "storage_location" in url: + return { + "path": "FAIRDataPipeline/FAIR-CLI/archive/refs/heads/main.zip", + "storage_root": "storage_root" + } + elif "storage_root" in url: + return { + "root": "https://github.com/" + } + elif "namespace" in url: + return { + "name": _dummy_data_product_namespace, + "url": "namespace" + } + elif "object" in url: + return { + "storage_location": "storage_location", + "url": "object" + } + + mocker.patch("fair.registry.requests.get", mock_get) + mocker.patch("fair.registry.requests.url_get", mock_url_get) + + _example_data_product = { + "version": _dummy_data_product_version, + "namespace": "namespace", + "name": _dummy_data_product_name, + "data_product": _dummy_data_product_name, + "object": "object" + } + + fdp_sync.fetch_data_product( + "", + tempd, + _example_data_product + ) diff --git a/tests/test_user_config.py b/tests/test_user_config.py index 1f6d2673..bcf67f86 100644 --- a/tests/test_user_config.py +++ b/tests/test_user_config.py @@ -1,41 +1,46 @@ import os.path import typing -from datetime import datetime +import yaml import pytest import pytest_mock import fair.user_config as fdp_user -from fair.common import CMD_MODE +import fair.common as fdp_com from . import conftest as conf -TEST_USER_CONFIG = os.path.join( - os.path.dirname(__file__), "data", "test_config.yaml" +TEST_CONFIG_WC = os.path.join( + os.path.dirname(__file__), + "data", + "test_wildcards_config.yaml" ) - @pytest.fixture -def make_config(local_config: typing.Tuple[str, str]): - _config = fdp_user.JobConfiguration( - TEST_USER_CONFIG, +def make_config(local_config: typing.Tuple[str, str], pyDataPipeline: str): + _cfg_path = os.path.join( + pyDataPipeline, + "simpleModel", + "ext", + "SEIRSconfig.yaml" ) + _config = fdp_user.JobConfiguration(_cfg_path) _config.update_from_fair(os.path.join(local_config[1], "project")) return _config -@pytest.mark.user_config +@pytest.mark.faircli_user_config def test_get_value( local_config: typing.Tuple[str, str], make_config: fdp_user.JobConfiguration, ): - assert make_config["run_metadata.description"] == "SEIRS Model R" + assert make_config["run_metadata.description"] == "SEIRS Model python" assert make_config["run_metadata.local_repo"] == os.path.join( local_config[1], "project" ) -@pytest.mark.user_config +@pytest.mark.faircli_user_config def test_set_value(make_config: fdp_user.JobConfiguration): make_config["run_metadata.description"] = "a new description" assert ( @@ -44,24 +49,23 @@ def test_set_value(make_config: fdp_user.JobConfiguration): ) -@pytest.mark.user_config +@pytest.mark.faircli_user_config def test_is_public(make_config: fdp_user.JobConfiguration): assert make_config.is_public_global make_config["run_metadata.public"] = False assert not make_config.is_public_global - -@pytest.mark.user_config +@pytest.mark.faircli_user_config def test_default_input_namespace(make_config: fdp_user.JobConfiguration): - assert make_config.default_input_namespace == "unit_testing" + assert make_config.default_input_namespace == "rfield" -@pytest.mark.user_config +@pytest.mark.faircli_user_config def test_default_output_namespace(make_config: fdp_user.JobConfiguration): assert make_config.default_output_namespace == "testing" -@pytest.mark.user_config +@pytest.mark.faircli_user_config def test_preparation( mocker: pytest_mock.MockerFixture, make_config: fdp_user.JobConfiguration, @@ -70,5 +74,100 @@ def test_preparation( ): mocker.patch("fair.common.registry_home", lambda: local_registry._install) with local_registry: - make_config.prepare(local_config[1], datetime.now(), CMD_MODE.PULL) - make_config.write("test.yaml") + os.makedirs(os.path.join(local_config[1], fdp_com.FAIR_FOLDER, "logs")) + make_config.prepare(fdp_com.CMD_MODE.RUN, True) + + _out_dir = os.path.join(conf.TEST_OUT_DIR, "test_preparation") + os.makedirs(_out_dir, exist_ok=True) + + make_config.write(os.path.join(_out_dir, "out.yaml")) + + +@pytest.mark.faircli_user_config +def test_wildcard_unpack_local( + local_config: typing.Tuple[str, str], + mocker: pytest_mock.MockerFixture, + local_registry: conf.RegistryTest +): + with local_registry: + os.makedirs(os.path.join(local_config[1], fdp_com.FAIR_FOLDER, "logs")) + _manage = os.path.join(local_registry._install, "manage.py") + local_registry._venv.run(f"python {_manage} add_example_data", capture=True) + mocker.patch("fair.registry.requests.local_token", lambda *args: local_registry._token) + _data = os.path.join(local_registry._install, "data") + _example_entries = conf.get_example_entries(local_registry._install) + + _out_dir = os.path.join(conf.TEST_OUT_DIR, "test_wildcard_unpack_local") + os.makedirs(_out_dir, exist_ok=True) + + _namespace, _path, _ = _example_entries[0] + + _split_key = _path.split('/')[2] + + _wildcard_path = _path.split(_split_key)[0] + "*" + + with open(TEST_CONFIG_WC) as cfg_file: + _cfg_str = cfg_file.read() + + _cfg_str = _cfg_str.replace("", _namespace) + _cfg_str = _cfg_str.replace("", _wildcard_path) + + _cfg = yaml.safe_load(_cfg_str) + _cfg["run_metadata"]["write_data_store"] = _data + + _new_cfg_path = os.path.join(_out_dir, "in.yaml") + + yaml.dump(_cfg, open(_new_cfg_path, 'w')) + + _config = fdp_user.JobConfiguration(_new_cfg_path) + _config.update_from_fair(os.path.join(local_config[1], "project")) + _config.prepare(fdp_com.CMD_MODE.RUN, True) + assert len(_config["read"]) > 1 + + _config.write(os.path.join(_out_dir, "out.yaml")) + + +@pytest.mark.faircli_user_config +def test_wildcard_unpack_remote( + local_config: typing.Tuple[str, str], + mocker: pytest_mock.MockerFixture, + local_registry: conf.RegistryTest, + remote_registry: conf.RegistryTest +): + with local_registry, remote_registry: + os.makedirs(os.path.join(local_config[1], fdp_com.FAIR_FOLDER, "logs")) + _manage = os.path.join(remote_registry._install, "manage.py") + remote_registry._venv.run(f"python {_manage} add_example_data", capture=True) + mocker.patch("fair.registry.requests.local_token", lambda *args: local_registry._token) + mocker.patch("fair.configuration.get_remote_token", lambda *args: remote_registry._token) + _data = os.path.join(local_registry._install, "data") + _example_entries = conf.get_example_entries(remote_registry._install) + + _out_dir = os.path.join(conf.TEST_OUT_DIR, "test_wildcard_unpack_remote") + os.makedirs(_out_dir, exist_ok=True) + + _namespace, _path, _ = _example_entries[0] + + _split_key = _path.split('/')[2] + + _wildcard_path = _path.split(_split_key)[0] + "*" + + with open(TEST_CONFIG_WC) as cfg_file: + _cfg_str = cfg_file.read() + + _cfg_str = _cfg_str.replace("", _namespace) + _cfg_str = _cfg_str.replace("", _wildcard_path) + + _cfg = yaml.safe_load(_cfg_str) + _cfg["run_metadata"]["write_data_store"] = _data + + _new_cfg_path = os.path.join(_out_dir, "in.yaml") + + yaml.dump(_cfg, open(_new_cfg_path, 'w')) + + _config = fdp_user.JobConfiguration(_new_cfg_path) + _config.update_from_fair(os.path.join(local_config[1], "project")) + _config.prepare(fdp_com.CMD_MODE.PULL, True, remote_registry._url, remote_registry._token) + assert len(_config["read"]) > 1 + + _config.write(os.path.join(_out_dir, "out.yaml")) diff --git a/tests/test_utilities.py b/tests/test_utilities.py index 8600db11..99c4342f 100644 --- a/tests/test_utilities.py +++ b/tests/test_utilities.py @@ -6,21 +6,21 @@ import fair.utilities as fdp_util -@pytest.mark.utilities +@pytest.mark.faircli_utilities def test_flatten_dict(): _input = {"X": {"Y": "Z"}, "A": "B", "C": {"D": {"E": "F"}}} _expect = {"X.Y": "Z", "A": "B", "C.D.E": "F"} assert fdp_util.flatten_dict(_input) == _expect -@pytest.mark.utilities +@pytest.mark.faircli_utilities def test_expand_dict(): _expect = {"X": {"Y": "Z"}, "A": "B", "C": {"D": {"E": "F"}}} _input = {"X.Y": "Z", "A": "B", "C.D.E": "F"} assert fdp_util.expand_dict(_input) == _expect -@pytest.mark.utilities +@pytest.mark.faircli_utilities def test_remove_dictlist_dupes(): _a = {"X": "Y", "A": "B"} _b = {"X": "B", "A": "Z"} @@ -30,14 +30,14 @@ def test_remove_dictlist_dupes(): assert fdp_util.remove_dictlist_dupes(_input) == _expect -@pytest.mark.utilities +@pytest.mark.faircli_utilities def test_json_datetime_encoder(): _input = {"A": datetime.datetime.strptime("10:04", "%H:%M")} _expect = {"A": "1900-01-01 10:04:00"} assert json.loads(fdp_util.JSONDateTimeEncoder().encode(_input)) == _expect -@pytest.mark.utilities +@pytest.mark.faircli_utilities @pytest.mark.parametrize( "test_input,expected", [("lallero", "lallero/"), ("lallero/", "lallero/")] ) @@ -46,7 +46,7 @@ def test_trailing_slash(test_input, expected): assert result == expected -@pytest.mark.utilities +@pytest.mark.faircli_utilities def test_api_url_check(): _test_url = "http://127.0.0.1:8000/api/test-url" _not_url = "notaurl" diff --git a/tests/test_versioning.py b/tests/test_versioning.py index 4c2ffee7..c62050d4 100644 --- a/tests/test_versioning.py +++ b/tests/test_versioning.py @@ -4,7 +4,7 @@ import fair.registry.versioning as fdp_ver -@pytest.mark.versioning +@pytest.mark.faircli_versioning def test_incrementer_parsing(): for key in fdp_ver.BUMP_FUNCS: assert ( @@ -13,12 +13,12 @@ def test_incrementer_parsing(): ) -@pytest.mark.versioning +@pytest.mark.faircli_versioning def test_remove_incrementing(): assert fdp_ver.undo_incrementer("${{MINOR}}") == "${{ LATEST }}" -@pytest.mark.versioning +@pytest.mark.faircli_versioning def test_get_latest(): assert fdp_ver.get_latest_version() == semver.VersionInfo(0, 0, 0) results = [ @@ -29,7 +29,7 @@ def test_get_latest(): assert fdp_ver.get_latest_version(results) == semver.VersionInfo(2, 1, 0) -@pytest.mark.versioning +@pytest.mark.faircli_versioning def test_default_bump(): assert fdp_ver.default_bump( semver.VersionInfo(0, 1, 0) diff --git a/tests/test_with_api.py b/tests/test_with_api.py index 507a8673..ae07d7c4 100644 --- a/tests/test_with_api.py +++ b/tests/test_with_api.py @@ -1,50 +1,123 @@ -import os.path import pathlib import typing import yaml +import shutil +import os import click.testing -import git import pytest import pytest_mock from fair.cli import cli -from fair.common import FAIR_FOLDER, default_data_dir -from fair.registry.requests import get -from tests.conftest import RegistryTest +from fair.common import FAIR_FOLDER +from fair.registry.requests import get, url_get +from tests.conftest import RegistryTest, get_example_entries import fair.registry.server as fdp_serv -PYTHON_API_GIT = "https://github.com/FAIRDataPipeline/pyDataPipeline.git" REPO_ROOT = pathlib.Path(os.path.dirname(__file__)).parent +PULL_TEST_CFG = os.path.join(os.path.dirname(__file__), "data", "test_pull_config.yaml") -@pytest.mark.with_api -@pytest.mark.dependency(name='pull') -def test_pull(local_config: typing.Tuple[str, str], +@pytest.mark.faircli_pull +@pytest.mark.dependency(name='pull_new') +def test_pull_new(local_config: typing.Tuple[str, str], local_registry: RegistryTest, remote_registry: RegistryTest, mocker: pytest_mock.MockerFixture, + pyDataPipeline: str, capsys): + _manage = os.path.join(remote_registry._install, "manage.py") mocker.patch("fair.configuration.get_remote_token", lambda *args: remote_registry._token) mocker.patch("fair.registry.requests.local_token", lambda *args: local_registry._token) mocker.patch("fair.registry.server.launch_server", lambda *args, **kwargs: True) mocker.patch("fair.registry.server.stop_server", lambda *args: True) + mocker.patch("fair.registry.sync.fetch_data_product", lambda *args, **kwargs: None) _cli_runner = click.testing.CliRunner() - _proj_dir = os.path.join(local_config[1], "code") - _repo = git.Repo.clone_from(PYTHON_API_GIT, to_path=_proj_dir) - _repo.git.checkout("dev") - with _cli_runner.isolated_filesystem(_proj_dir): + with _cli_runner.isolated_filesystem(pyDataPipeline): with remote_registry, local_registry: - os.makedirs(os.path.join(_proj_dir, FAIR_FOLDER), exist_ok=True) + remote_registry._venv.run(f"python {_manage} add_example_data", capture=True) + os.makedirs(os.path.join(pyDataPipeline, FAIR_FOLDER), exist_ok=True) _data = os.path.join(local_registry._install, "data") + os.makedirs(_data, exist_ok=True) + fdp_serv.update_registry_post_setup(pyDataPipeline, True) + with open(os.path.join(pyDataPipeline, FAIR_FOLDER, "staging"), "w") as sf: + yaml.dump({"data_product": {}, "file": {}, "job": {}}, sf) + mocker.patch("fair.common.staging_cache", lambda *args: os.path.join(pyDataPipeline, FAIR_FOLDER, "staging")) + mocker.patch("fair.configuration.get_local_data_store", lambda *args: _data) + _namespace, _path, _version = get_example_entries(remote_registry._install)[0] + + with open(PULL_TEST_CFG) as cfg_file: + _cfg_str = cfg_file.read() + + _cfg_str = _cfg_str.replace("", _namespace) + _cfg_str = _cfg_str.replace("", _version) + _cfg_str = _cfg_str.replace("", _path) + + _cfg = yaml.safe_load(_cfg_str) + + assert get( + "http://127.0.0.1:8001/api/", + "data_product", + remote_registry._token, + params={ + "name": _path, + } + ) + + _cfg["run_metadata"]["write_data_store"] = _data + _cfg["run_metadata"]["local_repo"] = pyDataPipeline + + _new_cfg_path = os.path.join(os.path.dirname(pyDataPipeline), 'config.yaml') + + with open(_new_cfg_path, "w") as cfg_file: + yaml.dump(_cfg, cfg_file) + + with capsys.disabled(): + print(f"\tRUNNING: fair pull {_new_cfg_path} --debug") + _res = _cli_runner.invoke(cli, ["pull", _new_cfg_path, "--debug"]) + + assert _res.exit_code == 0 + + assert get( + "http://127.0.0.1:8000/api/", + "data_product", + local_registry._token, + params={ + "name": _path, + } + ) + + +@pytest.mark.faircli_run +@pytest.mark.faircli_push +@pytest.mark.faircli_pull +@pytest.mark.dependency(name='pull_existing') +def test_pull_existing(local_config: typing.Tuple[str, str], + local_registry: RegistryTest, + remote_registry: RegistryTest, + mocker: pytest_mock.MockerFixture, + pyDataPipeline: str, + capsys): + mocker.patch("fair.configuration.get_remote_token", lambda *args: remote_registry._token) + mocker.patch("fair.registry.requests.local_token", lambda *args: local_registry._token) + mocker.patch("fair.registry.server.launch_server", lambda *args, **kwargs: True) + mocker.patch("fair.registry.server.stop_server", lambda *args: True) + mocker.patch("fair.registry.sync.fetch_data_product", lambda *args, **kwargs: None) + _cli_runner = click.testing.CliRunner() + with _cli_runner.isolated_filesystem(pyDataPipeline): + with remote_registry, local_registry: + os.makedirs(os.path.join(pyDataPipeline, FAIR_FOLDER), exist_ok=True) + _data = os.path.join(local_registry._install, "data") + if os.path.exists(_data): + shutil.rmtree(_data) os.makedirs(_data) - fdp_serv.update_registry_post_setup(_proj_dir, True) - with open(os.path.join(_proj_dir, FAIR_FOLDER, "staging"), "w") as sf: + fdp_serv.update_registry_post_setup(pyDataPipeline, True) + with open(os.path.join(pyDataPipeline, FAIR_FOLDER, "staging"), "w") as sf: yaml.dump({"data_product": {}, "file": {}, "job": {}}, sf) - mocker.patch("fair.common.staging_cache", lambda *args: os.path.join(_proj_dir, FAIR_FOLDER, "staging")) + mocker.patch("fair.common.staging_cache", lambda *args: os.path.join(pyDataPipeline, FAIR_FOLDER, "staging")) mocker.patch("fair.configuration.get_local_data_store", lambda *args: _data) _cfg_path = os.path.join( - _proj_dir, + pyDataPipeline, "simpleModel", "ext", "SEIRSconfig.yaml" @@ -53,40 +126,74 @@ def test_pull(local_config: typing.Tuple[str, str], _cfg = yaml.safe_load(cfg_file) _cfg["run_metadata"]["write_data_store"] = _data - with open(_cfg_path, "w") as cfg_file: + _cfg["run_metadata"]["local_repo"] = pyDataPipeline + + _new_cfg_path = os.path.join(os.path.dirname(pyDataPipeline), 'config.yaml') + + with open(_new_cfg_path, "w") as cfg_file: yaml.dump(_cfg, cfg_file) + with capsys.disabled(): - print(f"\tRUNNING: fair pull {_cfg_path} --debug") - _res = _cli_runner.invoke(cli, ["pull", _cfg_path, "--debug"]) + print(f"\tRUNNING: fair pull {_new_cfg_path} --debug") + _res = _cli_runner.invoke(cli, ["pull", _new_cfg_path, "--debug"]) assert _res.exit_code == 0 - assert get( + + _param_files = get( "http://127.0.0.1:8000/api/", "data_product", + local_registry._token, params={ "name": "SEIRS_model/parameters", + "version": "1.0.0" } ) + + assert _param_files + assert get( "http://127.0.0.1:8000/api/", "namespace", + local_registry._token, params={ - "name": "testing" + "name": "PSU" } ) - assert get( - "http://127.0.0.1:8000/api/", - "user_author" - ) - -@pytest.mark.with_api -@pytest.mark.dependency(name='run', depends=['pull']) +@pytest.mark.faircli_pull +@pytest.mark.skipif('CI' in os.environ, reason="Fails on GH CI") +@pytest.mark.dependency(name='check_local_files', depends=['pull_existing']) +def test_local_files_present( + local_registry: RegistryTest + ): + with local_registry: + _param_files = get( + "http://127.0.0.1:8000/api/", + "data_product", + local_registry._token, + params={ + "name": "SEIRS_model/parameters", + "version": "1.0.0" + } + ) + _param_file_obj = url_get(_param_files[0]["object"], local_registry._token) + _store = url_get(_param_file_obj["storage_location"], local_registry._token) + _path = _store["path"] + _root = url_get(_store["storage_root"], local_registry._token) + _root = _root["root"] + + assert os.path.exists(os.path.join(_root.replace("file://", ""), _path)) + + +@pytest.mark.faircli_run +@pytest.mark.faircli_push +@pytest.mark.dependency(name='run', depends=['pull_existing']) def test_run(local_config: typing.Tuple[str, str], local_registry: RegistryTest, remote_registry: RegistryTest, mocker: pytest_mock.MockerFixture, + pyDataPipeline: str, capsys): try: import fairdatapipeline @@ -97,61 +204,74 @@ def test_run(local_config: typing.Tuple[str, str], mocker.patch("fair.registry.server.launch_server", lambda *args, **kwargs: True) mocker.patch("fair.registry.server.stop_server", lambda *args: True) _cli_runner = click.testing.CliRunner() - _proj_dir = os.path.join(local_config[1], "code") - _repo = git.Repo.clone_from(PYTHON_API_GIT, to_path=_proj_dir) - _repo.git.checkout("dev") - with _cli_runner.isolated_filesystem(_proj_dir): + with _cli_runner.isolated_filesystem(pyDataPipeline): with remote_registry, local_registry: - os.makedirs(os.path.join(_proj_dir, FAIR_FOLDER), exist_ok=True) + os.makedirs(os.path.join(pyDataPipeline, FAIR_FOLDER), exist_ok=True) _data = os.path.join(local_registry._install, "data") mocker.patch("fair.configuration.get_local_data_store", lambda *args: _data) os.makedirs(_data, exist_ok=True) - assert os.path.exists( - os.path.join( - _data, - "testing", - "SEIRS_model", - "parameters", - "1.0.0.csv" + with open(os.path.join(pyDataPipeline, FAIR_FOLDER, "staging"), "w") as sf: + yaml.dump( + { + "data_product": { + "testing:SEIRS_model/parameters@v1.0.0": False + }, + "file": {}, + "job": {} + }, + sf ) + + mocker.patch( + "fair.common.staging_cache", + lambda *args: os.path.join(pyDataPipeline, FAIR_FOLDER, "staging") ) - with open(os.path.join(_proj_dir, FAIR_FOLDER, "staging"), "w") as sf: - yaml.dump({"data_product": {"testing:SEIRS_model/parameters@v1.0.0": False}, "file": {}, "job": {}}, sf) - mocker.patch("fair.common.staging_cache", lambda *args: os.path.join(_proj_dir, FAIR_FOLDER, "staging")) assert get( "http://127.0.0.1:8000/api/", - "user_author" + "user_author", + local_registry._token ) _cfg_path = os.path.join( - _proj_dir, + pyDataPipeline, "simpleModel", "ext", "SEIRSconfig.yaml" ) + _new_cfg_path = os.path.join(os.path.dirname(pyDataPipeline), 'config.yaml') + with open(_cfg_path) as cfg_file: _cfg = yaml.safe_load(cfg_file) - _cfg["run_metadata"]["script"] = _cfg["run_metadata"]["script"].replace("src", f"{_proj_dir}/src") + _cfg["run_metadata"]["local_repo"] = pyDataPipeline _cfg["run_metadata"]["write_data_store"] = _data - with open(_cfg_path, "w") as cfg_file: + with open(_new_cfg_path, "w") as cfg_file: yaml.dump(_cfg, cfg_file) + assert os.path.exists(os.path.join(pyDataPipeline, "simpleModel", "ext", "SEIRSModelRun.py")) + with capsys.disabled(): - print(f"\tRUNNING: fair run {_cfg_path} --debug") + print(f"\tRUNNING: fair run {_new_cfg_path} --debug") - _res = _cli_runner.invoke(cli, ["run", _cfg_path, "--debug", "--dirty"]) + _res = _cli_runner.invoke(cli, ["run", _new_cfg_path, "--debug", "--dirty"]) assert _res.exit_code == 0 + print( + get("http://127.0.0.1:8000/api/", + "data_product", + local_registry._token) + ) + assert get( "http://127.0.0.1:8000/api/", "data_product", + local_registry._token, params={ "name": "SEIRS_model/results/figure/python", "version": "0.0.1" @@ -159,36 +279,33 @@ def test_run(local_config: typing.Tuple[str, str], ) -@pytest.mark.with_api -@pytest.mark.dependency(name='push', depends=['pull']) +@pytest.mark.faircli_push +@pytest.mark.dependency(name='push', depends=['pull_existing']) def test_push_initial(local_config: typing.Tuple[str, str], local_registry: RegistryTest, remote_registry: RegistryTest, mocker: pytest_mock.MockerFixture, + pyDataPipeline: str, capsys): mocker.patch("fair.configuration.get_remote_token", lambda *args: remote_registry._token) mocker.patch("fair.registry.requests.local_token", lambda *args: local_registry._token) mocker.patch("fair.registry.server.launch_server", lambda *args, **kwargs: True) mocker.patch("fair.registry.server.stop_server", lambda *args: True) _cli_runner = click.testing.CliRunner() - _proj_dir = os.path.join(local_config[1], "code") - _repo = git.Repo.clone_from(PYTHON_API_GIT, to_path=_proj_dir) - _repo.git.checkout("dev") - with _cli_runner.isolated_filesystem(_proj_dir): + with _cli_runner.isolated_filesystem(pyDataPipeline): with remote_registry, local_registry: - os.makedirs(os.path.join(_proj_dir, FAIR_FOLDER), exist_ok=True) + os.makedirs(os.path.join(pyDataPipeline, FAIR_FOLDER), exist_ok=True) _data = os.path.join(local_registry._install, "data") mocker.patch("fair.configuration.get_local_data_store", lambda *args: _data) - os.makedirs(_data, exist_ok=True) - with open(os.path.join(_proj_dir, FAIR_FOLDER, "staging"), "w") as sf: - yaml.dump({"data_product": {"testing:SEIRS_model/parameters@v1.0.0": False}, "file": {}, "job": {}}, sf) - mocker.patch("fair.common.staging_cache", lambda *args: os.path.join(_proj_dir, FAIR_FOLDER, "staging")) - fdp_serv.update_registry_post_setup(_proj_dir, True) + with open(os.path.join(pyDataPipeline, FAIR_FOLDER, "staging"), "w") as sf: + yaml.dump({"data_product": {"PSU:SEIRS_model/parameters@v1.0.0": False}, "file": {}, "job": {}}, sf) + mocker.patch("fair.common.staging_cache", lambda *args: os.path.join(pyDataPipeline, FAIR_FOLDER, "staging")) + fdp_serv.update_registry_post_setup(pyDataPipeline, True) with capsys.disabled(): - print("\tRUNNING: fair add testing:SEIRS_model/parameters@v1.0.0") + print("\tRUNNING: fair add PSU:SEIRS_model/parameters@v1.0.0") - _res = _cli_runner.invoke(cli, ["add", "testing:SEIRS_model/parameters@v1.0.0"]) + _res = _cli_runner.invoke(cli, ["add", "PSU:SEIRS_model/parameters@v1.0.0"]) assert _res.exit_code == 0 @@ -202,34 +319,32 @@ def test_push_initial(local_config: typing.Tuple[str, str], assert get( "http://127.0.0.1:8001/api/", "data_product", + remote_registry._token, params={"name": "SEIRS_model/parameters", "version": "1.0.0"}, - token=remote_registry._token ) -@pytest.mark.with_api -@pytest.mark.dependency(name='push', depends=['pull', 'run']) +@pytest.mark.faircli_push +@pytest.mark.dependency(name='push', depends=['pull_existing', 'run']) def test_push_postrun(local_config: typing.Tuple[str, str], local_registry: RegistryTest, remote_registry: RegistryTest, mocker: pytest_mock.MockerFixture, + pyDataPipeline: str, capsys): mocker.patch("fair.configuration.get_remote_token", lambda *args: remote_registry._token) mocker.patch("fair.registry.requests.local_token", lambda *args: local_registry._token) mocker.patch("fair.registry.server.launch_server", lambda *args, **kwargs: True) mocker.patch("fair.registry.server.stop_server", lambda *args: True) _cli_runner = click.testing.CliRunner() - _proj_dir = os.path.join(local_config[1], "code") - _repo = git.Repo.clone_from(PYTHON_API_GIT, to_path=_proj_dir) - _repo.git.checkout("dev") - with _cli_runner.isolated_filesystem(_proj_dir): + with _cli_runner.isolated_filesystem(pyDataPipeline): with remote_registry, local_registry: - os.makedirs(os.path.join(_proj_dir, FAIR_FOLDER), exist_ok=True) - with open(os.path.join(_proj_dir, FAIR_FOLDER, "staging"), "w") as sf: + os.makedirs(os.path.join(pyDataPipeline, FAIR_FOLDER), exist_ok=True) + with open(os.path.join(pyDataPipeline, FAIR_FOLDER, "staging"), "w") as sf: yaml.dump({"data_product": {"testing:SEIRS_model/results/figure/python@v0.0.1": False}, "file": {}, "job": {}}, sf) - mocker.patch("fair.common.staging_cache", lambda *args: os.path.join(_proj_dir, FAIR_FOLDER, "staging")) - fdp_serv.update_registry_post_setup(_proj_dir, True) - with open(os.path.join(_proj_dir, FAIR_FOLDER, "staging")) as cfg: + mocker.patch("fair.common.staging_cache", lambda *args: os.path.join(pyDataPipeline, FAIR_FOLDER, "staging")) + fdp_serv.update_registry_post_setup(pyDataPipeline, True) + with open(os.path.join(pyDataPipeline, FAIR_FOLDER, "staging")) as cfg: _staging = yaml.safe_load(cfg) assert "testing:SEIRS_model/results/figure/python@v0.0.1" in _staging["data_product"] mocker.patch("fair.configuration.get_local_data_store", lambda *args: os.path.join(local_registry._install, "data")) @@ -239,6 +354,7 @@ def test_push_postrun(local_config: typing.Tuple[str, str], assert get( "http://127.0.0.1:8000/api/", "data_product", + local_registry._token, params={ "name": "SEIRS_model/results/figure/python", "version": "0.0.1" @@ -259,6 +375,6 @@ def test_push_postrun(local_config: typing.Tuple[str, str], assert get( "http://127.0.0.1:8001/api/", "data_product", + remote_registry._token, params={"name": "SEIRS_model/results/figure/python", "version": "0.0.1"}, - token=remote_registry._token )