Add push to remote functionality

.github/workflows/implementations.yml

@@ -29,10 +29,16 @@ jobs:
       - name: run Python Model with fair cli
         run: |
           poetry run fair registry install
+          poetry run fair registry install --directory ${GITHUB_WORKSPACE}/registry-rem
           poetry run fair init --ci
+          poetry run fair registry start
+          poetry run ${GITHUB_WORKSPACE}/registry-rem/scripts/start_fair_registry -p 8001
+          cp ${GITHUB_WORKSPACE}/registry-rem/token $PWD/token
           poetry run pip install .
           poetry run fair pull --debug src/org/fairdatapipeline/simpleModel/ext/SEIRSconfig.yaml
           poetry run fair run --dirty --debug src/org/fairdatapipeline/simpleModel/ext/SEIRSconfig.yaml
+          poetry run fair add testing:SEIRS_model/parameters@v1.0.0
+          poetry run fair push
         working-directory: python_example
 
   Java:

fair/cli.py

@@ -67,7 +67,7 @@ def status(verbose, debug) -> None:
         with fdp_session.FAIR(
             os.getcwd(), debug=debug, server_mode=fdp_svr.SwitchMode.CLI
         ) as fair_session:
-            fair_session.status(verbose)
+            fair_session.status_data_products()
     except fdp_exc.FAIRCLIException as e:
         if debug:
             raise e
@@ -302,19 +302,44 @@ def view(job_id: str, debug: bool) -> None:
 
 
 @cli.command()
-@click.argument("file_paths", type=click.Path(exists=True), nargs=-1)
+@click.argument("identifier")
 @click.option("--debug/--no-debug", help="Run in debug mode", default=False)
-def reset(file_paths: typing.List[str], debug: bool) -> None:
-    """Removes jobs from staging"""
-    pass
+@click.option("-j", "--job/--no-job", help="Stage entire job", default=False)
+def unstage(identifier: str, debug: bool, job: bool) -> None:
+    """Remove data products or jobs from staging"""
+    try:
+        with fdp_session.FAIR(os.getcwd(), debug=debug,) as fair_session:
+            fair_session.change_staging_state(
+                identifier,
+                "job" if job else "data_product",
+                stage=False,
+            )
+    except fdp_exc.FAIRCLIException as e:
+        if debug:
+            raise e
+        e.err_print()
+        if e.level.lower() == "error":
+            sys.exit(e.exit_code)
 
 
 @cli.command()
-@click.argument("job_ids", nargs=-1)
+@click.argument("identifier")
 @click.option("--debug/--no-debug", help="Run in debug mode", default=False)
-def addjob(job_ids: typing.List[str], debug: bool) -> None:
-    """Add a job to staging"""
-    pass
+@click.option("-j", "--job/--no-job", help="Stage entire job", default=False)
+def add(identifier: str, debug: bool, job: bool) -> None:
+    """Add a data product or job to staging"""
+    try:
+        with fdp_session.FAIR(os.getcwd(), debug=debug,) as fair_session:
+            fair_session.change_staging_state(
+                identifier,
+                "job" if job else "data_product",
+            )
+    except fdp_exc.FAIRCLIException as e:
+        if debug:
+            raise e
+        e.err_print()
+        if e.level.lower() == "error":
+            sys.exit(e.exit_code)
 
 
 @cli.command()

fair/configuration.py

@@ -285,7 +285,8 @@ def get_remote_token(repo_dir: str, remote: str = "origin") -> str:
 
     if not os.path.exists(_token_file):
         raise fdp_exc.FileNotFoundError(
-            f"Cannot read token for registry '{remote}', no such token file"
+            f"Cannot read token for registry '{remote}', token file '{_token_file}'"
+            " does not exist"
         )
 
     _token = open(_token_file).read().strip()
@@ -663,14 +664,15 @@ def global_config_query(registry: str = None) -> typing.Dict[str, typing.Any]:
         "Remote Data Storage Root", default=_remote_url.replace("api", "data")
     )
 
-    _default_token = os.path.join(registry, "remote-token")
-    _rem_key_file = click.prompt("Remote API Token File", default=_default_token)
+    _rem_key_file = click.prompt(
+        "Remote API Token File",
+    )
     _rem_key_file = os.path.expandvars(_rem_key_file)
 
-    # TODO fix search for valid token
-    while False and (
-        not os.path.exists(_rem_key_file) or not open(_rem_key_file).read().strip()
-    ):
+    while (
+        not os.path.exists(_rem_key_file)
+        or not open(_rem_key_file).read().strip()
+        ):
         click.echo(
             f"Token file '{_rem_key_file}' does not exist or is empty, "
             "please provide a valid token file."
@@ -829,11 +831,9 @@ def local_config_query(
         _def_rem_key = click.prompt("Remote API Token File", default=_def_rem_key)
         _def_rem_key = os.path.expandvars(_def_rem_key)
 
-        # TODO fix search for valid token
         while (
-            False
-            # not os.path.exists(_def_rem_key)
-            # or not open(_def_rem_key).read().strip()
+            not os.path.exists(_def_rem_key)
+            or not open(_def_rem_key).read().strip()
         ):
             click.echo(
                 f"Token file '{_def_rem_key}' does not exist or is empty, "
@@ -864,7 +864,8 @@ def local_config_query(
     # Local registry is a globally defined entity
     del _local_config["registries"]["local"]
 
-    _local_config["registries"]["origin"]["uri"] = _def_remote
+    _local_config['registries']['origin']['uri'] =  _def_remote
+    _local_config['registries']['origin']['token'] = _def_rem_key
 
     _local_config["user"] = _def_user
 

fair/registry/requests.py

@@ -90,7 +90,7 @@ def _access(
     uri: str,
     method: str,
     obj_path: str = None,
-    response_code: int = 200,
+    response_codes: typing.List[int] = [201, 200],
     token: str = None,
     headers: typing.Dict[str, typing.Any] = None,
     params: typing.Dict = None,
@@ -125,10 +125,12 @@ def _access(
 
     try:
         if method == "get":
+            logger.debug("Query parameters: %s", params)
             _request = requests.get(
                 _url, headers=_headers, params=params, *args, **kwargs
             )
         elif method == "post":
+            logger.debug("Post data: %s", data)
             _request = requests.post(_url, headers=_headers, data=data, *args, **kwargs)
         else:
             _request = getattr(requests, method)(
@@ -177,7 +179,7 @@ def _access(
             error_code=_request.status_code,
         )
 
-    if _request.status_code != response_code:
+    if _request.status_code not in response_codes:
         _info = ""
         if isinstance(_result, dict) and "detail" in _result:
             _info = _result["detail"]
@@ -228,7 +230,6 @@ def post(
         uri,
         "post",
         obj_path,
-        201,
         headers=headers,
         data=json.dumps(data, cls=fdp_util.JSONDateTimeEncoder),
         token=token,
@@ -350,7 +351,12 @@ def get(
         return _output
 
     return _access(
-        uri, "get", obj_path, 200, headers=headers, params=params, token=token
+        uri,
+        "get",
+        obj_path,
+        headers=headers,
+        params=params,
+        token=token
     )
 
 
@@ -388,19 +394,21 @@ def post_else_get(
         token = local_token()
 
     try:
+        logger.debug("Attempting to post an instance of '%s' to '%s'", obj_path, uri)
         _loc = post(uri, obj_path, data=data, token=token)
     except fdp_exc.RegistryAPICallError as e:
         # If the item is already in the registry then ignore the
         # conflict error and continue, else raise exception
         if e.error_code == 409:
-            _loc = get(uri, obj_path, params=params)
+            logger.debug("Object already exists, retrieving entry")
+            _loc = get(uri, obj_path, params=params, token=token)
         else:
             raise e
 
     if isinstance(_loc, list):
         if not _loc:
-            raise fdp_exc.RegistryAPICallError(
-                "Expected to receieve a URL location from registry post"
+            raise fdp_exc.RegistryError(
+                "Expected to receive a URL location from registry post"
             )
         _loc = _loc[0]
     if isinstance(_loc, dict):
@@ -428,7 +436,7 @@ def filter_object_dependencies(
         list of object type paths
     """
     try:
-        _actions = _access(uri, "options", obj_path, 200)["actions"]["POST"]
+        _actions = _access(uri, 'options', obj_path)['actions']['POST']
     except KeyError:
         # No 'actions' key means no dependencies
         return []
@@ -463,7 +471,7 @@ def get_filter_variables(uri: str, obj_path: str) -> typing.List[str]:
         list of filterable fields
     """
     try:
-        _filters = _access(uri, "options", obj_path, 200)["filter_fields"]
+        _filters = _access(uri, 'options', obj_path)['filter_fields']
     except KeyError:
         # No 'filter_fields' key means no filters
         return []

fair/registry/server.py

@@ -355,7 +355,7 @@ def install_registry(
         venv.create(
             venv_dir,
             with_pip=True,
-            prompt="TestRegistry",
+            prompt="RegistryTest",
         )
 
     _python_exe = "python.exe" if platform.system() == "Windows" else "python"
@@ -447,7 +447,7 @@ def update_registry_post_setup(repo_dir: str, global_setup: bool = False) -> Non
             fdp_conf.get_local_uri(), "users", params={"username": "admin"}
         )[0]["url"]
     except (KeyError, IndexError):
-        raise fdp_exc.RegistryAPICallError(
+        raise fdp_exc.RegistryError(
             "Failed to retrieve 'admin' user from registry database"
         )
 

fair/registry/storage.py

@@ -548,7 +548,7 @@ def get_storage_root_obj_address(
         if not _results:
             raise AssertionError
     except (AssertionError, fdp_exc.RegistryAPICallError):
-        raise fdp_exc.RegistryAPICallError(
+        raise fdp_exc.RegistryError(
             f"Cannot find a match for path '{address_str}' "
             f"from endpoint '{remote_uri}."
         )