Merge branch 'develop' into kzscisoft/fix-binary-wrong-python-venv

.github/workflows/fair-cli.yaml

@@ -30,8 +30,17 @@ jobs:
         run: python -m poetry install
       - name: Install Python API for API Tests
         run: python -m poetry run pip install git+https://github.com/FAIRDataPipeline/pyDataPipeline.git@dev
-      - name: Run Tests
-        run: python -m poetry run pytest --cov=fair --cov-report=xml --cov-report=term -s tests/
+      - name: Run Tests for Each Marker
+        run: |
+            for marker in $(poetry run pytest --markers | grep -oE "faircli_[a-zA-Z|_|0-9]+")
+            do
+              echo "Running tests for marker '$marker'"
+              python -m poetry run pytest -m $marker --cov=fair --cov-report=xml --cov-report=term --cov-append -s tests/
+              exit_code=$?
+              if [ "$exit_code" != "0" ]; then
+                echo "ERROR: Tests for marker '$marker' failed"
+              fi             
+            done
       - uses: codecov/codecov-action@v2
         with:
           token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/implementations.yml

@@ -18,35 +18,34 @@ jobs:
           python-version: "3.9"
           architecture: "x64"
 
+      - name: Install memcached
+        run: sudo apt install -y libmemcached-dev
+
       - name: Install and initialise FAIR CLI
         run: |
           pip install poetry
           poetry install
 
-      - name: Setup Python API
+      - name: Build Python API
+        run: |
+          poetry install
+          poetry build
+        working-directory: python_example
+
+      - name: run Python Model with fair cli
         run: |
-          rm pyproject.toml poetry.lock # Remove conflicting poetry setup in Python API
+          rm pyproject.toml poetry.lock
+          poetry run pip install --find-links=dist/ pydatapipeline
           poetry run fair registry install
           poetry run fair registry install --directory ${GITHUB_WORKSPACE}/registry-rem
           poetry run fair init --ci
           poetry run fair registry start
           poetry run ${GITHUB_WORKSPACE}/registry-rem/scripts/start_fair_registry -p 8001
           cp ${GITHUB_WORKSPACE}/registry-rem/token $PWD/token
-          poetry run pip install git+https://github.com/FAIRDataPipeline/pyDataPipeline.git@dev # Install the Python API via Poetry itself
-        working-directory: python_example
-
-      - name: Test Pull
-        run: poetry run fair pull --debug simpleModel/ext/SEIRSconfig.yaml
-        working-directory: python_example
-
-      - name: Test Run
-        run: poetry run fair run --dirty --debug simpleModel/ext/SEIRSconfig.yaml
-        working-directory: python_example
-
-      - name: Test Push
-        run: |
-          poetry run fair add testing:SEIRS_model/parameters@v1.0.0
-          poetry run fair push
+          poetry run fair pull --debug simpleModel/ext/SEIRSconfig.yaml
+          poetry run fair run --dirty --debug simpleModel/ext/SEIRSconfig.yaml
+          poetry run fair add PSU:SEIRS_model/parameters@v1.0.0
+          poetry run fair push --dirty
         working-directory: python_example
 
   Java:
@@ -78,18 +77,16 @@ jobs:
         pip install poetry
         poetry install
 
-    - name: Setup javaSimpleModel
+    - name: run javaSimpleModel with fair cli
       run: |
         poetry run fair registry install
+        poetry run fair registry install --directory ${GITHUB_WORKSPACE}/registry-rem
+        poetry run fair registry start
+        poetry run ${GITHUB_WORKSPACE}/registry-rem/scripts/start_fair_registry -p 8001
+        cp ${GITHUB_WORKSPACE}/registry-rem/token $PWD/token
         poetry run fair init --ci
-      working-directory: java_example
-
-    - name: Test Pull
-      run: poetry run fair pull --debug src/main/resources/seirs-config.yaml
-      working-directory: java_example
-
-    - name: Test Run
-      run: poetry run fair run --dirty --debug src/main/resources/seirs-config.yaml
+        poetry run fair pull --debug src/main/resources/seirs-config.yaml
+        poetry run fair run --dirty --debug src/main/resources/seirs-config.yaml
       working-directory: java_example 
 
   R:
@@ -156,18 +153,16 @@ jobs:
         pip install poetry
         poetry install
 
-    - name: Setup rSimpleModel
+    - name: run rSimpleModel with fair cli
       run: |
         poetry run fair registry install
+        poetry run fair registry install --directory ${GITHUB_WORKSPACE}/registry-rem
+        poetry run fair registry start
+        poetry run ${GITHUB_WORKSPACE}/registry-rem/scripts/start_fair_registry -p 8001
+        cp ${GITHUB_WORKSPACE}/registry-rem/token $PWD/token
         poetry run fair init --ci
-      working-directory: r_example 
-
-    - name: Test Pull
-      run: poetry run fair pull --debug inst/extdata/SEIRSconfig.yaml
-      working-directory: r_example 
-
-    - name: Test Run
-      run: poetry run fair run --dirty --debug inst/extdata/SEIRSconfig.yaml
+        poetry run fair pull --debug inst/extdata/SEIRSconfig.yaml
+        poetry run fair run --dirty --debug inst/extdata/SEIRSconfig.yaml
       working-directory: r_example 
 
   Julia:
@@ -196,16 +191,57 @@ jobs:
         pip install poetry
         poetry install
 
-    - name: Setup SEIRS Model
+    - name: Run SEIRS Model with fair cli
       run: |
         poetry run fair registry install
+        poetry run fair registry install --directory ${GITHUB_WORKSPACE}/registry-rem
+        poetry run fair registry start
+        poetry run ${GITHUB_WORKSPACE}/registry-rem/scripts/start_fair_registry -p 8001
+        cp ${GITHUB_WORKSPACE}/registry-rem/token $PWD/token
         poetry run fair init --ci
+        poetry run fair pull --debug examples/fdp/SEIRSconfig.yaml
+        poetry run fair run --dirty --debug examples/fdp/SEIRSconfig.yaml
       working-directory: julia_example
 
-    - name: Test Pull
-      run: poetry run fair pull --debug examples/fdp/SEIRSconfig.yaml
-      working-directory: julia_example
-
-    - name: Test Run
-      run: poetry run fair run --dirty --debug examples/fdp/SEIRSconfig.yaml
-      working-directory: julia_example
+  CPlusPlus:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Clone CPlusPlus Model
+      uses: actions/checkout@v2
+      with:
+        repository: FAIRDataPipeline/cppSimpleModel
+        path: cpp_example
+        ref: 'main'
+
+    - name: Install requirements
+      run: |
+        sudo apt install -y libmemcached-dev graphviz gnuplot
+        sudo apt install -y libjsoncpp-dev curl libcurl4-openssl-dev
+        sudo apt install -y libyaml-cpp-dev libhdf5-dev
+
+    - name: Install Model
+      run: |
+        cmake -Bbuild
+        cmake --build build
+      working-directory: cpp_example
+
+    - name: install fair-cli
+      run: |
+        pip install poetry
+        poetry install
+      
+    - name: Run Example model with fair-cli
+      run: |
+        poetry run fair registry install
+        poetry run fair registry install --directory ${GITHUB_WORKSPACE}/registry-rem
+        poetry run fair registry start
+        poetry run ${GITHUB_WORKSPACE}/registry-rem/scripts/start_fair_registry -p 8001
+        cp ${GITHUB_WORKSPACE}/registry-rem/token $PWD/token
+        poetry run fair init --ci
+        poetry run fair pull --debug data/seirs_config.yaml
+        poetry run fair run --dirty --debug data/seirs_config.yaml
+      working-directory: cpp_example
+
+

CHANGELOG.md

@@ -1,4 +1,6 @@
 # Unreleased
+- Wildcard '*' parsing introduced for data products.
+- Ability to `push` to a registry added.
 - Added `--dirty` option to `fair run` to allow running with uncommitted changes.
 - Added `config.yaml` file validation.
 - Added initialisation from existing registry.

fair/cli.py

@@ -271,7 +271,8 @@ def install(debug: bool, force: bool, directory: str):
     try:
         if debug:
             logging.getLogger("FAIRDataPipeline").setLevel(logging.DEBUG)
-        fdp_svr.install_registry(install_dir=directory, force=force)
+        _version = fdp_svr.install_registry(install_dir=directory, force=force)
+        click.echo(f"Installed registry version '{_version}'")
     except fdp_exc.FAIRCLIException as e:
         if debug:
             raise e
@@ -424,7 +425,6 @@ def run(config: str, script: str, debug: bool, ci: bool, dirty: bool):
     """Initialises a job with the option to specify a bash command"""
     # Allow no config to be specified, if that is the case use default local
     config = config[0] if config else fdp_com.local_user_config(os.getcwd())
-    _run_mode = fdp_run.CMD_MODE.RUN if not ci else fdp_run.CMD_MODE.PASS
     try:
         with fdp_session.FAIR(
             os.getcwd(),
@@ -433,7 +433,7 @@ def run(config: str, script: str, debug: bool, ci: bool, dirty: bool):
             server_mode=fdp_svr.SwitchMode.CLI,
             allow_dirty=dirty
         ) as fair_session:
-            _hash = fair_session.run_job(script, mode=_run_mode)
+            _hash = fair_session.run(script, passive=ci, allow_dirty=dirty)
             if ci:
                 click.echo(fdp_run.get_job_dir(_hash))
     except fdp_exc.FAIRCLIException as e:
@@ -533,12 +533,18 @@ def modify(ctx, label: str, url: str, debug: bool) -> None:
 @cli.command()
 @click.argument("remote", nargs=-1)
 @click.option("--debug/--no-debug", help="Run in debug mode", default=False)
-def push(remote: str, debug: bool):
+@click.option(
+    "--dirty/--clean", help="Allow running with uncommitted changes", default=False
+)
+def push(remote: str, debug: bool, dirty: bool):
     """Push data between the local and remote registry"""
     remote = "origin" if not remote else remote[0]
     try:
         with fdp_session.FAIR(
-            os.getcwd(), debug=debug, server_mode=fdp_svr.SwitchMode.CLI
+            os.getcwd(),
+            debug=debug,
+            server_mode=fdp_svr.SwitchMode.CLI,
+            allow_dirty=dirty
         ) as fair_session:
             fair_session.push(remote)
     except fdp_exc.FAIRCLIException as e:
@@ -572,15 +578,16 @@ def config_email(user_email: str) -> None:
 @click.option("--debug/--no-debug")
 def pull(config: str, debug: bool):
     """Update local registry from remotes and sources"""
-    config = config[0] if config != "" else fdp_com.local_user_config(os.getcwd())
+    config = config[0] if config else fdp_com.local_user_config(os.getcwd())
     try:
         with fdp_session.FAIR(
             os.getcwd(),
             config,
             server_mode=fdp_svr.SwitchMode.CLI,
             debug=debug,
+            allow_dirty=True
         ) as fair:
-            fair.run_job(mode=fdp_run.CMD_MODE.PULL)
+            fair.pull()
     except fdp_exc.FAIRCLIException as e:
         if debug:
             raise e

fair/exceptions.py

@@ -30,6 +30,9 @@
 
 import click
 import typing
+import json
+import logging
+
 
 
 class FAIRCLIException(Exception):
@@ -152,10 +155,11 @@ class ValidationError(FAIRCLIException):
     def __init__(self, info: typing.List[typing.Dict]) -> None:
         _invalid_data: typing.List[typing.Dict] = []
 
-        for data in info:
-            _location = ":".join(data["loc"])
-            _type = data["type"]
-            _msg = data["msg"]
+        for data in json.loads(info):
+            _location = map(str, data["loc"])
+            _location = ":".join(_location)
+            _type = str(data["type"])
+            _msg = str(data["msg"])
             _invalid_data.append(f"{_location:<50}  {_type:<20}  {_msg:<20}")
 
         _msg = "User 'config.yaml' file validation failed with:\n"

fair/identifiers.py

@@ -149,6 +149,8 @@ def check_id_permitted(identifier: str, retries: int = 5) -> bool:
     ----------
     identifier : str
         identifier URL candidate
+    retries: int
+        number of attempts
 
     Returns
     -------