Update CI, testing and dependencies

- directory tree test is skipped until protopipe 0.5.0 is available
- CI has been updated to only support conda-based installation also on
  macos
- yaml needed to be added to the test dependencies
- conda recipes moved back to root folder

Update CI

Add pyyaml as a dependency for testing

Update CI installation

Remove unused import

Fix CI installation order

Test CI  on macos-latest

Try giving CONDA permission to its own files

Update CI

- add macos-latest
- update conda env name

Forgot to unset env name

Use unambiguous conda env name

Import load_config from utils not from protopipe

also skip test that requires new protopipe

leave protopipe import in submit jobs

Test CI only on macos-latest

Test CI only with conda-based installation

Test CI also on ubuntu-latest but only conda

.github/install.sh

@@ -2,17 +2,19 @@
 
 if [[ "$INSTALL_METHOD" == "conda" ]]; then
   echo "Using conda"
+  sudo chown -R $USER $CONDA # Give CONDA permission to its own files
   source $CONDA/etc/profile.d/conda.sh
   conda config --set always_yes yes --set changeps1 no
   conda update -q conda  # get latest conda version
   # Useful for debugging any issues with conda
   conda info -a
 
-  sed -i -e "s/- python=.*/- python=$PYTHON_VERSION/g" environment.yaml
+  sed -i -e "s/- python=.*/- python=$PYTHON_VERSION/g" environment_development.yaml
   conda install -c conda-forge mamba
-  mamba env create -n protopipe-grid-interface --file environment.yaml
-  conda activate protopipe-grid-interface
+  mamba env create -n CI --file environment_development.yaml
+  conda activate CI
 else
   echo "Using pip"
   pip install -U pip setuptools wheel
+  # This installation is not currently supported
 fi

.github/workflows/ci.yml

@@ -8,15 +8,16 @@ on: [push, pull_request]
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
   tests:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
+        os: [ubuntu-latest, macos-latest]
         python-version: [3.8]
-        install-method: ["conda", "pip"]
+        install-method: ["conda"]
     steps:
       - uses: actions/checkout@v2
         with:
-            fetch-depth: 0
+          fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
@@ -30,15 +31,15 @@ jobs:
         run: |
           source .github/install.sh
           python --version
-          pip install pytest codecov pytest-cov tables
+          pip install .[all]
           pip freeze
       - name: Tests
         env:
           INSTALL_METHOD: ${{ matrix.install-method }}
         run: |
           if [[ "$INSTALL_METHOD" == "conda" ]]; then
             source $CONDA/etc/profile.d/conda.sh
-            conda activate protopipe-grid-interface
+            conda activate CI
             conda list
           fi
           pytest --cov=protopipe --cov-report=xml

protopipe_grid_interface/scripts/submit_jobs.py

@@ -14,7 +14,7 @@
 except ImportError:
     raise ImportError("protopipe is not installed in this environment.")
 
-from protopipe_grid_interface.utils import initialize_logger
+from protopipe_grid_interface.utils import initialize_logger, load_config
 
 try:
     from DIRAC.Core.Base import Script

protopipe_grid_interface/scripts/tests/test_directory_tree.py

@@ -1,14 +1,16 @@
 from pathlib import Path
 import subprocess
 from pkg_resources import resource_filename
-
+import pytest
 import yaml
 
-from protopipe.pipeline.io import load_config
+from protopipe_grid_interface.utils import load_config
 
 
 def test_directory_tree(tmpdir):
 
+    pytest.importorskip("protopipe", minversion="0.5.0")
+
     test_analysis_workflow = tmpdir.join("test_analysis_workflow.yaml")
 
     tree = {
@@ -31,7 +33,9 @@ def test_directory_tree(tmpdir):
     subprocess.run(
         [
             "python",
-            resource_filename("protopipe_grid_interface", "scripts/create_analysis_tree.py"),
+            resource_filename(
+                "protopipe_grid_interface", "scripts/create_analysis_tree.py"
+            ),
             "--analysis_name",
             "test_analysis",
             "--analysis_directory_tree",
@@ -41,7 +45,8 @@ def test_directory_tree(tmpdir):
             "home_test",
             "--output_path",
             tmpdir,
-        ], check=True
+        ],
+        check=True,
     )
 
     analysis_path = tmpdir / "shared_folder/analyses/test_analysis"
@@ -57,7 +62,7 @@ def test_directory_tree(tmpdir):
         assert metadata["GRID is DIRAC"] == True
         assert metadata["Home directory on the GRID"] == "home_test"
         assert metadata["analysis directory on the GRID from home"] == ""
-        
+
     # Check that the created directory tree is the expected one
     assert analysis_path.exists()
     assert (analysis_path / "configs").exists()
@@ -71,4 +76,3 @@ def test_directory_tree(tmpdir):
     assert (analysis_path / "data/DL3").exists()
     assert (analysis_path / "estimators/energy_regressor").exists()
     assert (analysis_path / "estimators/gamma_hadron_classifier").exists()
-

protopipe_grid_interface/scripts/tests/test_merge.py

@@ -5,7 +5,6 @@
 from pkg_resources import resource_filename
 
 import tables as tb
-import numpy as np
 
 
 def create_mock_file(tmpdir, filename):

protopipe_grid_interface/utils.py

@@ -172,3 +172,24 @@ def upload(indir, infile, outdir, se=DEFAULT_SE):
         log.error(f"STDERR: {e.stderr}")
 
     return None
+
+
+def load_config(name):
+    """Load a YAML configuration file.
+
+    Parameters
+    ----------
+    name: str or pathlib.Path
+
+    Returns
+    -------
+    cfg: object
+        Python object (usually a dictionary).
+    """
+    try:
+        with open(name, "r") as stream:
+            cfg = yaml.load(stream, Loader=yaml.FullLoader)
+    except FileNotFoundError as e:
+        raise e
+
+    return cfg

setup.py

@@ -7,6 +7,8 @@
         "pytest-cov",
         "pytest-dependency",
         "codecov",
+        "pyyaml",
+        "tables",
     ],
 }
 
@@ -31,7 +33,7 @@
     package_data={
         "protopipe_grid_interface": ["aux/standard_analysis_workflow.yaml", "grid.yaml"]
     },
-    install_requires=["DIRAC", "CTADIRAC", "PyYAML", "tables"],
+    install_requires=["DIRAC", "CTADIRAC", "pyyaml", "tables"],
     zip_safe=False,
     use_scm_version={
         "write_to": os.path.join("protopipe_grid_interface", "_version.py")