Merge branch 'refactor' into feature/noise-model

.github/workflows/build_and_publish.yaml

@@ -1,25 +1,30 @@
 name: Build and publish
 
-on: [push, workflow_dispatch]
+on: 
+  pull_request:
+  workflow_dispatch:
+  push:
+    branches:
+      - master
 
 jobs:
   build:
 
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
     - name: Get the master branch (for checks)
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         ref: 'refs/heads/master'
+    - uses: actions/checkout@v4
 
     - name: Set up Python 3.7
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: 3.7
     - name: Cache pip
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         # This path is specific to Ubuntu
         path: ~/.cache/pip
@@ -34,20 +39,22 @@ jobs:
         python -m pip install --upgrade pip
         pip install poetry
       # poetry recommends using curl to install - probably doesn't make a difference though?
+
+    - name: Build NuRadioMC # let's always build - maybe this will occasionally catch errors
+      run: poetry build
+
     - name: Check if version is updated
+      if: ${{ github.ref != 'refs/heads/master'}} # don't run if we're on master
       run: |
-        git checkout origin/master
+        git checkout master
         export master_version=$(poetry version)
-        git checkout $GITHUB_REF_NAME
+        git checkout $GITHUB_SHA
         if [ "$master_version" = "$(poetry version)" ]
         then
           echo Version number is the same as existing release. Please update the version number!
-          # exit 1
+          exit 1
         fi
 
-    - name: Build NuRadioMC # let's always build - maybe this will occasionally catch errors
-      run: poetry build
-
     - name: Publish distribution 📦 to Test PyPI
       if: ${{ github.ref == 'refs/heads/master'}} # only runs if the push is to master
       uses: pypa/gh-action-pypi-publish@master
@@ -59,4 +66,23 @@ jobs:
       if: ${{ github.ref == 'refs/heads/master'}} # only runs if the push is to master
       uses: pypa/gh-action-pypi-publish@master
       with:
-        password: ${{ secrets.PYPI_API_TOKEN }}
+        password: ${{ secrets.PYPI_API_TOKEN }}
+
+    - name: Make tag and release
+      if: ${{ github.ref == 'refs/heads/master'}} # only runs if the push is to master
+      env:
+        GH_TOKEN: ${{ github.token}}
+      run: |
+        export VERSION_NEW="$(poetry version -s)"
+        export NEW_TAG=v$VERSION_NEW
+        git tag $NEW_TAG -m "$NEW_TAG release"
+        git push origin $NEW_TAG
+
+        # we want to include the changelog since the last release
+        # First, we use gh release and some regex to get the latest tag
+        export LATEST_TAG=$(gh release list | sed -rn 's/.*Latest\s*(\S+).*/\1/p')
+        # Next, we use some more regex to select
+        # the section of the changelog between the new and latest versions:
+        git diff --output-indicator-new=\| $LATEST_TAG $NEW_TAG changelog.txt | sed -rn "/^\|.*$VERSION_NEW/,/^[^|]/{s/\|//p}" | sed -n "2,\$p" > /tmp/changelog_$NEW_TAG.md
+
+        gh release create $NEW_TAG --notes-file /tmp/changelog_$NEW_TAG.md --generate-notes

.github/workflows/deploy_documentation.yaml

@@ -1,41 +1,25 @@
 name: Deploy Documentation
 
 on: [push, pull_request, workflow_dispatch]
-jobs:
-  prepare:
-    runs-on: ubuntu-latest
-    outputs:
-      runtests: ${{ steps.check.outputs.runtests }}
-    steps:
-    - id: check
-      run: |
-        output='false'
-        if [[ "${{ github.event_name }}" != pull_request ]]
-        then
-          runtests='true'
-        elif [[ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.event.pull_request.base.repo.full_name }}" ]]
-        then
-          runtests='true'
-        fi
-        echo "runtests=${runtests}" >> $GITHUB_OUTPUT
 
+jobs:
   build:
-    needs: [prepare]
-    if: needs.prepare.outputs.runtests
     runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
+
 
     steps:
-      - uses: actions/checkout@v3 # get the develop branch
+      - uses: actions/checkout@v4 # get the develop branch
         with:
           ref: 'refs/heads/develop'
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python 3.7
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.7
 
       - name: Cache pip
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           # This path is specific to Ubuntu
           path: ~/.cache/pip

.github/workflows/python_version_compatibility.yaml

@@ -0,0 +1,23 @@
+name: Check compatibility
+
+on:
+  push:
+    branches:
+      - develop
+  workflow_dispatch:
+
+
+jobs:
+  version:
+    strategy:
+      fail-fast: false # continue even if one of the versions fails
+      matrix:
+        python_version: ["3.8", "3.9", "3.10", "3.11", "3.x"] # 3.x = latest available
+        os: [ubuntu-latest]
+        include:
+          - python_version: "3.6"
+            os: ubuntu-20.04 # python 3.6 is not available with 22.04 on github actions
+    uses: ./.github/workflows/run_tests.yaml
+    with:
+      python_version: ${{ matrix.python_version }}
+      os: ${{ matrix.os }}

.github/workflows/run_tests.yaml

@@ -1,38 +1,47 @@
 name: Unit tests
 
-on: [push, pull_request, workflow_dispatch]
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      python_version:
+        description: Specify Python version to run tests on
+        type: string
+        required: true
+        default: '3.7'
+      os:
+        description: OS to run tests on
+        type: string
+        default: ubuntu-latest
+  workflow_call:
+    inputs:
+      python_version:
+        description: Specify Python version to run tests on
+        type: string
+        required: true
+        default: '3.7'
+      os:
+        description: OS to run tests on
+        type: string
+        default: ubuntu-latest
 
-jobs:
-  prepare:
-    runs-on: ubuntu-latest
-    outputs:
-      runtests: ${{ steps.check.outputs.runtests }}
-    steps:
-    - id: check
-      run: |
-        output='false'
-        if [[ "${{ github.event_name }}" != pull_request ]]
-        then
-          runtests='true'
-        elif [[ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.event.pull_request.base.repo.full_name }}" ]]
-        then
-          runtests='true'
-        fi
-        echo "runtests=${runtests}" >> $GITHUB_OUTPUT
+env: # set the python_version to 3.7 if the workflow is not triggered by workflow_call or workflow_dispatch
+  PYTHON_VERSION: ${{ inputs.python_version || '3.7' }}
 
+jobs:
   build:
-    needs: [prepare]
-    if: needs.prepare.outputs.runtests
-    runs-on: ubuntu-latest
+    runs-on: ${{ inputs.os || 'ubuntu-latest' }}
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
 
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v4
+    - uses: actions/checkout@v4
+    - name: Set up Python [${{ env.PYTHON_VERSION }}]
+      uses: actions/setup-python@v5
       with:
-        python-version: 3.7
+        python-version: ${{ env.PYTHON_VERSION }}
     - name: Cache pip
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         # This path is specific to Ubuntu
         path: ~/.cache/pip
@@ -61,6 +70,17 @@ jobs:
       run: |
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 . --count --max-complexity=10 --max-line-length=127 --statistics  --exit-zero
+    - name: "Detector"
+      if: always()
+      run: |
+        export PYTHONPATH=$(pwd):$PYTHONPATH
+        NuRadioReco/detector/test/tests.sh
+    - name: "RNO-G data reader"
+      if: always()
+      run: |
+        pip install git+https://github.com/RNO-G/mattak.git
+        export PYTHONPATH=$(pwd):$PYTHONPATH
+        NuRadioReco/test/RNO_G/test_read_rnog_data.sh
     - name: "Single event test (South Pole)"
       if: always()
       run: |
@@ -116,7 +136,7 @@ jobs:
       run: |
         export GSLDIR=$(gsl-config --prefix)
         export PYTHONPATH=$(pwd):$PYTHONPATH
-        NuRadioMC/test/examples/test_emitter_example.sh
+        NuRadioMC/test/examples/test_cal_pulser_example.sh
     - name: "Test radio emitter pulser example"
       if: always()
       run: |