Draft: Licensing

.github/workflows/check_licenses.yml

@@ -0,0 +1,139 @@
+name: Check and update licenses
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches:  [ "main" ]
+      #    types: [opened, synchronized]
+permissions:
+  contents: read # we dont need to write
+
+jobs:
+  license_update:
+
+    strategy: 
+      fail-fast: false
+      matrix:
+        # we just do these two architectures for now as they are ones causing more discrepancies
+        include:
+         - runs_on: ubuntu-24.04-arm
+           EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/generic
+         - runs_on: ubuntu-24.04
+           EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/generic
+
+    runs-on: ${{ matrix.runs_on }}
+
+    steps:
+      -  uses: actions/checkout@v4
+      -  uses: eessi/github-action-eessi@v3
+
+      - name: Check for missing installlations
+        env:
+           PR_NUMBER: ${{ github.event.number }}
+
+        run: |
+          export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
+          source /cvmfs/software.eessi.io/versions/${EESSI_VERSION}/init/bash
+
+          # set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash),
+          # to prevent issues with checks in the Easybuild configuration that use this variable
+          export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*}
+          export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${EESSI_VERSION}
+          export EESSI_OS_TYPE=linux
+          env | grep ^EESSI | sort
+          module load EasyBuild
+
+          # create a temporary directory to store the output
+          LOCAL_TMPDIR=$(mktemp -d)
+          eb_missing_out=$LOCAL_TMPDIR/eb_missing.out
+          echo "eb_missing_out=$LOCAL_TMPDIR/eb_missing.out" >> $GITHUB_ENV
+          echo "Temporary directory created: ${eb_missing_out}"
+          file_list=$(curl -sS \
+                        -H "Accept: application/vnd.github+json" \
+                        "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?per_page=100" |
+                        jq -r '.[].filename | select(test("easystack"))')
+          echo "Files to check:"
+          echo $file_list
+
+          for easystack_file in $file_list; do
+              eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*.yml/\1/g')
+              echo "check missing installations for ${easystack_file} with EasyBuild ${eb_version}..."
+              module purge
+              module load EasyBuild/${eb_version}
+              module load EESSI-extend/${EESSI_VERSION}-easybuild
+              which eb
+              ${EB:-eb}  --version
+              ${EB:-eb}  --missing --easystack ${easystack_file} 2>&1 | tee ${eb_missing_out}
+
+              exit_code=${PIPESTATUS[0]}
+              echo "exit code for eb --missing --easystack ${easystack_file} is ${exit_code}"
+              grep " required modules missing:" ${eb_missing_out} # > /dev/null
+              exit_code=$?
+
+              if [[ ${exit_code} -eq 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2;
+              else
+                echo "no missing installations found for ${easystack_file}."
+                exit 0
+              fi  
+          done
+
+      - name: Check for modules existing in licenses.json file 
+        run: |
+          if [ -s licenses/licenses.json ]; then
+              echo "licenses.json file exists, checking for modules that are not in the file..."
+              echo "tmp file check:  ${eb_missing_out}"
+             # cat ${eb_missing_out}
+              grep -oP '^\* \K[^ ]+' "${eb_missing_out}" | sort -u  > missing.txt
+              echo "Modules to check"
+              cat missing.txt
+
+              # Check if module exists as key in JSON
+              while read -r module; do    
+                if ! jq -e --arg key "$module" 'has($key)' "licenses/licenses.json" > /dev/null; then
+                echo "$module" >> missing_modules.txt
+              fi
+              done < missing.txt
+
+              echo "Modules not in licenses.json: "
+              cat missing_modules.txt
+
+            else
+              echo "licenses.json file does not exist? what happened?"
+              exit 1
+            fi
+
+      - name : Search sources for missing modules
+        run: |
+                if [ -s missing_modules.txt ]; then
+                  echo "Searching sources for missing modules..."
+                  # Generates a "modules_results.json" file 
+                  module load Python-bundle-PyPI/2023.06-GCCcore-12.3.0
+                  python licenses/parsing_easyconfigs.py missing_modules.txt
+                  cat modules_results.json
+                fi
+
+      - name : Try to fetch the license 
+        run: |
+              if [ -s modules_results.json ]; then
+                echo "modules_results.json file exists, trying to fetch the license..."
+                module load Python-bundle-PyPI/2023.06-GCCcore-12.3.0
+                module load BeautifulSoup/4.12.2-GCCcore-12.3.0
+                python licenses/parse_licenses.py modules_results.json
+                cat licenses_test.json
+              else
+                echo "modules_results.json file does not exist, skipping license fetch."
+              fi
+
+      - name: Check for missing licenses
+        run: |
+              grep -3 "not found" licenses_test.json 
+              exit_code=$?
+              if [[ ${exit_code} -ne 0 ]]; then 
+                echo "No missing licenses found."
+              else
+                echo "Missing licenses found, please check the licenses.json file."
+                #exit 1
+              fi
+
+

easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml

@@ -45,3 +45,12 @@ easyconfigs:
       options:
         # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22469
         from-commit: fc22841fef99cbb2a221c18029b15e692e78c27c
+  - Scalasca-2.6.1-gompi-2023b.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21135
+        from-commit: c61e72465641ffb2852f60c5264386af690b2ea1
+  - LAMMPS-29Aug2024_update2-foss-2023b-kokkos-CUDA-12.4.0.eb:
+      options:
+        # testing purposes 
+        from-commit: cb372c4e096030c0bd285346b6b897842c998b92
+

licenses/README.md

@@ -0,0 +1,3 @@
+see https://spdx.org/licenses
+
+Python function to download SPDX list of licenses is available in `spdx.py`