From 85ca5a000653d3d5fe6f457309ec8b322fb79fca Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 12 Feb 2025 12:28:03 +0100 Subject: [PATCH 1/3] Add CI to compare stacks, checking for both modules and extensions --- .github/workflows/scripts/compare_stacks.py | 131 ++++++++++++++++++ .../workflows/scripts/compare_to_generic.sh | 26 ++++ .github/workflows/test_compare_stacks.yml | 51 +++++++ 3 files changed, 208 insertions(+) create mode 100644 .github/workflows/scripts/compare_stacks.py create mode 100755 .github/workflows/scripts/compare_to_generic.sh create mode 100644 .github/workflows/test_compare_stacks.yml diff --git a/.github/workflows/scripts/compare_stacks.py b/.github/workflows/scripts/compare_stacks.py new file mode 100644 index 0000000000..9e81f61538 --- /dev/null +++ b/.github/workflows/scripts/compare_stacks.py @@ -0,0 +1,131 @@ +import argparse +import os +import re +import glob +import json + + +def parse_module_file(module_file_path): + """ + Extracts module name, version, and extensions from a module file. + """ + module_name = os.path.basename(os.path.dirname(module_file_path)) + version = os.path.basename(module_file_path) + + try: + with open(module_file_path, "r") as file: + content = file.read() + + # Extract extensions from content using regex + match = re.search(r'extensions\("(.+)"\)', content) + extensions = [] + + if match: + # Split the list of packages by commas + packages = match.group(1) + for pkg in packages.split(","): + parts = pkg.split("/") + + # Check if the package is in the name/version format + if len(parts) == 2: + extensions.append((parts[0], parts[1])) + elif len(parts) == 1: + extensions.append((parts[0], "none")) + else: + print(f"Warning: Skipping invalid package format: {pkg}") + + return {(module_name, version): tuple(extensions)} + + except Exception as e: + print(f"Error parsing module file {module_file_path}: {e}") + return {(module_name, version): ()} + + +def get_available_modules(base_dir): + """ + Get the list of modules from all subdirectories inside the specified base directory. + """ + try: + modules = {} + # Only look for .lua files + for module_path in glob.glob(os.path.join(base_dir, "*/*.lua")): + modules.update(parse_module_file(module_path)) + return modules + + except Exception as e: + print(f"Error retrieving modules from {base_dir}: {e}") + return {} + + +def compare_stacks(dir1, dir2): + """ + Compare two sets of Lmod module files, including versions and extensions. + """ + modules1 = get_available_modules(dir1) + modules2 = get_available_modules(dir2) + + # Find differences between the two dictionaries + modules_removed = set(modules1.keys()) - set(modules2.keys()) + modules_added = set(modules2.keys()) - set(modules1.keys()) + matching_keys = set(modules1.keys()) & set(modules2.keys()) + + diff_results = { + "module_differences": { + "missing": list("/".join(module) for module in modules_removed), + "added": list("/".join(module) for module in modules_added), + }, + "extension_differences": [], + } + + # Compare extensions for matching keys + for key in matching_keys: + if modules1[key] != modules2[key]: + diff_results["extension_differences"].append( + { + "/".join(key): { + "missing": list( + "/".join(key) + for key in list(set(modules1[key]) - set(modules2[key])) + ), + "added": list( + "/".join(key) + for key in list(set(modules2[key]) - set(modules1[key])) + ), + } + } + ) + + return diff_results + + +def main(): + # Set up argument parser + parser = argparse.ArgumentParser(description="Compare two Lmod module directories") + parser.add_argument("path1", type=str, help="The first directory path") + parser.add_argument("path2", type=str, help="The second directory path") + + # Parse the arguments + args = parser.parse_args() + + # Validate the paths + for path in [args.path1, args.path2]: + if not os.path.exists(path): + print(f"Warning: Path does not exist: {path}") + + # Compare the stacks + diff_results = compare_stacks(args.path1, args.path2) + + # Print the differences + if any( + [ + diff_results["module_differences"]["missing"], + diff_results["module_differences"]["added"], + diff_results["extension_differences"], + ] + ): + print(json.dumps(diff_results, indent=2)) + exit(1) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/scripts/compare_to_generic.sh b/.github/workflows/scripts/compare_to_generic.sh new file mode 100755 index 0000000000..59a1397ec5 --- /dev/null +++ b/.github/workflows/scripts/compare_to_generic.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Take the arguments +base_dir=$1 +target_arch=$2 +modules_subdir="modules/all" +# Decide if we want x86_64 or aarch64 +arch=$(echo $target_arch | cut -d"/" -f1) +# Get the generic directory +source_of_truth="$arch/generic" +case $arch in + "x86_64") + echo "Using $source_of_truth as source of truth" + ;; + "aarch64") + echo "Using $source_of_truth as source of truth" + ;; + *) + echo "I don't understand the base architecture: $arch" + exit 1 + ;; +esac +source_of_truth_modules="$base_dir/$source_of_truth/$modules_subdir" +arch_modules="$base_dir/$target_arch/$modules_subdir" +echo "Comparing $arch_modules to $source_of_truth_modules" +python3 $script_dir/compare_stacks.py $source_of_truth_modules $arch_modules diff --git a/.github/workflows/test_compare_stacks.yml b/.github/workflows/test_compare_stacks.yml new file mode 100644 index 0000000000..8ad6bb6c8c --- /dev/null +++ b/.github/workflows/test_compare_stacks.yml @@ -0,0 +1,51 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Check for discrepancies between software stacks in software.eessi.io +on: + push: + branches: [ "*-software.eessi.io" ] + pull_request: + workflow_dispatch: +permissions: + contents: read # to fetch code (actions/checkout) +env: + EESSI_ACCELERATOR_TARGETS: | + x86_64/amd/zen2: + - nvidia/cc80 + x86_64/amd/zen3: + - nvidia/cc80 +jobs: + check_missing: + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + EESSI_VERSION: + - 2023.06 + COMPARISON_ARCH: + - aarch64/neoverse_n1 + - aarch64/neoverse_v1 + - x86_64/amd/zen2 + - x86_64/amd/zen3 + - x86_64/amd/zen4 + - x86_64/intel/haswell + - x86_64/intel/skylake_avx512 + steps: + - name: Check out software-layer repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Mount EESSI CernVM-FS pilot repository + uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0 + with: + cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb + cvmfs_http_proxy: DIRECT + cvmfs_repositories: software.eessi.io + + - name: Check + run: | + export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} + export EESSI_OS_TYPE=linux + env | grep ^EESSI | sort + + # Compare the requested architecture to the generic stack + # (assumes the general structure /cvmfs/software.eessi.io/versions/2023.06/software/linux/$COMPARISON_ARCH/modules/all) + .github/workflows/scripts/compare_to_generic.sh ${EESSI_PREFIX}/software/${EESSI_OS_TYPE} ${{matrix.COMPARISON_ARCH}} From d98f04b779ca860cfa3a6c3d9dee4201d5edf5d6 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 12 Feb 2025 12:34:39 +0100 Subject: [PATCH 2/3] Use EESSI action, improve names --- .github/workflows/test_compare_stacks.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test_compare_stacks.yml b/.github/workflows/test_compare_stacks.yml index 8ad6bb6c8c..bbb30a3f61 100644 --- a/.github/workflows/test_compare_stacks.yml +++ b/.github/workflows/test_compare_stacks.yml @@ -1,5 +1,5 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions -name: Check for discrepancies between software stacks in software.eessi.io +name: Check for discrepencies between software stacks in software.eessi.io on: push: branches: [ "*-software.eessi.io" ] @@ -14,7 +14,7 @@ env: x86_64/amd/zen3: - nvidia/cc80 jobs: - check_missing: + compare_stacks: runs-on: ubuntu-24.04 strategy: fail-fast: false @@ -34,13 +34,9 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Mount EESSI CernVM-FS pilot repository - uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0 - with: - cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb - cvmfs_http_proxy: DIRECT - cvmfs_repositories: software.eessi.io + uses: eessi/github-action-eessi@v3 - - name: Check + - name: Compare stacks run: | export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux From d7bf4d748a5b08310bdc8542a803e70cf83d9940 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 27 Feb 2025 16:33:12 +0100 Subject: [PATCH 3/3] Added sapphirerapids to check --- .github/workflows/test_compare_stacks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_compare_stacks.yml b/.github/workflows/test_compare_stacks.yml index bbb30a3f61..c47f174705 100644 --- a/.github/workflows/test_compare_stacks.yml +++ b/.github/workflows/test_compare_stacks.yml @@ -29,6 +29,7 @@ jobs: - x86_64/amd/zen4 - x86_64/intel/haswell - x86_64/intel/skylake_avx512 + - x86_64/intel/sapphirerapids steps: - name: Check out software-layer repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1