From 83159f5f5f30dd8333b7e069fbb256d8102c975d Mon Sep 17 00:00:00 2001 From: Naman Gera Date: Wed, 8 Jan 2025 14:48:51 +0000 Subject: [PATCH 1/8] Downloaded data from Zenodo --- .../workflows/httomolibgpu_tests_run_iris.yml | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/.github/workflows/httomolibgpu_tests_run_iris.yml b/.github/workflows/httomolibgpu_tests_run_iris.yml index 44b78a30..b988287b 100644 --- a/.github/workflows/httomolibgpu_tests_run_iris.yml +++ b/.github/workflows/httomolibgpu_tests_run_iris.yml @@ -31,6 +31,43 @@ jobs: post-cleanup: 'all' init-shell: bash + - name: Download test data from Zenodo + run: | + # Install required tools + yum install -y curl jq + + # Create a directory to store test data + mkdir -p tests/large_data_archive + cd tests/large_data_archive + + # Download and process all files from Zenodo + echo "Fetching files from Zenodo record 14338424..." + curl -s "https://zenodo.org/api/records/14338424" | \ + jq -r '.files.entries[] | [.key, .links.content, .checksum] | @tsv' | \ + while IFS=$'\t' read -r filename url checksum; do + echo "Downloading ${filename}..." + curl -L -o "${filename}" "${url}" + + # Extract MD5 from checksum string (removes 'md5:' prefix) + expected_md5=${checksum#md5:} + actual_md5=$(md5sum "${filename}" | cut -d' ' -f1) + + if [ "${actual_md5}" = "${expected_md5}" ]; then + echo "✓ Verified ${filename}" + else + echo "✗ Checksum verification failed for ${filename}" + echo "Expected: ${expected_md5}" + echo "Got: ${actual_md5}" + exit 1 + fi + done + + # Show downloaded files + echo -e "\nDownloaded files:" + ls -lh + + cd ../.. + - name: Install httomolibgpu run: | pip install .[dev] From d077dd9e56c04fc7854c00764ab78efc014c6c29 Mon Sep 17 00:00:00 2001 From: Naman Gera Date: Wed, 8 Jan 2025 14:57:30 +0000 Subject: [PATCH 2/8] Use `dnf` for installation --- .github/workflows/httomolibgpu_tests_run_iris.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/httomolibgpu_tests_run_iris.yml b/.github/workflows/httomolibgpu_tests_run_iris.yml index b988287b..b5ddd6ce 100644 --- a/.github/workflows/httomolibgpu_tests_run_iris.yml +++ b/.github/workflows/httomolibgpu_tests_run_iris.yml @@ -34,7 +34,8 @@ jobs: - name: Download test data from Zenodo run: | # Install required tools - yum install -y curl jq + dnf install -y epel-release + dnf install -y curl jq # Create a directory to store test data mkdir -p tests/large_data_archive From f2d3075899c3d222a3970b05cb822a1795855b49 Mon Sep 17 00:00:00 2001 From: Naman Gera Date: Thu, 9 Jan 2025 13:34:24 +0000 Subject: [PATCH 3/8] Use a python script instead --- .../workflows/httomolibgpu_tests_run_iris.yml | 35 +--------- .scripts/download_zenodo.py | 70 +++++++++++++++++++ 2 files changed, 73 insertions(+), 32 deletions(-) create mode 100755 .scripts/download_zenodo.py diff --git a/.github/workflows/httomolibgpu_tests_run_iris.yml b/.github/workflows/httomolibgpu_tests_run_iris.yml index b5ddd6ce..8a7bfa17 100644 --- a/.github/workflows/httomolibgpu_tests_run_iris.yml +++ b/.github/workflows/httomolibgpu_tests_run_iris.yml @@ -33,41 +33,12 @@ jobs: - name: Download test data from Zenodo run: | - # Install required tools - dnf install -y epel-release - dnf install -y curl jq + chmod +x ./.scripts/download_zenodo.py - # Create a directory to store test data - mkdir -p tests/large_data_archive - cd tests/large_data_archive + ./.scripts/download_zenodo.py tests/large_data_archive - # Download and process all files from Zenodo - echo "Fetching files from Zenodo record 14338424..." - curl -s "https://zenodo.org/api/records/14338424" | \ - jq -r '.files.entries[] | [.key, .links.content, .checksum] | @tsv' | \ - while IFS=$'\t' read -r filename url checksum; do - echo "Downloading ${filename}..." - curl -L -o "${filename}" "${url}" - - # Extract MD5 from checksum string (removes 'md5:' prefix) - expected_md5=${checksum#md5:} - actual_md5=$(md5sum "${filename}" | cut -d' ' -f1) - - if [ "${actual_md5}" = "${expected_md5}" ]; then - echo "✓ Verified ${filename}" - else - echo "✗ Checksum verification failed for ${filename}" - echo "Expected: ${expected_md5}" - echo "Got: ${actual_md5}" - exit 1 - fi - done - - # Show downloaded files echo -e "\nDownloaded files:" - ls -lh - - cd ../.. + ls -lh tests/large_data_archive - name: Install httomolibgpu run: | diff --git a/.scripts/download_zenodo.py b/.scripts/download_zenodo.py new file mode 100755 index 00000000..76de5010 --- /dev/null +++ b/.scripts/download_zenodo.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 + +import json +import urllib.request +import hashlib +import sys +import os +from pathlib import Path + + +def calculate_md5(filename): + """Calculate MD5 hash of a file.""" + md5_hash = hashlib.md5() + with open(filename, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + md5_hash.update(chunk) + return md5_hash.hexdigest() + + +def download_zenodo_files(output_dir: Path): + """ + Download all files from Zenodo record 14338424 and verify their checksums. + + Args: + output_dir: Directory where files should be downloaded + """ + try: + print("Fetching files from Zenodo record 14338424...") + with urllib.request.urlopen("https://zenodo.org/api/records/14338424") as response: + data = json.loads(response.read()) + + # Create output directory if it doesn't exist + output_dir.mkdir(parents=True, exist_ok=True) + + # Now 'files' is a list, not a dictionary + for file_info in data["files"]: + filename = file_info["key"] # The 'key' is the filename + output_file = output_dir / filename + print(f"Downloading {filename}...") + url = file_info["links"]["self"] # The link to download the file + + expected_md5 = file_info["checksum"].split(":")[1] # Extract MD5 hash + + # Download the file + urllib.request.urlretrieve(url, output_file) + + # Verify checksum + actual_md5 = calculate_md5(output_file) + if actual_md5 == expected_md5: + print(f"✓ Verified {filename}") + else: + print(f"✗ Checksum verification failed for {filename}") + print(f"Expected: {expected_md5}") + print(f"Got: {actual_md5}") + sys.exit(1) + + print("\nAll files downloaded and verified successfully!") + + except Exception as e: + print(f"Error: {str(e)}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Usage: download_zenodo.py ") + sys.exit(1) + + output_dir = Path(sys.argv[1]) + download_zenodo_files(output_dir) From 64b2b68720973d7b6fcbf3560bfb25084e4ba222 Mon Sep 17 00:00:00 2001 From: Naman Gera Date: Thu, 9 Jan 2025 14:23:30 +0000 Subject: [PATCH 4/8] Add tests from httomolib-data --- .../workflows/httomolibgpu_tests_run_iris.yml | 6 +- zenodo-tests/conftest.py | 67 ++++++++++++++++++ zenodo-tests/test_recon/__init__.py | 0 zenodo-tests/test_recon/test_rotation.py | 69 +++++++++++++++++++ 4 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 zenodo-tests/conftest.py create mode 100644 zenodo-tests/test_recon/__init__.py create mode 100644 zenodo-tests/test_recon/test_rotation.py diff --git a/.github/workflows/httomolibgpu_tests_run_iris.yml b/.github/workflows/httomolibgpu_tests_run_iris.yml index 8a7bfa17..9ef8ccc2 100644 --- a/.github/workflows/httomolibgpu_tests_run_iris.yml +++ b/.github/workflows/httomolibgpu_tests_run_iris.yml @@ -45,6 +45,10 @@ jobs: pip install .[dev] micromamba list - - name: Run tests + - name: Run unit tests on small data run: | pytest tests/ + + - name: Run Zenodo tests + run: | + pytest zenodo-tests/ diff --git a/zenodo-tests/conftest.py b/zenodo-tests/conftest.py new file mode 100644 index 00000000..c8576f02 --- /dev/null +++ b/zenodo-tests/conftest.py @@ -0,0 +1,67 @@ +import os +import cupy as cp +import numpy as np +import pytest + +CUR_DIR = os.path.abspath(os.path.dirname(__file__)) + + +@pytest.fixture(scope="session") +def test_data_path(): + return os.path.join(CUR_DIR, "large_data_archive") + + +@pytest.fixture(scope="session") +def data_i12LFOV_file(test_data_path): + in_file = os.path.join(test_data_path, "i12LFOV.npz") + return np.load(in_file) + + +@pytest.fixture(scope="session") +def data_i12_sandstone_file(test_data_path): + in_file = os.path.join(test_data_path, "i12_sandstone_50sinoslices.npz") + return np.load(in_file) + + +@pytest.fixture(scope="session") +def data_geant4sim_file(test_data_path): + in_file = os.path.join(test_data_path, "geant4_640_540_proj360.npz") + return np.load(in_file) + +@pytest.fixture +def i12LFOV_data(data_i12LFOV_file): + return ( + cp.asarray(data_i12LFOV_file["projdata"]), + data_i12LFOV_file["angles"], + cp.asarray(data_i12LFOV_file["flats"]), + cp.asarray(data_i12LFOV_file["darks"]), + ) + + +@pytest.fixture +def i12sandstone_data(data_i12_sandstone_file): + return ( + cp.asarray(data_i12_sandstone_file["projdata"]), + data_i12_sandstone_file["angles"], + cp.asarray(data_i12_sandstone_file["flats"]), + cp.asarray(data_i12_sandstone_file["darks"]), + ) + + +@pytest.fixture +def geantsim_data(data_geant4sim_file): + return ( + cp.asarray(data_geant4sim_file["projdata"]), + data_geant4sim_file["angles"], + cp.asarray(data_geant4sim_file["flats"]), + cp.asarray(data_geant4sim_file["darks"]), + ) + + +@pytest.fixture +def ensure_clean_memory(): + cp.get_default_memory_pool().free_all_blocks() + cp.get_default_pinned_memory_pool().free_all_blocks() + yield None + cp.get_default_memory_pool().free_all_blocks() + cp.get_default_pinned_memory_pool().free_all_blocks() diff --git a/zenodo-tests/test_recon/__init__.py b/zenodo-tests/test_recon/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/zenodo-tests/test_recon/test_rotation.py b/zenodo-tests/test_recon/test_rotation.py new file mode 100644 index 00000000..536a0ace --- /dev/null +++ b/zenodo-tests/test_recon/test_rotation.py @@ -0,0 +1,69 @@ +import cupy as cp +import numpy as np +import pytest + +from httomolibgpu.prep.normalize import normalize +from httomolibgpu.recon.rotation import find_center_vo + + +def test_center_vo_i12LFOV(i12LFOV_data, ensure_clean_memory): + projdata = i12LFOV_data[0] + flats = i12LFOV_data[2] + darks = i12LFOV_data[3] + del i12LFOV_data + + data_normalised = normalize(projdata, flats, darks, minus_log=False) + del flats, darks, projdata + + mid_slice = data_normalised.shape[1] // 2 + cor = find_center_vo(data_normalised[:, mid_slice, :]) + + assert cor == 1197.75 + assert cor.dtype == np.float32 + + +def test_center_vo_average_i12LFOV(i12LFOV_data, ensure_clean_memory): + projdata = i12LFOV_data[0] + flats = i12LFOV_data[2] + darks = i12LFOV_data[3] + del i12LFOV_data + + data_normalised = normalize(projdata, flats, darks, minus_log=False) + del flats, darks, projdata + + cor = find_center_vo(data_normalised[:, 10:25, :], average_radius=5) + + assert cor == 1199.25 + assert cor.dtype == np.float32 + + +def test_center_vo_i12_sandstone(i12sandstone_data, ensure_clean_memory): + projdata = i12sandstone_data[0] + flats = i12sandstone_data[2] + darks = i12sandstone_data[3] + del i12sandstone_data + + data_normalised = normalize(projdata, flats, darks, minus_log=True) + del flats, darks, projdata + + mid_slice = data_normalised.shape[1] // 2 + cor = find_center_vo(data_normalised[:, mid_slice, :]) + + assert cor == 1253.75 + assert cor.dtype == np.float32 + + +def test_center_vo_i12_geantsim(geantsim_data, ensure_clean_memory): + projdata = geantsim_data[0] + flats = geantsim_data[2] + darks = geantsim_data[3] + del geantsim_data + + data_normalised = normalize(projdata, flats, darks, minus_log=True) + del flats, darks, projdata + + mid_slice = data_normalised.shape[1] // 2 + cor = find_center_vo(data_normalised[:, mid_slice, :]) + + assert cor == 319.5 + assert cor.dtype == np.float32 From 9e1d38e4b07c41f7bef5a425fafa045e18703de2 Mon Sep 17 00:00:00 2001 From: Naman Gera Date: Thu, 9 Jan 2025 15:02:32 +0000 Subject: [PATCH 5/8] Set up `LD_LIBRARY_PATH` --- .github/workflows/httomolibgpu_tests_run_iris.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/httomolibgpu_tests_run_iris.yml b/.github/workflows/httomolibgpu_tests_run_iris.yml index 9ef8ccc2..0c4f1c87 100644 --- a/.github/workflows/httomolibgpu_tests_run_iris.yml +++ b/.github/workflows/httomolibgpu_tests_run_iris.yml @@ -14,6 +14,7 @@ jobs: image: nvidia/cuda:12.6.3-devel-ubi8 env: NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} + options: --gpus all --runtime=nvidia defaults: run: @@ -23,6 +24,10 @@ jobs: - name: Checkout repository code uses: actions/checkout@v4 + - name: Set up CUDA environment + run: | + echo "LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH" >> $GITHUB_ENV + - name: Create conda environment uses: mamba-org/setup-micromamba@v1 with: From 98f2500aa641cdf110baf1bb938f4099f1b550b4 Mon Sep 17 00:00:00 2001 From: Naman Gera Date: Fri, 10 Jan 2025 10:09:01 +0000 Subject: [PATCH 6/8] Download data files under zenodo-tests --- .github/workflows/httomolibgpu_tests_run_iris.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/httomolibgpu_tests_run_iris.yml b/.github/workflows/httomolibgpu_tests_run_iris.yml index 0c4f1c87..1c80d6a8 100644 --- a/.github/workflows/httomolibgpu_tests_run_iris.yml +++ b/.github/workflows/httomolibgpu_tests_run_iris.yml @@ -40,10 +40,10 @@ jobs: run: | chmod +x ./.scripts/download_zenodo.py - ./.scripts/download_zenodo.py tests/large_data_archive + ./.scripts/download_zenodo.py zenodo-tests/large_data_archive echo -e "\nDownloaded files:" - ls -lh tests/large_data_archive + ls -lh zenodo-tests/large_data_archive - name: Install httomolibgpu run: | From 5fdf7b69c372104394ca8483549c19003ae19f58 Mon Sep 17 00:00:00 2001 From: Naman Gera Date: Fri, 10 Jan 2025 10:47:30 +0000 Subject: [PATCH 7/8] Run Zenodo tests with a label --- .github/workflows/httomolibgpu_tests_run_iris.yml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/httomolibgpu_tests_run_iris.yml b/.github/workflows/httomolibgpu_tests_run_iris.yml index 1c80d6a8..2b196855 100644 --- a/.github/workflows/httomolibgpu_tests_run_iris.yml +++ b/.github/workflows/httomolibgpu_tests_run_iris.yml @@ -36,15 +36,6 @@ jobs: post-cleanup: 'all' init-shell: bash - - name: Download test data from Zenodo - run: | - chmod +x ./.scripts/download_zenodo.py - - ./.scripts/download_zenodo.py zenodo-tests/large_data_archive - - echo -e "\nDownloaded files:" - ls -lh zenodo-tests/large_data_archive - - name: Install httomolibgpu run: | pip install .[dev] @@ -54,6 +45,10 @@ jobs: run: | pytest tests/ - - name: Run Zenodo tests + # Optional: Run Zenodo tests only if PR has a label + - name: Download and run Zenodo tests + if: contains(github.event.pull_request.labels.*.name, 'run-zenodo-tests') run: | + chmod +x ./.scripts/download_zenodo.py + ./.scripts/download_zenodo.py zenodo-tests/large_data_archive pytest zenodo-tests/ From 2058f31d911f607efe39dc8df4320396250a55fc Mon Sep 17 00:00:00 2001 From: Naman Gera Date: Fri, 10 Jan 2025 11:10:03 +0000 Subject: [PATCH 8/8] Run all the tests on merge commits --- .github/workflows/main-checks.yml | 47 +++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/main-checks.yml diff --git a/.github/workflows/main-checks.yml b/.github/workflows/main-checks.yml new file mode 100644 index 00000000..0b6b6157 --- /dev/null +++ b/.github/workflows/main-checks.yml @@ -0,0 +1,47 @@ +name: Main Branch Tests +on: + push: + branches: + - main + +jobs: + iris-gpu: + runs-on: iris-gpu + container: + image: nvidia/cuda:12.6.3-devel-ubi8 + env: + NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} + options: --gpus all --runtime=nvidia + + defaults: + run: + shell: bash -l {0} + + steps: + - uses: actions/checkout@v4 + + - name: Set up CUDA environment + run: | + echo "LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH" >> $GITHUB_ENV + + - name: Create conda environment + uses: mamba-org/setup-micromamba@v1 + with: + environment-file: conda/environment.yml + environment-name: httomo + post-cleanup: 'all' + init-shell: bash + + - name: Download test data from Zenodo + run: | + chmod +x ./.scripts/download_zenodo.py + ./.scripts/download_zenodo.py zenodo-tests/large_data_archive + + - name: Install httomolibgpu + run: | + pip install .[dev] + micromamba list + + - name: Run all tests (including Zenodo) + run: | + pytest tests/ zenodo-tests/