diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index d9e904415..f18dd5653 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -58,7 +58,17 @@ jobs: files: | ./collectedValues outPath: collectedValues.tar.gz - + - name: upload reference values artifact + id: artifact-upload-step + if: ${{ steps.matlab-refs-cache.outputs.cache-hit != 'true' }} + uses: actions/upload-artifact@v4 + with: + name: matlab_reference_test_values + path: collectedValues.tar.gz + # overwrite: true + - name: Output artifact URL + if: ${{ steps.matlab-refs-cache.outputs.cache-hit != 'true' }} + run: echo 'Artifact URL is ${{ steps.artifact-upload-step.outputs.artifact-url }}' test: needs: collect_references strategy: diff --git a/.github/workflows/test_optional_dependencies.yml b/.github/workflows/test_optional_dependencies.yml new file mode 100644 index 000000000..223921685 --- /dev/null +++ b/.github/workflows/test_optional_dependencies.yml @@ -0,0 +1,23 @@ +name: test_optional_requirements + +on: [push, pull_request] + +jobs: + test_install: + strategy: + matrix: + os: [ "windows-latest", "ubuntu-latest" ] #, "macos-latest"] + python-version: [ "3.9", "3.10", "3.11" ] + extra_requirements: [ "test", "examples", "docs", "dev", "all" ] + runs-on: ${{matrix.os}} + steps: + - uses: actions/checkout@v4 + # Pull the cache based on the hash value + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install dependencies + run: | + pip install '.[${{ matrix.extra_requirements }}]' diff --git a/docs/development/development_environment.rst b/docs/development/development_environment.rst index ab59c788c..f35d42a77 100644 --- a/docs/development/development_environment.rst +++ b/docs/development/development_environment.rst @@ -2,31 +2,67 @@ Development Environment ======================= Currently, this package serves as an interface to the cpp binaries of k-Wave. -For this reason, the k-Wave binaries are packaged with the code in this repository. -The k-Wave binaries can currently be found on the `k-Wave download page `_. +For this reason, binaries are required to run simulations with `k-Wave-python`. +The binaries are downloaded by k-Wave-python when the package is run for the first time. -In order to correctly set up your development environment for this repository, clone the repository from github, and install the project dependencies. +To correctly set up your development environment for this repository, clone the repository from github, and install the project dependencies. .. code-block:: bash git clone https://github.com/waltsims/k-wave-python cd k-wave-python - pip install -e . + pip install -e '.[test,dev]' -Next, download all k-Wave binaries from the `k-Wave download page `_. +Test References +======================= -Lastly, place the contents of the linux-binaries, and windows-executables directories in the project directory structure under ``kwave/bin/linux/``, ``kwave/bin/darwin`` and ``kwave/bin/windows`` respectively. -You will have to create the directory structure yourself. +Tests compare the outputs of the python and the matlab interfaces. +These tests are located in the ``tests`` directory. The comparison between ``matlab`` and ``python`` outputs are done in two ways: -With this, you are ready to develop k-Wave-python. -If you have any issues or questions, please post them on the `k-Wave-python discussions page `_ to discuss. We look forward to interacting with you. +- **Unit testing**: k-Wave-python functions that have a direct counterpart in original k-Wave are tested by comparing the outputs of the two functions. + The output of the original k-Wave functions are stored in ``.mat`` files. + These files can be generated by running the corresponding MATLAB scripts located in the ``tests/matlab_test_data_collectors/matlab_collectors/`` directory by running ``tests/matlab_test_data_collectors/run_all_collectors.m``. + After running the scripts, the reference files can be found in ``tests/matlab_test_data_collectors/python_testes/collectedValues/``. + +.. note:: + If you do not have MATLAB installed to generate the reference files, you can download recently generated reference file outputs from the GitHub CI and place them in the ``python_testers/collectedValues/`` directory. + The latest reference files can be found in the artifacts of the latest CI run of ``pytest.yml`` (e.g. [here](https://github.com/waltsims/k-wave-python/actions/runs/7770639710/artifacts/1217868112)). +- **Integration testing**: k-Wave-python tests output .h5 files that are passed to the k-Wave binaries and ensures that they match the output of the original k-Wave. + This testing compares the output for many of the example scripts from the original k-Wave package. + Hash values of the reference output ``.h5`` file from MATLAB examples are generated and stored in ``.json`` files in ``tests/reference_outputs/``. + These ``.json`` files are stored in the code repository and do not need to be regenerated. + Since these files are generated from the original k-Wave package, they only need to be updated when a new release of k-Wave is made. -Test References -======================= +**Matlab reference file generation** is a bit involved process. Below are the steps that describe the process. -In order to ensure that the python interface is working correctly, we have created a set of tests that compare the output of the python interface to the output of the matlab interface. -These tests are located in the ``tests`` directory. The comparison between ``matlab`` and ``python`` outputs are done in two ways: +#. Open desired example in matlab, e.g. `example_pr_2D_TR_directional_sensors.m `_ +#. Find the lines where the call to one of the `kSpaceFirstOrder-family` function is made. For example, + + .. code-block:: python + + input_args = {'PMLInside', false, 'PMLSize', PML_size, 'PlotPML', false, 'Smooth', false}; + sensor_data = kspaceFirstOrder2D(kgrid, medium, source, sensor, input_args{:}); + +#. Update the ``input_args`` field by adding two new options - ``{'SaveToDisk', true, 'SaveToDiskExit': true}``. These options will ensure that we a ``.h5`` file will be created and saved in your ``tmp`` folder, while avoiding to run the actual simulation. +#. Run the modified example. You will find created files in your ``tmp`` folder. Usually exact file name depends on how many calls are made to the `kSpaceFirstOrder-family` function in the example: + * If there is only a single call, created file name will be ``example_input.h5`` + * If there are two or more calls, created files will have names like ``example_input_1.h5``, ``example_input_2.h5``, ``example_input_3.h5`` and so on +#. Now it is time to turn the ``.h5`` files to the hashed ``.json`` files. This can be done with the ``H5Summary``. + * If you have a single ``.h5`` file, adapt the lines below and run the script: + https://github.com/waltsims/k-wave-python/blob/1f9df5d987d0b3edb1a8a43fad0885d3d6079029/tests/h5_summary.py#L92-L95 + * For multiple files, adapt the lines below: + https://github.com/waltsims/k-wave-python/blob/1f9df5d987d0b3edb1a8a43fad0885d3d6079029/tests/h5_summary.py#L97-L106 + + +To run the tests, use the following command: + +.. code-block:: bash + + pytest + +To run the tests with coverage, use the following command: + +.. code-block:: bash -- Using ``.mat`` files. The files are generated by running Matlab scripts that are located in the ``tests/matlab_test_data_collectors`` directory. -- Using ``.json`` files which are generated by hashing the Matlab variables. These are located at ``tests/reference_outputs/``. In order to regenerate the ``.json`` files, check the ``H5Summary`` class and its ``.save(...)`` method. + coverage run diff --git a/tests/test_binary_present.py b/tests/test_binary_present.py deleted file mode 100644 index 5c3f8f0ee..000000000 --- a/tests/test_binary_present.py +++ /dev/null @@ -1,91 +0,0 @@ -import os -import os.path - -import pytest - - -# TODO: refactor this for new lazy install strategy - -@pytest.mark.skipif(os.environ.get("CI") == 'true', reason="Running in GitHub Workflow.") -def test_linux_afp_binaries_present(): - assert os.path.exists(os.path.join(os.getcwd(), 'kwave', 'bin', 'linux', 'acousticFieldPropagator-OMP')) - - -@pytest.mark.skipif(os.environ.get("CI") == 'true', reason="Running in GitHub Workflow.") -def test_linux_omp_binaries_present(): - assert os.path.exists(os.path.join(os.getcwd(), 'kwave', 'bin', 'linux', 'kspaceFirstOrder-OMP')) - - -@pytest.mark.skipif(os.environ.get("CI") == 'true', reason="Running in GitHub Workflow.") -def test_linux_cuda_binaries_present(): - assert os.path.exists(os.path.join(os.getcwd(), 'kwave', 'bin', 'linux', 'kspaceFirstOrder-CUDA')) - - -@pytest.mark.skipif(os.environ.get("CI") == 'true', reason="Running in GitHub Workflow.") -def test_windows_afp_binaries_present(): - assert os.path.exists(os.path.join(os.getcwd(), 'kwave', 'bin', 'windows', 'acousticFieldPropagator-OMP.exe')) - - -@pytest.mark.skipif(os.environ.get("CI") == 'true', reason="Running in GitHub Workflow.") -def test_windows_cuda_binaries_present(): - assert os.path.exists(os.path.join(os.getcwd(), 'kwave', 'bin', 'windows', 'kspaceFirstOrder-CUDA.exe')) - - -@pytest.mark.skipif(os.environ.get("CI") == 'true', reason="Running in GitHub Workflow.") -def test_windows_omp_binaries_present(): - assert os.path.exists(os.path.join(os.getcwd(), 'kwave', 'bin', 'windows', 'kspaceFirstOrder-OMP.exe')) - - -@pytest.mark.skipif(os.environ.get("CI") == 'true', reason="Running in GitHub Workflow.") -def test_windows_hdf5_binaries_present(): - assert os.path.exists(os.path.join(os.getcwd(), 'kwave', 'bin', 'windows', 'hdf5.dll')) - - -@pytest.mark.skipif(os.environ.get("CI") == 'true', reason="Running in GitHub Workflow.") -def test_windows_hdf5hl_binaries_present(): - assert os.path.exists(os.path.join(os.getcwd(), 'kwave', 'bin', 'windows', 'hdf5_hl.dll')) - - -@pytest.mark.skipif(os.environ.get("CI") == 'true', reason="Running in GitHub Workflow.") -def test_windows_cufft64_binaries_present(): - assert os.path.exists(os.path.join(os.getcwd(), 'kwave', 'bin', 'windows', 'cufft64_10.dll')) - - -@pytest.mark.skipif(os.environ.get("CI") == 'true', reason="Running in GitHub Workflow.") -def test_windows_libiomp_binaries_present(): - assert os.path.exists(os.path.join(os.getcwd(), 'kwave', 'bin', 'windows', 'libiomp5md.dll')) - - -@pytest.mark.skipif(os.environ.get("CI") == 'true', reason="Running in GitHub Workflow.") -def test_windows_libmmd_binaries_present(): - assert os.path.exists(os.path.join(os.getcwd(), 'kwave', 'bin', 'windows', 'libmmd.dll')) - - -@pytest.mark.skipif(os.environ.get("CI") == 'true', reason="Running in GitHub Workflow.") -def test_windows_msvcp_binaries_present(): - assert os.path.exists(os.path.join(os.getcwd(), 'kwave', 'bin', 'windows', 'msvcp140.dll')) - - -@pytest.mark.skipif(os.environ.get("CI") == 'true', reason="Running in GitHub Workflow.") -def test_windows_svmldispmd_binaries_present(): - assert os.path.exists(os.path.join(os.getcwd(), 'kwave', 'bin', 'windows', 'svml_dispmd.dll')) - - -@pytest.mark.skipif(os.environ.get("CI") == 'true', reason="Running in GitHub Workflow.") -def test_windows_szip_binaries_present(): - assert os.path.exists(os.path.join(os.getcwd(), 'kwave', 'bin', 'windows', 'szip.dll')) - - -@pytest.mark.skipif(os.environ.get("CI") == 'true', reason="Running in GitHub Workflow.") -def test_windows_vcruntime140_binaries_present(): - assert os.path.exists(os.path.join(os.getcwd(), 'kwave', 'bin', 'windows', 'vcruntime140.dll')) - - -@pytest.mark.skipif(os.environ.get("CI") == 'true', reason="Running in GitHub Workflow.") -def test_windows_zlib_binaries_present(): - assert os.path.exists(os.path.join(os.getcwd(), 'kwave', 'bin', 'windows', 'zlib.dll')) - - -@pytest.mark.skipif(os.environ.get("CI") == 'true', reason="Running in GitHub Workflow.") -def test_windows_cufft6410_binaries_present(): - assert os.path.exists(os.path.join(os.getcwd(), 'kwave', 'bin', 'windows', 'cufft64_10.dll')) diff --git a/tests/test_executor.py b/tests/test_executor.py deleted file mode 100644 index 0f29b3955..000000000 --- a/tests/test_executor.py +++ /dev/null @@ -1,34 +0,0 @@ -import sys -import unittest.mock -import logging -from kwave.executor import Executor -import pytest -import os - -check_is_linux = pytest.mark.skipif(not sys.platform.startswith('linux'), reason="Currently only implemented for linux.") - - -@check_is_linux -@pytest.mark.skipif(os.environ.get("CI") == 'true', reason="Running in GitHub Workflow.") -class TestExecutor(unittest.TestCase): - - @unittest.mock.patch('os.system') - def test_system_call_correct(self, os_system): - try: - ex = Executor('cpu') - input_filename = '/tmp/input.h5' - output_filename = '/tmp/output.h5' - try: - ex.run_simulation(input_filename, output_filename, options='') - except OSError: - logging.info("Caught 'Unable to open file' exception.") - - call_str = f"{ex.binary_path} -i {input_filename} -o {output_filename} " - os_system.assert_called_once_with(call_str) - except NotImplementedError as err: - if not sys.platform.startswith('darwin'): - raise err - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_readme.py b/tests/test_readme.py deleted file mode 100644 index dbd33c2fd..000000000 --- a/tests/test_readme.py +++ /dev/null @@ -1,44 +0,0 @@ -import os -import re -import subprocess -import sys -from pathlib import Path -from tempfile import mkdtemp - -import pytest -import requests - - -@pytest.mark.skipif(os.environ.get("CI") == 'true', reason="Running in GitHub Workflow.") -def test_readme(): - # Check if there is internet connectivity - try: - requests.get("https://google.com") - except requests.exceptions.ConnectionError: - pytest.skip("No internet connectivity") - - # Skip the test if the operating system is MacOS - if sys.platform.startswith('darwin'): - pytest.skip("This test cannot be run on MacOS") - - # Read the getting started section from the READMEfind a .md file - with open(Path('README.md'), 'r') as f: - readme = f.read() - - tempdir = mkdtemp() - cwd = os.getcwd() - os.chdir(tempdir) - # Use a regular expression to find code blocks in the getting started section - code_blocks = re.findall(r'```bash(.*?)```', readme, re.DOTALL) - - for block in code_blocks: - instruction = block - result = subprocess.run(['bash', '-c', instruction]) - try: - assert result.returncode == 0, f"instruction failed: {instruction}" - except AssertionError as e: - os.chdir(cwd) - raise e - - os.chdir(cwd) - pass