From 0d7b710da1be82597bc7ab64adaf7d5d5c1284fe Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 8 Aug 2024 13:10:19 +0200 Subject: [PATCH 01/11] copier update --- .copier-answers.yml | 6 +++--- .github/workflows/ci.yml | 2 ++ .github/workflows/docs.yml | 6 +++++- .github/workflows/nightly_at_main.yml | 1 + .github/workflows/nightly_at_release.yml | 1 + .github/workflows/test.yml | 18 ++++++++++++++++++ .github/workflows/unpinned.yml | 1 + .gitignore | 1 + docs/conf.py | 1 + requirements/make_base.py | 3 ++- tests/package_test.py | 13 +++++++++++++ tox.ini | 2 +- 12 files changed, 49 insertions(+), 6 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index e9f997f3..de550ae1 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,13 +1,13 @@ # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY -_commit: 6848c57 +_commit: 86a1e5c _src_path: gh:scipp/copier_template description: Diffraction data reduction for the European Spallation Source max_python: '3.12' min_python: '3.10' namespace_package: ess -nightly_deps: scipp,scippnexus,sciline,plopp,scippneutron +nightly_deps: scipp,scippnexus,sciline,plopp,scippneutron,essreduce orgname: scipp prettyname: ESSdiffraction projectname: essdiffraction -related_projects: Scipp,ScippNexus,ScippNeutron,Sciline,Plopp +related_projects: Scipp,ScippNexus,ScippNeutron,Sciline,Plopp,ESSreduce year: 2024 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44266a23..8234bc9f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,6 +46,7 @@ jobs: os-variant: ${{ matrix.os }} python-version: ${{ matrix.python.version }} tox-env: ${{ matrix.python.tox-env }} + secrets: inherit docs: needs: tests @@ -54,3 +55,4 @@ jobs: publish: false linkcheck: ${{ contains(matrix.variant.os, 'ubuntu') && github.ref == 'refs/heads/main' }} branch: ${{ github.head_ref == '' && github.ref_name || github.head_ref }} + secrets: inherit diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 98aaf568..a5ea2b05 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -42,6 +42,10 @@ jobs: docs: name: Build documentation runs-on: 'ubuntu-22.04' + env: + ESS_PROTECTED_FILESTORE_USERNAME: ${{ secrets.ESS_PROTECTED_FILESTORE_USERNAME }} + ESS_PROTECTED_FILESTORE_PASSWORD: ${{ secrets.ESS_PROTECTED_FILESTORE_PASSWORD }} + steps: - run: sudo apt install --yes graphviz pandoc - uses: actions/checkout@v4 @@ -65,7 +69,7 @@ jobs: name: docs_html path: html/ - - uses: JamesIves/github-pages-deploy-action@v4.6.1 + - uses: JamesIves/github-pages-deploy-action@v4.6.3 if: ${{ inputs.publish }} with: branch: gh-pages diff --git a/.github/workflows/nightly_at_main.yml b/.github/workflows/nightly_at_main.yml index 08fdddd2..c2b9d33a 100644 --- a/.github/workflows/nightly_at_main.yml +++ b/.github/workflows/nightly_at_main.yml @@ -31,3 +31,4 @@ jobs: os-variant: ${{ matrix.os }} python-version: ${{ matrix.python.version }} tox-env: ${{ matrix.python.tox-env }} + secrets: inherit diff --git a/.github/workflows/nightly_at_release.yml b/.github/workflows/nightly_at_release.yml index 373c4546..3faa1c23 100644 --- a/.github/workflows/nightly_at_release.yml +++ b/.github/workflows/nightly_at_release.yml @@ -38,3 +38,4 @@ jobs: python-version: ${{ matrix.python.version }} tox-env: ${{ matrix.python.tox-env }} checkout_ref: ${{ needs.setup.outputs.release_tag }} + secrets: inherit diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5f56a069..e98ed7cd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,8 +41,26 @@ on: type: string jobs: + package-test: + runs-on: ${{ inputs.os-variant }} + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.checkout_ref }} + - uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + - run: python -m pip install --upgrade pip + - run: python -m pip install . + - run: python tests/package_test.py + name: Run package tests + test: runs-on: ${{ inputs.os-variant }} + env: + ESS_PROTECTED_FILESTORE_USERNAME: ${{ secrets.ESS_PROTECTED_FILESTORE_USERNAME }} + ESS_PROTECTED_FILESTORE_PASSWORD: ${{ secrets.ESS_PROTECTED_FILESTORE_PASSWORD }} steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/unpinned.yml b/.github/workflows/unpinned.yml index 46a84c1c..3f49f722 100644 --- a/.github/workflows/unpinned.yml +++ b/.github/workflows/unpinned.yml @@ -38,3 +38,4 @@ jobs: python-version: ${{ matrix.python.version }} tox-env: ${{ matrix.python.tox-env }} checkout_ref: ${{ needs.setup.outputs.release_tag }} + secrets: inherit diff --git a/.gitignore b/.gitignore index 0f0541bc..46c5ae85 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ __pycache__/ .pytest_cache .mypy_cache docs/generated/ +.ruff_cache # Editor settings .idea/ diff --git a/docs/conf.py b/docs/conf.py index 288770a4..e444fb4e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -169,6 +169,7 @@ "image_dark": "_static/logo-dark.svg", }, "external_links": [ + {"name": "ESSreduce", "url": "https://scipp.github.io/essreduce"}, {"name": "Plopp", "url": "https://scipp.github.io/plopp"}, {"name": "Sciline", "url": "https://scipp.github.io/sciline"}, {"name": "Scipp", "url": "https://scipp.github.io"}, diff --git a/requirements/make_base.py b/requirements/make_base.py index 68a17e84..493ede16 100644 --- a/requirements/make_base.py +++ b/requirements/make_base.py @@ -55,7 +55,8 @@ def as_nightly(repo: str) -> str: version = f"cp{sys.version_info.major}{sys.version_info.minor}" base = "https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly" suffix = "manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - return "-".join([base, version, version, suffix]) + prefix = "scipp @ " + return prefix + "-".join([base, version, version, suffix]) return f"{repo} @ git+https://github.com/{org}/{repo}@main" diff --git a/tests/package_test.py b/tests/package_test.py index a389e10a..31b57c60 100644 --- a/tests/package_test.py +++ b/tests/package_test.py @@ -1,7 +1,20 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) + +"""Tests of package integrity. + +Note that addidional imports need to be added for repositories that +contain multiple packages. +""" + from ess import diffraction as pkg def test_has_version(): assert hasattr(pkg, '__version__') + + +# This is for CI package tests. They need to run tests with minimal dependencies, +# that is, without installing pytest. This code does not affect pytest. +if __name__ == '__main__': + test_has_version() diff --git a/tox.ini b/tox.ini index 5abdaa9b..c933dc84 100644 --- a/tox.ini +++ b/tox.ini @@ -67,5 +67,5 @@ deps = tomli skip_install = true changedir = requirements -commands = python ./make_base.py --nightly scipp,scippnexus,sciline,plopp,scippneutron +commands = python ./make_base.py --nightly scipp,scippnexus,sciline,plopp,scippneutron,essreduce pip-compile-multi -d . --backtracking From 16bfed30aab18cd4b4150adacf101b6370632aaf Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 8 Aug 2024 13:14:39 +0200 Subject: [PATCH 02/11] tox -e deps (updating ScippNexus) --- requirements/base.txt | 35 ++++++++++---------------------- requirements/basetest.txt | 6 +++--- requirements/ci.txt | 4 ++-- requirements/dev.txt | 12 +++++------ requirements/docs.txt | 42 +++++++++++++++++++++++++-------------- requirements/mypy.txt | 2 +- requirements/nightly.in | 4 ++-- requirements/nightly.txt | 28 +++++++------------------- requirements/static.txt | 4 ++-- 9 files changed, 60 insertions(+), 77 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index bfada6e6..b6b03576 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -7,10 +7,6 @@ # asttokens==2.4.1 # via stack-data -certifi==2024.7.4 - # via requests -charset-normalizer==3.3.2 - # via requests click==8.1.7 # via dask cloudpickle==3.0.0 @@ -23,13 +19,13 @@ cyclebane==24.6.0 # via sciline cycler==0.12.1 # via matplotlib -dask==2024.7.0 +dask==2024.8.0 # via -r base.in decorator==5.1.1 # via ipython -essreduce==24.7.1 +essreduce==24.8.0 # via -r base.in -exceptiongroup==1.2.1 +exceptiongroup==1.2.2 # via ipython executing==2.0.1 # via stack-data @@ -43,9 +39,7 @@ h5py==3.11.0 # via # scippneutron # scippnexus -idna==3.7 - # via requests -importlib-metadata==8.0.0 +importlib-metadata==8.2.0 # via dask ipydatawidgets==4.3.5 # via pythreejs @@ -63,7 +57,7 @@ kiwisolver==1.4.5 # via matplotlib locket==1.0.0 # via partd -matplotlib==3.9.1 +matplotlib==3.9.1.post1 # via # mpltoolbox # plopp @@ -73,7 +67,7 @@ mpltoolbox==24.5.1 # via scippneutron networkx==3.3 # via cyclebane -numpy==2.0.0 +numpy==2.0.1 # via # -r base.in # contourpy @@ -89,7 +83,6 @@ packaging==24.1 # via # dask # matplotlib - # pooch parso==0.8.4 # via jedi partd==1.4.2 @@ -98,19 +91,15 @@ pexpect==4.9.0 # via ipython pillow==10.4.0 # via matplotlib -platformdirs==4.2.2 - # via pooch plopp==24.6.0 # via # -r base.in # scippneutron -pooch==1.8.2 - # via scippneutron prompt-toolkit==3.0.47 # via ipython ptyprocess==0.7.0 # via pexpect -pure-eval==0.2.2 +pure-eval==0.2.3 # via stack-data pygments==2.18.0 # via ipython @@ -122,10 +111,8 @@ python-dateutil==2.9.0.post0 # scippnexus pythreejs==2.4.2 # via -r base.in -pyyaml==6.0.1 +pyyaml==6.0.2 # via dask -requests==2.32.3 - # via pooch sciline==24.6.2 # via -r base.in scipp==24.6.0 @@ -134,9 +121,9 @@ scipp==24.6.0 # essreduce # scippneutron # scippnexus -scippneutron==24.7.0 +scippneutron==24.8.0 # via -r base.in -scippnexus==24.6.0 +scippnexus==24.8.1 # via # -r base.in # essreduce @@ -167,8 +154,6 @@ traittypes==0.2.1 # via ipydatawidgets typing-extensions==4.12.2 # via ipython -urllib3==2.2.2 - # via requests wcwidth==0.2.13 # via prompt-toolkit widgetsnbextension==4.0.11 diff --git a/requirements/basetest.txt b/requirements/basetest.txt index 0a9b851c..600e0daa 100644 --- a/requirements/basetest.txt +++ b/requirements/basetest.txt @@ -5,11 +5,11 @@ # # pip-compile-multi # -exceptiongroup==1.2.1 +exceptiongroup==1.2.2 # via pytest iniconfig==2.0.0 # via pytest -numpy==2.0.0 +numpy==2.0.1 # via # -r basetest.in # pandas @@ -19,7 +19,7 @@ pandas==2.2.2 # via -r basetest.in pluggy==1.5.0 # via pytest -pytest==8.2.2 +pytest==8.3.2 # via -r basetest.in python-dateutil==2.9.0.post0 # via pandas diff --git a/requirements/ci.txt b/requirements/ci.txt index 4d81cbd2..6bb608a6 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -5,7 +5,7 @@ # # pip-compile-multi # -cachetools==5.3.3 +cachetools==5.4.0 # via tox certifi==2024.7.4 # via requests @@ -48,7 +48,7 @@ tomli==2.0.1 # via # pyproject-api # tox -tox==4.16.0 +tox==4.17.1 # via -r ci.in urllib3==2.2.2 # via requests diff --git a/requirements/dev.txt b/requirements/dev.txt index cabce220..4de7d28b 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -26,11 +26,11 @@ arrow==1.3.0 # via isoduration async-lru==2.0.4 # via jupyterlab -cffi==1.16.0 +cffi==1.17.0 # via argon2-cffi-bindings copier==9.3.1 # via -r dev.in -dunamai==1.21.2 +dunamai==1.22.0 # via copier fqdn==1.5.1 # via jsonschema @@ -59,7 +59,7 @@ jupyter-events==0.10.0 # via jupyter-server jupyter-lsp==2.2.5 # via jupyterlab -jupyter-server==2.14.1 +jupyter-server==2.14.2 # via # jupyter-lsp # jupyterlab @@ -67,9 +67,9 @@ jupyter-server==2.14.1 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.2.3 +jupyterlab==4.2.4 # via -r dev.in -jupyterlab-server==2.27.2 +jupyterlab-server==2.27.3 # via jupyterlab notebook-shim==0.2.4 # via jupyterlab @@ -123,7 +123,7 @@ webcolors==24.6.0 # via jsonschema websocket-client==1.8.0 # via jupyter-server -wheel==0.43.0 +wheel==0.44.0 # via pip-tools # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/docs.txt b/requirements/docs.txt index b0b04012..fa325730 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -8,9 +8,9 @@ -r base.txt accessible-pygments==0.0.5 # via pydata-sphinx-theme -alabaster==0.7.16 +alabaster==1.0.0 # via sphinx -attrs==23.2.0 +attrs==24.2.0 # via # jsonschema # referencing @@ -24,7 +24,11 @@ beautifulsoup4==4.12.3 # pydata-sphinx-theme bleach==6.1.0 # via nbconvert -debugpy==1.8.2 +certifi==2024.7.4 + # via requests +charset-normalizer==3.3.2 + # via requests +debugpy==1.8.5 # via ipykernel defusedxml==0.7.1 # via nbconvert @@ -38,6 +42,8 @@ docutils==0.21.2 # sphinxcontrib-bibtex fastjsonschema==2.20.0 # via nbformat +idna==3.7 + # via requests imagesize==1.4.1 # via sphinx ipykernel==6.29.5 @@ -85,7 +91,7 @@ mdurl==0.1.2 # via markdown-it-py mistune==3.0.2 # via nbconvert -myst-parser==3.0.1 +myst-parser==4.0.0 # via -r docs.in nbclient==0.10.0 # via nbconvert @@ -104,9 +110,11 @@ pandas==2.2.2 # via -r docs.in pandocfilters==1.5.1 # via nbconvert +platformdirs==4.2.2 + # via jupyter-core psutil==6.0.0 # via ipykernel -pyarrow==16.1.0 +pyarrow==17.0.0 # via -r docs.in pybtex==0.24.0 # via @@ -118,7 +126,7 @@ pydata-sphinx-theme==0.15.4 # via -r docs.in pytz==2024.1 # via pandas -pyzmq==26.0.3 +pyzmq==26.1.0 # via # ipykernel # jupyter-client @@ -126,7 +134,9 @@ referencing==0.35.1 # via # jsonschema # jsonschema-specifications -rpds-py==0.19.0 +requests==2.32.3 + # via sphinx +rpds-py==0.20.0 # via # jsonschema # referencing @@ -134,7 +144,7 @@ snowballstemmer==2.2.0 # via sphinx soupsieve==2.5 # via beautifulsoup4 -sphinx==7.3.7 +sphinx==8.0.2 # via # -r docs.in # myst-parser @@ -144,25 +154,25 @@ sphinx==7.3.7 # sphinx-copybutton # sphinx-design # sphinxcontrib-bibtex -sphinx-autodoc-typehints==2.2.2 +sphinx-autodoc-typehints==2.2.3 # via -r docs.in sphinx-copybutton==0.5.2 # via -r docs.in -sphinx-design==0.6.0 +sphinx-design==0.6.1 # via -r docs.in -sphinxcontrib-applehelp==1.0.8 +sphinxcontrib-applehelp==2.0.0 # via sphinx sphinxcontrib-bibtex==2.6.2 # via -r docs.in -sphinxcontrib-devhelp==1.0.6 +sphinxcontrib-devhelp==2.0.0 # via sphinx -sphinxcontrib-htmlhelp==2.0.5 +sphinxcontrib-htmlhelp==2.1.0 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.7 +sphinxcontrib-qthelp==2.0.0 # via sphinx -sphinxcontrib-serializinghtml==1.1.10 +sphinxcontrib-serializinghtml==2.0.0 # via sphinx tinycss2==1.3.0 # via nbconvert @@ -174,6 +184,8 @@ tornado==6.4.1 # jupyter-client tzdata==2024.1 # via pandas +urllib3==2.2.2 + # via requests webencodings==0.5.1 # via # bleach diff --git a/requirements/mypy.txt b/requirements/mypy.txt index 6afffb45..e0374a17 100644 --- a/requirements/mypy.txt +++ b/requirements/mypy.txt @@ -6,7 +6,7 @@ # pip-compile-multi # -r test.txt -mypy==1.10.1 +mypy==1.11.1 # via -r mypy.in mypy-extensions==1.0.0 # via mypy diff --git a/requirements/nightly.in b/requirements/nightly.in index 3ac68d04..a1ea3a88 100644 --- a/requirements/nightly.in +++ b/requirements/nightly.in @@ -2,12 +2,12 @@ # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! dask -essreduce>=24.07.1 graphviz numpy pythreejs -https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl +scipp @ https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl scippnexus @ git+https://github.com/scipp/scippnexus@main sciline @ git+https://github.com/scipp/sciline@main plopp @ git+https://github.com/scipp/plopp@main scippneutron @ git+https://github.com/scipp/scippneutron@main +essreduce @ git+https://github.com/scipp/essreduce@main diff --git a/requirements/nightly.txt b/requirements/nightly.txt index a88ebdc2..f892b2c9 100644 --- a/requirements/nightly.txt +++ b/requirements/nightly.txt @@ -1,4 +1,4 @@ -# SHA1:9db68cd4d40ba40501f7f36513adba4d274e18ae +# SHA1:14c661e4a5ccde65d56656d46e7d4e7077f040bf # # This file is autogenerated by pip-compile-multi # To update, run: @@ -8,10 +8,6 @@ -r basetest.txt asttokens==2.4.1 # via stack-data -certifi==2024.7.4 - # via requests -charset-normalizer==3.3.2 - # via requests click==8.1.7 # via dask cloudpickle==3.0.0 @@ -24,11 +20,11 @@ cyclebane==24.6.0 # via sciline cycler==0.12.1 # via matplotlib -dask==2024.7.0 +dask==2024.8.0 # via -r nightly.in decorator==5.1.1 # via ipython -essreduce==24.7.1 +essreduce @ git+https://github.com/scipp/essreduce@main # via -r nightly.in executing==2.0.1 # via stack-data @@ -42,9 +38,7 @@ h5py==3.11.0 # via # scippneutron # scippnexus -idna==3.7 - # via requests -importlib-metadata==8.0.0 +importlib-metadata==8.2.0 # via dask ipydatawidgets==4.3.5 # via pythreejs @@ -62,7 +56,7 @@ kiwisolver==1.4.5 # via matplotlib locket==1.0.0 # via partd -matplotlib==3.9.1 +matplotlib==3.9.1.post1 # via # mpltoolbox # plopp @@ -80,19 +74,15 @@ pexpect==4.9.0 # via ipython pillow==10.4.0 # via matplotlib -platformdirs==4.2.2 - # via pooch plopp @ git+https://github.com/scipp/plopp@main # via # -r nightly.in # scippneutron -pooch==1.8.2 - # via scippneutron prompt-toolkit==3.0.47 # via ipython ptyprocess==0.7.0 # via pexpect -pure-eval==0.2.2 +pure-eval==0.2.3 # via stack-data pygments==2.18.0 # via ipython @@ -100,10 +90,8 @@ pyparsing==3.1.2 # via matplotlib pythreejs==2.4.2 # via -r nightly.in -pyyaml==6.0.1 +pyyaml==6.0.2 # via dask -requests==2.32.3 - # via pooch sciline @ git+https://github.com/scipp/sciline@main # via -r nightly.in scipp @ https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl @@ -141,8 +129,6 @@ traittypes==0.2.1 # via ipydatawidgets typing-extensions==4.12.2 # via ipython -urllib3==2.2.2 - # via requests wcwidth==0.2.13 # via prompt-toolkit widgetsnbextension==4.0.11 diff --git a/requirements/static.txt b/requirements/static.txt index e106d602..85da246d 100644 --- a/requirements/static.txt +++ b/requirements/static.txt @@ -17,9 +17,9 @@ nodeenv==1.9.1 # via pre-commit platformdirs==4.2.2 # via virtualenv -pre-commit==3.7.1 +pre-commit==3.8.0 # via -r static.in -pyyaml==6.0.1 +pyyaml==6.0.2 # via pre-commit virtualenv==20.26.3 # via pre-commit From 05b35793b1d14fe41271da5bd822612a4f752849 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 8 Aug 2024 13:35:45 +0200 Subject: [PATCH 03/11] Avoid depending on pooch, which was implicit via ScippNeutron --- requirements/basetest.in | 1 + requirements/basetest.txt | 20 +++++++++++++++++-- src/ess/dream/__init__.py | 2 -- src/ess/snspowder/powgen/__init__.py | 8 ++------ src/ess/snspowder/powgen/workflow.py | 6 +++++- tests/dream/io/nexus_test.py | 2 +- .../snspowder/powgen/powgen_reduction_test.py | 3 ++- 7 files changed, 29 insertions(+), 13 deletions(-) diff --git a/requirements/basetest.in b/requirements/basetest.in index ef9d834f..18c6cbba 100644 --- a/requirements/basetest.in +++ b/requirements/basetest.in @@ -3,4 +3,5 @@ numpy pandas +pooch pytest diff --git a/requirements/basetest.txt b/requirements/basetest.txt index 600e0daa..29cc9ca2 100644 --- a/requirements/basetest.txt +++ b/requirements/basetest.txt @@ -1,12 +1,18 @@ -# SHA1:ddff3f126978358816fc9518b85c5efc1ff00444 +# SHA1:71bfa26144d7bd59b7128e5f32727ebae47f0168 # # This file is autogenerated by pip-compile-multi # To update, run: # # pip-compile-multi # +certifi==2024.7.4 + # via requests +charset-normalizer==3.3.2 + # via requests exceptiongroup==1.2.2 # via pytest +idna==3.7 + # via requests iniconfig==2.0.0 # via pytest numpy==2.0.1 @@ -14,20 +20,30 @@ numpy==2.0.1 # -r basetest.in # pandas packaging==24.1 - # via pytest + # via + # pooch + # pytest pandas==2.2.2 # via -r basetest.in +platformdirs==4.2.2 + # via pooch pluggy==1.5.0 # via pytest +pooch==1.8.2 + # via -r basetest.in pytest==8.3.2 # via -r basetest.in python-dateutil==2.9.0.post0 # via pandas pytz==2024.1 # via pandas +requests==2.32.3 + # via pooch six==1.16.0 # via python-dateutil tomli==2.0.1 # via pytest tzdata==2024.1 # via pandas +urllib3==2.2.2 + # via requests diff --git a/src/ess/dream/__init__.py b/src/ess/dream/__init__.py index 15f44b9e..d003b17f 100644 --- a/src/ess/dream/__init__.py +++ b/src/ess/dream/__init__.py @@ -7,7 +7,6 @@ import importlib.metadata -from . import data from .instrument_view import instrument_view from .io import load_geant4_csv, nexus from .workflow import DreamGeant4Workflow, default_parameters @@ -24,7 +23,6 @@ __all__ = [ 'DreamGeant4Workflow', - 'data', 'default_parameters', 'beamline', 'instrument_view', diff --git a/src/ess/snspowder/powgen/__init__.py b/src/ess/snspowder/powgen/__init__.py index 36af63d0..c8640b5f 100644 --- a/src/ess/snspowder/powgen/__init__.py +++ b/src/ess/snspowder/powgen/__init__.py @@ -7,22 +7,18 @@ the ``dream`` module when that is available. """ -from . import beamline, data, peaks +from . import beamline, peaks from .calibration import load_calibration from .instrument_view import instrument_view from .workflow import PowgenWorkflow, default_parameters -providers = ( - *beamline.providers, - *data.providers, -) +providers = (*beamline.providers,) """Sciline Providers for POWGEN-specific functionality.""" __all__ = [ 'PowgenWorkflow', 'beamline', - 'data', 'default_parameters', 'instrument_view', 'load_calibration', diff --git a/src/ess/snspowder/powgen/workflow.py b/src/ess/snspowder/powgen/workflow.py index 64a7dc2e..141d7f44 100644 --- a/src/ess/snspowder/powgen/workflow.py +++ b/src/ess/snspowder/powgen/workflow.py @@ -6,7 +6,7 @@ from ess.powder import providers as powder_providers from ess.powder.types import NeXusDetectorName -from . import beamline, data +from . import beamline def default_parameters() -> dict: @@ -17,6 +17,10 @@ def PowgenWorkflow() -> sciline.Pipeline: """ Workflow with default parameters for the Powgen SNS instrument. """ + # The package does not depend on pooch which is needed for the tutorial + # data. Delay import until workflow is actually used. + from . import data + return sciline.Pipeline( providers=powder_providers + beamline.providers + data.providers, params=default_parameters(), diff --git a/tests/dream/io/nexus_test.py b/tests/dream/io/nexus_test.py index 762a7a86..c248ef06 100644 --- a/tests/dream/io/nexus_test.py +++ b/tests/dream/io/nexus_test.py @@ -4,7 +4,7 @@ import sciline from ess import dream -from ess.dream import nexus +from ess.dream import data, nexus # noqa: F401 from ess.powder.types import ( Filename, NeXusDetectorName, diff --git a/tests/snspowder/powgen/powgen_reduction_test.py b/tests/snspowder/powgen/powgen_reduction_test.py index f4f90497..2fb1bd8f 100644 --- a/tests/snspowder/powgen/powgen_reduction_test.py +++ b/tests/snspowder/powgen/powgen_reduction_test.py @@ -6,6 +6,7 @@ import scipp as sc from ess import powder from ess.snspowder import powgen +from ess.snspowder.powgen import data # noqa: F401 from ess.powder.types import ( CalibrationFilename, @@ -30,7 +31,7 @@ def providers(): from ess import powder - return [*powder.providers, *powgen.providers] + return [*powder.providers, *powgen.providers, *powgen.data.providers] @pytest.fixture() From f0a2f56f88841fadffd9a9a9f9533822c97e5641 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 8 Aug 2024 13:53:30 +0200 Subject: [PATCH 04/11] Fix docs --- docs/user-guide/dream/dream-data-reduction.ipynb | 16 +++++++++++++--- .../user-guide/dream/dream-instrument-view.ipynb | 9 +++++++-- .../sns-instruments/POWGEN_data_reduction.ipynb | 12 +++++++++--- requirements/docs.in | 1 + requirements/docs.txt | 12 +++++++++--- tests/dream/io/nexus_test.py | 3 ++- tests/snspowder/powgen/powgen_reduction_test.py | 2 +- 7 files changed, 42 insertions(+), 13 deletions(-) diff --git a/docs/user-guide/dream/dream-data-reduction.ipynb b/docs/user-guide/dream/dream-data-reduction.ipynb index 99dd14c8..f91fb0fc 100644 --- a/docs/user-guide/dream/dream-data-reduction.ipynb +++ b/docs/user-guide/dream/dream-data-reduction.ipynb @@ -5,7 +5,11 @@ "id": "f47beab6-47c9-4cfb-a70e-c00bc8daebef", "metadata": {}, "source": [ - "# DREAM data reduction" + "# DREAM data reduction\n", + "\n", + "We begin with relevant imports.\n", + "We will be using tutorial data downloaded with `pooch`.\n", + "If you get an error about a missing module `pooch`, you can install it with `!pip install pooch`:" ] }, { @@ -17,7 +21,10 @@ "source": [ "import scipp as sc\n", "import scippneutron as scn\n", + "import scippneutron.io\n", + "\n", "from ess import dream, powder\n", + "import ess.dream.data # noqa: F401\n", "from ess.powder.types import *" ] }, @@ -67,7 +74,9 @@ "# Edges for binning in d-spacing\n", "workflow[DspacingBins] = sc.linspace(\"dspacing\", 0.0, 2.3434, 201, unit=\"angstrom\")\n", "# Mask in time-of-flight to crop to valid range\n", - "workflow[TofMask] = lambda x: (x < sc.scalar(0.0, unit=\"ns\")) | (x > sc.scalar(86e6, unit=\"ns\"))\n", + "workflow[TofMask] = lambda x: (x < sc.scalar(0.0, unit=\"ns\")) | (\n", + " x > sc.scalar(86e6, unit=\"ns\")\n", + ")\n", "workflow[TwoThetaMask] = None\n", "workflow[WavelengthMask] = None\n", "# No pixel masks\n", @@ -265,7 +274,8 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" + "pygments_lexer": "ipython3", + "version": "3.10.14" } }, "nbformat": 4, diff --git a/docs/user-guide/dream/dream-instrument-view.ipynb b/docs/user-guide/dream/dream-instrument-view.ipynb index ad4dc3fa..eb792111 100644 --- a/docs/user-guide/dream/dream-instrument-view.ipynb +++ b/docs/user-guide/dream/dream-instrument-view.ipynb @@ -10,7 +10,11 @@ "This notebook is a simple example of how to use the instrument view for the DREAM instrument.\n", "\n", "- The DREAM-specific instrument view is capable of slicing the data with a slider widget along a dimension (e.g. `tof`) by using the `dim` argument.\n", - "- There are also checkboxes to hide/show the different elements that make up the DREAM detectors." + "- There are also checkboxes to hide/show the different elements that make up the DREAM detectors.\n", + "\n", + "We begin with relevant imports.\n", + "We will be using tutorial data downloaded with `pooch`.\n", + "If you get an error about a missing module `pooch`, you can install it with `!pip install pooch`:" ] }, { @@ -21,7 +25,8 @@ "outputs": [], "source": [ "import scipp as sc\n", - "from ess import dream" + "from ess import dream\n", + "import ess.dream.data # noqa: F401" ] }, { diff --git a/docs/user-guide/sns-instruments/POWGEN_data_reduction.ipynb b/docs/user-guide/sns-instruments/POWGEN_data_reduction.ipynb index b44983dc..cd6415ed 100644 --- a/docs/user-guide/sns-instruments/POWGEN_data_reduction.ipynb +++ b/docs/user-guide/sns-instruments/POWGEN_data_reduction.ipynb @@ -12,7 +12,9 @@ "This notebook gives a concise overview of how to use the ESSDiffraction package with Sciline.\n", "It uses a simple reduction workflow for the SNS [POWGEN](https://sns.gov/powgen) experiment.\n", "\n", - "We begin with relevant imports:" + "We begin with relevant imports.\n", + "We will be using tutorial data downloaded with `pooch`.\n", + "If you get an error about a missing module `pooch`, you can install it with `!pip install pooch`:" ] }, { @@ -24,9 +26,11 @@ "source": [ "import scipp as sc\n", "import scippneutron as scn\n", + "import scippneutron.io\n", "\n", "from ess import powder\n", "from ess.snspowder import powgen\n", + "import ess.snspowder.powgen.data # noqa: F401\n", "from ess.powder.types import *" ] }, @@ -74,7 +78,9 @@ "# Edges for binning in d-spacing\n", "workflow[DspacingBins] = sc.linspace(\"dspacing\", 0.0, 2.3434, 201, unit=\"angstrom\")\n", "# Mask in time-of-flight to crop to valid range\n", - "workflow[TofMask] = lambda x: (x < sc.scalar(0.0, unit=\"us\")) | (x > sc.scalar(16666.67, unit=\"us\"))\n", + "workflow[TofMask] = lambda x: (x < sc.scalar(0.0, unit=\"us\")) | (\n", + " x > sc.scalar(16666.67, unit=\"us\")\n", + ")\n", "workflow[TwoThetaMask] = None\n", "workflow[WavelengthMask] = None\n", "# No pixel masks\n", @@ -388,7 +394,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.10.14" } }, "nbformat": 4, diff --git a/requirements/docs.in b/requirements/docs.in index a1110dd5..52af4c7b 100644 --- a/requirements/docs.in +++ b/requirements/docs.in @@ -5,6 +5,7 @@ ipython!=8.7.0 # Breaks syntax highlighting in Jupyter code cells. myst-parser nbsphinx pandas +pooch pydata-sphinx-theme>=0.14 sphinx sphinx-autodoc-typehints diff --git a/requirements/docs.txt b/requirements/docs.txt index fa325730..a7952ab0 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,4 +1,4 @@ -# SHA1:a0b29b772e4f1fe4102ea0ecf6978eae404c635e +# SHA1:f334aa080c9558edc1a060050a0ddfab6eb8d408 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -111,7 +111,11 @@ pandas==2.2.2 pandocfilters==1.5.1 # via nbconvert platformdirs==4.2.2 - # via jupyter-core + # via + # jupyter-core + # pooch +pooch==1.8.2 + # via -r docs.in psutil==6.0.0 # via ipykernel pyarrow==17.0.0 @@ -135,7 +139,9 @@ referencing==0.35.1 # jsonschema # jsonschema-specifications requests==2.32.3 - # via sphinx + # via + # pooch + # sphinx rpds-py==0.20.0 # via # jsonschema diff --git a/tests/dream/io/nexus_test.py b/tests/dream/io/nexus_test.py index c248ef06..2e684279 100644 --- a/tests/dream/io/nexus_test.py +++ b/tests/dream/io/nexus_test.py @@ -4,7 +4,8 @@ import sciline from ess import dream -from ess.dream import data, nexus # noqa: F401 +import ess.dream.data # noqa: F401 +from ess.dream import nexus from ess.powder.types import ( Filename, NeXusDetectorName, diff --git a/tests/snspowder/powgen/powgen_reduction_test.py b/tests/snspowder/powgen/powgen_reduction_test.py index 2fb1bd8f..7313ace6 100644 --- a/tests/snspowder/powgen/powgen_reduction_test.py +++ b/tests/snspowder/powgen/powgen_reduction_test.py @@ -1,12 +1,12 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +import ess.snspowder.powgen.data # noqa: F401 import pytest import sciline import scipp as sc from ess import powder from ess.snspowder import powgen -from ess.snspowder.powgen import data # noqa: F401 from ess.powder.types import ( CalibrationFilename, From 350f970a6cf110820cd1cccca4f8d8f33899eb87 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 7 Aug 2024 13:52:03 +0200 Subject: [PATCH 05/11] Add common Nexus loaders --- src/ess/dream/io/geant4.py | 16 +-- src/ess/dream/io/nexus.py | 192 +++++++++++++++++++++++++++++----- src/ess/powder/types.py | 61 +++++++++-- tests/dream/io/geant4_test.py | 4 +- tests/dream/io/nexus_test.py | 32 +++--- 5 files changed, 249 insertions(+), 56 deletions(-) diff --git a/src/ess/dream/io/geant4.py b/src/ess/dream/io/geant4.py index d5060e4a..d9aec0ed 100644 --- a/src/ess/dream/io/geant4.py +++ b/src/ess/dream/io/geant4.py @@ -10,10 +10,10 @@ CalibrationData, CalibrationFilename, Filename, + NeXusDetector, NeXusDetectorDimensions, NeXusDetectorName, RawDetector, - RawDetectorData, RawSample, RawSource, ReducibleDetectorData, @@ -64,16 +64,16 @@ def load_geant4_csv(file_path: Filename[RunType]) -> AllRawDetectors[RunType]: def extract_geant4_detector( detectors: AllRawDetectors[RunType], detector_name: NeXusDetectorName -) -> RawDetector[RunType]: +) -> NeXusDetector[RunType]: """Extract a single detector from a loaded GEANT4 simulation.""" - return RawDetector[RunType](detectors["instrument"][detector_name]) + return NeXusDetector[RunType](detectors["instrument"][detector_name]) def extract_geant4_detector_data( - detector: RawDetector[RunType], -) -> RawDetectorData[RunType]: + detector: NeXusDetector[RunType], +) -> RawDetector[RunType]: """Extract the histogram or event data from a loaded GEANT4 detector.""" - return RawDetectorData[RunType](extract_detector_data(detector)) + return RawDetector[RunType](extract_detector_data(detector)) def _load_raw_events(file_path: str) -> sc.DataArray: @@ -176,7 +176,7 @@ def get_sample_position(raw_sample: RawSample[RunType]) -> SamplePosition[RunTyp def patch_detector_data( - detector_data: RawDetectorData[RunType], + detector_data: RawDetector[RunType], source_position: SourcePosition[RunType], sample_position: SamplePosition[RunType], ) -> ReducibleDetectorData[RunType]: @@ -188,7 +188,7 @@ def patch_detector_data( def geant4_detector_dimensions( - data: RawDetectorData[SampleRun], + data: RawDetector[SampleRun], ) -> NeXusDetectorDimensions: # For geant4 data, we group by detector identifier, so the data already has # logical dimensions, so we simply return the dimensions of the detector. diff --git a/src/ess/dream/io/nexus.py b/src/ess/dream/io/nexus.py index 98fb618f..96d7bc29 100644 --- a/src/ess/dream/io/nexus.py +++ b/src/ess/dream/io/nexus.py @@ -13,14 +13,25 @@ but it is not possible to reshape the data into all the logical dimensions. """ +import warnings +from typing import Any + import scipp as sc +import scippnexus as snx from ess.reduce import nexus from ess.powder.types import ( + DetectorEventData, Filename, - LoadedNeXusDetector, + MonitorEventData, + MonitorType, + NeXusDetector, NeXusDetectorName, - RawDetectorData, + NeXusMonitor, + NeXusMonitorName, + RawDetector, + RawMonitor, + RawMonitorData, RawSample, RawSource, ReducibleDetectorData, @@ -84,10 +95,62 @@ def load_nexus_source(file_path: Filename[RunType]) -> RawSource[RunType]: def load_nexus_detector( file_path: Filename[RunType], detector_name: NeXusDetectorName -) -> LoadedNeXusDetector[RunType]: - out = nexus.load_detector(file_path=file_path, detector_name=detector_name) - out.pop("pixel_shape", None) - return LoadedNeXusDetector[RunType](out) +) -> NeXusDetector[RunType]: + definitions = snx.base_definitions() + definitions["NXdetector"] = FilteredDetector + # Events will be loaded later. Should we set something else as data instead, or + # use different NeXus definitions to completely bypass the (empty) event load? + dg = nexus.load_detector( + file_path=file_path, + detector_name=detector_name, + selection={'event_time_zero': slice(0, 0)}, + definitions=definitions, + ) + # The name is required later, e.g., for determining logical detector shape + dg['detector_name'] = detector_name + return NeXusDetector[RunType](dg) + + +def load_nexus_monitor( + file_path: Filename[RunType], monitor_name: NeXusMonitorName[MonitorType] +) -> NeXusMonitor[RunType, MonitorType]: + # It would be simpler to use something like + # selection={'event_time_zero': slice(0, 0)}, + # to avoid loading events, but currently we have files with empty NXevent_data + # groups so that does not work. Instead, skip event loading and create empty dummy. + definitions = snx.base_definitions() + definitions["NXmonitor"] = NXmonitor_no_events + # TODO There is a another problem with the DREAM files: + # Transformaiton chains depend on transformations outside the current group, so + # loading a monitor in an isolated manner is not possible + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + category=UserWarning, + message="Failed to load", + ) + monitor = nexus.load_monitor( + file_path=file_path, + monitor_name=monitor_name, + definitions=definitions, + ) + empty_events = sc.DataArray( + sc.empty(dims=['event'], shape=[0], dtype='float32', unit='counts'), + coords={'event_time_offset': sc.array(dims=['event'], values=[], unit='ns')}, + ) + monitor[f'{monitor_name}_events'] = sc.DataArray( + sc.bins( + dim='event', + data=empty_events, + begin=sc.empty(dims=['event_time_zero'], shape=[0], unit=None), + ), + coords={ + 'event_time_zero': sc.datetimes( + dims=['event_time_zero'], values=[], unit='ns' + ) + }, + ) + return NeXusMonitor[RunType, MonitorType](monitor) def get_source_position( @@ -103,42 +166,121 @@ def get_sample_position( def get_detector_data( - detector: LoadedNeXusDetector[RunType], - detector_name: NeXusDetectorName, -) -> RawDetectorData[RunType]: + detector: NeXusDetector[RunType], +) -> RawDetector[RunType]: da = nexus.extract_detector_data(detector) - if detector_name in DETECTOR_BANK_SIZES: - da = da.fold(dim="detector_number", sizes=DETECTOR_BANK_SIZES[detector_name]) - return RawDetectorData[RunType](da) + if (sizes := DETECTOR_BANK_SIZES.get(detector['detector_name'])) is not None: + da = da.fold(dim="detector_number", sizes=sizes) + return RawDetector[RunType](da) + + +def get_monitor_data( + monitor: NeXusMonitor[RunType, MonitorType], + source_position: SourcePosition[RunType], +) -> RawMonitor[RunType, MonitorType]: + return RawMonitor[RunType, MonitorType]( + nexus.extract_monitor_data(monitor).assign_coords( + position=monitor['position'], source_position=source_position + ) + ) -def patch_detector_data( - detector_data: RawDetectorData[RunType], +def assemble_detector_data( + detector: RawDetector[RunType], + event_data: DetectorEventData[RunType], source_position: SourcePosition[RunType], sample_position: SamplePosition[RunType], ) -> ReducibleDetectorData[RunType]: """ - Patch a detector data object with source and sample positions. + Assemble a detector data object with source and sample positions and event data. Also adds variances to the event data if they are missing. """ - out = detector_data.copy(deep=False) + grouped = nexus.group_event_data( + event_data=event_data, detector_number=detector.coords['detector_number'] + ) + detector.data = grouped.data + return ReducibleDetectorData[RunType]( + _add_variances(da=detector).assign_coords( + source_position=source_position, sample_position=sample_position + ) + ) + + +def assemble_monitor_data( + monitor_data: RawMonitor[RunType, MonitorType], + event_data: MonitorEventData[RunType, MonitorType], +) -> RawMonitorData[RunType, MonitorType]: + meta = monitor_data.drop_coords('event_time_zero') + da = event_data.assign_coords(meta.coords).assign_masks(meta.masks) + return RawMonitorData[RunType, MonitorType](_add_variances(da=da)) + + +def _skip( + _: str, obj: snx.Field | snx.Group, classes: tuple[snx.NXobject, ...] +) -> bool: + return isinstance(obj, snx.Group) and (obj.nx_class in classes) + + +class FilteredDetector(snx.NXdetector): + def __init__( + self, attrs: dict[str, Any], children: dict[str, snx.Field | snx.Group] + ): + children = { + name: child + for name, child in children.items() + if not _skip(name, child, classes=(snx.NXoff_geometry,)) + } + super().__init__(attrs=attrs, children=children) + + +class NXmonitor_no_events(snx.NXmonitor): + def __init__( + self, attrs: dict[str, Any], children: dict[str, snx.Field | snx.Group] + ): + children = { + name: child + for name, child in children.items() + if not _skip(name, child, classes=(snx.NXevent_data,)) + } + super().__init__(attrs=attrs, children=children) + + +def load_detector_event_data( + file_path: Filename[RunType], detector_name: NeXusDetectorName +) -> DetectorEventData[RunType]: + da = nexus.load_event_data(file_path=file_path, component_name=detector_name) + return DetectorEventData[RunType](da) + + +def load_monitor_event_data( + file_path: Filename[RunType], monitor_name: NeXusMonitorName[MonitorType] +) -> MonitorEventData[RunType, MonitorType]: + da = nexus.load_event_data(file_path=file_path, component_name=monitor_name) + return MonitorEventData[RunType, MonitorType](da) + + +def _add_variances(da: sc.DataArray) -> sc.DataArray: + out = da.copy(deep=False) if out.bins is not None: - content = out.bins.constituents["data"] + content = out.bins.constituents['data'] if content.variances is None: content.variances = content.values - out.coords["sample_position"] = sample_position - out.coords["source_position"] = source_position - return ReducibleDetectorData[RunType](out) + return out providers = ( + assemble_detector_data, + assemble_monitor_data, + get_detector_data, + get_monitor_data, + get_sample_position, + get_source_position, + load_detector_event_data, + load_monitor_event_data, + load_nexus_detector, + load_nexus_monitor, load_nexus_sample, load_nexus_source, - load_nexus_detector, - get_source_position, - get_sample_position, - get_detector_data, - patch_detector_data, ) """ Providers for loading and processing DREAM NeXus data. diff --git a/src/ess/powder/types.py b/src/ess/powder/types.py index 658bcc1a..e9adb4de 100644 --- a/src/ess/powder/types.py +++ b/src/ess/powder/types.py @@ -28,6 +28,13 @@ RunType = TypeVar("RunType", EmptyInstrumentRun, SampleRun, VanadiumRun) """TypeVar used for specifying the run.""" +# 1.2 Monitor types +Monitor1 = NewType('Monitor1', int) +"""Placeholder for monitor 1.""" +Monitor2 = NewType('Monitor2', int) +"""Placeholder for monitor 2.""" +MonitorType = TypeVar('MonitorType', Monitor1, Monitor2) +"""TypeVar used for identifying a monitor""" # 2 Workflow parameters @@ -38,6 +45,11 @@ NeXusDetectorName = NewType("NeXusDetectorName", str) """Name of detector entry in NeXus file""" + +class NeXusMonitorName(sciline.Scope[MonitorType, str], str): + """Name of Incident|Transmission monitor in NeXus file""" + + DspacingBins = NewType("DSpacingBins", sc.Variable) """Bin edges for d-spacing.""" @@ -126,9 +138,46 @@ class FocussedDataDspacingTwoTheta(sciline.Scope[RunType, sc.DataArray], sc.Data """Data that has been normalized by a vanadium run, and grouped into 2theta bins.""" -class LoadedNeXusDetector(sciline.Scope[RunType, sc.DataGroup], sc.DataGroup): - """Detector data, loaded from a NeXus file, containing not only neutron events - but also pixel shape information, transformations, ...""" +class NeXusDetector(sciline.Scope[RunType, sc.DataGroup], sc.DataGroup): + """ + Detector loaded from a NeXus file, without event data. + + Contains detector numbers, pixel shape information, transformations, ... + """ + + +class NeXusMonitor( + sciline.ScopeTwoParams[RunType, MonitorType, sc.DataGroup], sc.DataGroup +): + """ + Monitor loaded from a NeXus file, without event data. + + Contains detector numbers, pixel shape information, transformations, ... + """ + + +class DetectorEventData(sciline.Scope[RunType, sc.DataArray], sc.DataArray): + """Event data loaded from a detector in a NeXus file""" + + +class MonitorEventData( + sciline.ScopeTwoParams[RunType, MonitorType, sc.DataArray], sc.DataArray +): + """Event data loaded from a monitor in a NeXus file""" + + +class RawMonitor( + sciline.ScopeTwoParams[RunType, MonitorType, sc.DataArray], sc.DataArray +): + """Raw monitor data""" + + +class RawMonitorData( + sciline.ScopeTwoParams[RunType, MonitorType, sc.DataArray], sc.DataArray +): + """Raw monitor data where variances and necessary coordinates + (e.g. source position) have been added, and where optionally some + user configuration was applied to some of the coordinates.""" class MaskedData(sciline.Scope[RunType, sc.DataArray], sc.DataArray): @@ -156,11 +205,7 @@ class RawDataAndMetadata(sciline.Scope[RunType, sc.DataGroup], sc.DataGroup): """Raw data and associated metadata.""" -class RawDetector(sciline.Scope[RunType, sc.DataGroup], sc.DataGroup): - """Full raw data for a detector.""" - - -class RawDetectorData(sciline.Scope[RunType, sc.DataArray], sc.DataArray): +class RawDetector(sciline.Scope[RunType, sc.DataArray], sc.DataArray): """Data (events / histogram) extracted from a RawDetector.""" diff --git a/tests/dream/io/geant4_test.py b/tests/dream/io/geant4_test.py index 74522663..70adecb0 100644 --- a/tests/dream/io/geant4_test.py +++ b/tests/dream/io/geant4_test.py @@ -11,7 +11,7 @@ import scipp.testing from ess.dream import data, load_geant4_csv -from ess.powder.types import Filename, NeXusDetectorName, RawDetectorData, SampleRun +from ess.powder.types import Filename, NeXusDetectorName, RawDetector, SampleRun @pytest.fixture(scope="module") @@ -180,6 +180,6 @@ def test_geant4_in_pipeline(file_path, file): NeXusDetectorName: NeXusDetectorName("mantle"), }, ) - detector = pipeline.compute(RawDetectorData[SampleRun]) + detector = pipeline.compute(RawDetector[SampleRun]) expected = load_geant4_csv(file)["instrument"]["mantle"]["events"] sc.testing.assert_identical(detector, expected) diff --git a/tests/dream/io/nexus_test.py b/tests/dream/io/nexus_test.py index 2e684279..8c7e8b9d 100644 --- a/tests/dream/io/nexus_test.py +++ b/tests/dream/io/nexus_test.py @@ -8,8 +8,11 @@ from ess.dream import nexus from ess.powder.types import ( Filename, + Monitor1, NeXusDetectorName, - RawDetectorData, + NeXusMonitorName, + RawDetector, + RawMonitor, ReducibleDetectorData, SampleRun, ) @@ -20,15 +23,7 @@ @pytest.fixture() def providers(): - return ( - nexus.dummy_load_sample, - nexus.load_nexus_source, - nexus.load_nexus_detector, - nexus.get_sample_position, - nexus.get_source_position, - nexus.get_detector_data, - nexus.patch_detector_data, - ) + return (*nexus.providers, nexus.dummy_load_sample) @pytest.fixture( @@ -50,7 +45,7 @@ def params(request): def test_can_load_nexus_detector_data(providers, params): pipeline = sciline.Pipeline(params=params, providers=providers) - result = pipeline.compute(RawDetectorData[SampleRun]) + result = pipeline.compute(RawDetector[SampleRun]) assert ( set(result.dims) == hr_sans_dims if params[NeXusDetectorName] @@ -60,6 +55,17 @@ def test_can_load_nexus_detector_data(providers, params): ) else bank_dims ) + assert result.bins.size().sum().value == 0 + + +def test_can_load_nexus_monitor_data(providers): + pipeline = sciline.Pipeline(providers=providers) + pipeline[Filename[SampleRun]] = dream.data.get_path( + 'DREAM_nexus_sorted-2023-12-07.nxs' + ) + pipeline[NeXusMonitorName[Monitor1]] = 'monitor_cave' + result = pipeline.compute(RawMonitor[SampleRun, Monitor1]) + assert result.bins.size().sum().value == 0 def test_load_fails_with_bad_detector_name(providers): @@ -69,10 +75,10 @@ def test_load_fails_with_bad_detector_name(providers): } pipeline = sciline.Pipeline(params=params, providers=providers) with pytest.raises(KeyError, match='bad_detector'): - pipeline.compute(RawDetectorData[SampleRun]) + pipeline.compute(RawDetector[SampleRun]) -def test_patch_nexus_detector_data(providers, params): +def test_assemble_nexus_detector_data(providers, params): pipeline = sciline.Pipeline(params=params, providers=providers) result = pipeline.compute(ReducibleDetectorData[SampleRun]) assert ( From 8a38782a25f94fcb23bc02e9c9d427de3cb3ad78 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 7 Aug 2024 14:08:11 +0200 Subject: [PATCH 06/11] Cleanup --- src/ess/dream/io/nexus.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/ess/dream/io/nexus.py b/src/ess/dream/io/nexus.py index 96d7bc29..6e248f5b 100644 --- a/src/ess/dream/io/nexus.py +++ b/src/ess/dream/io/nexus.py @@ -120,9 +120,6 @@ def load_nexus_monitor( # groups so that does not work. Instead, skip event loading and create empty dummy. definitions = snx.base_definitions() definitions["NXmonitor"] = NXmonitor_no_events - # TODO There is a another problem with the DREAM files: - # Transformaiton chains depend on transformations outside the current group, so - # loading a monitor in an isolated manner is not possible with warnings.catch_warnings(): warnings.filterwarnings( "ignore", From 84db083cd22ab36a231ee0a5be8034d2071e890d Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 7 Aug 2024 14:25:22 +0200 Subject: [PATCH 07/11] Move common NeXus stuff to ess.powder.nexus --- src/ess/dream/io/nexus.py | 233 +------------------------------- src/ess/powder/__init__.py | 2 + src/ess/powder/nexus.py | 251 +++++++++++++++++++++++++++++++++++ src/ess/powder/types.py | 2 + tests/dream/io/nexus_test.py | 4 +- 5 files changed, 262 insertions(+), 230 deletions(-) create mode 100644 src/ess/powder/nexus.py diff --git a/src/ess/dream/io/nexus.py b/src/ess/dream/io/nexus.py index 6e248f5b..942ce438 100644 --- a/src/ess/dream/io/nexus.py +++ b/src/ess/dream/io/nexus.py @@ -13,32 +13,7 @@ but it is not possible to reshape the data into all the logical dimensions. """ -import warnings -from typing import Any - -import scipp as sc -import scippnexus as snx -from ess.reduce import nexus - -from ess.powder.types import ( - DetectorEventData, - Filename, - MonitorEventData, - MonitorType, - NeXusDetector, - NeXusDetectorName, - NeXusMonitor, - NeXusMonitorName, - RawDetector, - RawMonitor, - RawMonitorData, - RawSample, - RawSource, - ReducibleDetectorData, - RunType, - SamplePosition, - SourcePosition, -) +from ess import powder DETECTOR_BANK_SIZES = { "endcap_backward_detector": { @@ -76,209 +51,11 @@ } -def load_nexus_sample(file_path: Filename[RunType]) -> RawSample[RunType]: - return RawSample[RunType](nexus.load_sample(file_path)) - - -def dummy_load_sample(file_path: Filename[RunType]) -> RawSample[RunType]: - """ - In test files there is not always a sample, so we need a dummy. - """ - return RawSample[RunType]( - sc.DataGroup({'position': sc.vector(value=[0, 0, 0], unit='m')}) - ) - - -def load_nexus_source(file_path: Filename[RunType]) -> RawSource[RunType]: - return RawSource[RunType](nexus.load_source(file_path)) - - -def load_nexus_detector( - file_path: Filename[RunType], detector_name: NeXusDetectorName -) -> NeXusDetector[RunType]: - definitions = snx.base_definitions() - definitions["NXdetector"] = FilteredDetector - # Events will be loaded later. Should we set something else as data instead, or - # use different NeXus definitions to completely bypass the (empty) event load? - dg = nexus.load_detector( - file_path=file_path, - detector_name=detector_name, - selection={'event_time_zero': slice(0, 0)}, - definitions=definitions, - ) - # The name is required later, e.g., for determining logical detector shape - dg['detector_name'] = detector_name - return NeXusDetector[RunType](dg) - - -def load_nexus_monitor( - file_path: Filename[RunType], monitor_name: NeXusMonitorName[MonitorType] -) -> NeXusMonitor[RunType, MonitorType]: - # It would be simpler to use something like - # selection={'event_time_zero': slice(0, 0)}, - # to avoid loading events, but currently we have files with empty NXevent_data - # groups so that does not work. Instead, skip event loading and create empty dummy. - definitions = snx.base_definitions() - definitions["NXmonitor"] = NXmonitor_no_events - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", - category=UserWarning, - message="Failed to load", - ) - monitor = nexus.load_monitor( - file_path=file_path, - monitor_name=monitor_name, - definitions=definitions, - ) - empty_events = sc.DataArray( - sc.empty(dims=['event'], shape=[0], dtype='float32', unit='counts'), - coords={'event_time_offset': sc.array(dims=['event'], values=[], unit='ns')}, - ) - monitor[f'{monitor_name}_events'] = sc.DataArray( - sc.bins( - dim='event', - data=empty_events, - begin=sc.empty(dims=['event_time_zero'], shape=[0], unit=None), - ), - coords={ - 'event_time_zero': sc.datetimes( - dims=['event_time_zero'], values=[], unit='ns' - ) - }, - ) - return NeXusMonitor[RunType, MonitorType](monitor) - - -def get_source_position( - raw_source: RawSource[RunType], -) -> SourcePosition[RunType]: - return SourcePosition[RunType](raw_source["position"]) - - -def get_sample_position( - raw_sample: RawSample[RunType], -) -> SamplePosition[RunType]: - return SamplePosition[RunType](raw_sample["position"]) - - -def get_detector_data( - detector: NeXusDetector[RunType], -) -> RawDetector[RunType]: - da = nexus.extract_detector_data(detector) - if (sizes := DETECTOR_BANK_SIZES.get(detector['detector_name'])) is not None: - da = da.fold(dim="detector_number", sizes=sizes) - return RawDetector[RunType](da) - - -def get_monitor_data( - monitor: NeXusMonitor[RunType, MonitorType], - source_position: SourcePosition[RunType], -) -> RawMonitor[RunType, MonitorType]: - return RawMonitor[RunType, MonitorType]( - nexus.extract_monitor_data(monitor).assign_coords( - position=monitor['position'], source_position=source_position - ) - ) - - -def assemble_detector_data( - detector: RawDetector[RunType], - event_data: DetectorEventData[RunType], - source_position: SourcePosition[RunType], - sample_position: SamplePosition[RunType], -) -> ReducibleDetectorData[RunType]: - """ - Assemble a detector data object with source and sample positions and event data. - Also adds variances to the event data if they are missing. - """ - grouped = nexus.group_event_data( - event_data=event_data, detector_number=detector.coords['detector_number'] - ) - detector.data = grouped.data - return ReducibleDetectorData[RunType]( - _add_variances(da=detector).assign_coords( - source_position=source_position, sample_position=sample_position - ) - ) - - -def assemble_monitor_data( - monitor_data: RawMonitor[RunType, MonitorType], - event_data: MonitorEventData[RunType, MonitorType], -) -> RawMonitorData[RunType, MonitorType]: - meta = monitor_data.drop_coords('event_time_zero') - da = event_data.assign_coords(meta.coords).assign_masks(meta.masks) - return RawMonitorData[RunType, MonitorType](_add_variances(da=da)) - - -def _skip( - _: str, obj: snx.Field | snx.Group, classes: tuple[snx.NXobject, ...] -) -> bool: - return isinstance(obj, snx.Group) and (obj.nx_class in classes) - - -class FilteredDetector(snx.NXdetector): - def __init__( - self, attrs: dict[str, Any], children: dict[str, snx.Field | snx.Group] - ): - children = { - name: child - for name, child in children.items() - if not _skip(name, child, classes=(snx.NXoff_geometry,)) - } - super().__init__(attrs=attrs, children=children) - - -class NXmonitor_no_events(snx.NXmonitor): - def __init__( - self, attrs: dict[str, Any], children: dict[str, snx.Field | snx.Group] - ): - children = { - name: child - for name, child in children.items() - if not _skip(name, child, classes=(snx.NXevent_data,)) - } - super().__init__(attrs=attrs, children=children) - - -def load_detector_event_data( - file_path: Filename[RunType], detector_name: NeXusDetectorName -) -> DetectorEventData[RunType]: - da = nexus.load_event_data(file_path=file_path, component_name=detector_name) - return DetectorEventData[RunType](da) - - -def load_monitor_event_data( - file_path: Filename[RunType], monitor_name: NeXusMonitorName[MonitorType] -) -> MonitorEventData[RunType, MonitorType]: - da = nexus.load_event_data(file_path=file_path, component_name=monitor_name) - return MonitorEventData[RunType, MonitorType](da) - - -def _add_variances(da: sc.DataArray) -> sc.DataArray: - out = da.copy(deep=False) - if out.bins is not None: - content = out.bins.constituents['data'] - if content.variances is None: - content.variances = content.values - return out +def dream_detector_bank_sizes() -> powder.types.DetectorBankSizes | None: + return powder.types.DetectorBankSizes(DETECTOR_BANK_SIZES) -providers = ( - assemble_detector_data, - assemble_monitor_data, - get_detector_data, - get_monitor_data, - get_sample_position, - get_source_position, - load_detector_event_data, - load_monitor_event_data, - load_nexus_detector, - load_nexus_monitor, - load_nexus_sample, - load_nexus_source, -) +providers = (*powder.nexus.providers, dream_detector_bank_sizes) """ -Providers for loading and processing DREAM NeXus data. +Providers for loading and processing NeXus data. """ diff --git a/src/ess/powder/__init__.py b/src/ess/powder/__init__.py index 50ecb914..38ba4624 100644 --- a/src/ess/powder/__init__.py +++ b/src/ess/powder/__init__.py @@ -15,6 +15,7 @@ smoothing, ) from .masking import with_pixel_mask_filenames +from . import nexus try: __version__ = importlib.metadata.version(__package__ or __name__) @@ -39,6 +40,7 @@ "filtering", "grouping", "masking", + "nexus", "providers", "smoothing", "with_pixel_mask_filenames", diff --git a/src/ess/powder/nexus.py b/src/ess/powder/nexus.py new file mode 100644 index 00000000..6cba8944 --- /dev/null +++ b/src/ess/powder/nexus.py @@ -0,0 +1,251 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) + +"""NeXus input/output for DREAM. + +Notes on the detector dimensions (2024-05-22): + +See https://confluence.esss.lu.se/pages/viewpage.action?pageId=462000005 +and the ICD DREAM interface specification for details. + +- The high-resolution and SANS detectors have a very odd numbering scheme. + The scheme attempts to follows some sort of physical ordering in space (x,y,z), + but it is not possible to reshape the data into all the logical dimensions. +""" + +import warnings +from typing import Any + +import scipp as sc +import scippnexus as snx +from ess.reduce import nexus + +from ess.powder.types import ( + DetectorBankSizes, + DetectorEventData, + Filename, + MonitorEventData, + MonitorType, + NeXusDetector, + NeXusDetectorName, + NeXusMonitor, + NeXusMonitorName, + RawDetector, + RawMonitor, + RawMonitorData, + RawSample, + RawSource, + ReducibleDetectorData, + RunType, + SamplePosition, + SourcePosition, +) + + +def load_nexus_sample(file_path: Filename[RunType]) -> RawSample[RunType]: + return RawSample[RunType](nexus.load_sample(file_path)) + + +def dummy_load_sample(file_path: Filename[RunType]) -> RawSample[RunType]: + """ + In test files there is not always a sample, so we need a dummy. + """ + return RawSample[RunType]( + sc.DataGroup({'position': sc.vector(value=[0, 0, 0], unit='m')}) + ) + + +def load_nexus_source(file_path: Filename[RunType]) -> RawSource[RunType]: + return RawSource[RunType](nexus.load_source(file_path)) + + +def load_nexus_detector( + file_path: Filename[RunType], detector_name: NeXusDetectorName +) -> NeXusDetector[RunType]: + definitions = snx.base_definitions() + definitions["NXdetector"] = FilteredDetector + # Events will be loaded later. Should we set something else as data instead, or + # use different NeXus definitions to completely bypass the (empty) event load? + dg = nexus.load_detector( + file_path=file_path, + detector_name=detector_name, + selection={'event_time_zero': slice(0, 0)}, + definitions=definitions, + ) + # The name is required later, e.g., for determining logical detector shape + dg['detector_name'] = detector_name + return NeXusDetector[RunType](dg) + + +def load_nexus_monitor( + file_path: Filename[RunType], monitor_name: NeXusMonitorName[MonitorType] +) -> NeXusMonitor[RunType, MonitorType]: + # It would be simpler to use something like + # selection={'event_time_zero': slice(0, 0)}, + # to avoid loading events, but currently we have files with empty NXevent_data + # groups so that does not work. Instead, skip event loading and create empty dummy. + definitions = snx.base_definitions() + definitions["NXmonitor"] = NXmonitor_no_events + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + category=UserWarning, + message="Failed to load", + ) + monitor = nexus.load_monitor( + file_path=file_path, + monitor_name=monitor_name, + definitions=definitions, + ) + empty_events = sc.DataArray( + sc.empty(dims=['event'], shape=[0], dtype='float32', unit='counts'), + coords={'event_time_offset': sc.array(dims=['event'], values=[], unit='ns')}, + ) + monitor[f'{monitor_name}_events'] = sc.DataArray( + sc.bins( + dim='event', + data=empty_events, + begin=sc.empty(dims=['event_time_zero'], shape=[0], unit=None), + ), + coords={ + 'event_time_zero': sc.datetimes( + dims=['event_time_zero'], values=[], unit='ns' + ) + }, + ) + return NeXusMonitor[RunType, MonitorType](monitor) + + +def get_source_position( + raw_source: RawSource[RunType], +) -> SourcePosition[RunType]: + return SourcePosition[RunType](raw_source["position"]) + + +def get_sample_position( + raw_sample: RawSample[RunType], +) -> SamplePosition[RunType]: + return SamplePosition[RunType](raw_sample["position"]) + + +def get_detector_data( + detector: NeXusDetector[RunType], + bank_sizes: DetectorBankSizes | None = None, +) -> RawDetector[RunType]: + da = nexus.extract_detector_data(detector) + if (sizes := (bank_sizes or {}).get(detector['detector_name'])) is not None: + da = da.fold(dim="detector_number", sizes=sizes) + return RawDetector[RunType](da) + + +def get_monitor_data( + monitor: NeXusMonitor[RunType, MonitorType], + source_position: SourcePosition[RunType], +) -> RawMonitor[RunType, MonitorType]: + return RawMonitor[RunType, MonitorType]( + nexus.extract_monitor_data(monitor).assign_coords( + position=monitor['position'], source_position=source_position + ) + ) + + +def assemble_detector_data( + detector: RawDetector[RunType], + event_data: DetectorEventData[RunType], + source_position: SourcePosition[RunType], + sample_position: SamplePosition[RunType], +) -> ReducibleDetectorData[RunType]: + """ + Assemble a detector data object with source and sample positions and event data. + Also adds variances to the event data if they are missing. + """ + grouped = nexus.group_event_data( + event_data=event_data, detector_number=detector.coords['detector_number'] + ) + detector.data = grouped.data + return ReducibleDetectorData[RunType]( + _add_variances(da=detector).assign_coords( + source_position=source_position, sample_position=sample_position + ) + ) + + +def assemble_monitor_data( + monitor_data: RawMonitor[RunType, MonitorType], + event_data: MonitorEventData[RunType, MonitorType], +) -> RawMonitorData[RunType, MonitorType]: + meta = monitor_data.drop_coords('event_time_zero') + da = event_data.assign_coords(meta.coords).assign_masks(meta.masks) + return RawMonitorData[RunType, MonitorType](_add_variances(da=da)) + + +def _skip( + _: str, obj: snx.Field | snx.Group, classes: tuple[snx.NXobject, ...] +) -> bool: + return isinstance(obj, snx.Group) and (obj.nx_class in classes) + + +class FilteredDetector(snx.NXdetector): + def __init__( + self, attrs: dict[str, Any], children: dict[str, snx.Field | snx.Group] + ): + children = { + name: child + for name, child in children.items() + if not _skip(name, child, classes=(snx.NXoff_geometry,)) + } + super().__init__(attrs=attrs, children=children) + + +class NXmonitor_no_events(snx.NXmonitor): + def __init__( + self, attrs: dict[str, Any], children: dict[str, snx.Field | snx.Group] + ): + children = { + name: child + for name, child in children.items() + if not _skip(name, child, classes=(snx.NXevent_data,)) + } + super().__init__(attrs=attrs, children=children) + + +def load_detector_event_data( + file_path: Filename[RunType], detector_name: NeXusDetectorName +) -> DetectorEventData[RunType]: + da = nexus.load_event_data(file_path=file_path, component_name=detector_name) + return DetectorEventData[RunType](da) + + +def load_monitor_event_data( + file_path: Filename[RunType], monitor_name: NeXusMonitorName[MonitorType] +) -> MonitorEventData[RunType, MonitorType]: + da = nexus.load_event_data(file_path=file_path, component_name=monitor_name) + return MonitorEventData[RunType, MonitorType](da) + + +def _add_variances(da: sc.DataArray) -> sc.DataArray: + out = da.copy(deep=False) + if out.bins is not None: + content = out.bins.constituents['data'] + if content.variances is None: + content.variances = content.values + return out + + +providers = ( + assemble_detector_data, + assemble_monitor_data, + get_detector_data, + get_monitor_data, + get_sample_position, + get_source_position, + load_detector_event_data, + load_monitor_event_data, + load_nexus_detector, + load_nexus_monitor, + load_nexus_sample, + load_nexus_source, +) +""" +Providers for loading and processing NeXus data. +""" diff --git a/src/ess/powder/types.py b/src/ess/powder/types.py index e9adb4de..5697475d 100644 --- a/src/ess/powder/types.py +++ b/src/ess/powder/types.py @@ -38,6 +38,8 @@ # 2 Workflow parameters +DetectorBankSizes = NewType("DetectorBankSizes", dict[str, dict[str, int | Any]]) + CalibrationFilename = NewType("CalibrationFilename", str | None) """Filename of the instrument calibration file.""" diff --git a/tests/dream/io/nexus_test.py b/tests/dream/io/nexus_test.py index 8c7e8b9d..db8c9f25 100644 --- a/tests/dream/io/nexus_test.py +++ b/tests/dream/io/nexus_test.py @@ -2,7 +2,7 @@ # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) import pytest import sciline -from ess import dream +from ess import dream, powder import ess.dream.data # noqa: F401 from ess.dream import nexus @@ -23,7 +23,7 @@ @pytest.fixture() def providers(): - return (*nexus.providers, nexus.dummy_load_sample) + return (*nexus.providers, powder.nexus.dummy_load_sample) @pytest.fixture( From e5a50c551f63318473c56000c159cc1891404a95 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 7 Aug 2024 14:28:41 +0200 Subject: [PATCH 08/11] Format --- src/ess/dream/io/nexus.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/ess/dream/io/nexus.py b/src/ess/dream/io/nexus.py index 942ce438..3e9755a3 100644 --- a/src/ess/dream/io/nexus.py +++ b/src/ess/dream/io/nexus.py @@ -37,16 +37,10 @@ "strip": 256, "counter": 2, }, - "high_resolution_detector": { - "strip": 32, - "other": -1, - }, + "high_resolution_detector": {"strip": 32, "other": -1}, "sans_detector": lambda x: x.fold( dim="detector_number", - sizes={ - "strip": 32, - "other": -1, - }, + sizes={"strip": 32, "other": -1}, ), } From 4fe18da28f951adfc24beb12145e86588e3dcbb5 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 8 Aug 2024 11:51:14 +0200 Subject: [PATCH 09/11] Don't make empty bins, add docstring --- src/ess/powder/nexus.py | 67 +++++++++++++++++++----------------- tests/dream/io/nexus_test.py | 6 ++-- 2 files changed, 40 insertions(+), 33 deletions(-) diff --git a/src/ess/powder/nexus.py b/src/ess/powder/nexus.py index 6cba8944..479c09d0 100644 --- a/src/ess/powder/nexus.py +++ b/src/ess/powder/nexus.py @@ -13,7 +13,6 @@ but it is not possible to reshape the data into all the logical dimensions. """ -import warnings from typing import Any import scipp as sc @@ -62,14 +61,31 @@ def load_nexus_source(file_path: Filename[RunType]) -> RawSource[RunType]: def load_nexus_detector( file_path: Filename[RunType], detector_name: NeXusDetectorName ) -> NeXusDetector[RunType]: + """ + Load detector from NeXus, but with event data replaced by placeholders. + + Currently the placeholder is the detector number, but this may change in the future. + + The returned object is a scipp.DataGroup, as it may contain additional information + about the detector that cannot be represented as a single scipp.DataArray. Most + downstream code will only be interested in the contained scipp.DataArray so this + needs to be extracted. However, other processing steps may require the additional + information, so it is kept in the DataGroup. + + Loading thus proceeds in three steps: + + 1. This function loads the detector, but replaces the event data with placeholders. + 2. :py:func:`get_detector_data` drops the additional information, returning only + the contained scipp.DataArray, reshaped to the logical detector shape. + This will generally contain coordinates as well as pixel masks. + 3. :py:func:`assemble_detector_data` replaces placeholder data values with the + event data, and adds source and sample positions. + """ definitions = snx.base_definitions() definitions["NXdetector"] = FilteredDetector - # Events will be loaded later. Should we set something else as data instead, or - # use different NeXus definitions to completely bypass the (empty) event load? dg = nexus.load_detector( file_path=file_path, detector_name=detector_name, - selection={'event_time_zero': slice(0, 0)}, definitions=definitions, ) # The name is required later, e.g., for determining logical detector shape @@ -86,32 +102,8 @@ def load_nexus_monitor( # groups so that does not work. Instead, skip event loading and create empty dummy. definitions = snx.base_definitions() definitions["NXmonitor"] = NXmonitor_no_events - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", - category=UserWarning, - message="Failed to load", - ) - monitor = nexus.load_monitor( - file_path=file_path, - monitor_name=monitor_name, - definitions=definitions, - ) - empty_events = sc.DataArray( - sc.empty(dims=['event'], shape=[0], dtype='float32', unit='counts'), - coords={'event_time_offset': sc.array(dims=['event'], values=[], unit='ns')}, - ) - monitor[f'{monitor_name}_events'] = sc.DataArray( - sc.bins( - dim='event', - data=empty_events, - begin=sc.empty(dims=['event_time_zero'], shape=[0], unit=None), - ), - coords={ - 'event_time_zero': sc.datetimes( - dims=['event_time_zero'], values=[], unit='ns' - ) - }, + monitor = nexus.load_monitor( + file_path=file_path, monitor_name=monitor_name, definitions=definitions ) return NeXusMonitor[RunType, MonitorType](monitor) @@ -192,8 +184,9 @@ def __init__( children = { name: child for name, child in children.items() - if not _skip(name, child, classes=(snx.NXoff_geometry,)) + if not _skip(name, child, classes=(snx.NXoff_geometry, snx.NXevent_data)) } + children['data'] = children['detector_number'] super().__init__(attrs=attrs, children=children) @@ -206,6 +199,18 @@ def __init__( for name, child in children.items() if not _skip(name, child, classes=(snx.NXevent_data,)) } + + class DummyField: + def __init__(self): + self.attrs = {} + self.sizes = {'event_time_zero': 0} + self.dims = ('event_time_zero',) + self.shape = (0,) + + def __getitem__(self, key: Any) -> sc.Variable: + return sc.empty(dims=self.dims, shape=self.shape, unit=None) + + children['data'] = DummyField() super().__init__(attrs=attrs, children=children) diff --git a/tests/dream/io/nexus_test.py b/tests/dream/io/nexus_test.py index db8c9f25..7b28e4f1 100644 --- a/tests/dream/io/nexus_test.py +++ b/tests/dream/io/nexus_test.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) import pytest import sciline +import scipp as sc from ess import dream, powder import ess.dream.data # noqa: F401 @@ -55,7 +56,8 @@ def test_can_load_nexus_detector_data(providers, params): ) else bank_dims ) - assert result.bins.size().sum().value == 0 + + assert sc.identical(result.data, result.coords['detector_number']) def test_can_load_nexus_monitor_data(providers): @@ -65,7 +67,7 @@ def test_can_load_nexus_monitor_data(providers): ) pipeline[NeXusMonitorName[Monitor1]] = 'monitor_cave' result = pipeline.compute(RawMonitor[SampleRun, Monitor1]) - assert result.bins.size().sum().value == 0 + assert result.sizes == {'event_time_zero': 0} def test_load_fails_with_bad_detector_name(providers): From baf4ddd96195115da96c9216947e38cdf764d8ce Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 8 Aug 2024 13:03:20 +0200 Subject: [PATCH 10/11] Cleanup and docs --- src/ess/powder/nexus.py | 149 ++++++++++++++++++++++++---------------- 1 file changed, 91 insertions(+), 58 deletions(-) diff --git a/src/ess/powder/nexus.py b/src/ess/powder/nexus.py index 479c09d0..46728900 100644 --- a/src/ess/powder/nexus.py +++ b/src/ess/powder/nexus.py @@ -1,17 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) -"""NeXus input/output for DREAM. - -Notes on the detector dimensions (2024-05-22): - -See https://confluence.esss.lu.se/pages/viewpage.action?pageId=462000005 -and the ICD DREAM interface specification for details. - -- The high-resolution and SANS detectors have a very odd numbering scheme. - The scheme attempts to follows some sort of physical ordering in space (x,y,z), - but it is not possible to reshape the data into all the logical dimensions. -""" +"""NeXus input/output for ESS powder reduction.""" from typing import Any @@ -75,14 +65,14 @@ def load_nexus_detector( Loading thus proceeds in three steps: 1. This function loads the detector, but replaces the event data with placeholders. - 2. :py:func:`get_detector_data` drops the additional information, returning only + 2. :py:func:`get_detector_array` drops the additional information, returning only the contained scipp.DataArray, reshaped to the logical detector shape. This will generally contain coordinates as well as pixel masks. 3. :py:func:`assemble_detector_data` replaces placeholder data values with the event data, and adds source and sample positions. """ definitions = snx.base_definitions() - definitions["NXdetector"] = FilteredDetector + definitions["NXdetector"] = _StrippedDetector dg = nexus.load_detector( file_path=file_path, detector_name=detector_name, @@ -96,12 +86,28 @@ def load_nexus_detector( def load_nexus_monitor( file_path: Filename[RunType], monitor_name: NeXusMonitorName[MonitorType] ) -> NeXusMonitor[RunType, MonitorType]: - # It would be simpler to use something like - # selection={'event_time_zero': slice(0, 0)}, - # to avoid loading events, but currently we have files with empty NXevent_data - # groups so that does not work. Instead, skip event loading and create empty dummy. + """ + Load monitor from NeXus, but with event data replaced by placeholders. + + Currently the placeholder is a size-0 array, but this may change in the future. + + The returned object is a scipp.DataGroup, as it may contain additional information + about the monitor that cannot be represented as a single scipp.DataArray. Most + downstream code will only be interested in the contained scipp.DataArray so this + needs to be extracted. However, other processing steps may require the additional + information, so it is kept in the DataGroup. + + Loading thus proceeds in three steps: + + 1. This function loads the monitor, but replaces the event data with placeholders. + 2. :py:func:`get_monitor_array` drops the additional information, returning only + the contained scipp.DataArray. + This will generally contain coordinates as well as pixel masks. + 3. :py:func:`assemble_monitor_data` replaces placeholder data values with the + event data, and adds source and sample positions. + """ definitions = snx.base_definitions() - definitions["NXmonitor"] = NXmonitor_no_events + definitions["NXmonitor"] = _StrippedMonitor monitor = nexus.load_monitor( file_path=file_path, monitor_name=monitor_name, definitions=definitions ) @@ -120,20 +126,34 @@ def get_sample_position( return SamplePosition[RunType](raw_sample["position"]) -def get_detector_data( +def get_detector_signal_array( detector: NeXusDetector[RunType], bank_sizes: DetectorBankSizes | None = None, ) -> RawDetector[RunType]: + """ + Extract the data array corresponding to a detector's signal field. + + The returned data array includes coords and masks pertaining directly to the + signal values array, but not additional information about the detector. The + data array is reshaped to the logical detector shape, which by folding the data + array along the detector_number dimension. + """ da = nexus.extract_detector_data(detector) if (sizes := (bank_sizes or {}).get(detector['detector_name'])) is not None: da = da.fold(dim="detector_number", sizes=sizes) return RawDetector[RunType](da) -def get_monitor_data( +def get_monitor_signal_array( monitor: NeXusMonitor[RunType, MonitorType], source_position: SourcePosition[RunType], ) -> RawMonitor[RunType, MonitorType]: + """ + Extract the data array corresponding to a monitor's signal field. + + The returned data array includes coords pertaining directly to the + signal values array, but not additional information about the monitor. + """ return RawMonitor[RunType, MonitorType]( nexus.extract_monitor_data(monitor).assign_coords( position=monitor['position'], source_position=source_position @@ -148,69 +168,82 @@ def assemble_detector_data( sample_position: SamplePosition[RunType], ) -> ReducibleDetectorData[RunType]: """ - Assemble a detector data object with source and sample positions and event data. + Assemble a detector data array with event data and source- and sample-position. + Also adds variances to the event data if they are missing. """ grouped = nexus.group_event_data( event_data=event_data, detector_number=detector.coords['detector_number'] ) - detector.data = grouped.data return ReducibleDetectorData[RunType]( - _add_variances(da=detector).assign_coords( - source_position=source_position, sample_position=sample_position - ) + _add_variances(grouped) + .assign_coords(source_position=source_position, sample_position=sample_position) + .assign_coords(detector.coords) + .assign_masks(detector.masks) ) def assemble_monitor_data( - monitor_data: RawMonitor[RunType, MonitorType], + monitor: RawMonitor[RunType, MonitorType], event_data: MonitorEventData[RunType, MonitorType], ) -> RawMonitorData[RunType, MonitorType]: - meta = monitor_data.drop_coords('event_time_zero') - da = event_data.assign_coords(meta.coords).assign_masks(meta.masks) + """ + Assemble a monitor data array with event data. + + Also adds variances to the event data if they are missing. + """ + da = event_data.assign_coords(monitor.coords).assign_masks(monitor.masks) return RawMonitorData[RunType, MonitorType](_add_variances(da=da)) -def _skip( - _: str, obj: snx.Field | snx.Group, classes: tuple[snx.NXobject, ...] -) -> bool: - return isinstance(obj, snx.Group) and (obj.nx_class in classes) +def _drop( + children: dict[str, snx.Field | snx.Group], classes: tuple[snx.NXobject, ...] +) -> dict[str, snx.Field | snx.Group]: + return { + name: child + for name, child in children.items() + if not (isinstance(child, snx.Group) and (child.nx_class in classes)) + } + +class _StrippedDetector(snx.NXdetector): + """Detector definition without large geometry or event data for ScippNexus. + + Drops NXoff_geometry and NXevent_data groups, data is replaced by detector_number. + """ -class FilteredDetector(snx.NXdetector): def __init__( self, attrs: dict[str, Any], children: dict[str, snx.Field | snx.Group] ): - children = { - name: child - for name, child in children.items() - if not _skip(name, child, classes=(snx.NXoff_geometry, snx.NXevent_data)) - } + children = _drop(children, (snx.NXoff_geometry, snx.NXevent_data)) children['data'] = children['detector_number'] super().__init__(attrs=attrs, children=children) -class NXmonitor_no_events(snx.NXmonitor): +class _DummyField: + """Dummy field that can replace snx.Field in NXmonitor.""" + + def __init__(self): + self.attrs = {} + self.sizes = {'event_time_zero': 0} + self.dims = ('event_time_zero',) + self.shape = (0,) + + def __getitem__(self, key: Any) -> sc.Variable: + return sc.empty(dims=self.dims, shape=self.shape, unit=None) + + +class _StrippedMonitor(snx.NXmonitor): + """Monitor definition without event data for ScippNexus. + + Drops NXevent_data group, data is replaced by a dummy field. + """ + def __init__( self, attrs: dict[str, Any], children: dict[str, snx.Field | snx.Group] ): - children = { - name: child - for name, child in children.items() - if not _skip(name, child, classes=(snx.NXevent_data,)) - } - - class DummyField: - def __init__(self): - self.attrs = {} - self.sizes = {'event_time_zero': 0} - self.dims = ('event_time_zero',) - self.shape = (0,) - - def __getitem__(self, key: Any) -> sc.Variable: - return sc.empty(dims=self.dims, shape=self.shape, unit=None) - - children['data'] = DummyField() + children = _drop(children, (snx.NXevent_data,)) + children['data'] = _DummyField() super().__init__(attrs=attrs, children=children) @@ -240,8 +273,8 @@ def _add_variances(da: sc.DataArray) -> sc.DataArray: providers = ( assemble_detector_data, assemble_monitor_data, - get_detector_data, - get_monitor_data, + get_detector_signal_array, + get_monitor_signal_array, get_sample_position, get_source_position, load_detector_event_data, From 78b985a69834fc7129023e3e478423f56680d38d Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 8 Aug 2024 14:41:16 +0200 Subject: [PATCH 11/11] Do not use make_binned --- tests/powder/filtering_test.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/powder/filtering_test.py b/tests/powder/filtering_test.py index 95661cae..a32968a0 100644 --- a/tests/powder/filtering_test.py +++ b/tests/powder/filtering_test.py @@ -39,12 +39,8 @@ def make_data_with_pulse_time(rng, n_event) -> sc.DataArray: ), }, ) - return sc.binning.make_binned( - events, - edges=[ - sc.array(dims=['tof'], values=[10, 500, 1000], unit='us', dtype='int64') - ], - groups=[sc.arange('spectrum', 0, 10, unit=None, dtype='int64')], + return events.group(sc.arange('spectrum', 0, 10, unit=None, dtype='int64')).bin( + tof=sc.array(dims=['tof'], values=[10, 500, 1000], unit='us', dtype='int64') )