diff --git a/requirements/base.txt b/requirements/base.txt index 05116451..cf700d5a 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -17,24 +17,22 @@ click==8.1.7 # via dask cloudpickle==3.0.0 # via dask -comm==0.2.0 +comm==0.2.1 # via ipywidgets contourpy==1.2.0 # via matplotlib cycler==0.12.1 # via matplotlib -dask==2023.12.0 +dask==2023.12.1 # via -r base.in decorator==5.1.1 # via ipython executing==2.0.1 # via stack-data -fonttools==4.46.0 +fonttools==4.47.0 # via matplotlib -fsspec==2023.12.1 +fsspec==2023.12.2 # via dask -graphlib-backport==1.0.3 - # via sciline graphviz==0.20.1 # via -r base.in h5py==3.10.0 @@ -43,8 +41,10 @@ h5py==3.10.0 # scippnexus idna==3.6 # via requests -importlib-metadata==7.0.0 +importlib-metadata==7.0.1 # via dask +importlib-resources==6.1.1 + # via matplotlib ipydatawidgets==4.3.5 # via pythreejs ipython==8.9.0 @@ -67,7 +67,7 @@ matplotlib==3.8.2 # via plopp matplotlib-inline==0.1.6 # via ipython -numpy==1.26.2 +numpy==1.26.3 # via # contourpy # h5py @@ -90,7 +90,7 @@ pexpect==4.9.0 # via ipython pickleshare==0.7.5 # via ipython -pillow==10.1.0 +pillow==10.2.0 # via matplotlib platformdirs==4.1.0 # via pooch @@ -120,7 +120,7 @@ pyyaml==6.0.1 # via dask requests==2.31.0 # via pooch -sciline==23.9.1 +sciline==24.1.0 # via -r base.in scipp==23.12.0 # via @@ -147,7 +147,7 @@ toolz==0.12.0 # via # dask # partd -traitlets==5.14.0 +traitlets==5.14.1 # via # comm # ipython @@ -164,4 +164,6 @@ wcwidth==0.2.12 widgetsnbextension==4.0.9 # via ipywidgets zipp==3.17.0 - # via importlib-metadata + # via + # importlib-metadata + # importlib-resources diff --git a/requirements/basetest.in b/requirements/basetest.in index e4a48b29..d5fe7467 100644 --- a/requirements/basetest.in +++ b/requirements/basetest.in @@ -1,4 +1,5 @@ # Dependencies that are only used by tests. # Do not make an environment from this file, use test.txt instead! +pandas pytest diff --git a/requirements/basetest.txt b/requirements/basetest.txt index 6bf43fa2..74195a90 100644 --- a/requirements/basetest.txt +++ b/requirements/basetest.txt @@ -1,4 +1,4 @@ -# SHA1:0eaa389e1fdb3a1917c0f987514bd561be5718ee +# SHA1:2da4cc17c82e9ac0b3764abd55420a1e9d217c9d # # This file is autogenerated by pip-compile-multi # To update, run: @@ -9,11 +9,23 @@ exceptiongroup==1.2.0 # via pytest iniconfig==2.0.0 # via pytest +numpy==1.26.3 + # via pandas packaging==23.2 # via pytest +pandas==2.1.4 + # via -r basetest.in pluggy==1.3.0 # via pytest -pytest==7.4.3 +pytest==7.4.4 # via -r basetest.in +python-dateutil==2.8.2 + # via pandas +pytz==2023.3.post1 + # via pandas +six==1.16.0 + # via python-dateutil tomli==2.0.1 # via pytest +tzdata==2023.4 + # via pandas diff --git a/requirements/ci.txt b/requirements/ci.txt index 279a1737..e33fbbcf 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -15,7 +15,7 @@ charset-normalizer==3.3.2 # via requests colorama==0.4.6 # via tox -distlib==0.3.7 +distlib==0.3.8 # via virtualenv filelock==3.13.1 # via diff --git a/requirements/dev.txt b/requirements/dev.txt index c418cb82..dc88310c 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -14,7 +14,7 @@ -r wheels.txt annotated-types==0.6.0 # via pydantic -anyio==4.1.0 +anyio==4.2.0 # via jupyter-server argon2-cffi==23.1.0 # via jupyter-server @@ -57,9 +57,9 @@ jupyter-server==2.12.1 # jupyterlab # jupyterlab-server # notebook-shim -jupyter-server-terminals==0.4.4 +jupyter-server-terminals==0.5.1 # via jupyter-server -jupyterlab==4.0.9 +jupyterlab==4.0.10 # via -r dev.in jupyterlab-server==2.25.2 # via jupyterlab @@ -67,7 +67,7 @@ notebook-shim==0.2.3 # via jupyterlab overrides==7.4.0 # via jupyter-server -pathspec==0.11.2 +pathspec==0.12.1 # via copier pip-compile-multi==2.6.3 # via -r dev.in @@ -79,13 +79,13 @@ prometheus-client==0.19.0 # via jupyter-server pycparser==2.21 # via cffi -pydantic==2.5.2 +pydantic==2.5.3 # via copier -pydantic-core==2.14.5 +pydantic-core==2.14.6 # via pydantic python-json-logger==2.0.7 # via jupyter-events -pyyaml-include==1.3.1 +pyyaml-include==1.3.2 # via copier questionary==2.0.1 # via copier diff --git a/requirements/docs.txt b/requirements/docs.txt index f7c92213..f2c7518b 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -10,11 +10,11 @@ accessible-pygments==0.0.4 # via pydata-sphinx-theme alabaster==0.7.13 # via sphinx -attrs==23.1.0 +attrs==23.2.0 # via # jsonschema # referencing -babel==2.13.1 +babel==2.14.0 # via # pydata-sphinx-theme # sphinx @@ -34,11 +34,11 @@ docutils==0.20.1 # nbsphinx # pydata-sphinx-theme # sphinx -fastjsonschema==2.19.0 +fastjsonschema==2.19.1 # via nbformat imagesize==1.4.1 # via sphinx -ipykernel==6.27.1 +ipykernel==6.28.0 # via -r docs.in ipympl==0.9.3 # via -r docs.in @@ -52,13 +52,13 @@ jinja2==3.1.2 # sphinx jsonschema==4.20.0 # via nbformat -jsonschema-specifications==2023.11.2 +jsonschema-specifications==2023.12.1 # via jsonschema jupyter-client==8.6.0 # via # ipykernel # nbclient -jupyter-core==5.5.0 +jupyter-core==5.6.1 # via # ipykernel # jupyter-client @@ -85,7 +85,7 @@ myst-parser==2.0.0 # via -r docs.in nbclient==0.9.0 # via nbconvert -nbconvert==7.12.0 +nbconvert==7.14.0 # via nbsphinx nbformat==5.9.2 # via @@ -98,7 +98,7 @@ nest-asyncio==1.5.8 # via ipykernel pandocfilters==1.5.0 # via nbconvert -psutil==5.9.6 +psutil==5.9.7 # via ipykernel pydata-sphinx-theme==0.14.4 # via -r docs.in @@ -106,11 +106,11 @@ pyzmq==25.1.2 # via # ipykernel # jupyter-client -referencing==0.31.1 +referencing==0.32.0 # via # jsonschema # jsonschema-specifications -rpds-py==0.13.2 +rpds-py==0.16.2 # via # jsonschema # referencing @@ -156,7 +156,7 @@ tornado==6.4 # via # ipykernel # jupyter-client -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via pydata-sphinx-theme webencodings==0.5.1 # via diff --git a/requirements/mypy.txt b/requirements/mypy.txt index f98e1a53..ac285686 100644 --- a/requirements/mypy.txt +++ b/requirements/mypy.txt @@ -6,9 +6,9 @@ # pip-compile-multi # -r test.txt -mypy==1.7.1 +mypy==1.8.0 # via -r mypy.in mypy-extensions==1.0.0 # via mypy -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via mypy diff --git a/requirements/nightly.txt b/requirements/nightly.txt index a814b6f9..c0b5a2ac 100644 --- a/requirements/nightly.txt +++ b/requirements/nightly.txt @@ -16,26 +16,22 @@ click==8.1.7 # via dask cloudpickle==3.0.0 # via dask -comm==0.2.0 +comm==0.2.1 # via ipywidgets contourpy==1.2.0 # via matplotlib cycler==0.12.1 # via matplotlib -dask==2023.12.0 +dask==2023.12.1 # via -r nightly.in decorator==5.1.1 # via ipython executing==2.0.1 # via stack-data -fonttools==4.46.0 +fonttools==4.47.0 # via matplotlib -fsspec==2023.12.1 +fsspec==2023.12.2 # via dask -graphlib-backport==1.0.3 - # via - # sciline - # scipp graphviz==0.20.1 # via -r nightly.in h5py==3.10.0 @@ -44,7 +40,7 @@ h5py==3.10.0 # scippnexus idna==3.6 # via requests -importlib-metadata==7.0.0 +importlib-metadata==7.0.1 # via dask importlib-resources==6.1.1 # via matplotlib @@ -68,23 +64,13 @@ matplotlib==3.8.2 # via plopp matplotlib-inline==0.1.6 # via ipython -numpy==1.26.2 - # via - # contourpy - # h5py - # ipydatawidgets - # matplotlib - # pythreejs - # scipp - # scippneutron - # scipy parso==0.8.3 # via jedi partd==1.4.1 # via dask pexpect==4.9.0 # via ipython -pillow==10.1.0 +pillow==10.2.0 # via matplotlib platformdirs==4.1.0 # via pooch @@ -92,7 +78,7 @@ plopp @ git+https://github.com/scipp/plopp@main # via -r nightly.in pooch==1.8.0 # via scippneutron -prompt-toolkit==3.0.41 +prompt-toolkit==3.0.43 # via ipython ptyprocess==0.7.0 # via pexpect @@ -102,10 +88,6 @@ pygments==2.17.2 # via ipython pyparsing==3.1.1 # via matplotlib -python-dateutil==2.8.2 - # via - # matplotlib - # scippnexus pythreejs==2.4.2 # via -r nightly.in pyyaml==6.0.1 @@ -129,17 +111,13 @@ scipy==1.11.4 # via # scippneutron # scippnexus -six==1.16.0 - # via - # asttokens - # python-dateutil stack-data==0.6.3 # via ipython toolz==0.12.0 # via # dask # partd -traitlets==5.14.0 +traitlets==5.14.1 # via # comm # ipython @@ -149,7 +127,7 @@ traitlets==5.14.0 # traittypes traittypes==0.2.1 # via ipydatawidgets -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via ipython urllib3==2.1.0 # via requests diff --git a/requirements/static.txt b/requirements/static.txt index 38c7aa83..fa660fba 100644 --- a/requirements/static.txt +++ b/requirements/static.txt @@ -7,7 +7,7 @@ # cfgv==3.4.0 # via pre-commit -distlib==0.3.7 +distlib==0.3.8 # via virtualenv filelock==3.13.1 # via virtualenv @@ -17,7 +17,7 @@ nodeenv==1.8.0 # via pre-commit platformdirs==4.1.0 # via virtualenv -pre-commit==3.5.0 +pre-commit==3.6.0 # via -r static.in pyyaml==6.0.1 # via pre-commit diff --git a/requirements/wheels.txt b/requirements/wheels.txt index d5378cf7..c26530af 100644 --- a/requirements/wheels.txt +++ b/requirements/wheels.txt @@ -7,7 +7,7 @@ # build==1.0.3 # via -r wheels.in -importlib-metadata==7.0.0 +importlib-metadata==7.0.1 # via build packaging==23.2 # via build diff --git a/src/ess/dream/__init__.py b/src/ess/dream/__init__.py index b621a546..99d88e5e 100644 --- a/src/ess/dream/__init__.py +++ b/src/ess/dream/__init__.py @@ -7,7 +7,7 @@ import importlib.metadata from . import data -from .io import fold_nexus_detectors, load_nexus +from .io import fold_nexus_detectors, load_geant4_csv, load_nexus try: __version__ = importlib.metadata.version(__package__ or __name__) @@ -19,5 +19,6 @@ __all__ = [ "data", "fold_nexus_detectors", + "load_geant4_csv", "load_nexus", ] diff --git a/src/ess/dream/data.py b/src/ess/dream/data.py index 8b09dd43..806facdc 100644 --- a/src/ess/dream/data.py +++ b/src/ess/dream/data.py @@ -14,6 +14,7 @@ def _make_pooch(): base_url='https://public.esss.dk/groups/scipp/ess/dream/{version}/', version=_version, registry={ + 'data_dream_with_sectors.csv.zip': 'md5:52ae6eb3705e5e54306a001bc0ae85d8', 'DREAM_nexus_sorted-2023-12-07.nxs': 'md5:22824e14f6eb950d24a720b2a0e2cb66', }, ) diff --git a/src/ess/dream/io/__init__.py b/src/ess/dream/io/__init__.py new file mode 100644 index 00000000..605738ed --- /dev/null +++ b/src/ess/dream/io/__init__.py @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) + +"""Input/output for DREAM.""" + +from .geant4 import load_geant4_csv +from .nexus import fold_nexus_detectors, load_nexus + +__all__ = ["fold_nexus_detectors", "load_geant4_csv", "load_nexus"] diff --git a/src/ess/dream/io/geant4.py b/src/ess/dream/io/geant4.py new file mode 100644 index 00000000..8205f079 --- /dev/null +++ b/src/ess/dream/io/geant4.py @@ -0,0 +1,117 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) + +import os +from io import BytesIO, StringIO +from typing import Dict, Optional, Union + +import numpy as np +import scipp as sc + +MANTLE_DETECTOR_ID = sc.index(7) +HIGH_RES_DETECTOR_ID = sc.index(8) +ENDCAPS_DETECTOR_IDS = tuple(map(sc.index, (3, 4, 5, 6))) + + +def load_geant4_csv( + filename: Union[str, os.PathLike, StringIO, BytesIO] +) -> sc.DataGroup: + """Load a GEANT4 CSV file for DREAM. + + Parameters + ---------- + filename: + Path to the GEANT4 CSV file. + + Returns + ------- + : + A :class:`scipp.DataGroup` containing the loaded events. + """ + events = _load_raw_events(filename) + detectors = _split_detectors(events) + for det in detectors.values(): + _adjust_coords(det) + detectors = _group(detectors) + + return sc.DataGroup({'instrument': sc.DataGroup(detectors)}) + + +def _load_raw_events( + filename: Union[str, os.PathLike, StringIO, BytesIO] +) -> sc.DataArray: + table = sc.io.load_csv(filename, sep='\t', header_parser='bracket', data_columns=[]) + table = table.rename_dims(row='event') + return sc.DataArray( + sc.ones(sizes=table.sizes, with_variances=True, unit='counts'), + coords=table.coords, + ) + + +def _adjust_coords(da: sc.DataArray) -> None: + da.coords['wavelength'] = da.coords.pop('lambda') + da.coords['position'] = sc.spatial.as_vectors( + da.coords.pop('x_pos'), da.coords.pop('y_pos'), da.coords.pop('z_pos') + ) + + +def _group(detectors: Dict[str, sc.DataArray]) -> Dict[str, sc.DataArray]: + elements = ('module', 'segment', 'counter', 'wire', 'strip') + + def group(key: str, da: sc.DataArray) -> sc.DataArray: + if key == 'high_resolution': + # Only the HR detector has sectors. + return da.group('sector', *elements) + res = da.group(*elements) + res.bins.coords.pop('sector', None) + return res + + return {key: group(key, da) for key, da in detectors.items()} + + +def _split_detectors( + data: sc.DataArray, detector_id_name: str = 'det ID' +) -> Dict[str, sc.DataArray]: + groups = data.group( + sc.concat( + [MANTLE_DETECTOR_ID, HIGH_RES_DETECTOR_ID, *ENDCAPS_DETECTOR_IDS], + dim=detector_id_name, + ) + ) + detectors = {} + if ( + mantle := _extract_detector(groups, detector_id_name, MANTLE_DETECTOR_ID) + ) is not None: + detectors['mantle'] = mantle.copy() + if ( + high_res := _extract_detector(groups, detector_id_name, HIGH_RES_DETECTOR_ID) + ) is not None: + detectors['high_resolution'] = high_res.copy() + + endcaps_list = [ + det + for i in ENDCAPS_DETECTOR_IDS + if (det := _extract_detector(groups, detector_id_name, i)) is not None + ] + if endcaps_list: + endcaps = sc.concat(endcaps_list, data.dim) + endcaps = endcaps.bin( + z_pos=sc.array( + dims=['z_pos'], + values=[-np.inf, 0.0, np.inf], + unit=endcaps.coords['z_pos'].unit, + ) + ) + detectors['endcap_backward'] = endcaps[0].bins.concat().value.copy() + detectors['endcap_forward'] = endcaps[1].bins.concat().value.copy() + + return detectors + + +def _extract_detector( + detector_groups: sc.DataArray, detector_id_name: str, detector_id: sc.Variable +) -> Optional[sc.DataArray]: + try: + return detector_groups[detector_id_name, detector_id].value + except IndexError: + return None diff --git a/src/ess/dream/io.py b/src/ess/dream/io/nexus.py similarity index 99% rename from src/ess/dream/io.py rename to src/ess/dream/io/nexus.py index b0ad7e38..dd485e61 100644 --- a/src/ess/dream/io.py +++ b/src/ess/dream/io/nexus.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +"""NeXus input/output for DREAM.""" + import os from typing import Any, Dict, Union diff --git a/tests/dream/io/geant4_test.py b/tests/dream/io/geant4_test.py new file mode 100644 index 00000000..a9a94465 --- /dev/null +++ b/tests/dream/io/geant4_test.py @@ -0,0 +1,126 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) + +import zipfile +from io import BytesIO +from typing import Optional, Set + +import numpy as np +import pytest +import scipp as sc +import scipp.testing + +from ess.dream import data, load_geant4_csv + + +@pytest.fixture(scope='module') +def file_path(): + return data.get_path('data_dream_with_sectors.csv.zip') + + +# Load file into memory only once +@pytest.fixture(scope='module') +def load_file(file_path): + with zipfile.ZipFile(file_path, 'r') as archive: + return archive.read(archive.namelist()[0]) + + +@pytest.fixture(scope='function') +def file(load_file): + return BytesIO(load_file) + + +def assert_index_coord( + coord: sc.Variable, *, values: Optional[Set[int]] = None +) -> None: + assert coord.ndim == 1 + assert coord.unit is None + assert coord.dtype == 'int64' + if values is not None: + assert set(np.unique(coord.values)) == values + + +def test_load_geant4_csv_loads_expected_structure(file): + loaded = load_geant4_csv(file) + assert isinstance(loaded, sc.DataGroup) + assert loaded.keys() == {'instrument'} + + instrument = loaded['instrument'] + assert isinstance(instrument, sc.DataGroup) + assert instrument.keys() == { + 'mantle', + 'high_resolution', + 'endcap_forward', + 'endcap_backward', + } + + +@pytest.mark.parametrize( + 'key', ('mantle', 'high_resolution', 'endcap_forward', 'endcap_backward') +) +def test_load_gean4_csv_set_weights_to_one(file, key): + detector = load_geant4_csv(file)['instrument'][key] + events = detector.bins.constituents['data'].data + sc.testing.assert_identical( + events, sc.ones(sizes=events.sizes, with_variances=True, unit='counts') + ) + + +def test_load_geant4_csv_mantle_has_expected_coords(file): + # Only testing ranges that will not change in the future + mantle = load_geant4_csv(file)['instrument']['mantle'] + assert_index_coord(mantle.coords['module']) + assert_index_coord(mantle.coords['segment']) + assert_index_coord(mantle.coords['counter']) + assert_index_coord(mantle.coords['wire'], values=set(range(1, 33))) + assert_index_coord(mantle.coords['strip'], values=set(range(1, 257))) + assert 'sector' not in mantle.coords + + assert 'sector' not in mantle.bins.coords + assert 'tof' in mantle.bins.coords + assert 'wavelength' in mantle.bins.coords + assert 'position' in mantle.bins.coords + + +def test_load_geant4_csv_endcap_backward_has_expected_coords(file): + endcap = load_geant4_csv(file)['instrument']['endcap_backward'] + assert_index_coord(endcap.coords['module']) + assert_index_coord(endcap.coords['segment']) + assert_index_coord(endcap.coords['counter']) + assert_index_coord(endcap.coords['wire'], values=set(range(1, 17))) + assert_index_coord(endcap.coords['strip'], values=set(range(1, 17))) + assert 'sector' not in endcap.coords + + assert 'sector' not in endcap.bins.coords + assert 'tof' in endcap.bins.coords + assert 'wavelength' in endcap.bins.coords + assert 'position' in endcap.bins.coords + + +def test_load_geant4_csv_endcap_forward_has_expected_coords(file): + endcap = load_geant4_csv(file)['instrument']['endcap_forward'] + assert_index_coord(endcap.coords['module']) + assert_index_coord(endcap.coords['segment']) + assert_index_coord(endcap.coords['counter']) + assert_index_coord(endcap.coords['wire'], values=set(range(1, 17))) + assert_index_coord(endcap.coords['strip'], values=set(range(1, 17))) + assert 'sector' not in endcap.coords + + assert 'sector' not in endcap.bins.coords + assert 'tof' in endcap.bins.coords + assert 'wavelength' in endcap.bins.coords + assert 'position' in endcap.bins.coords + + +def test_load_geant4_csv_high_resolution_has_expected_coords(file): + hr = load_geant4_csv(file)['instrument']['high_resolution'] + assert_index_coord(hr.coords['module']) + assert_index_coord(hr.coords['segment']) + assert_index_coord(hr.coords['counter']) + assert_index_coord(hr.coords['wire'], values=set(range(1, 17))) + assert_index_coord(hr.coords['strip'], values=set(range(1, 33))) + assert_index_coord(hr.coords['sector'], values=set(range(1, 5))) + + assert 'tof' in hr.bins.coords + assert 'wavelength' in hr.bins.coords + assert 'position' in hr.bins.coords diff --git a/tests/dream/io_test.py b/tests/dream/io/nexus_test.py similarity index 100% rename from tests/dream/io_test.py rename to tests/dream/io/nexus_test.py