From 4e8e3a0459f05173b08c0686342203adc0e1db32 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Mon, 26 May 2025 15:55:50 +0200 Subject: [PATCH 01/11] Fix run mapping functionality Fixes #211. --- src/ess/sans/workflow.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ess/sans/workflow.py b/src/ess/sans/workflow.py index e01b81aa..ecb6721a 100644 --- a/src/ess/sans/workflow.py +++ b/src/ess/sans/workflow.py @@ -12,7 +12,7 @@ from . import common, conversions, i_of_q, masking, normalization from .types import ( BackgroundRun, - CleanSummedQ, + ReducedQ, CorrectForGravity, Denominator, DetectorBankSizes, @@ -99,8 +99,8 @@ def _set_runs( pipeline = pipeline.copy() runs = pd.DataFrame({Filename[key]: runs}).rename_axis(axis_name) for part in (Numerator, Denominator): - pipeline[CleanSummedQ[key, part]] = ( - pipeline[CleanSummedQ[key, part]] + pipeline[ReducedQ[key, part]] = ( + pipeline[ReducedQ[key, part]] .map(runs) .reduce(index=axis_name, func=merge_contributions) ) From 9e64ef1c92ed70c238021504ce00be0b2313ba39 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Mon, 26 May 2025 13:58:25 +0000 Subject: [PATCH 02/11] Apply automatic formatting --- src/ess/sans/workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ess/sans/workflow.py b/src/ess/sans/workflow.py index ecb6721a..602e0816 100644 --- a/src/ess/sans/workflow.py +++ b/src/ess/sans/workflow.py @@ -12,7 +12,6 @@ from . import common, conversions, i_of_q, masking, normalization from .types import ( BackgroundRun, - ReducedQ, CorrectForGravity, Denominator, DetectorBankSizes, @@ -24,6 +23,7 @@ NeXusDetectorName, Numerator, PixelMaskFilename, + ReducedQ, SampleRun, TransformationPath, Transmission, From dbe6701179dabe634cdfa60519518c571cfab7a8 Mon Sep 17 00:00:00 2001 From: Simon Heybrock <12912489+SimonHeybrock@users.noreply.github.com> Date: Wed, 4 Jun 2025 05:01:24 +0200 Subject: [PATCH 03/11] Update tests --- tests/loki/iofq_test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/loki/iofq_test.py b/tests/loki/iofq_test.py index e979ce07..41917697 100644 --- a/tests/loki/iofq_test.py +++ b/tests/loki/iofq_test.py @@ -16,7 +16,6 @@ BackgroundSubtractedIofQ, BackgroundSubtractedIofQxy, BeamCenter, - CleanSummedQ, CleanWavelength, CorrectForGravity, Denominator, @@ -28,6 +27,7 @@ QBins, QxBins, QyBins, + ReducedQ, ReturnEvents, SampleRun, UncertaintyBroadcastMode, @@ -252,17 +252,17 @@ def test_pipeline_IofQ_merging_events_yields_consistent_results(): assert all(sc.variances(iofq1.data) > sc.variances(iofq3.data)) assert sc.allclose( sc.values( - pipeline_single.compute(CleanSummedQ[SampleRun, Numerator]).hist().data + pipeline_single.compute(ReducedQ[SampleRun, Numerator]).hist().data ) * N, sc.values( - pipeline_triple.compute(CleanSummedQ[SampleRun, Numerator]).hist().data + pipeline_triple.compute(ReducedQ[SampleRun, Numerator]).hist().data ), ) assert sc.allclose( - sc.values(pipeline_single.compute(CleanSummedQ[SampleRun, Denominator]).data) + sc.values(pipeline_single.compute(ReducedQ[SampleRun, Denominator]).data) * N, - sc.values(pipeline_triple.compute(CleanSummedQ[SampleRun, Denominator]).data), + sc.values(pipeline_triple.compute(ReducedQ[SampleRun, Denominator]).data), ) From 9c223c3da95cdac698647ac394f8bc054adc0b36 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Wed, 4 Jun 2025 03:02:05 +0000 Subject: [PATCH 04/11] Apply automatic formatting --- tests/loki/iofq_test.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tests/loki/iofq_test.py b/tests/loki/iofq_test.py index 41917697..5917603d 100644 --- a/tests/loki/iofq_test.py +++ b/tests/loki/iofq_test.py @@ -251,17 +251,12 @@ def test_pipeline_IofQ_merging_events_yields_consistent_results(): assert sc.identical(iofq1.coords['Q'], iofq3.coords['Q']) assert all(sc.variances(iofq1.data) > sc.variances(iofq3.data)) assert sc.allclose( - sc.values( - pipeline_single.compute(ReducedQ[SampleRun, Numerator]).hist().data - ) + sc.values(pipeline_single.compute(ReducedQ[SampleRun, Numerator]).hist().data) * N, - sc.values( - pipeline_triple.compute(ReducedQ[SampleRun, Numerator]).hist().data - ), + sc.values(pipeline_triple.compute(ReducedQ[SampleRun, Numerator]).hist().data), ) assert sc.allclose( - sc.values(pipeline_single.compute(ReducedQ[SampleRun, Denominator]).data) - * N, + sc.values(pipeline_single.compute(ReducedQ[SampleRun, Denominator]).data) * N, sc.values(pipeline_triple.compute(ReducedQ[SampleRun, Denominator]).data), ) From e385213ee3ba4a410640b63d5260bb3081935dbb Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 4 Jun 2025 07:36:04 +0200 Subject: [PATCH 05/11] Reduce WavelengthScaledQ instead of ReducedQ so direct-beam iteration works --- src/ess/sans/workflow.py | 6 +++--- tests/loki/iofq_test.py | 19 ++++++++++++++----- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/ess/sans/workflow.py b/src/ess/sans/workflow.py index 602e0816..2459bb2c 100644 --- a/src/ess/sans/workflow.py +++ b/src/ess/sans/workflow.py @@ -23,13 +23,13 @@ NeXusDetectorName, Numerator, PixelMaskFilename, - ReducedQ, SampleRun, TransformationPath, Transmission, TransmissionRun, WavelengthBands, WavelengthMask, + WavelengthScaledQ, ) @@ -99,8 +99,8 @@ def _set_runs( pipeline = pipeline.copy() runs = pd.DataFrame({Filename[key]: runs}).rename_axis(axis_name) for part in (Numerator, Denominator): - pipeline[ReducedQ[key, part]] = ( - pipeline[ReducedQ[key, part]] + pipeline[WavelengthScaledQ[key, part]] = ( + pipeline[WavelengthScaledQ[key, part]] .map(runs) .reduce(index=axis_name, func=merge_contributions) ) diff --git a/tests/loki/iofq_test.py b/tests/loki/iofq_test.py index 5917603d..8d2af5e5 100644 --- a/tests/loki/iofq_test.py +++ b/tests/loki/iofq_test.py @@ -27,12 +27,12 @@ QBins, QxBins, QyBins, - ReducedQ, ReturnEvents, SampleRun, UncertaintyBroadcastMode, WavelengthBands, WavelengthBins, + WavelengthScaledQ, ) sys.path.insert(0, str(Path(__file__).resolve().parent)) @@ -251,13 +251,22 @@ def test_pipeline_IofQ_merging_events_yields_consistent_results(): assert sc.identical(iofq1.coords['Q'], iofq3.coords['Q']) assert all(sc.variances(iofq1.data) > sc.variances(iofq3.data)) assert sc.allclose( - sc.values(pipeline_single.compute(ReducedQ[SampleRun, Numerator]).hist().data) + sc.values( + pipeline_single.compute(WavelengthScaledQ[SampleRun, Numerator]).hist().data + ) * N, - sc.values(pipeline_triple.compute(ReducedQ[SampleRun, Numerator]).hist().data), + sc.values( + pipeline_triple.compute(WavelengthScaledQ[SampleRun, Numerator]).hist().data + ), ) assert sc.allclose( - sc.values(pipeline_single.compute(ReducedQ[SampleRun, Denominator]).data) * N, - sc.values(pipeline_triple.compute(ReducedQ[SampleRun, Denominator]).data), + sc.values( + pipeline_single.compute(WavelengthScaledQ[SampleRun, Denominator]).data + ) + * N, + sc.values( + pipeline_triple.compute(WavelengthScaledQ[SampleRun, Denominator]).data + ), ) From fe57ac7b61b1578e7eff874bf1f1c9de5a691777 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Fri, 6 Jun 2025 06:40:23 +0200 Subject: [PATCH 06/11] Support ISIS histogram files --- src/ess/isissans/__init__.py | 9 +++++- src/ess/isissans/general.py | 53 ++++++++++++++++++++++++++++++++++-- src/ess/isissans/mantidio.py | 26 ++++++++++++++++-- 3 files changed, 81 insertions(+), 7 deletions(-) diff --git a/src/ess/isissans/__init__.py b/src/ess/isissans/__init__.py index 9f45f337..b7b0c96c 100644 --- a/src/ess/isissans/__init__.py +++ b/src/ess/isissans/__init__.py @@ -4,7 +4,13 @@ import importlib.metadata from . import general, io, sans2d, zoom -from .general import DetectorBankOffset, MonitorOffset, SampleOffset, default_parameters +from .general import ( + DetectorBankOffset, + MonitorOffset, + MonitorSpectrumNumber, + SampleOffset, + default_parameters, +) from .io import CalibrationFilename from .visualization import plot_flat_detector_xy @@ -21,6 +27,7 @@ 'CalibrationFilename', 'DetectorBankOffset', 'MonitorOffset', + 'MonitorSpectrumNumber', 'SampleOffset', 'default_parameters', 'io', diff --git a/src/ess/isissans/general.py b/src/ess/isissans/general.py index a2e6e57c..fb9436ad 100644 --- a/src/ess/isissans/general.py +++ b/src/ess/isissans/general.py @@ -4,7 +4,8 @@ Providers for the ISIS instruments. """ -from typing import NewType +from dataclasses import dataclass +from typing import Generic, NewType import sciline import scipp as sc @@ -45,6 +46,17 @@ class MonitorOffset(sciline.Scope[MonitorType, sc.Variable], sc.Variable): """ +@dataclass +class MonitorSpectrumNumber(Generic[MonitorType]): + """ + Spectrum number for monitor data. + + This is used to identify the monitor in ISIS histogram data files. + """ + + value: int + + DetectorBankOffset = NewType('DetectorBankOffset', sc.Variable) SampleOffset = NewType('SampleOffset', sc.Variable) @@ -61,6 +73,10 @@ def default_parameters() -> dict: SampleOffset: SampleOffset(sc.vector([0, 0, 0], unit='m')), NonBackgroundWavelengthRange: None, Period: None, + # Not used for event files, setting default so the workflow works. If histogram + # data is used, the user should set this to the correct value. + MonitorSpectrumNumber[Incident]: MonitorSpectrumNumber[Incident](-1), + MonitorSpectrumNumber[Transmission]: MonitorSpectrumNumber[Transmission](-1), } @@ -132,12 +148,43 @@ def get_calibrated_isis_detector( def get_monitor_data( - dg: LoadedFileContents[RunType], nexus_name: NeXusMonitorName[MonitorType] + dg: LoadedFileContents[RunType], + nexus_name: NeXusMonitorName[MonitorType], + spectrum_number: MonitorSpectrumNumber[MonitorType], ) -> NeXusComponent[MonitorType, RunType]: + """ + Extract monitor data that was loaded together with detector data. + + If the raw files is histogram data, Mantid stores this as a Workspace2D, where some + or all spectra corresponds to monitors. + + Parameters + ---------- + dg: + Data loaded with Mantid and converted to Scipp. + nexus_name: + Name of the monitor in the NeXus file, e.g. 'incident_monitor' or + 'transmission_monitor'. USed when raw data is an EventWorkspace, where + monitors are stored in a group with this name. + spectrum_number: + Spectrum number of the monitor in the NeXus file, e.g. 0 for incident monitor + or 1 for transmission monitor. This is used when the raw data is a + Workspace2D, where the monitor data is stored in a spectrum with this number. + + Returns + ------- + : + Monitor data extracted from the loaded file. + """ # The generic NeXus workflow will try to extract 'data' from this, which is exactly # what we also have in the Mantid data. We use the generic workflow since it also # applies offsets, etc. - monitor = dg['monitors'][nexus_name]['data'] + if 'monitors' in dg: + # From EventWorkspace + monitor = dg['monitors'][nexus_name]['data'] + else: + # From Workspace2D + monitor = sc.values(dg["data"]["spectrum", sc.index(spectrum_number.value)]) return NeXusComponent[MonitorType, RunType]( sc.DataGroup(data=monitor, position=monitor.coords['position']) ) diff --git a/src/ess/isissans/mantidio.py b/src/ess/isissans/mantidio.py index 6026f949..5cad7457 100644 --- a/src/ess/isissans/mantidio.py +++ b/src/ess/isissans/mantidio.py @@ -4,6 +4,7 @@ File loading functions for ISIS data using Mantid. """ +import threading from typing import NewType, NoReturn import sciline @@ -99,12 +100,29 @@ def from_data_workspace( def load_run(filename: Filename[RunType], period: Period) -> DataWorkspace[RunType]: - loaded = _mantid_simpleapi.Load( - Filename=str(filename), LoadMonitors=True, StoreInADS=False - ) + # Loading many small files with Mantid is, for some reason, very slow when using + # the default number of threads in the Dask threaded scheduler (1 thread worked + # best, 2 is a bit slower but still fast). We can either limit that thread count, + # or add a lock here, which is more specific. + with load_run.lock: + loaded = _mantid_simpleapi.Load(Filename=str(filename), StoreInADS=False) + try: + loaded = _mantid_simpleapi.Load( + Filename=str(filename), LoadMonitors=True, StoreInADS=False + ) + except TypeError: + # Not loaded using LoadEventNexus, so LoadMonitor option is not available. + loaded = _mantid_simpleapi.Load(Filename=str(filename), StoreInADS=False) if isinstance(loaded, _mantid_api.Workspace): # A single workspace data_ws = loaded + if isinstance(data_ws, _mantid_api.WorkspaceGroup): + if period is None: + raise ValueError( + f'Needs {Period} to be set to know what ' + 'section of the event data to load' + ) + data_ws = data_ws.getItem(period) else: # Separate data and monitor workspaces data_ws = loaded.OutputWorkspace @@ -121,6 +139,8 @@ def load_run(filename: Filename[RunType], period: Period) -> DataWorkspace[RunTy return DataWorkspace[RunType](data_ws) +load_run.lock = threading.Lock() + providers = ( from_data_workspace, load_calibration, From 03a07fd20ea313bb9115ffbcd8fbe1d94db00b55 Mon Sep 17 00:00:00 2001 From: Simon Heybrock <12912489+SimonHeybrock@users.noreply.github.com> Date: Tue, 10 Jun 2025 09:20:42 +0200 Subject: [PATCH 07/11] Update src/ess/isissans/general.py Co-authored-by: Jan-Lukas Wynen --- src/ess/isissans/general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ess/isissans/general.py b/src/ess/isissans/general.py index fb9436ad..99b775ee 100644 --- a/src/ess/isissans/general.py +++ b/src/ess/isissans/general.py @@ -155,7 +155,7 @@ def get_monitor_data( """ Extract monitor data that was loaded together with detector data. - If the raw files is histogram data, Mantid stores this as a Workspace2D, where some + If the raw file is histogram data, Mantid stores this as a Workspace2D, where some or all spectra corresponds to monitors. Parameters From 5fa9cc98538189b921b8a714d491ee7f693e0412 Mon Sep 17 00:00:00 2001 From: Simon Heybrock <12912489+SimonHeybrock@users.noreply.github.com> Date: Tue, 10 Jun 2025 09:20:53 +0200 Subject: [PATCH 08/11] Update src/ess/isissans/general.py Co-authored-by: Jan-Lukas Wynen --- src/ess/isissans/general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ess/isissans/general.py b/src/ess/isissans/general.py index 99b775ee..ee68a617 100644 --- a/src/ess/isissans/general.py +++ b/src/ess/isissans/general.py @@ -156,7 +156,7 @@ def get_monitor_data( Extract monitor data that was loaded together with detector data. If the raw file is histogram data, Mantid stores this as a Workspace2D, where some - or all spectra corresponds to monitors. + or all spectra correspond to monitors. Parameters ---------- From 6002f268baf3be44722f32bbb7aa4b1f49f9e35b Mon Sep 17 00:00:00 2001 From: Simon Heybrock <12912489+SimonHeybrock@users.noreply.github.com> Date: Tue, 10 Jun 2025 09:21:00 +0200 Subject: [PATCH 09/11] Update src/ess/isissans/general.py Co-authored-by: Jan-Lukas Wynen --- src/ess/isissans/general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ess/isissans/general.py b/src/ess/isissans/general.py index ee68a617..7b8a883d 100644 --- a/src/ess/isissans/general.py +++ b/src/ess/isissans/general.py @@ -164,7 +164,7 @@ def get_monitor_data( Data loaded with Mantid and converted to Scipp. nexus_name: Name of the monitor in the NeXus file, e.g. 'incident_monitor' or - 'transmission_monitor'. USed when raw data is an EventWorkspace, where + 'transmission_monitor'. Used when raw data is an EventWorkspace, where monitors are stored in a group with this name. spectrum_number: Spectrum number of the monitor in the NeXus file, e.g. 0 for incident monitor From c16e87ccfdd868cd7260bbabc3b8b0a0a4b941c5 Mon Sep 17 00:00:00 2001 From: Simon Heybrock <12912489+SimonHeybrock@users.noreply.github.com> Date: Tue, 10 Jun 2025 09:23:01 +0200 Subject: [PATCH 10/11] Update src/ess/isissans/general.py --- src/ess/isissans/general.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ess/isissans/general.py b/src/ess/isissans/general.py index 7b8a883d..f5231d96 100644 --- a/src/ess/isissans/general.py +++ b/src/ess/isissans/general.py @@ -167,8 +167,8 @@ def get_monitor_data( 'transmission_monitor'. Used when raw data is an EventWorkspace, where monitors are stored in a group with this name. spectrum_number: - Spectrum number of the monitor in the NeXus file, e.g. 0 for incident monitor - or 1 for transmission monitor. This is used when the raw data is a + Spectrum number of the monitor in the NeXus file, e.g., 3 for incident monitor + or 4 for transmission monitor. This is used when the raw data is a Workspace2D, where the monitor data is stored in a spectrum with this number. Returns From 3ffad5aa61e3d4d046ee3fdbd2db58557f29ea39 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Tue, 10 Jun 2025 09:26:41 +0200 Subject: [PATCH 11/11] More try/except into lock --- src/ess/isissans/mantidio.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/ess/isissans/mantidio.py b/src/ess/isissans/mantidio.py index 5cad7457..0c20abac 100644 --- a/src/ess/isissans/mantidio.py +++ b/src/ess/isissans/mantidio.py @@ -105,14 +105,13 @@ def load_run(filename: Filename[RunType], period: Period) -> DataWorkspace[RunTy # best, 2 is a bit slower but still fast). We can either limit that thread count, # or add a lock here, which is more specific. with load_run.lock: - loaded = _mantid_simpleapi.Load(Filename=str(filename), StoreInADS=False) - try: - loaded = _mantid_simpleapi.Load( - Filename=str(filename), LoadMonitors=True, StoreInADS=False - ) - except TypeError: - # Not loaded using LoadEventNexus, so LoadMonitor option is not available. - loaded = _mantid_simpleapi.Load(Filename=str(filename), StoreInADS=False) + try: + loaded = _mantid_simpleapi.Load( + Filename=str(filename), LoadMonitors=True, StoreInADS=False + ) + except TypeError: + # Not loaded using LoadEventNexus, so LoadMonitor option is not available. + loaded = _mantid_simpleapi.Load(Filename=str(filename), StoreInADS=False) if isinstance(loaded, _mantid_api.Workspace): # A single workspace data_ws = loaded