From edcf4c2ab6122b9c9d25009fba85025997f72397 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Mon, 5 May 2025 15:10:55 +0200 Subject: [PATCH 01/75] feat: more flexible helper for batch reduction --- src/ess/amor/load.py | 2 +- src/ess/amor/types.py | 4 - src/ess/reflectometry/tools.py | 117 +++++++++++++++++++++--------- src/ess/reflectometry/types.py | 4 + src/ess/reflectometry/workflow.py | 2 +- tests/tools_test.py | 57 +++++++++++++-- 6 files changed, 141 insertions(+), 45 deletions(-) diff --git a/src/ess/amor/load.py b/src/ess/amor/load.py index ea103497..47cb65aa 100644 --- a/src/ess/amor/load.py +++ b/src/ess/amor/load.py @@ -10,6 +10,7 @@ LoadedNeXusDetector, NeXusDetectorName, ProtonCurrent, + RawChopper, RawDetectorData, RunType, SampleRotation, @@ -21,7 +22,6 @@ ChopperFrequency, ChopperPhase, ChopperSeparation, - RawChopper, ) diff --git a/src/ess/amor/types.py b/src/ess/amor/types.py index 92eb704e..bb801398 100644 --- a/src/ess/amor/types.py +++ b/src/ess/amor/types.py @@ -27,8 +27,4 @@ class ChopperSeparation(sciline.Scope[RunType, sc.Variable], sc.Variable): """Distance between the two choppers.""" -class RawChopper(sciline.Scope[RunType, sc.DataGroup], sc.DataGroup): - """Chopper data loaded from nexus file.""" - - GravityToggle = NewType("GravityToggle", bool) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 6302824e..ee82396c 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -8,10 +8,15 @@ import sciline import scipp as sc import scipy.optimize as opt -from orsopy.fileio.orso import OrsoDataset from ess.reflectometry import orso -from ess.reflectometry.types import ReflectivityOverQ +from ess.reflectometry.types import ( + Filename, + ReducibleData, + ReflectivityOverQ, + SampleRun, +) +from ess.reflectometry.workflow import with_filenames _STD_TO_FWHM = sc.scalar(2.0) * sc.sqrt(sc.scalar(2.0) * sc.log(sc.scalar(2.0))) @@ -280,18 +285,18 @@ def combine_curves( ) -def orso_datasets_from_measurements( +def from_measurements( workflow: sciline.Pipeline, - runs: Sequence[Mapping[type, Any]], + runs: Sequence[Mapping[type, Any]] | Mapping[Any, Mapping[type, Any]], + target: type | Sequence[type] = orso.OrsoIofQDataset, *, - scale_to_overlap: bool = True, -) -> list[OrsoDataset]: - '''Produces a list of ORSO datasets containing one - reflectivity curve for each of the provided runs. + scale_to_overlap: bool | tuple[sc.Variable, sc.Variable] = False, +) -> list | Mapping: + '''Produces a list of datasets containing for each of the provided runs. Each entry of :code:`runs` is a mapping of parameters and values needed to produce the dataset. - Optionally, the reflectivity curves can be scaled to overlap in + Optionally, the results can be scaled so that the reflectivity curves overlap in the regions where they have the same Q-value. Parameters @@ -300,38 +305,82 @@ def orso_datasets_from_measurements( The sciline workflow used to compute `ReflectivityOverQ` for each of the runs. runs: - The sciline parameters to be used for each run + The sciline parameters to be used for each run. + + target: + The domain type to compute for each run. scale_to_overlap: - If True the curves will be scaled to overlap. - Note that the curve of the first run is unscaled and - the rest are scaled to match it. + If not ``None`` the curves will be scaled to overlap. + If a tuple then this argument will be passed as the ``critical_edge_interval`` + to the ``scale_reflectivity_curves_to_overlap`` function. Returns --------- list of the computed ORSO datasets, containing one reflectivity curve each ''' - reflectivity_curves = [] - for parameters in runs: - wf = workflow.copy() - for name, value in parameters.items(): - wf[name] = value - reflectivity_curves.append(wf.compute(ReflectivityOverQ)) - - scale_factors = ( - scale_reflectivity_curves_to_overlap([r.hist() for r in reflectivity_curves])[1] - if scale_to_overlap - else (1,) * len(runs) - ) + names = runs.keys() if hasattr(runs, 'keys') else None + runs = runs.values() if hasattr(runs, 'values') else runs + + def init_workflow(workflow, parameters): + if Filename[SampleRun] in parameters: + if isinstance(parameters[Filename[SampleRun]], list | tuple): + wf = with_filenames( + workflow, + SampleRun, + parameters[Filename[SampleRun]], + ) + else: + wf = workflow.copy() + wf[Filename[SampleRun]] = parameters[Filename[SampleRun]] + else: + wf = workflow.copy() + for tp, value in parameters.items(): + if tp is Filename[SampleRun]: + continue + wf[tp] = value + return wf + + if scale_to_overlap: + results = [] + for parameters in runs: + wf = init_workflow(workflow, parameters) + results.append(wf.compute((ReflectivityOverQ, ReducibleData[SampleRun]))) + + scale_factors = scale_reflectivity_curves_to_overlap( + [r[ReflectivityOverQ].hist() for r in results], + critical_edge_interval=scale_to_overlap + if isinstance(scale_to_overlap, list | tuple) + else None, + )[1] datasets = [] - for parameters, curve, scale_factor in zip( - runs, reflectivity_curves, scale_factors, strict=True - ): - wf = workflow.copy() - for name, value in parameters.items(): - wf[name] = value - wf[ReflectivityOverQ] = scale_factor * curve - dataset = wf.compute(orso.OrsoIofQDataset) + for i, parameters in enumerate(runs): + wf = init_workflow(workflow, parameters) + if scale_to_overlap: + # Optimization in case we can avoid re-doing some work: + # Check if any of the targets need ReducibleData if + # ReflectivityOverQ already exists. + # If they don't, we can avoid recomputing ReducibleData. + targets = target if hasattr(target, '__len__') else (target,) + if any( + _workflow_needs_quantity_A_even_if_quantitiy_B_is_set( + wf[t], ReducibleData[SampleRun], ReflectivityOverQ + ) + for t in targets + ): + wf[ReducibleData[SampleRun]] = ( + scale_factors[i] * results[i][ReducibleData[SampleRun]] + ) + else: + wf[ReflectivityOverQ] = scale_factors[i] * results[i][ReflectivityOverQ] + + dataset = wf.compute(target) datasets.append(dataset) - return datasets + return datasets if names is None else dict(zip(names, datasets, strict=True)) + + +def _workflow_needs_quantity_A_even_if_quantitiy_B_is_set(workflow, A, B): + wf = workflow.copy() + wf[B] = 'Not important' + return A in wf.underlying_graph diff --git a/src/ess/reflectometry/types.py b/src/ess/reflectometry/types.py index 4451fe48..6f02b70e 100644 --- a/src/ess/reflectometry/types.py +++ b/src/ess/reflectometry/types.py @@ -36,6 +36,10 @@ class LoadedNeXusDetector(sciline.Scope[RunType, sc.DataGroup], sc.DataGroup): """NXdetector loaded from file""" +class RawChopper(sciline.Scope[RunType, sc.DataGroup], sc.DataGroup): + """Chopper data loaded from nexus file.""" + + class ReducibleData(sciline.Scope[RunType, sc.DataArray], sc.DataArray): """Event data with common coordinates added""" diff --git a/src/ess/reflectometry/workflow.py b/src/ess/reflectometry/workflow.py index 2c0fa9be..4f40570f 100644 --- a/src/ess/reflectometry/workflow.py +++ b/src/ess/reflectometry/workflow.py @@ -6,7 +6,6 @@ import sciline import scipp as sc -from ess.amor.types import RawChopper from ess.reflectometry.orso import ( OrsoExperiment, OrsoOwner, @@ -15,6 +14,7 @@ ) from ess.reflectometry.types import ( Filename, + RawChopper, ReducibleData, RunType, SampleRotation, diff --git a/tests/tools_test.py b/tests/tools_test.py index 03c07aaf..ce71cbe9 100644 --- a/tests/tools_test.py +++ b/tests/tools_test.py @@ -8,11 +8,10 @@ from orsopy.fileio import Orso, OrsoDataset from scipp.testing import assert_allclose -from ess.reflectometry.orso import OrsoIofQDataset from ess.reflectometry.tools import ( combine_curves, + from_measurements, linlogspace, - orso_datasets_from_measurements, scale_reflectivity_curves_to_overlap, ) from ess.reflectometry.types import Filename, ReflectivityOverQ, SampleRun @@ -225,11 +224,11 @@ def test_linlogspace_bad_input(): @pytest.mark.filterwarnings("ignore:No suitable") -def test_orso_datasets_tool(): +def test_from_measurements_tool(): def normalized_ioq(filename: Filename[SampleRun]) -> ReflectivityOverQ: return filename - def orso_dataset(filename: Filename[SampleRun]) -> OrsoIofQDataset: + def orso_dataset(filename: Filename[SampleRun]) -> OrsoDataset: class Reduction: corrections = [] # noqa: RUF012 @@ -240,10 +239,58 @@ class Reduction: workflow = sl.Pipeline( [normalized_ioq, orso_dataset], params={Filename[SampleRun]: 'default'} ) - datasets = orso_datasets_from_measurements( + datasets = from_measurements( workflow, [{}, {Filename[SampleRun]: 'special'}], + target=OrsoDataset, scale_to_overlap=False, ) assert len(datasets) == 2 assert tuple(d.info.name for d in datasets) == ('default.orso', 'special.orso') + + +@pytest.mark.filterwarnings("ignore:No suitable") +def test_from_measurements_tool_uses_expected_parameters_from_each_run(): + def normalized_ioq(filename: Filename[SampleRun]) -> ReflectivityOverQ: + return filename + + def orso_dataset(filename: Filename[SampleRun]) -> OrsoDataset: + class Reduction: + corrections = [] # noqa: RUF012 + + return OrsoDataset( + Orso({}, Reduction, [], name=f'{filename}.orso'), np.ones((0, 0)) + ) + + workflow = sl.Pipeline( + [normalized_ioq, orso_dataset], params={Filename[SampleRun]: 'default'} + ) + datasets = from_measurements( + workflow, + [{}, {Filename[SampleRun]: 'special'}], + target=OrsoDataset, + scale_to_overlap=False, + ) + assert len(datasets) == 2 + assert tuple(d.info.name for d in datasets) == ('default.orso', 'special.orso') + + +@pytest.mark.parametrize('targets', [(int,), (float, int)]) +@pytest.mark.parametrize( + 'params', [[{str: '1'}, {str: '2'}], {'a': {str: '1'}, 'b': {str: '2'}}] +) +def test_from_measurements_tool_returns_mapping_if_passed_mapping(params, targets): + def A(x: str) -> float: + return float(x) + + def B(x: str) -> int: + return int(x) + + workflow = sl.Pipeline([A, B]) + datasets = from_measurements( + workflow, + params, + target=targets, + ) + assert len(datasets) == len(params) + assert type(datasets) is type(params) From 7dbbf2ce0e07972aef0c2edb85e907e3e7dadc46 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Mon, 5 May 2025 15:16:36 +0200 Subject: [PATCH 02/75] fix: remove duplicate --- tests/tools_test.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/tests/tools_test.py b/tests/tools_test.py index ce71cbe9..40b7dc39 100644 --- a/tests/tools_test.py +++ b/tests/tools_test.py @@ -223,32 +223,6 @@ def test_linlogspace_bad_input(): ) -@pytest.mark.filterwarnings("ignore:No suitable") -def test_from_measurements_tool(): - def normalized_ioq(filename: Filename[SampleRun]) -> ReflectivityOverQ: - return filename - - def orso_dataset(filename: Filename[SampleRun]) -> OrsoDataset: - class Reduction: - corrections = [] # noqa: RUF012 - - return OrsoDataset( - Orso({}, Reduction, [], name=f'{filename}.orso'), np.ones((0, 0)) - ) - - workflow = sl.Pipeline( - [normalized_ioq, orso_dataset], params={Filename[SampleRun]: 'default'} - ) - datasets = from_measurements( - workflow, - [{}, {Filename[SampleRun]: 'special'}], - target=OrsoDataset, - scale_to_overlap=False, - ) - assert len(datasets) == 2 - assert tuple(d.info.name for d in datasets) == ('default.orso', 'special.orso') - - @pytest.mark.filterwarnings("ignore:No suitable") def test_from_measurements_tool_uses_expected_parameters_from_each_run(): def normalized_ioq(filename: Filename[SampleRun]) -> ReflectivityOverQ: From 25add6ba22d92e401a2311465f80af2af2cc1c88 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 6 May 2025 09:38:44 +0200 Subject: [PATCH 03/75] update docs --- docs/user-guide/amor/amor-reduction.ipynb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/user-guide/amor/amor-reduction.ipynb b/docs/user-guide/amor/amor-reduction.ipynb index 3331c4de..643572eb 100644 --- a/docs/user-guide/amor/amor-reduction.ipynb +++ b/docs/user-guide/amor/amor-reduction.ipynb @@ -465,13 +465,15 @@ "metadata": {}, "outputs": [], "source": [ - "from ess.reflectometry.tools import orso_datasets_from_measurements\n", + "from ess.reflectometry.tools import from_measurements\n", "\n", - "datasets = orso_datasets_from_measurements(\n", + "datasets = from_measurements(\n", " workflow,\n", " runs.values(),\n", + " target=orso.OrsoIofQDataset,\n", " # Optionally scale the curves to overlap using `scale_reflectivity_curves_to_overlap`\n", - " scale_to_overlap=True\n", + " # with the same \"critical edge interval\" that was used before\n", + " scale_to_overlap=(sc.scalar(0.01, unit='1/angstrom'), sc.scalar(0.014, unit='1/angstrom'))\n", ")" ] }, From eda422e85d4540fa8e8ffbf087442f571bc0284d Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 6 May 2025 09:42:20 +0200 Subject: [PATCH 04/75] spelling --- src/ess/reflectometry/tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index ee82396c..1ae5b37f 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -364,7 +364,7 @@ def init_workflow(workflow, parameters): # If they don't, we can avoid recomputing ReducibleData. targets = target if hasattr(target, '__len__') else (target,) if any( - _workflow_needs_quantity_A_even_if_quantitiy_B_is_set( + _workflow_needs_quantity_A_even_if_quantity_B_is_set( wf[t], ReducibleData[SampleRun], ReflectivityOverQ ) for t in targets @@ -380,7 +380,7 @@ def init_workflow(workflow, parameters): return datasets if names is None else dict(zip(names, datasets, strict=True)) -def _workflow_needs_quantity_A_even_if_quantitiy_B_is_set(workflow, A, B): +def _workflow_needs_quantity_A_even_if_quantity_B_is_set(workflow, A, B): wf = workflow.copy() wf[B] = 'Not important' return A in wf.underlying_graph From 98a938925db2c71985a45544d37c3b7878989ed8 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 6 May 2025 13:11:22 +0200 Subject: [PATCH 05/75] fix: add theta to reference, can be useful in some contexts --- src/ess/amor/normalization.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ess/amor/normalization.py b/src/ess/amor/normalization.py index eb97d188..d4c450b0 100644 --- a/src/ess/amor/normalization.py +++ b/src/ess/amor/normalization.py @@ -88,6 +88,7 @@ def evaluate_reference_at_sample_coords( ref = ref.transform_coords( ( "Q", + "theta", "wavelength_resolution", "sample_size_resolution", "angular_resolution", From 743743a7874d0355bea2fe0fe084adfa066f05ef Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 6 May 2025 13:56:05 +0200 Subject: [PATCH 06/75] fix: handle case when SampleRotation etc are set in workflow --- src/ess/reflectometry/workflow.py | 38 +++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/src/ess/reflectometry/workflow.py b/src/ess/reflectometry/workflow.py index 4f40570f..76dfe734 100644 --- a/src/ess/reflectometry/workflow.py +++ b/src/ess/reflectometry/workflow.py @@ -13,6 +13,7 @@ OrsoSampleFilenames, ) from ess.reflectometry.types import ( + DetectorRotation, Filename, RawChopper, ReducibleData, @@ -62,15 +63,34 @@ def with_filenames( mapped = wf.map(df) - wf[ReducibleData[runtype]] = mapped[ReducibleData[runtype]].reduce( - index=axis_name, func=_concatenate_event_lists - ) - wf[RawChopper[runtype]] = mapped[RawChopper[runtype]].reduce( - index=axis_name, func=_any_value - ) - wf[SampleRotation[runtype]] = mapped[SampleRotation[runtype]].reduce( - index=axis_name, func=_any_value - ) + try: + wf[ReducibleData[runtype]] = mapped[ReducibleData[runtype]].reduce( + index=axis_name, func=_concatenate_event_lists + ) + except ValueError: + # ReducibleData[runtype] is independent of Filename[runtype] + pass + try: + wf[RawChopper[runtype]] = mapped[RawChopper[runtype]].reduce( + index=axis_name, func=_any_value + ) + except ValueError: + # RawChopper[runtype] is independent of Filename[runtype] + pass + try: + wf[SampleRotation[runtype]] = mapped[SampleRotation[runtype]].reduce( + index=axis_name, func=_any_value + ) + except ValueError: + # SampleRotation[runtype] is independent of Filename[runtype] + pass + try: + wf[DetectorRotation[runtype]] = mapped[DetectorRotation[runtype]].reduce( + index=axis_name, func=_any_value + ) + except ValueError: + # DetectorRotation[runtype] is independent of Filename[runtype] + pass if runtype is SampleRun: wf[OrsoSample] = mapped[OrsoSample].reduce(index=axis_name, func=_any_value) From 45bcd0ea969bbb998ead0f1651be7dd704048308 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 6 May 2025 14:04:30 +0200 Subject: [PATCH 07/75] fix: add parameters before setting filenames --- src/ess/reflectometry/tools.py | 17 ++++----- tests/amor/tools_test.py | 68 ++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 9 deletions(-) create mode 100644 tests/amor/tools_test.py diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 1ae5b37f..c06d6c3b 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -308,7 +308,7 @@ def from_measurements( The sciline parameters to be used for each run. target: - The domain type to compute for each run. + The domain type(s) to compute for each run. scale_to_overlap: If not ``None`` the curves will be scaled to overlap. @@ -323,22 +323,21 @@ def from_measurements( runs = runs.values() if hasattr(runs, 'values') else runs def init_workflow(workflow, parameters): + wf = workflow.copy() + for tp, value in parameters.items(): + if tp is Filename[SampleRun]: + continue + wf[tp] = value + if Filename[SampleRun] in parameters: if isinstance(parameters[Filename[SampleRun]], list | tuple): wf = with_filenames( - workflow, + wf, SampleRun, parameters[Filename[SampleRun]], ) else: - wf = workflow.copy() wf[Filename[SampleRun]] = parameters[Filename[SampleRun]] - else: - wf = workflow.copy() - for tp, value in parameters.items(): - if tp is Filename[SampleRun]: - continue - wf[tp] = value return wf if scale_to_overlap: diff --git a/tests/amor/tools_test.py b/tests/amor/tools_test.py new file mode 100644 index 00000000..0dd6fef2 --- /dev/null +++ b/tests/amor/tools_test.py @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) +import pytest +import sciline +import scipp as sc +from scipp.testing import assert_allclose + +from amor.pipeline_test import amor_pipeline # noqa: F401 +from ess.amor import data +from ess.amor.types import ChopperPhase +from ess.reflectometry.tools import from_measurements +from ess.reflectometry.types import ( + DetectorRotation, + Filename, + QBins, + ReducedReference, + ReferenceRun, + ReflectivityOverQ, + SampleRotation, + SampleRun, +) + +# The files used in the AMOR reduction workflow have some scippnexus warnings +pytestmark = pytest.mark.filterwarnings( + "ignore:.*Invalid transformation, .*missing attribute 'vector':UserWarning", +) + + +@pytest.fixture +def pipeline_with_1632_reference(amor_pipeline): # noqa: F811 + amor_pipeline[ChopperPhase[ReferenceRun]] = sc.scalar(7.5, unit='deg') + amor_pipeline[ChopperPhase[SampleRun]] = sc.scalar(7.5, unit='deg') + amor_pipeline[Filename[ReferenceRun]] = data.amor_run('1632') + amor_pipeline[ReducedReference] = amor_pipeline.compute(ReducedReference) + return amor_pipeline + + +@pytestmark +def test_from_measurements_tool_concatenates_event_lists( + pipeline_with_1632_reference: sciline.Pipeline, +): + pl = pipeline_with_1632_reference + + run = { + Filename[SampleRun]: list(map(data.amor_run, (1636, 1639, 1641))), + QBins: sc.geomspace( + dim='Q', start=0.062, stop=0.18, num=391, unit='1/angstrom' + ), + DetectorRotation[SampleRun]: sc.scalar(0.140167, unit='rad'), + SampleRotation[SampleRun]: sc.scalar(0.0680678, unit='rad'), + } + results = from_measurements( + pl, + [run], + target=ReflectivityOverQ, + scale_to_overlap=False, + ) + + results2 = [] + for fname in run[Filename[SampleRun]]: + pl.copy() + pl[Filename[SampleRun]] = fname + pl[QBins] = run[QBins] + pl[DetectorRotation[SampleRun]] = run[DetectorRotation[SampleRun]] + pl[SampleRotation[SampleRun]] = run[SampleRotation[SampleRun]] + results2.append(pl.compute(ReflectivityOverQ).hist().data) + + assert_allclose(sum(results2), results[0].hist().data) From 7fb806bf3802c14906d08d6d7f919dcc3750ce70 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 6 May 2025 14:07:48 +0200 Subject: [PATCH 08/75] docs: fix --- src/ess/reflectometry/tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index c06d6c3b..0ba11287 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -311,9 +311,9 @@ def from_measurements( The domain type(s) to compute for each run. scale_to_overlap: - If not ``None`` the curves will be scaled to overlap. + If ``True`` the curves will be scaled to overlap. If a tuple then this argument will be passed as the ``critical_edge_interval`` - to the ``scale_reflectivity_curves_to_overlap`` function. + argument to the ``scale_reflectivity_curves_to_overlap`` function. Returns --------- From f097188e186670854875395e79500d218ce4555c Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 6 May 2025 14:08:54 +0200 Subject: [PATCH 09/75] tests --- tests/tools_test.py | 78 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/tests/tools_test.py b/tests/tools_test.py index 40b7dc39..7f1c81b5 100644 --- a/tests/tools_test.py +++ b/tests/tools_test.py @@ -14,7 +14,12 @@ linlogspace, scale_reflectivity_curves_to_overlap, ) -from ess.reflectometry.types import Filename, ReflectivityOverQ, SampleRun +from ess.reflectometry.types import ( + Filename, + ReducibleData, + ReflectivityOverQ, + SampleRun, +) def curve(d, qmin, qmax): @@ -268,3 +273,74 @@ def B(x: str) -> int: ) assert len(datasets) == len(params) assert type(datasets) is type(params) + + +def test_from_measurements_tool_does_not_recompute_reflectivity(): + R = sc.DataArray( + sc.ones(dims=['Q'], shape=(50,), with_variances=True), + coords={'Q': sc.linspace('Q', 0.1, 1, 50)}, + ).bin(Q=10) + + times_evaluated = 0 + + def reflectivity() -> ReflectivityOverQ: + nonlocal times_evaluated + times_evaluated += 1 + return ReflectivityOverQ(R) + + def reducible_data() -> ReducibleData[SampleRun]: + return 'Not important' + + pl = sl.Pipeline([reflectivity, reducible_data]) + + from_measurements( + pl, + [{}, {}], + target=(ReflectivityOverQ,), + scale_to_overlap=True, + ) + assert times_evaluated == 2 + + +def test_from_measurements_tool_applies_scaling_to_reflectivityoverq(): + R1 = sc.DataArray( + sc.ones(dims=['Q'], shape=(50,), with_variances=True), + coords={'Q': sc.linspace('Q', 0.1, 1, 50)}, + ).bin(Q=10) + R2 = 0.5 * R1 + + def reducible_data() -> ReducibleData[SampleRun]: + return 'Not important' + + pl = sl.Pipeline([reducible_data]) + + results = from_measurements( + pl, + [{ReflectivityOverQ: R1}, {ReflectivityOverQ: R2}], + target=(ReflectivityOverQ,), + scale_to_overlap=(sc.scalar(0.0), sc.scalar(1.0)), + ) + assert_allclose(results[0][ReflectivityOverQ], results[1][ReflectivityOverQ]) + + +def test_from_measurements_tool_applies_scaling_to_reducibledata(): + R1 = sc.DataArray( + sc.ones(dims=['Q'], shape=(50,), with_variances=True), + coords={'Q': sc.linspace('Q', 0.1, 1, 50)}, + ).bin(Q=10) + R2 = 0.5 * R1 + + def reducible_data() -> ReducibleData[SampleRun]: + return sc.scalar(1) + + pl = sl.Pipeline([reducible_data]) + + results = from_measurements( + pl, + [{ReflectivityOverQ: R1}, {ReflectivityOverQ: R2}], + target=(ReducibleData[SampleRun],), + scale_to_overlap=(sc.scalar(0.0), sc.scalar(1.0)), + ) + assert_allclose( + results[0][ReducibleData[SampleRun]], 0.5 * results[1][ReducibleData[SampleRun]] + ) From f158ac28c4f8826333d214d6aeb5b2cee6dd7788 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Thu, 3 Jul 2025 23:30:15 +0200 Subject: [PATCH 10/75] add scaling factor to Amor workflow --- src/ess/amor/__init__.py | 2 ++ src/ess/amor/workflow.py | 18 +++++++++++++++--- src/ess/reflectometry/types.py | 8 ++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/ess/amor/__init__.py b/src/ess/amor/__init__.py index 4148b94b..5c67959f 100644 --- a/src/ess/amor/__init__.py +++ b/src/ess/amor/__init__.py @@ -16,6 +16,7 @@ Position, RunType, SampleRotationOffset, + ScalingFactorForOverlap, ) from . import ( conversions, @@ -74,6 +75,7 @@ def default_parameters() -> dict: ), GravityToggle: True, SampleRotationOffset[RunType]: sc.scalar(0.0, unit='deg'), + ScalingFactorForOverlap[RunType]: 1.0, } diff --git a/src/ess/amor/workflow.py b/src/ess/amor/workflow.py index 828abe4f..68dc0644 100644 --- a/src/ess/amor/workflow.py +++ b/src/ess/amor/workflow.py @@ -12,6 +12,8 @@ ProtonCurrent, ReducibleData, RunType, + ScalingFactorForOverlap, + UnscaledReducibleData, WavelengthBins, YIndexLimits, ZIndexLimits, @@ -27,7 +29,7 @@ def add_coords_masks_and_apply_corrections( wbins: WavelengthBins, proton_current: ProtonCurrent[RunType], graph: CoordTransformationGraph, -) -> ReducibleData[RunType]: +) -> UnscaledReducibleData[RunType]: """ Computes coordinates, masks and corrections that are the same for the sample measurement and the reference measurement. @@ -43,7 +45,17 @@ def add_coords_masks_and_apply_corrections( da = add_proton_current_mask(da) da = correct_by_proton_current(da) - return ReducibleData[RunType](da) + return UnscaledReducibleData[RunType](da) + + +def scale_raw_reducible_data( + da: UnscaledReducibleData[RunType], + scale: ScalingFactorForOverlap[RunType], +) -> ReducibleData[RunType]: + """ + Scales the raw data by a given factor. + """ + return ReducibleData[RunType](da * scale) -providers = (add_coords_masks_and_apply_corrections,) +providers = (add_coords_masks_and_apply_corrections, scale_raw_reducible_data) diff --git a/src/ess/reflectometry/types.py b/src/ess/reflectometry/types.py index f1eea80a..6959534b 100644 --- a/src/ess/reflectometry/types.py +++ b/src/ess/reflectometry/types.py @@ -28,10 +28,18 @@ class RawChopper(sciline.Scope[RunType, sc.DataGroup], sc.DataGroup): """Chopper data loaded from nexus file.""" +class UnscaledReducibleData(sciline.Scope[RunType, sc.DataArray], sc.DataArray): + """""" + + class ReducibleData(sciline.Scope[RunType, sc.DataArray], sc.DataArray): """Event data with common coordinates added""" +class ScalingFactorForOverlap(sciline.Scope[RunType, float], float): + """""" + + ReducedReference = NewType("ReducedReference", sc.DataArray) """Intensity distribution on the detector for a sample with :math`R(Q) = 1`""" From 268f48781044fc34fe0f95d91cde1f189c5fa495 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Thu, 3 Jul 2025 23:50:06 +0200 Subject: [PATCH 11/75] map on unscaled data --- src/ess/reflectometry/workflow.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ess/reflectometry/workflow.py b/src/ess/reflectometry/workflow.py index 76dfe734..e960da99 100644 --- a/src/ess/reflectometry/workflow.py +++ b/src/ess/reflectometry/workflow.py @@ -16,10 +16,10 @@ DetectorRotation, Filename, RawChopper, - ReducibleData, RunType, SampleRotation, SampleRun, + UnscaledReducibleData, ) @@ -64,11 +64,11 @@ def with_filenames( mapped = wf.map(df) try: - wf[ReducibleData[runtype]] = mapped[ReducibleData[runtype]].reduce( - index=axis_name, func=_concatenate_event_lists - ) + wf[UnscaledReducibleData[runtype]] = mapped[ + UnscaledReducibleData[runtype] + ].reduce(index=axis_name, func=_concatenate_event_lists) except ValueError: - # ReducibleData[runtype] is independent of Filename[runtype] + # UnscaledReducibleData[runtype] is independent of Filename[runtype] pass try: wf[RawChopper[runtype]] = mapped[RawChopper[runtype]].reduce( From eb56472c8fbba36ba9fdccecc925a86908359235 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Tue, 15 Jul 2025 16:18:04 +0200 Subject: [PATCH 12/75] add batch_processor and WorkflowCollection to the tools --- src/ess/reflectometry/__init__.py | 17 +- src/ess/reflectometry/tools.py | 309 ++++++++++++++++++++---------- 2 files changed, 226 insertions(+), 100 deletions(-) diff --git a/src/ess/reflectometry/__init__.py b/src/ess/reflectometry/__init__.py index 7dd54dce..906c8271 100644 --- a/src/ess/reflectometry/__init__.py +++ b/src/ess/reflectometry/__init__.py @@ -12,6 +12,7 @@ from . import conversions, corrections, figures, normalization, orso from .load import load_reference, save_reference +from .tools import batch_processor providers = ( *corrections.providers, @@ -31,9 +32,23 @@ del importlib - __all__ = [ + "__version__", + "batch_processor", + "conversions", + "corrections", "figures", "load_reference", + "normalization", + "orso", + "providers", "save_reference", ] + +# __all__ = [ +# "__version__", +# "batch_processor", +# "figures", +# "load_reference", +# "save_reference", +# ] diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index cfbd36a2..4afda64a 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -12,9 +12,12 @@ from ess.reflectometry import orso from ess.reflectometry.types import ( Filename, + QBins, ReducibleData, ReflectivityOverQ, SampleRun, + ScalingFactorForOverlap, + UnscaledReducibleData, ) from ess.reflectometry.workflow import with_filenames @@ -165,44 +168,65 @@ def _interpolate_on_qgrid(curves, grid): def scale_reflectivity_curves_to_overlap( - curves: Sequence[sc.DataArray], + wf_collection: Sequence[sc.DataArray], critical_edge_interval: tuple[sc.Variable, sc.Variable] | None = None, + cache_intermediate_results: bool = True, ) -> tuple[list[sc.DataArray], list[sc.Variable]]: - '''Make the curves overlap by scaling all except the first by a factor. + ''' + Set the ``ScalingFactorForOverlap`` parameter on the provided workflows + in a way that would makes the 1D reflectivity curves overlap. + + If :code:`critical_edge_interval` is not provided, all workflows are scaled except + the data with the lowest Q-range, which is considered to be the reference curve. The scaling factors are determined by a maximum likelihood estimate (assuming the errors are normal distributed). - If :code:`critical_edge_interval` is provided then all curves are scaled. + If :code:`critical_edge_interval` is provided then all data are scaled. - All curves must be have the same unit for data and the Q-coordinate. + All reflectivity curves must be have the same unit for data and the Q-coordinate. Parameters --------- - curves: - the reflectivity curves that should be scaled together + wf_collection: + The collection of workflows that can compute the ``ReflectivityOverQ``. critical_edge_interval: - a tuple denoting an interval that is known to belong + A tuple denoting an interval that is known to belong to the critical edge, i.e. where the reflectivity is known to be 1. + cache_intermediate_results: + If ``True`` the intermediate results ``UnscaledReducibleData`` will be cached + (this is the base for all types that are downstream of the scaling factor). Returns --------- : A list of scaled reflectivity curves and a list of the scaling factors. ''' - if critical_edge_interval is not None: - q = next(iter(curves)).coords['Q'] - N = ( - ((q >= critical_edge_interval[0]) & (q < critical_edge_interval[1])) - .sum() - .value + # if critical_edge_interval is not None: + # # Find q bins with the lowest Q start point + # qmin = min( + # wf.compute(QBins).min() for wf in wf_collection.values()) + # q = next(iter(wf_collection.values())).compute(QBins) + # N = ( + # ((q >= critical_edge_interval[0]) & (q < critical_edge_interval[1])) + # .sum() + # .value + # ) + # edge = sc.DataArray( + # data=sc.ones(dims=('Q',), shape=(N,), with_variances=True), + # coords={'Q': sc.linspace('Q', *critical_edge_interval, N + 1)}, + # ) + # curves, factors = scale_reflectivity_curves_to_overlap([edge, *curves]) + # return curves[1:], factors[1:] + + wfc = wf_collection.copy() + if cache_intermediate_results: + wfc[UnscaledReducibleData[SampleRun]] = wfc.compute( + UnscaledReducibleData[SampleRun] ) - edge = sc.DataArray( - data=sc.ones(dims=('Q',), shape=(N,), with_variances=True), - coords={'Q': sc.linspace('Q', *critical_edge_interval, N + 1)}, - ) - curves, factors = scale_reflectivity_curves_to_overlap([edge, *curves]) - return curves[1:], factors[1:] + + curves = {key: r.hist() for key, r in wfc.compute(ReflectivityOverQ).items()} + if len({c.data.unit for c in curves}) != 1: raise ValueError('The reflectivity curves must have the same unit') if len({c.coords['Q'].unit for c in curves}) != 1: @@ -226,10 +250,13 @@ def cost(scaling_factors): sol = opt.minimize(cost, [1.0] * (len(curves) - 1)) scaling_factors = (1.0, *map(float, sol.x)) - return [ - scaling_factor * curve - for scaling_factor, curve in zip(scaling_factors, curves, strict=True) - ], scaling_factors + + wfc[ScalingFactorForOverlap[SampleRun]] = scaling_factors + return wfc + # return [ + # scaling_factor * curve + # for scaling_factor, curve in zip(scaling_factors, curves, strict=True) + # ], scaling_factors def combine_curves( @@ -284,44 +311,71 @@ def combine_curves( ) -def from_measurements( - workflow: sciline.Pipeline, - runs: Sequence[Mapping[type, Any]] | Mapping[Any, Mapping[type, Any]], - target: type | Sequence[type] = orso.OrsoIofQDataset, - *, - scale_to_overlap: bool | tuple[sc.Variable, sc.Variable] = False, -) -> list | Mapping: - '''Produces a list of datasets containing for each of the provided runs. - Each entry of :code:`runs` is a mapping of parameters and - values needed to produce the dataset. +class WorkflowCollection: + """ + A collection of sciline workflows that can be used to compute multiple + targets from multiple workflows. + It can also be used to set parameters for all workflows in a single shot. + """ - Optionally, the results can be scaled so that the reflectivity curves overlap in - the regions where they have the same Q-value. + def __init__(self, workflows: Mapping[str, sciline.Pipeline]): + self._workflows = {name: pl.copy() for name, pl in workflows.items()} - Parameters - ----------- - workflow: - The sciline workflow used to compute `ReflectivityOverQ` for each of the runs. + def __setitem__(self, key: type, value: Any | Mapping[type, Any]): + if hasattr(value, 'items'): + for name, v in value.items(): + self._workflows[name][key] = v + else: + for pl in self._workflows.values(): + pl[key] = value - runs: - The sciline parameters to be used for each run. + def compute(self, target: type | Sequence[type]) -> Mapping[str, Any]: + return {name: pl.compute(target) for name, pl in self._workflows.items()} - target: - The domain type(s) to compute for each run. + def copy(self) -> 'WorkflowCollection': + return self.__class__(self._workflows) - scale_to_overlap: - If ``True`` the curves will be scaled to overlap. - If a tuple then this argument will be passed as the ``critical_edge_interval`` - argument to the ``scale_reflectivity_curves_to_overlap`` function. + def keys(self) -> Sequence[str]: + return self._workflows.keys() - Returns - --------- - list of the computed ORSO datasets, containing one reflectivity curve each - ''' - names = runs.keys() if hasattr(runs, 'keys') else None - runs = runs.values() if hasattr(runs, 'values') else runs + def values(self) -> Sequence[sciline.Pipeline]: + return self._workflows.values() - def init_workflow(workflow, parameters): + def items(self) -> Sequence[tuple[str, sciline.Pipeline]]: + return self._workflows.items() + + def add(self, name: str, workflow: sciline.Pipeline): + """ + Adds a new workflow to the collection. + """ + self._workflows[name] = workflow.copy() + + def remove(self, name: str): + """ + Removes a workflow from the collection by its name. + """ + del self._workflows[name] + + +def batch_processor( + workflow: sciline.Pipeline, runs: Mapping[Any, Mapping[type, Any]] +) -> WorkflowCollection: + """ + Creates a collection of sciline workflows from the provided runs. + + Runs can be provided as a mapping of names to parameters or as a sequence + of mappings of parameters and values. + + Parameters + ---------- + workflow: + The sciline workflow used to compute the targets for each of the runs. + runs: + The sciline parameters to be used for each run. + TODO: explain how grouping works depending on the type of `runs`. + """ + workflows = {} + for name, parameters in runs.items(): wf = workflow.copy() for tp, value in parameters.items(): if tp is Filename[SampleRun]: @@ -337,48 +391,105 @@ def init_workflow(workflow, parameters): ) else: wf[Filename[SampleRun]] = parameters[Filename[SampleRun]] - return wf - - if scale_to_overlap: - results = [] - for parameters in runs: - wf = init_workflow(workflow, parameters) - results.append(wf.compute((ReflectivityOverQ, ReducibleData[SampleRun]))) - - scale_factors = scale_reflectivity_curves_to_overlap( - [r[ReflectivityOverQ].hist() for r in results], - critical_edge_interval=scale_to_overlap - if isinstance(scale_to_overlap, list | tuple) - else None, - )[1] - - datasets = [] - for i, parameters in enumerate(runs): - wf = init_workflow(workflow, parameters) - if scale_to_overlap: - # Optimization in case we can avoid re-doing some work: - # Check if any of the targets need ReducibleData if - # ReflectivityOverQ already exists. - # If they don't, we can avoid recomputing ReducibleData. - targets = target if hasattr(target, '__len__') else (target,) - if any( - _workflow_needs_quantity_A_even_if_quantity_B_is_set( - wf[t], ReducibleData[SampleRun], ReflectivityOverQ - ) - for t in targets - ): - wf[ReducibleData[SampleRun]] = ( - scale_factors[i] * results[i][ReducibleData[SampleRun]] - ) - else: - wf[ReflectivityOverQ] = scale_factors[i] * results[i][ReflectivityOverQ] - - dataset = wf.compute(target) - datasets.append(dataset) - return datasets if names is None else dict(zip(names, datasets, strict=True)) - - -def _workflow_needs_quantity_A_even_if_quantity_B_is_set(workflow, A, B): - wf = workflow.copy() - wf[B] = 'Not important' - return A in wf.underlying_graph + workflows[name] = wf + return WorkflowCollection(workflows) + + +# def from_measurements( +# workflow: sciline.Pipeline, +# runs: Sequence[Mapping[type, Any]] | Mapping[Any, Mapping[type, Any]], +# target: type | Sequence[type] = orso.OrsoIofQDataset, +# *, +# scale_to_overlap: bool | tuple[sc.Variable, sc.Variable] = False, +# ) -> list | Mapping: +# '''Produces a list of datasets containing for each of the provided runs. +# Each entry of :code:`runs` is a mapping of parameters and +# values needed to produce the dataset. + +# Optionally, the results can be scaled so that the reflectivity curves overlap in +# the regions where they have the same Q-value. + +# Parameters +# ----------- +# workflow: +# The sciline workflow used to compute `ReflectivityOverQ` for each of the runs. + +# runs: +# The sciline parameters to be used for each run. + +# target: +# The domain type(s) to compute for each run. + +# scale_to_overlap: +# If ``True`` the curves will be scaled to overlap. +# If a tuple then this argument will be passed as the ``critical_edge_interval`` +# argument to the ``scale_reflectivity_curves_to_overlap`` function. + +# Returns +# --------- +# list of the computed ORSO datasets, containing one reflectivity curve each +# ''' +# names = runs.keys() if hasattr(runs, 'keys') else None +# runs = runs.values() if hasattr(runs, 'values') else runs + +# def init_workflow(workflow, parameters): +# wf = workflow.copy() +# for tp, value in parameters.items(): +# if tp is Filename[SampleRun]: +# continue +# wf[tp] = value + +# if Filename[SampleRun] in parameters: +# if isinstance(parameters[Filename[SampleRun]], list | tuple): +# wf = with_filenames( +# wf, +# SampleRun, +# parameters[Filename[SampleRun]], +# ) +# else: +# wf[Filename[SampleRun]] = parameters[Filename[SampleRun]] +# return wf + +# if scale_to_overlap: +# results = [] +# for parameters in runs: +# wf = init_workflow(workflow, parameters) +# results.append(wf.compute((ReflectivityOverQ, ReducibleData[SampleRun]))) + +# scale_factors = scale_reflectivity_curves_to_overlap( +# [r[ReflectivityOverQ].hist() for r in results], +# critical_edge_interval=scale_to_overlap +# if isinstance(scale_to_overlap, list | tuple) +# else None, +# )[1] + +# datasets = [] +# for i, parameters in enumerate(runs): +# wf = init_workflow(workflow, parameters) +# if scale_to_overlap: +# # Optimization in case we can avoid re-doing some work: +# # Check if any of the targets need ReducibleData if +# # ReflectivityOverQ already exists. +# # If they don't, we can avoid recomputing ReducibleData. +# targets = target if hasattr(target, '__len__') else (target,) +# if any( +# _workflow_needs_quantity_A_even_if_quantity_B_is_set( +# wf[t], ReducibleData[SampleRun], ReflectivityOverQ +# ) +# for t in targets +# ): +# wf[ReducibleData[SampleRun]] = ( +# scale_factors[i] * results[i][ReducibleData[SampleRun]] +# ) +# else: +# wf[ReflectivityOverQ] = scale_factors[i] * results[i][ReflectivityOverQ] + +# dataset = wf.compute(target) +# datasets.append(dataset) +# return datasets if names is None else dict(zip(names, datasets, strict=True)) + + +# def _workflow_needs_quantity_A_even_if_quantity_B_is_set(workflow, A, B): +# wf = workflow.copy() +# wf[B] = 'Not important' +# return A in wf.underlying_graph From 9637acd5a26bc5e5345dee416d96958a93f89afd Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Tue, 15 Jul 2025 17:13:54 +0200 Subject: [PATCH 13/75] udpate Amor notebook to use workflow collection --- docs/user-guide/amor/amor-reduction.ipynb | 127 ++++++++-------------- src/ess/reflectometry/tools.py | 123 +++------------------ 2 files changed, 63 insertions(+), 187 deletions(-) diff --git a/docs/user-guide/amor/amor-reduction.ipynb b/docs/user-guide/amor/amor-reduction.ipynb index c3918f94..c3293114 100644 --- a/docs/user-guide/amor/amor-reduction.ipynb +++ b/docs/user-guide/amor/amor-reduction.ipynb @@ -30,11 +30,12 @@ "from ess.amor import data # noqa: F401\n", "from ess.reflectometry.types import *\n", "from ess.amor.types import *\n", + "from ess.reflectometry import batch_processor\n", "\n", "# The files used in this tutorial have some issues that makes scippnexus\n", "# raise warnings when loading them. To avoid noise in the notebook the warnings are silenced.\n", "warnings.filterwarnings('ignore', 'Failed to convert .* into a transformation')\n", - "warnings.filterwarnings('ignore', 'Invalid transformation, missing attribute')" + "warnings.filterwarnings('ignore', 'Invalid transformation')" ] }, { @@ -124,19 +125,17 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "## Computing sample reflectivity\n", + "## Computing sample reflectivity from batch reduction\n", "\n", "We now compute the sample reflectivity from 4 runs that used different sample rotation angles.\n", - "The measurements at different rotation angles cover different ranges of $Q$." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ + "The measurements at different rotation angles cover different ranges of $Q$.\n", + "\n", + "We set up a batch reduction helper (using the `batch_processor` function) which makes it easy to process multiple runs at once.\n", + "\n", "In this tutorial we use some Amor data files we have received.\n", "The file paths to the tutorial files are obtained by calling:" ] @@ -184,15 +183,15 @@ " },\n", "}\n", "\n", + "batch = batch_processor(workflow, runs)\n", + "display(batch.keys())\n", "\n", - "reflectivity = {}\n", - "for run_number, params in runs.items():\n", - " wf = workflow.copy()\n", - " for key, value in params.items():\n", - " wf[key] = value\n", - " reflectivity[run_number] = wf.compute(ReflectivityOverQ).hist()\n", - "\n", - "sc.plot(reflectivity, norm='log', vmin=1e-4)" + "# Compute R(Q) for all runs\n", + "reflectivity = batch.compute(ReflectivityOverQ)\n", + "sc.plot(\n", + " {key: r.hist() for key, r in reflectivity.items()},\n", + " norm='log', vmin=1e-4\n", + ")" ] }, { @@ -212,13 +211,17 @@ "source": [ "from ess.reflectometry.tools import scale_reflectivity_curves_to_overlap\n", "\n", - "scaled_reflectivity_curves, scale_factors = scale_reflectivity_curves_to_overlap(\n", - " reflectivity.values(),\n", - " # Optionally specify a Q-interval where the reflectivity is known to be 1.0\n", - " critical_edge_interval=(sc.scalar(0.01, unit='1/angstrom'), sc.scalar(0.014, unit='1/angstrom'))\n", + "# Pass the batch workflow collection and get a new workflow collection as output,\n", + "# with the correct scaling factors applied.\n", + "scaled_wf = scale_reflectivity_curves_to_overlap(\n", + " batch,\n", + " # TODO: implement handling of the critical_edge_interval\n", + " # critical_edge_interval=sc.scalar(0.01, unit='1/angstrom'), sc.scalar(0.014, unit='1/angstrom')\n", ")\n", "\n", - "sc.plot(dict(zip(reflectivity.keys(), scaled_reflectivity_curves, strict=True)), norm='log', vmin=1e-5)" + "scaled_r = {key: r.hist() for key, r in scaled_wf.compute(ReflectivityOverQ).items()}\n", + "\n", + "sc.plot(scaled_r, norm='log', vmin=1e-5)" ] }, { @@ -235,7 +238,7 @@ "outputs": [], "source": [ "from ess.reflectometry.tools import combine_curves\n", - "combined = combine_curves(scaled_reflectivity_curves, workflow.compute(QBins))\n", + "combined = combine_curves(scaled_r.values(), workflow.compute(QBins))\n", "combined.plot(norm='log')" ] }, @@ -265,26 +268,8 @@ "metadata": {}, "outputs": [], "source": [ - "# Start by computing the `ReflectivityData` for each of the files\n", - "diagnostics = {}\n", - "for run_number, params in runs.items():\n", - " wf = workflow.copy()\n", - " for key, value in params.items():\n", - " wf[key] = value\n", - " diagnostics[run_number] = wf.compute((ReflectivityOverZW, ThetaBins[SampleRun]))\n", - "\n", - "# Scale the results using the scale factors computed earlier\n", - "for run_number, scale_factor in zip(reflectivity.keys(), scale_factors, strict=True):\n", - " diagnostics[run_number][ReflectivityOverZW] *= scale_factor" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "diagnostics['608'][ReflectivityOverZW].hist().flatten(('blade', 'wire'), to='z').plot(norm='log')" + "diagnostics = scaled_wf.compute(ReflectivityOverZW)\n", + "diagnostics['608'].hist().flatten(('blade', 'wire'), to='z').plot(norm='log')" ] }, { @@ -304,8 +289,8 @@ "from ess.reflectometry.figures import wavelength_theta_figure\n", "\n", "wavelength_theta_figure(\n", - " [result[ReflectivityOverZW] for result in diagnostics.values()],\n", - " theta_bins=[result[ThetaBins[SampleRun]] for result in diagnostics.values()],\n", + " diagnostics.values(),\n", + " theta_bins=scaled_wf.compute(ThetaBins[SampleRun]).values(),\n", " q_edges_to_display=(sc.scalar(0.018, unit='1/angstrom'), sc.scalar(0.113, unit='1/angstrom'))\n", ")" ] @@ -334,8 +319,8 @@ "from ess.reflectometry.figures import q_theta_figure\n", "\n", "q_theta_figure(\n", - " [res[ReflectivityOverZW] for res in diagnostics.values()],\n", - " theta_bins=[res[ThetaBins[SampleRun]] for res in diagnostics.values()],\n", + " diagnostics.values(),\n", + " theta_bins=scaled_wf.compute(ThetaBins[SampleRun]).values(),\n", " q_bins=workflow.compute(QBins)\n", ")" ] @@ -380,8 +365,7 @@ "We can save the computed $I(Q)$ to an [ORSO](https://www.reflectometry.org) [.ort](https://github.com/reflectivity/file_format/blob/master/specification.md) file using the [orsopy](https://orsopy.readthedocs.io/en/latest/index.html) package.\n", "\n", "First, we need to collect the metadata for that file.\n", - "To this end, we build a pipeline with additional providers.\n", - "We also insert a parameter to indicate the creator of the processed data." + "To this end, we insert a parameter to indicate the creator of the processed data." ] }, { @@ -400,7 +384,7 @@ "metadata": {}, "outputs": [], "source": [ - "workflow[orso.OrsoCreator] = orso.OrsoCreator(\n", + "scaled_wf[orso.OrsoCreator] = orso.OrsoCreator(\n", " fileio.base.Person(\n", " name='Max Mustermann',\n", " affiliation='European Spallation Source ERIC',\n", @@ -409,20 +393,11 @@ ")" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "workflow.visualize(orso.OrsoIofQDataset, graph_attr={'rankdir': 'LR'})" - ] - }, { "cell_type": "markdown", "metadata": {}, "source": [ - "We build our ORSO dataset from the computed $I(Q)$ and the ORSO metadata:" + "We can visualize the workflow for a single run (`'608'`):" ] }, { @@ -431,15 +406,14 @@ "metadata": {}, "outputs": [], "source": [ - "iofq_dataset = workflow.compute(orso.OrsoIofQDataset)\n", - "iofq_dataset" + "scaled_wf['608'].visualize(orso.OrsoIofQDataset, graph_attr={'rankdir': 'LR'})" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "We also add the URL of this notebook to make it easier to reproduce the data:" + "We build our ORSO dataset from the computed $I(Q)$ and the ORSO metadata:" ] }, { @@ -448,17 +422,14 @@ "metadata": {}, "outputs": [], "source": [ - "iofq_dataset.info.reduction.script = (\n", - " 'https://scipp.github.io/essreflectometry/examples/amor.html'\n", - ")" + "iofq_datasets = scaled_wf.compute(orso.OrsoIofQDataset)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Now let's repeat this for all the sample measurements!\n", - "To do that we can use an utility in `ess.reflectometry.tools`:" + "We also add the URL of this notebook to make it easier to reproduce the data:" ] }, { @@ -467,16 +438,10 @@ "metadata": {}, "outputs": [], "source": [ - "from ess.reflectometry.tools import from_measurements\n", - "\n", - "datasets = from_measurements(\n", - " workflow,\n", - " runs.values(),\n", - " target=orso.OrsoIofQDataset,\n", - " # Optionally scale the curves to overlap using `scale_reflectivity_curves_to_overlap`\n", - " # with the same \"critical edge interval\" that was used before\n", - " scale_to_overlap=(sc.scalar(0.01, unit='1/angstrom'), sc.scalar(0.014, unit='1/angstrom'))\n", - ")" + "for ds in iofq_datasets.values():\n", + " ds.info.reduction.script = (\n", + " 'https://scipp.github.io/essreflectometry/user-guide/amor/amor-reduction.html'\n", + " )" ] }, { @@ -484,7 +449,7 @@ "metadata": {}, "source": [ "Finally, we can save the data to a file.\n", - "Note that `iofq_dataset` is an [orsopy.fileio.orso.OrsoDataset](https://orsopy.readthedocs.io/en/latest/orsopy.fileio.orso.html#orsopy.fileio.orso.OrsoDataset)." + "Note that `iofq_datasets` contains [orsopy.fileio.orso.OrsoDataset](https://orsopy.readthedocs.io/en/latest/orsopy.fileio.orso.html#orsopy.fileio.orso.OrsoDataset)s." ] }, { @@ -493,7 +458,7 @@ "metadata": {}, "outputs": [], "source": [ - "fileio.orso.save_orso(datasets=datasets, fname='amor_reduced_iofq.ort')" + "fileio.orso.save_orso(datasets=list(iofq_datasets.values()), fname='amor_reduced_iofq.ort')" ] }, { @@ -529,7 +494,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.14" + "version": "3.12.7" } }, "nbformat": 4, diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 4afda64a..7092a550 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -9,11 +9,11 @@ import scipp as sc import scipy.optimize as opt -from ess.reflectometry import orso +# from ess.reflectometry import orso from ess.reflectometry.types import ( Filename, QBins, - ReducibleData, + # ReducibleData, ReflectivityOverQ, SampleRun, ScalingFactorForOverlap, @@ -225,7 +225,7 @@ def scale_reflectivity_curves_to_overlap( UnscaledReducibleData[SampleRun] ) - curves = {key: r.hist() for key, r in wfc.compute(ReflectivityOverQ).items()} + curves = [r.hist() for r in wfc.compute(ReflectivityOverQ).values()] if len({c.data.unit for c in curves}) != 1: raise ValueError('The reflectivity curves must have the same unit') @@ -251,7 +251,10 @@ def cost(scaling_factors): sol = opt.minimize(cost, [1.0] * (len(curves) - 1)) scaling_factors = (1.0, *map(float, sol.x)) - wfc[ScalingFactorForOverlap[SampleRun]] = scaling_factors + wfc[ScalingFactorForOverlap[SampleRun]] = dict( + zip(wfc.keys(), scaling_factors, strict=True) + ) + return wfc # return [ # scaling_factor * curve @@ -329,8 +332,16 @@ def __setitem__(self, key: type, value: Any | Mapping[type, Any]): for pl in self._workflows.values(): pl[key] = value - def compute(self, target: type | Sequence[type]) -> Mapping[str, Any]: - return {name: pl.compute(target) for name, pl in self._workflows.items()} + def __getitem__(self, name: str) -> sciline.Pipeline: + """ + Returns a single workflow from the collection given by its name. + """ + return self._workflows[name] + + def compute(self, target: type | Sequence[type], **kwargs) -> Mapping[str, Any]: + return { + name: pl.compute(target, **kwargs) for name, pl in self._workflows.items() + } def copy(self) -> 'WorkflowCollection': return self.__class__(self._workflows) @@ -393,103 +404,3 @@ def batch_processor( wf[Filename[SampleRun]] = parameters[Filename[SampleRun]] workflows[name] = wf return WorkflowCollection(workflows) - - -# def from_measurements( -# workflow: sciline.Pipeline, -# runs: Sequence[Mapping[type, Any]] | Mapping[Any, Mapping[type, Any]], -# target: type | Sequence[type] = orso.OrsoIofQDataset, -# *, -# scale_to_overlap: bool | tuple[sc.Variable, sc.Variable] = False, -# ) -> list | Mapping: -# '''Produces a list of datasets containing for each of the provided runs. -# Each entry of :code:`runs` is a mapping of parameters and -# values needed to produce the dataset. - -# Optionally, the results can be scaled so that the reflectivity curves overlap in -# the regions where they have the same Q-value. - -# Parameters -# ----------- -# workflow: -# The sciline workflow used to compute `ReflectivityOverQ` for each of the runs. - -# runs: -# The sciline parameters to be used for each run. - -# target: -# The domain type(s) to compute for each run. - -# scale_to_overlap: -# If ``True`` the curves will be scaled to overlap. -# If a tuple then this argument will be passed as the ``critical_edge_interval`` -# argument to the ``scale_reflectivity_curves_to_overlap`` function. - -# Returns -# --------- -# list of the computed ORSO datasets, containing one reflectivity curve each -# ''' -# names = runs.keys() if hasattr(runs, 'keys') else None -# runs = runs.values() if hasattr(runs, 'values') else runs - -# def init_workflow(workflow, parameters): -# wf = workflow.copy() -# for tp, value in parameters.items(): -# if tp is Filename[SampleRun]: -# continue -# wf[tp] = value - -# if Filename[SampleRun] in parameters: -# if isinstance(parameters[Filename[SampleRun]], list | tuple): -# wf = with_filenames( -# wf, -# SampleRun, -# parameters[Filename[SampleRun]], -# ) -# else: -# wf[Filename[SampleRun]] = parameters[Filename[SampleRun]] -# return wf - -# if scale_to_overlap: -# results = [] -# for parameters in runs: -# wf = init_workflow(workflow, parameters) -# results.append(wf.compute((ReflectivityOverQ, ReducibleData[SampleRun]))) - -# scale_factors = scale_reflectivity_curves_to_overlap( -# [r[ReflectivityOverQ].hist() for r in results], -# critical_edge_interval=scale_to_overlap -# if isinstance(scale_to_overlap, list | tuple) -# else None, -# )[1] - -# datasets = [] -# for i, parameters in enumerate(runs): -# wf = init_workflow(workflow, parameters) -# if scale_to_overlap: -# # Optimization in case we can avoid re-doing some work: -# # Check if any of the targets need ReducibleData if -# # ReflectivityOverQ already exists. -# # If they don't, we can avoid recomputing ReducibleData. -# targets = target if hasattr(target, '__len__') else (target,) -# if any( -# _workflow_needs_quantity_A_even_if_quantity_B_is_set( -# wf[t], ReducibleData[SampleRun], ReflectivityOverQ -# ) -# for t in targets -# ): -# wf[ReducibleData[SampleRun]] = ( -# scale_factors[i] * results[i][ReducibleData[SampleRun]] -# ) -# else: -# wf[ReflectivityOverQ] = scale_factors[i] * results[i][ReflectivityOverQ] - -# dataset = wf.compute(target) -# datasets.append(dataset) -# return datasets if names is None else dict(zip(names, datasets, strict=True)) - - -# def _workflow_needs_quantity_A_even_if_quantity_B_is_set(workflow, A, B): -# wf = workflow.copy() -# wf[B] = 'Not important' -# return A in wf.underlying_graph From 2d2871dc960a598232d1bd4d0c324efb9af7c62a Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Tue, 15 Jul 2025 18:09:06 +0200 Subject: [PATCH 14/75] fix critical_edge parameter in scaling --- docs/user-guide/amor/amor-reduction.ipynb | 3 +- src/ess/reflectometry/tools.py | 78 ++++++++++++++--------- 2 files changed, 50 insertions(+), 31 deletions(-) diff --git a/docs/user-guide/amor/amor-reduction.ipynb b/docs/user-guide/amor/amor-reduction.ipynb index c3293114..18d9224e 100644 --- a/docs/user-guide/amor/amor-reduction.ipynb +++ b/docs/user-guide/amor/amor-reduction.ipynb @@ -215,8 +215,7 @@ "# with the correct scaling factors applied.\n", "scaled_wf = scale_reflectivity_curves_to_overlap(\n", " batch,\n", - " # TODO: implement handling of the critical_edge_interval\n", - " # critical_edge_interval=sc.scalar(0.01, unit='1/angstrom'), sc.scalar(0.014, unit='1/angstrom')\n", + " critical_edge_interval=(sc.scalar(0.01, unit='1/angstrom'), sc.scalar(0.014, unit='1/angstrom'))\n", ")\n", "\n", "scaled_r = {key: r.hist() for key, r in scaled_wf.compute(ReflectivityOverQ).items()}\n", diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 7092a550..89114ee1 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -2,7 +2,7 @@ # Copyright (c) 2025 Scipp contributors (https://github.com/scipp) from collections.abc import Mapping, Sequence from itertools import chain -from typing import Any +from typing import Any, NewType import numpy as np import sciline @@ -13,7 +13,6 @@ from ess.reflectometry.types import ( Filename, QBins, - # ReducibleData, ReflectivityOverQ, SampleRun, ScalingFactorForOverlap, @@ -167,6 +166,10 @@ def _interpolate_on_qgrid(curves, grid): ) +CriticalEdgeKey = NewType('CriticalEdgeKey', None) +"""A unique key used to store a 'fake' critical edge in a workflow collection.""" + + def scale_reflectivity_curves_to_overlap( wf_collection: Sequence[sc.DataArray], critical_edge_interval: tuple[sc.Variable, sc.Variable] | None = None, @@ -202,22 +205,29 @@ def scale_reflectivity_curves_to_overlap( : A list of scaled reflectivity curves and a list of the scaling factors. ''' - # if critical_edge_interval is not None: - # # Find q bins with the lowest Q start point - # qmin = min( - # wf.compute(QBins).min() for wf in wf_collection.values()) - # q = next(iter(wf_collection.values())).compute(QBins) - # N = ( - # ((q >= critical_edge_interval[0]) & (q < critical_edge_interval[1])) - # .sum() - # .value - # ) - # edge = sc.DataArray( - # data=sc.ones(dims=('Q',), shape=(N,), with_variances=True), - # coords={'Q': sc.linspace('Q', *critical_edge_interval, N + 1)}, - # ) - # curves, factors = scale_reflectivity_curves_to_overlap([edge, *curves]) - # return curves[1:], factors[1:] + if critical_edge_interval is not None: + # Find q bins with the lowest Q start point + q = min( + (wf.compute(QBins) for wf in wf_collection.values()), + key=lambda q_: q_.min(), + ) + N = ( + ((q >= critical_edge_interval[0]) & (q < critical_edge_interval[1])) + .sum() + .value + ) + edge = sc.DataArray( + data=sc.ones(dims=('Q',), shape=(N,), with_variances=True), + coords={'Q': sc.linspace('Q', *critical_edge_interval, N + 1)}, + ) + wfc = wf_collection.copy() + underlying_wf = next(iter(wfc.values())) + edge_wf = underlying_wf.copy() + edge_wf[ReflectivityOverQ] = edge + wfc.add(CriticalEdgeKey, edge_wf) + return scale_reflectivity_curves_to_overlap( + wfc, cache_intermediate_results=cache_intermediate_results + ) wfc = wf_collection.copy() if cache_intermediate_results: @@ -225,17 +235,28 @@ def scale_reflectivity_curves_to_overlap( UnscaledReducibleData[SampleRun] ) - curves = [r.hist() for r in wfc.compute(ReflectivityOverQ).values()] + reflectivities = wfc.compute(ReflectivityOverQ) - if len({c.data.unit for c in curves}) != 1: + # First sort the dict of reflectivities by the Q min value + curves = { + k: v.hist() if v.bins is not None else v + for k, v in sorted( + reflectivities.items(), key=lambda item: item[1].coords['Q'].min().value + ) + } + # Now place the critical edge at the beginning, if it exists + if CriticalEdgeKey in curves.keys(): + curves = {CriticalEdgeKey: curves[CriticalEdgeKey]} | curves + + if len({c.data.unit for c in curves.values()}) != 1: raise ValueError('The reflectivity curves must have the same unit') - if len({c.coords['Q'].unit for c in curves}) != 1: + if len({c.coords['Q'].unit for c in curves.values()}) != 1: raise ValueError('The Q-coordinates must have the same unit for each curve') - qgrid = _create_qgrid_where_overlapping([c.coords['Q'] for c in curves]) + qgrid = _create_qgrid_where_overlapping([c.coords['Q'] for c in curves.values()]) - r = _interpolate_on_qgrid(map(sc.values, curves), qgrid).values - v = _interpolate_on_qgrid(map(sc.variances, curves), qgrid).values + r = _interpolate_on_qgrid(map(sc.values, curves.values()), qgrid).values + v = _interpolate_on_qgrid(map(sc.variances, curves.values()), qgrid).values def cost(scaling_factors): scaling_factors = np.concatenate([[1.0], scaling_factors])[:, None] @@ -252,14 +273,13 @@ def cost(scaling_factors): scaling_factors = (1.0, *map(float, sol.x)) wfc[ScalingFactorForOverlap[SampleRun]] = dict( - zip(wfc.keys(), scaling_factors, strict=True) + zip(curves.keys(), scaling_factors, strict=True) ) + if CriticalEdgeKey in wfc.keys(): + wfc.remove(CriticalEdgeKey) + return wfc - # return [ - # scaling_factor * curve - # for scaling_factor, curve in zip(scaling_factors, curves, strict=True) - # ], scaling_factors def combine_curves( From 2a5120dc559608aaacd54fdb53a77f61db6194a2 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Tue, 15 Jul 2025 18:47:16 +0200 Subject: [PATCH 15/75] remove commented code --- src/ess/reflectometry/__init__.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/ess/reflectometry/__init__.py b/src/ess/reflectometry/__init__.py index 906c8271..3b6d18b2 100644 --- a/src/ess/reflectometry/__init__.py +++ b/src/ess/reflectometry/__init__.py @@ -44,11 +44,3 @@ "providers", "save_reference", ] - -# __all__ = [ -# "__version__", -# "batch_processor", -# "figures", -# "load_reference", -# "save_reference", -# ] From aced05051ea456b752b588cfb9024e02aaa80a3d Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Tue, 5 Aug 2025 15:59:35 +0200 Subject: [PATCH 16/75] add tests for wf collection --- tests/{ => reflectometry}/corrections_test.py | 0 tests/{ => reflectometry}/orso_test.py | 0 tests/{ => reflectometry}/tools_test.py | 0 .../reflectometry/workflow_collection_test.py | 136 ++++++++++++++++++ 4 files changed, 136 insertions(+) rename tests/{ => reflectometry}/corrections_test.py (100%) rename tests/{ => reflectometry}/orso_test.py (100%) rename tests/{ => reflectometry}/tools_test.py (100%) create mode 100644 tests/reflectometry/workflow_collection_test.py diff --git a/tests/corrections_test.py b/tests/reflectometry/corrections_test.py similarity index 100% rename from tests/corrections_test.py rename to tests/reflectometry/corrections_test.py diff --git a/tests/orso_test.py b/tests/reflectometry/orso_test.py similarity index 100% rename from tests/orso_test.py rename to tests/reflectometry/orso_test.py diff --git a/tests/tools_test.py b/tests/reflectometry/tools_test.py similarity index 100% rename from tests/tools_test.py rename to tests/reflectometry/tools_test.py diff --git a/tests/reflectometry/workflow_collection_test.py b/tests/reflectometry/workflow_collection_test.py new file mode 100644 index 00000000..bb1ff529 --- /dev/null +++ b/tests/reflectometry/workflow_collection_test.py @@ -0,0 +1,136 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2025 Scipp contributors (https://github.com/scipp) + +import sciline as sl + +from ess.reflectometry.tools import WorkflowCollection + + +def int_to_float(x: int) -> float: + return 0.5 * x + + +def int_float_to_str(x: int, y: float) -> str: + return f"{x};{y}" + + +def test_compute() -> None: + wf = sl.Pipeline([int_to_float, int_float_to_str]) + wfa = wf.copy() + wfa[int] = 3 + wfb = wf.copy() + wfb[int] = 4 + coll = WorkflowCollection({'a': wfa, 'b': wfb}) + + assert coll.compute(float) == {'a': 1.5, 'b': 2.0} + assert coll.compute(str) == {'a': '3;1.5', 'b': '4;2.0'} + + +def test_compute_multiple() -> None: + wf = sl.Pipeline([int_to_float, int_float_to_str]) + wfa = wf.copy() + wfa[int] = 3 + wfb = wf.copy() + wfb[int] = 4 + coll = WorkflowCollection({'a': wfa, 'b': wfb}) + + result = coll.compute([float, str]) + + assert result['a'] == {float: 1.5, str: '3;1.5'} + assert result['b'] == {float: 2.0, str: '4;2.0'} + + +def test_setitem_mapping() -> None: + wf = sl.Pipeline([int_to_float, int_float_to_str]) + wfa = wf.copy() + wfa[int] = 3 + wfb = wf.copy() + wfb[int] = 4 + coll = WorkflowCollection({'a': wfa, 'b': wfb}) + + coll[int] = {'a': 7, 'b': 8} + + assert coll.compute(float) == {'a': 3.5, 'b': 4.0} + assert coll.compute(str) == {'a': '7;3.5', 'b': '8;4.0'} + + +def test_setitem_single_value() -> None: + wf = sl.Pipeline([int_to_float, int_float_to_str]) + wfa = wf.copy() + wfa[int] = 3 + wfb = wf.copy() + wfb[int] = 4 + coll = WorkflowCollection({'a': wfa, 'b': wfb}) + + coll[int] = 5 + + assert coll.compute(float) == {'a': 2.5, 'b': 2.5} + assert coll.compute(str) == {'a': '5;2.5', 'b': '5;2.5'} + + +def test_copy() -> None: + wf = sl.Pipeline([int_to_float, int_float_to_str]) + wfa = wf.copy() + wfa[int] = 3 + wfb = wf.copy() + wfb[int] = 4 + coll = WorkflowCollection({'a': wfa, 'b': wfb}) + + coll_copy = coll.copy() + + assert coll_copy.compute(float) == {'a': 1.5, 'b': 2.0} + assert coll_copy.compute(str) == {'a': '3;1.5', 'b': '4;2.0'} + + coll_copy[int] = {'a': 7, 'b': 8} + assert coll.compute(float) == {'a': 1.5, 'b': 2.0} + assert coll.compute(str) == {'a': '3;1.5', 'b': '4;2.0'} + assert coll_copy.compute(float) == {'a': 3.5, 'b': 4.0} + assert coll_copy.compute(str) == {'a': '7;3.5', 'b': '8;4.0'} + + +def test_add_workflow() -> None: + wf = sl.Pipeline([int_to_float, int_float_to_str]) + wfa = wf.copy() + wfa[int] = 3 + wfb = wf.copy() + wfb[int] = 4 + coll = WorkflowCollection({'a': wfa, 'b': wfb}) + + wfc = wf.copy() + wfc[int] = 5 + coll.add('c', wfc) + + assert coll.compute(float) == {'a': 1.5, 'b': 2.0, 'c': 2.5} + assert coll.compute(str) == {'a': '3;1.5', 'b': '4;2.0', 'c': '5;2.5'} + + +def test_add_workflow_with_existing_key() -> None: + wf = sl.Pipeline([int_to_float, int_float_to_str]) + wfa = wf.copy() + wfa[int] = 3 + wfb = wf.copy() + wfb[int] = 4 + coll = WorkflowCollection({'a': wfa, 'b': wfb}) + + wfc = wf.copy() + wfc[int] = 5 + coll.add('a', wfc) + + assert coll.compute(float) == {'a': 2.5, 'b': 2.0} + assert coll.compute(str) == {'a': '5;2.5', 'b': '4;2.0'} + assert 'c' not in coll.keys() # 'c' should not exist + + +def test_remove_workflow() -> None: + wf = sl.Pipeline([int_to_float, int_float_to_str]) + wfa = wf.copy() + wfa[int] = 3 + wfb = wf.copy() + wfb[int] = 4 + coll = WorkflowCollection({'a': wfa, 'b': wfb}) + + coll.remove('b') + + assert 'b' not in coll.keys() + assert coll.compute(float) == {'a': 1.5} + assert coll.compute(str) == {'a': '3;1.5'} From 504405bae9e1be4707be4789d5a92b655fe67a07 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Tue, 5 Aug 2025 16:08:22 +0200 Subject: [PATCH 17/75] improve docstring --- src/ess/reflectometry/tools.py | 38 +++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 89114ee1..dfc594b9 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -394,8 +394,36 @@ def batch_processor( """ Creates a collection of sciline workflows from the provided runs. - Runs can be provided as a mapping of names to parameters or as a sequence - of mappings of parameters and values. + Example: + + ``` + from ess.reflectometry import amor, tools + + workflow = amor.AmorWorkflow() + + runs = { + '608': { + SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'), + Filename[SampleRun]: amor.data.amor_run(608), + }, + '609': { + SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'), + Filename[SampleRun]: amor.data.amor_run(609), + }, + '610': { + SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'), + Filename[SampleRun]: amor.data.amor_run(610), + }, + '611': { + SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'), + Filename[SampleRun]: amor.data.amor_run(611), + }, + } + + batch = tools.batch_processor(workflow, runs) + + results = batch.compute(ReflectivityOverQ) + ``` Parameters ---------- @@ -403,7 +431,11 @@ def batch_processor( The sciline workflow used to compute the targets for each of the runs. runs: The sciline parameters to be used for each run. - TODO: explain how grouping works depending on the type of `runs`. + Should be a mapping where the keys are the names of the runs + and the values are mappings of type to value pairs. + In addition, if one of the values for ``Filename[SampleRun]`` + is a list or a tuple, then the events from the files + will be concatenated into a single event list. """ workflows = {} for name, parameters in runs.items(): From 7e0c24d15291a7305b1a0f0a41700fdbd699955b Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Wed, 6 Aug 2025 19:08:48 +0200 Subject: [PATCH 18/75] start fixing existing unit tests --- tests/reflectometry/tools_test.py | 102 ++++++++++++++++++++++++------ 1 file changed, 84 insertions(+), 18 deletions(-) diff --git a/tests/reflectometry/tools_test.py b/tests/reflectometry/tools_test.py index 7f1c81b5..90e78f2d 100644 --- a/tests/reflectometry/tools_test.py +++ b/tests/reflectometry/tools_test.py @@ -9,41 +9,107 @@ from scipp.testing import assert_allclose from ess.reflectometry.tools import ( + WorkflowCollection, + batch_processor, combine_curves, - from_measurements, linlogspace, scale_reflectivity_curves_to_overlap, ) from ess.reflectometry.types import ( Filename, ReducibleData, + ReferenceRun, ReflectivityOverQ, + RunType, SampleRun, + ScalingFactorForOverlap, + UnscaledReducibleData, ) - -def curve(d, qmin, qmax): - return sc.DataArray(data=d, coords={'Q': sc.linspace('Q', qmin, qmax, len(d) + 1)}) +# def curve(d, qmin, qmax): +# return sc.DataArray(data=d, coords={'Q': sc.linspace('Q', qmin, qmax, len(d) + 1)}) + + +def make_sample_events(scale, qmin, qmax): + n1 = 10 + n2 = 15 + qbins = sc.linspace('Q', qmin, qmax, n1 + n2 + 1) + data = sc.DataArray( + data=sc.concat( + ( + sc.ones(dims=['Q'], shape=[10], with_variances=True), + 0.5 * sc.ones(dims=['Q'], shape=[15], with_variances=True), + ), + dim='Q', + ) + * scale, + coords={'Q': sc.midpoints(qbins, 'Q')}, + ) + data.variances[:] = 0.1 + return data.bin(Q=qbins) -def test_reflectivity_curve_scaling(): - data = sc.concat( - ( - sc.ones(dims=['Q'], shape=[10], with_variances=True), - 0.5 * sc.ones(dims=['Q'], shape=[15], with_variances=True), - ), - dim='Q', +def make_reference_events(qmin, qmax): + n = 25 + qbins = sc.linspace('Q', qmin, qmax, n + 1) + data = sc.DataArray( + data=sc.ones(dims=['Q'], shape=[n], with_variances=True), + coords={'Q': sc.midpoints(qbins, 'Q')}, ) data.variances[:] = 0.1 + return data.bin(Q=qbins) - curves, factors = scale_reflectivity_curves_to_overlap( - (curve(data, 0, 0.3), curve(0.8 * data, 0.2, 0.7), curve(0.1 * data, 0.6, 1.0)), - ) - assert_allclose(curves[0].data, data, rtol=sc.scalar(1e-5)) - assert_allclose(curves[1].data, 0.5 * data, rtol=sc.scalar(1e-5)) - assert_allclose(curves[2].data, 0.25 * data, rtol=sc.scalar(1e-5)) - np_assert_allclose((1, 0.5 / 0.8, 0.25 / 0.1), factors, 1e-4) +def make_workflow(): + def apply_scaling( + da: UnscaledReducibleData[RunType], + scale: ScalingFactorForOverlap[RunType], + ) -> ReducibleData[RunType]: + """ + Scales the raw data by a given factor. + """ + return ReducibleData[RunType](da * scale) + + def reflectivity( + sample: ReducibleData[SampleRun], reference: ReducibleData[ReferenceRun] + ) -> ReflectivityOverQ: + return ReflectivityOverQ(sample.hist() / reference.hist()) + + return sl.Pipeline([apply_scaling, reflectivity]) + + +def test_reflectivity_curve_scaling(): + wf = make_workflow() + wf[ScalingFactorForOverlap[SampleRun]] = 1.0 + wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 + runs = { + 'a': { + UnscaledReducibleData[SampleRun]: make_sample_events(1, 0, 0.3), + UnscaledReducibleData[ReferenceRun]: make_reference_events(0, 0.3), + }, + 'b': { + UnscaledReducibleData[SampleRun]: make_sample_events(0.8, 0.2, 0.7), + UnscaledReducibleData[ReferenceRun]: make_reference_events(0.2, 0.7), + }, + 'c': { + UnscaledReducibleData[SampleRun]: make_sample_events(0.1, 0.6, 1.0), + UnscaledReducibleData[ReferenceRun]: make_reference_events(0.6, 1.0), + }, + } + workflows = {} + for name, params in runs.items(): + workflows[name] = wf.copy() + for key, value in params.items(): + workflows[name][key] = value + wfc = WorkflowCollection(workflows) + + scaled_wf = scale_reflectivity_curves_to_overlap(wfc) + + factors = scaled_wf.compute(ScalingFactorForOverlap[SampleRun]) + + assert np.isclose(factors['a'], 1.0) + assert np.isclose(factors['b'], 0.5 / 0.8) + assert np.isclose(factors['c'], 0.25 / 0.1) def test_reflectivity_curve_scaling_with_critical_edge(): From 4bd66d34dd9d6464bc8cf1439f897bfc1bcbaba3 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Wed, 6 Aug 2025 20:05:23 +0200 Subject: [PATCH 19/75] refactor first tests slightly --- tests/reflectometry/tools_test.py | 97 ++++++++++++++++++------------- 1 file changed, 58 insertions(+), 39 deletions(-) diff --git a/tests/reflectometry/tools_test.py b/tests/reflectometry/tools_test.py index 90e78f2d..a620a3f6 100644 --- a/tests/reflectometry/tools_test.py +++ b/tests/reflectometry/tools_test.py @@ -17,6 +17,7 @@ ) from ess.reflectometry.types import ( Filename, + QBins, ReducibleData, ReferenceRun, ReflectivityOverQ, @@ -71,9 +72,11 @@ def apply_scaling( return ReducibleData[RunType](da * scale) def reflectivity( - sample: ReducibleData[SampleRun], reference: ReducibleData[ReferenceRun] + sample: ReducibleData[SampleRun], + reference: ReducibleData[ReferenceRun], + qbins: QBins, ) -> ReflectivityOverQ: - return ReflectivityOverQ(sample.hist() / reference.hist()) + return ReflectivityOverQ(sample.hist(Q=qbins) / reference.hist(Q=qbins)) return sl.Pipeline([apply_scaling, reflectivity]) @@ -82,25 +85,16 @@ def test_reflectivity_curve_scaling(): wf = make_workflow() wf[ScalingFactorForOverlap[SampleRun]] = 1.0 wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 - runs = { - 'a': { - UnscaledReducibleData[SampleRun]: make_sample_events(1, 0, 0.3), - UnscaledReducibleData[ReferenceRun]: make_reference_events(0, 0.3), - }, - 'b': { - UnscaledReducibleData[SampleRun]: make_sample_events(0.8, 0.2, 0.7), - UnscaledReducibleData[ReferenceRun]: make_reference_events(0.2, 0.7), - }, - 'c': { - UnscaledReducibleData[SampleRun]: make_sample_events(0.1, 0.6, 1.0), - UnscaledReducibleData[ReferenceRun]: make_reference_events(0.6, 1.0), - }, - } + params = {'a': (1, 0, 0.3), 'b': (0.8, 0.2, 0.7), 'c': (0.1, 0.6, 1.0)} workflows = {} - for name, params in runs.items(): - workflows[name] = wf.copy() - for key, value in params.items(): - workflows[name][key] = value + for k, v in params.items(): + sample = make_sample_events(*v) + reference = make_reference_events(v[1], v[2]) + workflows[k] = wf.copy() + workflows[k][UnscaledReducibleData[SampleRun]] = sample + workflows[k][UnscaledReducibleData[ReferenceRun]] = reference + workflows[k][QBins] = sample.coords['Q'] + wfc = WorkflowCollection(workflows) scaled_wf = scale_reflectivity_curves_to_overlap(wfc) @@ -113,28 +107,53 @@ def test_reflectivity_curve_scaling(): def test_reflectivity_curve_scaling_with_critical_edge(): - data = sc.concat( - ( - sc.ones(dims=['Q'], shape=[10], with_variances=True), - 0.5 * sc.ones(dims=['Q'], shape=[15], with_variances=True), - ), - dim='Q', - ) - data.variances[:] = 0.1 + wf = make_workflow() + wf[ScalingFactorForOverlap[SampleRun]] = 1.0 + wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 + params = {'a': (2, 0, 0.3), 'b': (0.8, 0.2, 0.7), 'c': (0.1, 0.6, 1.0)} + workflows = {} + for k, v in params.items(): + sample = make_sample_events(*v) + reference = make_reference_events(v[1], v[2]) + workflows[k] = wf.copy() + workflows[k][UnscaledReducibleData[SampleRun]] = sample + workflows[k][UnscaledReducibleData[ReferenceRun]] = reference + workflows[k][QBins] = sample.coords['Q'] - curves, factors = scale_reflectivity_curves_to_overlap( - ( - 2 * curve(data, 0, 0.3), - curve(0.8 * data, 0.2, 0.7), - curve(0.1 * data, 0.6, 1.0), - ), - critical_edge_interval=(sc.scalar(0.01), sc.scalar(0.05)), + wfc = WorkflowCollection(workflows) + + scaled_wf = scale_reflectivity_curves_to_overlap( + wfc, critical_edge_interval=(sc.scalar(0.01), sc.scalar(0.05)) ) - assert_allclose(curves[0].data, data, rtol=sc.scalar(1e-5)) - assert_allclose(curves[1].data, 0.5 * data, rtol=sc.scalar(1e-5)) - assert_allclose(curves[2].data, 0.25 * data, rtol=sc.scalar(1e-5)) - np_assert_allclose((0.5, 0.5 / 0.8, 0.25 / 0.1), factors, 1e-4) + factors = scaled_wf.compute(ScalingFactorForOverlap[SampleRun]) + + assert np.isclose(factors['a'], 0.5) + assert np.isclose(factors['b'], 0.5 / 0.8) + assert np.isclose(factors['c'], 0.25 / 0.1) + + # data = sc.concat( + # ( + # sc.ones(dims=['Q'], shape=[10], with_variances=True), + # 0.5 * sc.ones(dims=['Q'], shape=[15], with_variances=True), + # ), + # dim='Q', + # ) + # data.variances[:] = 0.1 + + # curves, factors = scale_reflectivity_curves_to_overlap( + # ( + # 2 * curve(data, 0, 0.3), + # curve(0.8 * data, 0.2, 0.7), + # curve(0.1 * data, 0.6, 1.0), + # ), + # critical_edge_interval=(sc.scalar(0.01), sc.scalar(0.05)), + # ) + + # assert_allclose(curves[0].data, data, rtol=sc.scalar(1e-5)) + # assert_allclose(curves[1].data, 0.5 * data, rtol=sc.scalar(1e-5)) + # assert_allclose(curves[2].data, 0.25 * data, rtol=sc.scalar(1e-5)) + # np_assert_allclose((0.5, 0.5 / 0.8, 0.25 / 0.1), factors, 1e-4) def test_combined_curves(): From 9f1428831e3f73a43de20a1e1df1da876f85c700 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Thu, 7 Aug 2025 12:04:41 +0200 Subject: [PATCH 20/75] simplify critical edge handling --- src/ess/reflectometry/tools.py | 63 ++++++++++++++-------------------- 1 file changed, 26 insertions(+), 37 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index dfc594b9..739fca52 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2025 Scipp contributors (https://github.com/scipp) +import uuid from collections.abc import Mapping, Sequence from itertools import chain from typing import Any, NewType @@ -166,10 +167,6 @@ def _interpolate_on_qgrid(curves, grid): ) -CriticalEdgeKey = NewType('CriticalEdgeKey', None) -"""A unique key used to store a 'fake' critical edge in a workflow collection.""" - - def scale_reflectivity_curves_to_overlap( wf_collection: Sequence[sc.DataArray], critical_edge_interval: tuple[sc.Variable, sc.Variable] | None = None, @@ -205,30 +202,6 @@ def scale_reflectivity_curves_to_overlap( : A list of scaled reflectivity curves and a list of the scaling factors. ''' - if critical_edge_interval is not None: - # Find q bins with the lowest Q start point - q = min( - (wf.compute(QBins) for wf in wf_collection.values()), - key=lambda q_: q_.min(), - ) - N = ( - ((q >= critical_edge_interval[0]) & (q < critical_edge_interval[1])) - .sum() - .value - ) - edge = sc.DataArray( - data=sc.ones(dims=('Q',), shape=(N,), with_variances=True), - coords={'Q': sc.linspace('Q', *critical_edge_interval, N + 1)}, - ) - wfc = wf_collection.copy() - underlying_wf = next(iter(wfc.values())) - edge_wf = underlying_wf.copy() - edge_wf[ReflectivityOverQ] = edge - wfc.add(CriticalEdgeKey, edge_wf) - return scale_reflectivity_curves_to_overlap( - wfc, cache_intermediate_results=cache_intermediate_results - ) - wfc = wf_collection.copy() if cache_intermediate_results: wfc[UnscaledReducibleData[SampleRun]] = wfc.compute( @@ -244,9 +217,26 @@ def scale_reflectivity_curves_to_overlap( reflectivities.items(), key=lambda item: item[1].coords['Q'].min().value ) } - # Now place the critical edge at the beginning, if it exists - if CriticalEdgeKey in curves.keys(): - curves = {CriticalEdgeKey: curves[CriticalEdgeKey]} | curves + + critical_edge_key = None + if critical_edge_interval is not None: + critical_edge_key = uuid.uuid4().hex + # Find q bins with the lowest Q start point + q = min( + (wf.compute(QBins) for wf in wf_collection.values()), + key=lambda q_: q_.min(), + ) + N = ( + ((q >= critical_edge_interval[0]) & (q < critical_edge_interval[1])) + .sum() + .value + ) + edge = sc.DataArray( + data=sc.ones(dims=('Q',), shape=(N,), with_variances=True), + coords={'Q': sc.linspace('Q', *critical_edge_interval, N + 1)}, + ) + # Now place the critical edge at the beginning + curves = {critical_edge_key: edge} | curves if len({c.data.unit for c in curves.values()}) != 1: raise ValueError('The reflectivity curves must have the same unit') @@ -272,12 +262,11 @@ def cost(scaling_factors): sol = opt.minimize(cost, [1.0] * (len(curves) - 1)) scaling_factors = (1.0, *map(float, sol.x)) - wfc[ScalingFactorForOverlap[SampleRun]] = dict( - zip(curves.keys(), scaling_factors, strict=True) - ) - - if CriticalEdgeKey in wfc.keys(): - wfc.remove(CriticalEdgeKey) + wfc[ScalingFactorForOverlap[SampleRun]] = { + k: v + for k, v in zip(curves.keys(), scaling_factors, strict=True) + if k != critical_edge_key + } return wfc From fff9b4e6def680b623ffc7cd4cc87f2cb2cbe62d Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Thu, 7 Aug 2025 14:11:59 +0200 Subject: [PATCH 21/75] update/fix tools tests --- src/ess/reflectometry/tools.py | 9 +- src/ess/reflectometry/workflow.py | 43 +++-- tests/reflectometry/tools_test.py | 284 +++++++++++++++--------------- 3 files changed, 173 insertions(+), 163 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 739fca52..587686bc 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -3,7 +3,7 @@ import uuid from collections.abc import Mapping, Sequence from itertools import chain -from typing import Any, NewType +from typing import Any import numpy as np import sciline @@ -14,6 +14,7 @@ from ess.reflectometry.types import ( Filename, QBins, + ReferenceRun, ReflectivityOverQ, SampleRun, ScalingFactorForOverlap, @@ -207,6 +208,9 @@ def scale_reflectivity_curves_to_overlap( wfc[UnscaledReducibleData[SampleRun]] = wfc.compute( UnscaledReducibleData[SampleRun] ) + wfc[UnscaledReducibleData[ReferenceRun]] = wfc.compute( + UnscaledReducibleData[ReferenceRun] + ) reflectivities = wfc.compute(ReflectivityOverQ) @@ -218,9 +222,8 @@ def scale_reflectivity_curves_to_overlap( ) } - critical_edge_key = None + critical_edge_key = uuid.uuid4().hex if critical_edge_interval is not None: - critical_edge_key = uuid.uuid4().hex # Find q bins with the lowest Q start point q = min( (wf.compute(QBins) for wf in wf_collection.values()), diff --git a/src/ess/reflectometry/workflow.py b/src/ess/reflectometry/workflow.py index e960da99..8a203eb5 100644 --- a/src/ess/reflectometry/workflow.py +++ b/src/ess/reflectometry/workflow.py @@ -67,41 +67,50 @@ def with_filenames( wf[UnscaledReducibleData[runtype]] = mapped[ UnscaledReducibleData[runtype] ].reduce(index=axis_name, func=_concatenate_event_lists) - except ValueError: - # UnscaledReducibleData[runtype] is independent of Filename[runtype] + except (ValueError, KeyError): + # UnscaledReducibleData[runtype] is independent of Filename[runtype] or is not + # present in the workflow. pass try: wf[RawChopper[runtype]] = mapped[RawChopper[runtype]].reduce( index=axis_name, func=_any_value ) - except ValueError: - # RawChopper[runtype] is independent of Filename[runtype] + except (ValueError, KeyError): + # RawChopper[runtype] is independent of Filename[runtype] or is not + # present in the workflow. pass try: wf[SampleRotation[runtype]] = mapped[SampleRotation[runtype]].reduce( index=axis_name, func=_any_value ) - except ValueError: - # SampleRotation[runtype] is independent of Filename[runtype] + except (ValueError, KeyError): + # SampleRotation[runtype] is independent of Filename[runtype] or is not + # present in the workflow. pass try: wf[DetectorRotation[runtype]] = mapped[DetectorRotation[runtype]].reduce( index=axis_name, func=_any_value ) - except ValueError: - # DetectorRotation[runtype] is independent of Filename[runtype] + except (ValueError, KeyError): + # DetectorRotation[runtype] is independent of Filename[runtype] or is not + # present in the workflow. pass if runtype is SampleRun: - wf[OrsoSample] = mapped[OrsoSample].reduce(index=axis_name, func=_any_value) - wf[OrsoExperiment] = mapped[OrsoExperiment].reduce( - index=axis_name, func=_any_value - ) - wf[OrsoOwner] = mapped[OrsoOwner].reduce(index=axis_name, func=lambda x, *_: x) - wf[OrsoSampleFilenames] = mapped[OrsoSampleFilenames].reduce( + if OrsoSample in wf.underlying_graph: + wf[OrsoSample] = mapped[OrsoSample].reduce(index=axis_name, func=_any_value) + if OrsoExperiment in wf.underlying_graph: + wf[OrsoExperiment] = mapped[OrsoExperiment].reduce( + index=axis_name, func=_any_value + ) + if OrsoOwner in wf.underlying_graph: + wf[OrsoOwner] = mapped[OrsoOwner].reduce( + index=axis_name, func=lambda x, *_: x + ) + if OrsoSampleFilenames in wf.underlying_graph: # When we don't map over filenames # each OrsoSampleFilenames is a list with a single entry. - index=axis_name, - func=_concatenate_lists, - ) + wf[OrsoSampleFilenames] = mapped[OrsoSampleFilenames].reduce( + index=axis_name, func=_concatenate_lists + ) return wf diff --git a/tests/reflectometry/tools_test.py b/tests/reflectometry/tools_test.py index a620a3f6..4e089684 100644 --- a/tests/reflectometry/tools_test.py +++ b/tests/reflectometry/tools_test.py @@ -1,10 +1,11 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +# Copyright (c) 2025 Scipp contributors (https://github.com/scipp) + import numpy as np import pytest import sciline as sl import scipp as sc -from numpy.testing import assert_allclose as np_assert_allclose +from numpy.testing import assert_almost_equal from orsopy.fileio import Orso, OrsoDataset from scipp.testing import assert_allclose @@ -27,9 +28,6 @@ UnscaledReducibleData, ) -# def curve(d, qmin, qmax): -# return sc.DataArray(data=d, coords={'Q': sc.linspace('Q', qmin, qmax, len(d) + 1)}) - def make_sample_events(scale, qmin, qmax): n1 = 10 @@ -47,6 +45,7 @@ def make_sample_events(scale, qmin, qmax): coords={'Q': sc.midpoints(qbins, 'Q')}, ) data.variances[:] = 0.1 + data.unit = 'counts' return data.bin(Q=qbins) @@ -58,10 +57,29 @@ def make_reference_events(qmin, qmax): coords={'Q': sc.midpoints(qbins, 'Q')}, ) data.variances[:] = 0.1 + data.unit = 'counts' return data.bin(Q=qbins) +# class RawData(sl.Scope[RunType, sc.DataArray], sc.DataArray): +# """A type alias for raw data arrays used in the test workflow.""" + + def make_workflow(): + def sample_data_from_filename( + filename: Filename[SampleRun], + ) -> UnscaledReducibleData[SampleRun]: + return UnscaledReducibleData[SampleRun]( + make_sample_events(*(float(x) for x in filename.split('_'))) + ) + + def reference_data_from_filename( + filename: Filename[ReferenceRun], + ) -> UnscaledReducibleData[ReferenceRun]: + return UnscaledReducibleData[ReferenceRun]( + make_reference_events(*(float(x) for x in filename.split('_'))) + ) + def apply_scaling( da: UnscaledReducibleData[RunType], scale: ScalingFactorForOverlap[RunType], @@ -78,22 +96,27 @@ def reflectivity( ) -> ReflectivityOverQ: return ReflectivityOverQ(sample.hist(Q=qbins) / reference.hist(Q=qbins)) - return sl.Pipeline([apply_scaling, reflectivity]) + return sl.Pipeline( + [ + sample_data_from_filename, + reference_data_from_filename, + apply_scaling, + reflectivity, + ] + ) def test_reflectivity_curve_scaling(): wf = make_workflow() wf[ScalingFactorForOverlap[SampleRun]] = 1.0 wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 - params = {'a': (1, 0, 0.3), 'b': (0.8, 0.2, 0.7), 'c': (0.1, 0.6, 1.0)} + params = {'a': (1.0, 0, 0.3), 'b': (0.8, 0.2, 0.7), 'c': (0.1, 0.6, 1.0)} workflows = {} for k, v in params.items(): - sample = make_sample_events(*v) - reference = make_reference_events(v[1], v[2]) workflows[k] = wf.copy() - workflows[k][UnscaledReducibleData[SampleRun]] = sample - workflows[k][UnscaledReducibleData[ReferenceRun]] = reference - workflows[k][QBins] = sample.coords['Q'] + workflows[k][Filename[SampleRun]] = "_".join(map(str, v)) + workflows[k][Filename[ReferenceRun]] = "_".join(map(str, v[1:])) + workflows[k][QBins] = make_reference_events(*v[1:]).coords['Q'] wfc = WorkflowCollection(workflows) @@ -113,12 +136,10 @@ def test_reflectivity_curve_scaling_with_critical_edge(): params = {'a': (2, 0, 0.3), 'b': (0.8, 0.2, 0.7), 'c': (0.1, 0.6, 1.0)} workflows = {} for k, v in params.items(): - sample = make_sample_events(*v) - reference = make_reference_events(v[1], v[2]) workflows[k] = wf.copy() - workflows[k][UnscaledReducibleData[SampleRun]] = sample - workflows[k][UnscaledReducibleData[ReferenceRun]] = reference - workflows[k][QBins] = sample.coords['Q'] + workflows[k][Filename[SampleRun]] = "_".join(map(str, v)) + workflows[k][Filename[ReferenceRun]] = "_".join(map(str, v[1:])) + workflows[k][QBins] = make_reference_events(*v[1:]).coords['Q'] wfc = WorkflowCollection(workflows) @@ -132,44 +153,92 @@ def test_reflectivity_curve_scaling_with_critical_edge(): assert np.isclose(factors['b'], 0.5 / 0.8) assert np.isclose(factors['c'], 0.25 / 0.1) - # data = sc.concat( - # ( - # sc.ones(dims=['Q'], shape=[10], with_variances=True), - # 0.5 * sc.ones(dims=['Q'], shape=[15], with_variances=True), - # ), - # dim='Q', - # ) - # data.variances[:] = 0.1 - - # curves, factors = scale_reflectivity_curves_to_overlap( - # ( - # 2 * curve(data, 0, 0.3), - # curve(0.8 * data, 0.2, 0.7), - # curve(0.1 * data, 0.6, 1.0), - # ), - # critical_edge_interval=(sc.scalar(0.01), sc.scalar(0.05)), - # ) - - # assert_allclose(curves[0].data, data, rtol=sc.scalar(1e-5)) - # assert_allclose(curves[1].data, 0.5 * data, rtol=sc.scalar(1e-5)) - # assert_allclose(curves[2].data, 0.25 * data, rtol=sc.scalar(1e-5)) - # np_assert_allclose((0.5, 0.5 / 0.8, 0.25 / 0.1), factors, 1e-4) + +def test_reflectivity_curve_scaling_caches_intermediate_results(): + sample_count = 0 + reference_count = 0 + + def sample_data_from_filename( + filename: Filename[SampleRun], + ) -> UnscaledReducibleData[SampleRun]: + nonlocal sample_count + sample_count += 1 + return UnscaledReducibleData[SampleRun]( + make_sample_events(*(float(x) for x in filename.split('_'))) + ) + + def reference_data_from_filename( + filename: Filename[ReferenceRun], + ) -> UnscaledReducibleData[ReferenceRun]: + nonlocal reference_count + reference_count += 1 + return UnscaledReducibleData[ReferenceRun]( + make_reference_events(*(float(x) for x in filename.split('_'))) + ) + + def apply_scaling( + da: UnscaledReducibleData[RunType], + scale: ScalingFactorForOverlap[RunType], + ) -> ReducibleData[RunType]: + """ + Scales the raw data by a given factor. + """ + return ReducibleData[RunType](da * scale) + + def reflectivity( + sample: ReducibleData[SampleRun], + reference: ReducibleData[ReferenceRun], + qbins: QBins, + ) -> ReflectivityOverQ: + return ReflectivityOverQ(sample.hist(Q=qbins) / reference.hist(Q=qbins)) + + wf = sl.Pipeline( + [ + sample_data_from_filename, + reference_data_from_filename, + apply_scaling, + reflectivity, + ] + ) + wf[ScalingFactorForOverlap[SampleRun]] = 1.0 + wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 + params = {'a': (1.0, 0, 0.3), 'b': (0.8, 0.2, 0.7), 'c': (0.1, 0.6, 1.0)} + workflows = {} + for k, v in params.items(): + workflows[k] = wf.copy() + workflows[k][Filename[SampleRun]] = "_".join(map(str, v)) + workflows[k][Filename[ReferenceRun]] = "_".join(map(str, v[1:])) + workflows[k][QBins] = make_reference_events(*v[1:]).coords['Q'] + + wfc = WorkflowCollection(workflows) + + scaled_wf = scale_reflectivity_curves_to_overlap( + wfc, cache_intermediate_results=False + ) + scaled_wf.compute(ReflectivityOverQ) + # We expect 6 counts: 3 for each of the 3 runs * 2 for computing ReflectivityOverQ + # inside the scaling function and one more time for the final computation just above + assert sample_count == 6 + assert reference_count == 6 + + sample_count = 0 + reference_count = 0 + + scaled_wf = scale_reflectivity_curves_to_overlap( + wfc, cache_intermediate_results=True + ) + scaled_wf.compute(ReflectivityOverQ) + # We expect 3 counts: 1 for each of the 3 runs * 1 for computing ReflectivityOverQ + assert sample_count == 3 + assert reference_count == 3 def test_combined_curves(): qgrid = sc.linspace('Q', 0, 1, 26) - data = sc.concat( - ( - sc.ones(dims=['Q'], shape=[10], with_variances=True), - 0.5 * sc.ones(dims=['Q'], shape=[15], with_variances=True), - ), - dim='Q', - ) - data.variances[:] = 0.1 curves = ( - curve(data, 0, 0.3), - curve(0.5 * data, 0.2, 0.7), - curve(0.25 * data, 0.6, 1.0), + make_sample_events(1.0, 0, 0.3).hist(), + 0.5 * make_sample_events(1.0, 0.2, 0.7).hist(), + 0.25 * make_sample_events(1.0, 0.6, 1.0).hist(), ) combined = combine_curves(curves, qgrid) @@ -231,6 +300,7 @@ def test_combined_curves(): 0.00625, 0.00625, ], + unit='counts', ), ) @@ -314,7 +384,7 @@ def test_linlogspace_bad_input(): @pytest.mark.filterwarnings("ignore:No suitable") -def test_from_measurements_tool_uses_expected_parameters_from_each_run(): +def test_batch_processor_tool_uses_expected_parameters_from_each_run(): def normalized_ioq(filename: Filename[SampleRun]) -> ReflectivityOverQ: return filename @@ -329,103 +399,31 @@ class Reduction: workflow = sl.Pipeline( [normalized_ioq, orso_dataset], params={Filename[SampleRun]: 'default'} ) - datasets = from_measurements( - workflow, - [{}, {Filename[SampleRun]: 'special'}], - target=OrsoDataset, - scale_to_overlap=False, - ) - assert len(datasets) == 2 - assert tuple(d.info.name for d in datasets) == ('default.orso', 'special.orso') - - -@pytest.mark.parametrize('targets', [(int,), (float, int)]) -@pytest.mark.parametrize( - 'params', [[{str: '1'}, {str: '2'}], {'a': {str: '1'}, 'b': {str: '2'}}] -) -def test_from_measurements_tool_returns_mapping_if_passed_mapping(params, targets): - def A(x: str) -> float: - return float(x) - - def B(x: str) -> int: - return int(x) - - workflow = sl.Pipeline([A, B]) - datasets = from_measurements( - workflow, - params, - target=targets, - ) - assert len(datasets) == len(params) - assert type(datasets) is type(params) + batch = batch_processor(workflow, {'a': {}, 'b': {Filename[SampleRun]: 'special'}}) -def test_from_measurements_tool_does_not_recompute_reflectivity(): - R = sc.DataArray( - sc.ones(dims=['Q'], shape=(50,), with_variances=True), - coords={'Q': sc.linspace('Q', 0.1, 1, 50)}, - ).bin(Q=10) + results = batch.compute(OrsoDataset) + assert len(results) == 2 + assert results['a'].info.name == 'default.orso' + assert results['b'].info.name == 'special.orso' - times_evaluated = 0 - def reflectivity() -> ReflectivityOverQ: - nonlocal times_evaluated - times_evaluated += 1 - return ReflectivityOverQ(R) - - def reducible_data() -> ReducibleData[SampleRun]: - return 'Not important' - - pl = sl.Pipeline([reflectivity, reducible_data]) - - from_measurements( - pl, - [{}, {}], - target=(ReflectivityOverQ,), - scale_to_overlap=True, - ) - assert times_evaluated == 2 - - -def test_from_measurements_tool_applies_scaling_to_reflectivityoverq(): - R1 = sc.DataArray( - sc.ones(dims=['Q'], shape=(50,), with_variances=True), - coords={'Q': sc.linspace('Q', 0.1, 1, 50)}, - ).bin(Q=10) - R2 = 0.5 * R1 - - def reducible_data() -> ReducibleData[SampleRun]: - return 'Not important' - - pl = sl.Pipeline([reducible_data]) - - results = from_measurements( - pl, - [{ReflectivityOverQ: R1}, {ReflectivityOverQ: R2}], - target=(ReflectivityOverQ,), - scale_to_overlap=(sc.scalar(0.0), sc.scalar(1.0)), - ) - assert_allclose(results[0][ReflectivityOverQ], results[1][ReflectivityOverQ]) - - -def test_from_measurements_tool_applies_scaling_to_reducibledata(): - R1 = sc.DataArray( - sc.ones(dims=['Q'], shape=(50,), with_variances=True), - coords={'Q': sc.linspace('Q', 0.1, 1, 50)}, - ).bin(Q=10) - R2 = 0.5 * R1 +def test_batch_processor_tool_merges_event_lists(): + wf = make_workflow() + wf[ScalingFactorForOverlap[SampleRun]] = 1.0 + wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 - def reducible_data() -> ReducibleData[SampleRun]: - return sc.scalar(1) + runs = { + 'a': {Filename[SampleRun]: ('1.0_0.0_0.3', '1.5_0.0_0.3')}, + 'b': {Filename[SampleRun]: '0.8_0.2_0.7'}, + 'c': {Filename[SampleRun]: ('0.1_0.6_1.0', '0.2_0.6_1.0')}, + } + batch = batch_processor(wf, runs) - pl = sl.Pipeline([reducible_data]) + results = batch.compute(UnscaledReducibleData[SampleRun]) - results = from_measurements( - pl, - [{ReflectivityOverQ: R1}, {ReflectivityOverQ: R2}], - target=(ReducibleData[SampleRun],), - scale_to_overlap=(sc.scalar(0.0), sc.scalar(1.0)), - ) - assert_allclose( - results[0][ReducibleData[SampleRun]], 0.5 * results[1][ReducibleData[SampleRun]] + assert_almost_equal(results['a'].sum().value, 10 + 15 * 0.5 + (10 + 15 * 0.5) * 1.5) + assert_almost_equal(results['b'].sum().value, 10 * 0.8 + 15 * 0.5 * 0.8) + assert_almost_equal( + results['c'].sum().value, (10 + 15 * 0.5) * 0.1 + (10 + 15 * 0.5) * 0.2 ) From a8d1b3a014a0d575093f3df3789eedd2cb716254 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Thu, 7 Aug 2025 14:43:24 +0200 Subject: [PATCH 22/75] fix scaling for a single workflow and fix amor pipeline tests --- src/ess/reflectometry/tools.py | 133 ++++++++++++++++-------------- tests/amor/pipeline_test.py | 13 +-- tests/amor/tools_test.py | 68 --------------- tests/reflectometry/tools_test.py | 17 ++++ 4 files changed, 96 insertions(+), 135 deletions(-) delete mode 100644 tests/amor/tools_test.py diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 587686bc..d15d7e6a 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -6,7 +6,7 @@ from typing import Any import numpy as np -import sciline +import sciline as sl import scipp as sc import scipy.optimize as opt @@ -109,6 +109,60 @@ def linlogspace( return sc.concat(grids, dim) +class WorkflowCollection: + """ + A collection of sciline workflows that can be used to compute multiple + targets from multiple workflows. + It can also be used to set parameters for all workflows in a single shot. + """ + + def __init__(self, workflows: Mapping[str, sl.Pipeline]): + self._workflows = {name: pl.copy() for name, pl in workflows.items()} + + def __setitem__(self, key: type, value: Any | Mapping[type, Any]): + if hasattr(value, 'items'): + for name, v in value.items(): + self._workflows[name][key] = v + else: + for pl in self._workflows.values(): + pl[key] = value + + def __getitem__(self, name: str) -> sl.Pipeline: + """ + Returns a single workflow from the collection given by its name. + """ + return self._workflows[name] + + def compute(self, target: type | Sequence[type], **kwargs) -> Mapping[str, Any]: + return { + name: pl.compute(target, **kwargs) for name, pl in self._workflows.items() + } + + def copy(self) -> 'WorkflowCollection': + return self.__class__(self._workflows) + + def keys(self) -> Sequence[str]: + return self._workflows.keys() + + def values(self) -> Sequence[sl.Pipeline]: + return self._workflows.values() + + def items(self) -> Sequence[tuple[str, sl.Pipeline]]: + return self._workflows.items() + + def add(self, name: str, workflow: sl.Pipeline): + """ + Adds a new workflow to the collection. + """ + self._workflows[name] = workflow.copy() + + def remove(self, name: str): + """ + Removes a workflow from the collection by its name. + """ + del self._workflows[name] + + def _sort_by(a, by): return [x for x, _ in sorted(zip(a, by, strict=True), key=lambda x: x[1])] @@ -169,13 +223,14 @@ def _interpolate_on_qgrid(curves, grid): def scale_reflectivity_curves_to_overlap( - wf_collection: Sequence[sc.DataArray], + workflows: WorkflowCollection | sl.Pipeline, critical_edge_interval: tuple[sc.Variable, sc.Variable] | None = None, cache_intermediate_results: bool = True, ) -> tuple[list[sc.DataArray], list[sc.Variable]]: ''' Set the ``ScalingFactorForOverlap`` parameter on the provided workflows in a way that would makes the 1D reflectivity curves overlap. + One can supply either a collection of workflows or a single workflow. If :code:`critical_edge_interval` is not provided, all workflows are scaled except the data with the lowest Q-range, which is considered to be the reference curve. @@ -188,8 +243,8 @@ def scale_reflectivity_curves_to_overlap( Parameters --------- - wf_collection: - The collection of workflows that can compute the ``ReflectivityOverQ``. + workflows: + The workflow or collection of workflows that can compute ``ReflectivityOverQ``. critical_edge_interval: A tuple denoting an interval that is known to belong to the critical edge, i.e. where the reflectivity is @@ -203,7 +258,17 @@ def scale_reflectivity_curves_to_overlap( : A list of scaled reflectivity curves and a list of the scaling factors. ''' - wfc = wf_collection.copy() + if isinstance(workflows, sl.Pipeline): + # If a single workflow is provided, convert it to a collection + wfc = WorkflowCollection({"": workflows}) + out = scale_reflectivity_curves_to_overlap( + wfc, + critical_edge_interval=critical_edge_interval, + cache_intermediate_results=cache_intermediate_results, + ) + return out[""] + + wfc = workflows.copy() if cache_intermediate_results: wfc[UnscaledReducibleData[SampleRun]] = wfc.compute( UnscaledReducibleData[SampleRun] @@ -226,7 +291,7 @@ def scale_reflectivity_curves_to_overlap( if critical_edge_interval is not None: # Find q bins with the lowest Q start point q = min( - (wf.compute(QBins) for wf in wf_collection.values()), + (wf.compute(QBins) for wf in workflows.values()), key=lambda q_: q_.min(), ) N = ( @@ -326,62 +391,8 @@ def combine_curves( ) -class WorkflowCollection: - """ - A collection of sciline workflows that can be used to compute multiple - targets from multiple workflows. - It can also be used to set parameters for all workflows in a single shot. - """ - - def __init__(self, workflows: Mapping[str, sciline.Pipeline]): - self._workflows = {name: pl.copy() for name, pl in workflows.items()} - - def __setitem__(self, key: type, value: Any | Mapping[type, Any]): - if hasattr(value, 'items'): - for name, v in value.items(): - self._workflows[name][key] = v - else: - for pl in self._workflows.values(): - pl[key] = value - - def __getitem__(self, name: str) -> sciline.Pipeline: - """ - Returns a single workflow from the collection given by its name. - """ - return self._workflows[name] - - def compute(self, target: type | Sequence[type], **kwargs) -> Mapping[str, Any]: - return { - name: pl.compute(target, **kwargs) for name, pl in self._workflows.items() - } - - def copy(self) -> 'WorkflowCollection': - return self.__class__(self._workflows) - - def keys(self) -> Sequence[str]: - return self._workflows.keys() - - def values(self) -> Sequence[sciline.Pipeline]: - return self._workflows.values() - - def items(self) -> Sequence[tuple[str, sciline.Pipeline]]: - return self._workflows.items() - - def add(self, name: str, workflow: sciline.Pipeline): - """ - Adds a new workflow to the collection. - """ - self._workflows[name] = workflow.copy() - - def remove(self, name: str): - """ - Removes a workflow from the collection by its name. - """ - del self._workflows[name] - - def batch_processor( - workflow: sciline.Pipeline, runs: Mapping[Any, Mapping[type, Any]] + workflow: sl.Pipeline, runs: Mapping[Any, Mapping[type, Any]] ) -> WorkflowCollection: """ Creates a collection of sciline workflows from the provided runs. diff --git a/tests/amor/pipeline_test.py b/tests/amor/pipeline_test.py index b6ca1504..c03e4774 100644 --- a/tests/amor/pipeline_test.py +++ b/tests/amor/pipeline_test.py @@ -127,16 +127,17 @@ def test_save_reduced_orso_file(output_folder: Path): ) wf[Filename[ReferenceRun]] = data.amor_run(4152) wf[QBins] = sc.geomspace(dim="Q", start=0.01, stop=0.06, num=201, unit="1/angstrom") - r = wf.compute(ReflectivityOverQ) - _, (s,) = scale_reflectivity_curves_to_overlap( - [r.hist()], + # r = wf.compute(ReflectivityOverQ) + + scaled_wf = scale_reflectivity_curves_to_overlap( + wf, critical_edge_interval=( sc.scalar(0.01, unit='1/angstrom'), sc.scalar(0.014, unit='1/angstrom'), ), ) - wf[ReflectivityOverQ] = s * r - wf[orso.OrsoCreator] = orso.OrsoCreator( + # wf[ReflectivityOverQ] = s * r + scaled_wf[orso.OrsoCreator] = orso.OrsoCreator( fileio.base.Person( name="Max Mustermann", affiliation="European Spallation Source ERIC", @@ -144,7 +145,7 @@ def test_save_reduced_orso_file(output_folder: Path): ) ) fileio.orso.save_orso( - datasets=[wf.compute(orso.OrsoIofQDataset)], + datasets=[scaled_wf.compute(orso.OrsoIofQDataset)], fname=output_folder / 'amor_reduced_iofq.ort', ) diff --git a/tests/amor/tools_test.py b/tests/amor/tools_test.py deleted file mode 100644 index 0dd6fef2..00000000 --- a/tests/amor/tools_test.py +++ /dev/null @@ -1,68 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) -import pytest -import sciline -import scipp as sc -from scipp.testing import assert_allclose - -from amor.pipeline_test import amor_pipeline # noqa: F401 -from ess.amor import data -from ess.amor.types import ChopperPhase -from ess.reflectometry.tools import from_measurements -from ess.reflectometry.types import ( - DetectorRotation, - Filename, - QBins, - ReducedReference, - ReferenceRun, - ReflectivityOverQ, - SampleRotation, - SampleRun, -) - -# The files used in the AMOR reduction workflow have some scippnexus warnings -pytestmark = pytest.mark.filterwarnings( - "ignore:.*Invalid transformation, .*missing attribute 'vector':UserWarning", -) - - -@pytest.fixture -def pipeline_with_1632_reference(amor_pipeline): # noqa: F811 - amor_pipeline[ChopperPhase[ReferenceRun]] = sc.scalar(7.5, unit='deg') - amor_pipeline[ChopperPhase[SampleRun]] = sc.scalar(7.5, unit='deg') - amor_pipeline[Filename[ReferenceRun]] = data.amor_run('1632') - amor_pipeline[ReducedReference] = amor_pipeline.compute(ReducedReference) - return amor_pipeline - - -@pytestmark -def test_from_measurements_tool_concatenates_event_lists( - pipeline_with_1632_reference: sciline.Pipeline, -): - pl = pipeline_with_1632_reference - - run = { - Filename[SampleRun]: list(map(data.amor_run, (1636, 1639, 1641))), - QBins: sc.geomspace( - dim='Q', start=0.062, stop=0.18, num=391, unit='1/angstrom' - ), - DetectorRotation[SampleRun]: sc.scalar(0.140167, unit='rad'), - SampleRotation[SampleRun]: sc.scalar(0.0680678, unit='rad'), - } - results = from_measurements( - pl, - [run], - target=ReflectivityOverQ, - scale_to_overlap=False, - ) - - results2 = [] - for fname in run[Filename[SampleRun]]: - pl.copy() - pl[Filename[SampleRun]] = fname - pl[QBins] = run[QBins] - pl[DetectorRotation[SampleRun]] = run[DetectorRotation[SampleRun]] - pl[SampleRotation[SampleRun]] = run[SampleRotation[SampleRun]] - results2.append(pl.compute(ReflectivityOverQ).hist().data) - - assert_allclose(sum(results2), results[0].hist().data) diff --git a/tests/reflectometry/tools_test.py b/tests/reflectometry/tools_test.py index 4e089684..4b19524c 100644 --- a/tests/reflectometry/tools_test.py +++ b/tests/reflectometry/tools_test.py @@ -154,6 +154,23 @@ def test_reflectivity_curve_scaling_with_critical_edge(): assert np.isclose(factors['c'], 0.25 / 0.1) +def test_reflectivity_curve_scaling_works_with_single_workflow_and_critical_edge(): + wf = make_workflow() + wf[ScalingFactorForOverlap[SampleRun]] = 1.0 + wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 + wf[Filename[SampleRun]] = '2.5_0.4_0.8' + wf[Filename[ReferenceRun]] = '0.4_0.8' + wf[QBins] = make_reference_events(0.4, 0.8).coords['Q'] + + scaled_wf = scale_reflectivity_curves_to_overlap( + wf, critical_edge_interval=(sc.scalar(0.0), sc.scalar(0.5)) + ) + + factor = scaled_wf.compute(ScalingFactorForOverlap[SampleRun]) + + assert np.isclose(factor, 0.4) + + def test_reflectivity_curve_scaling_caches_intermediate_results(): sample_count = 0 reference_count = 0 From 75675654d1224d45039cf1a46d589312727d2ec5 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Thu, 7 Aug 2025 14:49:04 +0200 Subject: [PATCH 23/75] cleanup --- src/ess/reflectometry/tools.py | 1 - tests/amor/pipeline_test.py | 2 -- tests/reflectometry/tools_test.py | 4 ---- 3 files changed, 7 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index d15d7e6a..18d7ac02 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -10,7 +10,6 @@ import scipp as sc import scipy.optimize as opt -# from ess.reflectometry import orso from ess.reflectometry.types import ( Filename, QBins, diff --git a/tests/amor/pipeline_test.py b/tests/amor/pipeline_test.py index c03e4774..9b5f01b3 100644 --- a/tests/amor/pipeline_test.py +++ b/tests/amor/pipeline_test.py @@ -127,7 +127,6 @@ def test_save_reduced_orso_file(output_folder: Path): ) wf[Filename[ReferenceRun]] = data.amor_run(4152) wf[QBins] = sc.geomspace(dim="Q", start=0.01, stop=0.06, num=201, unit="1/angstrom") - # r = wf.compute(ReflectivityOverQ) scaled_wf = scale_reflectivity_curves_to_overlap( wf, @@ -136,7 +135,6 @@ def test_save_reduced_orso_file(output_folder: Path): sc.scalar(0.014, unit='1/angstrom'), ), ) - # wf[ReflectivityOverQ] = s * r scaled_wf[orso.OrsoCreator] = orso.OrsoCreator( fileio.base.Person( name="Max Mustermann", diff --git a/tests/reflectometry/tools_test.py b/tests/reflectometry/tools_test.py index 4b19524c..72e108df 100644 --- a/tests/reflectometry/tools_test.py +++ b/tests/reflectometry/tools_test.py @@ -61,10 +61,6 @@ def make_reference_events(qmin, qmax): return data.bin(Q=qbins) -# class RawData(sl.Scope[RunType, sc.DataArray], sc.DataArray): -# """A type alias for raw data arrays used in the test workflow.""" - - def make_workflow(): def sample_data_from_filename( filename: Filename[SampleRun], From 7f884fb3fe1db11895d2edd647fb3f0beab6b4d3 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Thu, 7 Aug 2025 14:55:59 +0200 Subject: [PATCH 24/75] do not fail if UnscaledReducibleData is not in graph --- src/ess/reflectometry/tools.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 18d7ac02..434200aa 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -269,12 +269,18 @@ def scale_reflectivity_curves_to_overlap( wfc = workflows.copy() if cache_intermediate_results: - wfc[UnscaledReducibleData[SampleRun]] = wfc.compute( - UnscaledReducibleData[SampleRun] - ) - wfc[UnscaledReducibleData[ReferenceRun]] = wfc.compute( - UnscaledReducibleData[ReferenceRun] - ) + try: + wfc[UnscaledReducibleData[SampleRun]] = wfc.compute( + UnscaledReducibleData[SampleRun] + ) + except sl.UnsatisfiedRequirement: + pass + try: + wfc[UnscaledReducibleData[ReferenceRun]] = wfc.compute( + UnscaledReducibleData[ReferenceRun] + ) + except sl.UnsatisfiedRequirement: + pass reflectivities = wfc.compute(ReflectivityOverQ) From 2dea8de9bc081e6a92192c6f95e757e3fc8ebd9f Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Wed, 13 Aug 2025 16:06:35 +0200 Subject: [PATCH 25/75] modify the WorkflowCollection to just use cyclebane mapping under the hood and make its interface more like the Pipeline interface rather than a mix between Pipeline and a dict --- src/ess/reflectometry/tools.py | 168 ++++++++++++++---------------- tests/reflectometry/tools_test.py | 86 +++++++-------- 2 files changed, 120 insertions(+), 134 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 434200aa..a5416484 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -11,7 +11,6 @@ import scipy.optimize as opt from ess.reflectometry.types import ( - Filename, QBins, ReferenceRun, ReflectivityOverQ, @@ -19,7 +18,6 @@ ScalingFactorForOverlap, UnscaledReducibleData, ) -from ess.reflectometry.workflow import with_filenames _STD_TO_FWHM = sc.scalar(2.0) * sc.sqrt(sc.scalar(2.0) * sc.log(sc.scalar(2.0))) @@ -111,55 +109,50 @@ def linlogspace( class WorkflowCollection: """ A collection of sciline workflows that can be used to compute multiple - targets from multiple workflows. - It can also be used to set parameters for all workflows in a single shot. + targets from mapping a workflow over a parameter table. + It can also be used to set parameters for all mapped nodes in a single shot. """ - def __init__(self, workflows: Mapping[str, sl.Pipeline]): - self._workflows = {name: pl.copy() for name, pl in workflows.items()} + def __init__(self, workflow: sl.Pipeline, param_table): + self._original_workflow = workflow.copy() + self.param_table = param_table + self._mapped_workflow = self._original_workflow.map(self.param_table) - def __setitem__(self, key: type, value: Any | Mapping[type, Any]): - if hasattr(value, 'items'): - for name, v in value.items(): - self._workflows[name][key] = v + def __setitem__(self, key, value): + if key in self.param_table: + ind = list(self.param_table.keys()).index(key) + self.param_table.iloc[:, ind] = value + self._mapped_workflow = self._original_workflow.map(self.param_table) else: - for pl in self._workflows.values(): - pl[key] = value - - def __getitem__(self, name: str) -> sl.Pipeline: - """ - Returns a single workflow from the collection given by its name. - """ - return self._workflows[name] + self.param_table.insert(len(self.param_table.columns), key, value) + self._original_workflow[key] = None + self._mapped_workflow = self._original_workflow.map(self.param_table) def compute(self, target: type | Sequence[type], **kwargs) -> Mapping[str, Any]: - return { - name: pl.compute(target, **kwargs) for name, pl in self._workflows.items() - } - - def copy(self) -> 'WorkflowCollection': - return self.__class__(self._workflows) - - def keys(self) -> Sequence[str]: - return self._workflows.keys() + try: + # TODO: Sciline here returns a pandas Series. + # Should we convert it to a dict instead? + return sl.compute_mapped(self._mapped_workflow, target, **kwargs) + except ValueError: + return self._mapped_workflow.compute(target, **kwargs) - def values(self) -> Sequence[sl.Pipeline]: - return self._workflows.values() + def get(self, targets, **kwargs): + try: + targets = sl.get_mapped_node_names(self._mapped_workflow, targets) + return self._mapped_workflow.get(targets, **kwargs) + except ValueError: + return self._mapped_workflow.get(targets, **kwargs) - def items(self) -> Sequence[tuple[str, sl.Pipeline]]: - return self._workflows.items() + # TODO: implement the group() method to group by params in the parameter table - def add(self, name: str, workflow: sl.Pipeline): - """ - Adds a new workflow to the collection. - """ - self._workflows[name] = workflow.copy() + def visualize(self, targets, **kwargs): + targets = sl.get_mapped_node_names(self._mapped_workflow, targets) + return self._mapped_workflow.visualize(targets, **kwargs) - def remove(self, name: str): - """ - Removes a workflow from the collection by its name. - """ - del self._workflows[name] + def copy(self) -> 'WorkflowCollection': + return self.__class__( + workflow=self._original_workflow, param_table=self.param_table + ) def _sort_by(a, by): @@ -222,7 +215,7 @@ def _interpolate_on_qgrid(curves, grid): def scale_reflectivity_curves_to_overlap( - workflows: WorkflowCollection | sl.Pipeline, + workflow: WorkflowCollection | sl.Pipeline, critical_edge_interval: tuple[sc.Variable, sc.Variable] | None = None, cache_intermediate_results: bool = True, ) -> tuple[list[sc.DataArray], list[sc.Variable]]: @@ -257,17 +250,9 @@ def scale_reflectivity_curves_to_overlap( : A list of scaled reflectivity curves and a list of the scaling factors. ''' - if isinstance(workflows, sl.Pipeline): - # If a single workflow is provided, convert it to a collection - wfc = WorkflowCollection({"": workflows}) - out = scale_reflectivity_curves_to_overlap( - wfc, - critical_edge_interval=critical_edge_interval, - cache_intermediate_results=cache_intermediate_results, - ) - return out[""] + not_collection = isinstance(workflow, sl.Pipeline) - wfc = workflows.copy() + wfc = workflow.copy() if cache_intermediate_results: try: wfc[UnscaledReducibleData[SampleRun]] = wfc.compute( @@ -283,6 +268,8 @@ def scale_reflectivity_curves_to_overlap( pass reflectivities = wfc.compute(ReflectivityOverQ) + if not_collection: + reflectivities = {"": reflectivities} # First sort the dict of reflectivities by the Q min value curves = { @@ -294,20 +281,21 @@ def scale_reflectivity_curves_to_overlap( critical_edge_key = uuid.uuid4().hex if critical_edge_interval is not None: - # Find q bins with the lowest Q start point - q = min( - (wf.compute(QBins) for wf in workflows.values()), - key=lambda q_: q_.min(), - ) - N = ( - ((q >= critical_edge_interval[0]) & (q < critical_edge_interval[1])) - .sum() - .value - ) + q = wfc.compute(QBins) + if hasattr(q, "items"): + # If QBins is a mapping, find the one with the lowest Q start + # Note the conversion to a dict, because if pandas is used for the mapping, + # it will return a Series, whose `.values` attribute is not callable. + q = min(dict(q).values(), key=lambda q_: q_.min()) + + # TODO: This is slightly different from before: it extracts the bins from the + # QBins variable that cover the critical edge interval. This means that the + # resulting curve will not necessarily begin and end exactly at the values + # specified, but rather at the closest bin edges. edge = sc.DataArray( - data=sc.ones(dims=('Q',), shape=(N,), with_variances=True), - coords={'Q': sc.linspace('Q', *critical_edge_interval, N + 1)}, - ) + data=sc.ones(sizes={q.dim: q.sizes[q.dim] - 1}, with_variances=True), + coords={q.dim: q}, + )[q.dim, critical_edge_interval[0] : critical_edge_interval[1]] # Now place the critical edge at the beginning curves = {critical_edge_key: edge} | curves @@ -335,11 +323,14 @@ def cost(scaling_factors): sol = opt.minimize(cost, [1.0] * (len(curves) - 1)) scaling_factors = (1.0, *map(float, sol.x)) - wfc[ScalingFactorForOverlap[SampleRun]] = { + results = { k: v for k, v in zip(curves.keys(), scaling_factors, strict=True) if k != critical_edge_key } + if not_collection: + results = results[""] + wfc[ScalingFactorForOverlap[SampleRun]] = results return wfc @@ -397,10 +388,10 @@ def combine_curves( def batch_processor( - workflow: sl.Pipeline, runs: Mapping[Any, Mapping[type, Any]] + workflow: sl.Pipeline, params: Mapping[Any, Mapping[type, Any]] ) -> WorkflowCollection: """ - Creates a collection of sciline workflows from the provided runs. + Maps the provided workflow over the provided params. Example: @@ -415,7 +406,7 @@ def batch_processor( Filename[SampleRun]: amor.data.amor_run(608), }, '609': { - SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'), + SampleRotationOffset[SampleRun]: sc.scalar(0.06, unit='deg'), Filename[SampleRun]: amor.data.amor_run(609), }, '610': { @@ -423,7 +414,7 @@ def batch_processor( Filename[SampleRun]: amor.data.amor_run(610), }, '611': { - SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'), + SampleRotationOffset[SampleRun]: sc.scalar(0.07, unit='deg'), Filename[SampleRun]: amor.data.amor_run(611), }, } @@ -437,30 +428,23 @@ def batch_processor( ---------- workflow: The sciline workflow used to compute the targets for each of the runs. - runs: + params: The sciline parameters to be used for each run. Should be a mapping where the keys are the names of the runs and the values are mappings of type to value pairs. - In addition, if one of the values for ``Filename[SampleRun]`` - is a list or a tuple, then the events from the files - will be concatenated into a single event list. """ - workflows = {} - for name, parameters in runs.items(): - wf = workflow.copy() - for tp, value in parameters.items(): - if tp is Filename[SampleRun]: - continue - wf[tp] = value - - if Filename[SampleRun] in parameters: - if isinstance(parameters[Filename[SampleRun]], list | tuple): - wf = with_filenames( - wf, - SampleRun, - parameters[Filename[SampleRun]], - ) + import pandas as pd + + all_types = {t for v in params.values() for t in v.keys()} + data = {t: [] for t in all_types} + for param in params.values(): + for t in all_types: + if t in param: + data[t].append(param[t]) else: - wf[Filename[SampleRun]] = parameters[Filename[SampleRun]] - workflows[name] = wf - return WorkflowCollection(workflows) + # Set the default value + data[t].append(workflow.compute(t)) + + param_table = pd.DataFrame(data, index=params.keys()).rename_axis(index='run_id') + + return WorkflowCollection(workflow, param_table) diff --git a/tests/reflectometry/tools_test.py b/tests/reflectometry/tools_test.py index 72e108df..256b3d76 100644 --- a/tests/reflectometry/tools_test.py +++ b/tests/reflectometry/tools_test.py @@ -5,12 +5,10 @@ import pytest import sciline as sl import scipp as sc -from numpy.testing import assert_almost_equal from orsopy.fileio import Orso, OrsoDataset from scipp.testing import assert_allclose from ess.reflectometry.tools import ( - WorkflowCollection, batch_processor, combine_curves, linlogspace, @@ -107,14 +105,15 @@ def test_reflectivity_curve_scaling(): wf[ScalingFactorForOverlap[SampleRun]] = 1.0 wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 params = {'a': (1.0, 0, 0.3), 'b': (0.8, 0.2, 0.7), 'c': (0.1, 0.6, 1.0)} - workflows = {} - for k, v in params.items(): - workflows[k] = wf.copy() - workflows[k][Filename[SampleRun]] = "_".join(map(str, v)) - workflows[k][Filename[ReferenceRun]] = "_".join(map(str, v[1:])) - workflows[k][QBins] = make_reference_events(*v[1:]).coords['Q'] - - wfc = WorkflowCollection(workflows) + table = { + k: { + Filename[SampleRun]: "_".join(map(str, v)), + Filename[ReferenceRun]: "_".join(map(str, v[1:])), + QBins: make_reference_events(*v[1:]).coords['Q'], + } + for k, v in params.items() + } + wfc = batch_processor(wf, table) scaled_wf = scale_reflectivity_curves_to_overlap(wfc) @@ -130,14 +129,15 @@ def test_reflectivity_curve_scaling_with_critical_edge(): wf[ScalingFactorForOverlap[SampleRun]] = 1.0 wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 params = {'a': (2, 0, 0.3), 'b': (0.8, 0.2, 0.7), 'c': (0.1, 0.6, 1.0)} - workflows = {} - for k, v in params.items(): - workflows[k] = wf.copy() - workflows[k][Filename[SampleRun]] = "_".join(map(str, v)) - workflows[k][Filename[ReferenceRun]] = "_".join(map(str, v[1:])) - workflows[k][QBins] = make_reference_events(*v[1:]).coords['Q'] - - wfc = WorkflowCollection(workflows) + table = { + k: { + Filename[SampleRun]: "_".join(map(str, v)), + Filename[ReferenceRun]: "_".join(map(str, v[1:])), + QBins: make_reference_events(*v[1:]).coords['Q'], + } + for k, v in params.items() + } + wfc = batch_processor(wf, table) scaled_wf = scale_reflectivity_curves_to_overlap( wfc, critical_edge_interval=(sc.scalar(0.01), sc.scalar(0.05)) @@ -216,14 +216,15 @@ def reflectivity( wf[ScalingFactorForOverlap[SampleRun]] = 1.0 wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 params = {'a': (1.0, 0, 0.3), 'b': (0.8, 0.2, 0.7), 'c': (0.1, 0.6, 1.0)} - workflows = {} - for k, v in params.items(): - workflows[k] = wf.copy() - workflows[k][Filename[SampleRun]] = "_".join(map(str, v)) - workflows[k][Filename[ReferenceRun]] = "_".join(map(str, v[1:])) - workflows[k][QBins] = make_reference_events(*v[1:]).coords['Q'] - - wfc = WorkflowCollection(workflows) + table = { + k: { + Filename[SampleRun]: "_".join(map(str, v)), + Filename[ReferenceRun]: "_".join(map(str, v[1:])), + QBins: make_reference_events(*v[1:]).coords['Q'], + } + for k, v in params.items() + } + wfc = batch_processor(wf, table) scaled_wf = scale_reflectivity_curves_to_overlap( wfc, cache_intermediate_results=False @@ -421,22 +422,23 @@ class Reduction: assert results['b'].info.name == 'special.orso' -def test_batch_processor_tool_merges_event_lists(): - wf = make_workflow() - wf[ScalingFactorForOverlap[SampleRun]] = 1.0 - wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 +# TODO: need to implement groupby in the mapping +# def test_batch_processor_tool_merges_event_lists(): +# wf = make_workflow() +# wf[ScalingFactorForOverlap[SampleRun]] = 1.0 +# wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 - runs = { - 'a': {Filename[SampleRun]: ('1.0_0.0_0.3', '1.5_0.0_0.3')}, - 'b': {Filename[SampleRun]: '0.8_0.2_0.7'}, - 'c': {Filename[SampleRun]: ('0.1_0.6_1.0', '0.2_0.6_1.0')}, - } - batch = batch_processor(wf, runs) +# runs = { +# 'a': {Filename[SampleRun]: ('1.0_0.0_0.3', '1.5_0.0_0.3')}, +# 'b': {Filename[SampleRun]: '0.8_0.2_0.7'}, +# 'c': {Filename[SampleRun]: ('0.1_0.6_1.0', '0.2_0.6_1.0')}, +# } +# batch = batch_processor(wf, runs) - results = batch.compute(UnscaledReducibleData[SampleRun]) +# results = batch.compute(UnscaledReducibleData[SampleRun]) - assert_almost_equal(results['a'].sum().value, 10 + 15 * 0.5 + (10 + 15 * 0.5) * 1.5) - assert_almost_equal(results['b'].sum().value, 10 * 0.8 + 15 * 0.5 * 0.8) - assert_almost_equal( - results['c'].sum().value, (10 + 15 * 0.5) * 0.1 + (10 + 15 * 0.5) * 0.2 - ) +# assert_almost_equal(results['a'].sum().value, 10 + 15 * 0.5 + (10 + 15 * 0.5) * 1.5) +# assert_almost_equal(results['b'].sum().value, 10 * 0.8 + 15 * 0.5 * 0.8) +# assert_almost_equal( +# results['c'].sum().value, (10 + 15 * 0.5) * 0.1 + (10 + 15 * 0.5) * 0.2 +# ) From f181b41d42bee5f633c3eec5b8802da70f2c2282 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Wed, 13 Aug 2025 16:07:18 +0200 Subject: [PATCH 26/75] formatting --- tests/reflectometry/tools_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/reflectometry/tools_test.py b/tests/reflectometry/tools_test.py index 256b3d76..70e8838e 100644 --- a/tests/reflectometry/tools_test.py +++ b/tests/reflectometry/tools_test.py @@ -437,7 +437,8 @@ class Reduction: # results = batch.compute(UnscaledReducibleData[SampleRun]) -# assert_almost_equal(results['a'].sum().value, 10 + 15 * 0.5 + (10 + 15 * 0.5) * 1.5) +# assert_almost_equal(results['a'].sum().value, +# 10 + 15 * 0.5 + (10 + 15 * 0.5) * 1.5) # assert_almost_equal(results['b'].sum().value, 10 * 0.8 + 15 * 0.5 * 0.8) # assert_almost_equal( # results['c'].sum().value, (10 + 15 * 0.5) * 0.1 + (10 + 15 * 0.5) * 0.2 From a23c007e51646b0f63bd8a6e61ea6003c168238e Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Wed, 13 Aug 2025 16:10:42 +0200 Subject: [PATCH 27/75] update amor notebook --- docs/user-guide/amor/amor-reduction.ipynb | 27 ++++++++++++++--------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/docs/user-guide/amor/amor-reduction.ipynb b/docs/user-guide/amor/amor-reduction.ipynb index 18d9224e..c0c08ec0 100644 --- a/docs/user-guide/amor/amor-reduction.ipynb +++ b/docs/user-guide/amor/amor-reduction.ipynb @@ -184,8 +184,15 @@ "}\n", "\n", "batch = batch_processor(workflow, runs)\n", - "display(batch.keys())\n", - "\n", + "batch.param_table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "# Compute R(Q) for all runs\n", "reflectivity = batch.compute(ReflectivityOverQ)\n", "sc.plot(\n", @@ -288,8 +295,8 @@ "from ess.reflectometry.figures import wavelength_theta_figure\n", "\n", "wavelength_theta_figure(\n", - " diagnostics.values(),\n", - " theta_bins=scaled_wf.compute(ThetaBins[SampleRun]).values(),\n", + " diagnostics.values,\n", + " theta_bins=scaled_wf.compute(ThetaBins[SampleRun]).values,\n", " q_edges_to_display=(sc.scalar(0.018, unit='1/angstrom'), sc.scalar(0.113, unit='1/angstrom'))\n", ")" ] @@ -318,8 +325,8 @@ "from ess.reflectometry.figures import q_theta_figure\n", "\n", "q_theta_figure(\n", - " diagnostics.values(),\n", - " theta_bins=scaled_wf.compute(ThetaBins[SampleRun]).values(),\n", + " diagnostics.values,\n", + " theta_bins=scaled_wf.compute(ThetaBins[SampleRun]).values,\n", " q_bins=workflow.compute(QBins)\n", ")" ] @@ -396,7 +403,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We can visualize the workflow for a single run (`'608'`):" + "We can visualize the workflow for the `OrsoIofQDataset`:" ] }, { @@ -405,7 +412,7 @@ "metadata": {}, "outputs": [], "source": [ - "scaled_wf['608'].visualize(orso.OrsoIofQDataset, graph_attr={'rankdir': 'LR'})" + "scaled_wf.visualize(orso.OrsoIofQDataset, graph_attr={'rankdir': 'LR'})" ] }, { @@ -437,7 +444,7 @@ "metadata": {}, "outputs": [], "source": [ - "for ds in iofq_datasets.values():\n", + "for ds in iofq_datasets.values:\n", " ds.info.reduction.script = (\n", " 'https://scipp.github.io/essreflectometry/user-guide/amor/amor-reduction.html'\n", " )" @@ -457,7 +464,7 @@ "metadata": {}, "outputs": [], "source": [ - "fileio.orso.save_orso(datasets=list(iofq_datasets.values()), fname='amor_reduced_iofq.ort')" + "fileio.orso.save_orso(datasets=iofq_datasets.values, fname='amor_reduced_iofq.ort')" ] }, { From ccfb1b74f50ebc4de73504eed6b0755bf027c577 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Fri, 15 Aug 2025 11:12:12 +0200 Subject: [PATCH 28/75] debugging compute multiple --- src/ess/reflectometry/tools.py | 87 ++++++++++++++++--- .../reflectometry/workflow_collection_test.py | 34 +++++--- 2 files changed, 95 insertions(+), 26 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index a5416484..247fdd7d 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -128,20 +128,79 @@ def __setitem__(self, key, value): self._original_workflow[key] = None self._mapped_workflow = self._original_workflow.map(self.param_table) - def compute(self, target: type | Sequence[type], **kwargs) -> Mapping[str, Any]: - try: - # TODO: Sciline here returns a pandas Series. - # Should we convert it to a dict instead? - return sl.compute_mapped(self._mapped_workflow, target, **kwargs) - except ValueError: - return self._mapped_workflow.compute(target, **kwargs) - - def get(self, targets, **kwargs): - try: - targets = sl.get_mapped_node_names(self._mapped_workflow, targets) - return self._mapped_workflow.get(targets, **kwargs) - except ValueError: - return self._mapped_workflow.get(targets, **kwargs) + def compute(self, keys: type | Sequence[type], **kwargs) -> Mapping[str, Any]: + from sciline.pipeline import _is_multiple_keys + + out = {} + if _is_multiple_keys(keys): + for key in keys: + if sl.is_mapped_node(self._mapped_workflow, key): + targets = [ + n + for x in key + for n in sl.get_mapped_node_names(self._mapped_workflow, x) + ] + results = self._mapped_workflow.compute(targets, **kwargs) + for node, v in results.items(): + key = node.index.values[0] + if key not in out: + out[key] = {node.name: [v]} + else: + out[key][node.name] = v + + # if sl.is_mapped_node(target): + # from sciline.pipeline import _is_multiple_keys + + # targets = [ + # n + # for x in target + # for n in sl.get_mapped_node_names(self._mapped_workflow, x) + # ] + # results = self._mapped_workflow.compute(targets, **kwargs) + # out = {} + # for node, v in results.items(): + # key = node.index.values[0] + # if key not in out: + # out[key] = {node.name: [v]} + # else: + # out[key][node.name] = v + # return out + # else: + # return dict(sl.compute_mapped(self._mapped_workflow, target, **kwargs)) + # else: + # return self._mapped_workflow.compute(target, **kwargs) + + # def get(self, keys, **kwargs): + # if _is_multiple_keys(target): + + # if sl.is_mapped_node(target): + # from sciline.pipeline import _is_multiple_keys + + # if _is_multiple_keys(target): + # targets = [ + # n + # for x in target + # for n in sl.get_mapped_node_names(self._mapped_workflow, x) + # ] + + # results = self._mapped_workflow.compute(targets, **kwargs) + # out = {} + # for node, v in results.items(): + # key = node.index.values[0] + # if key not in out: + # out[key] = {node.name: [v]} + # else: + # out[key][node.name] = v + # return out + # else: + # return dict(sl.compute_mapped(self._mapped_workflow, target, **kwargs)) + # else: + # return self._mapped_workflow.compute(target, **kwargs) + # # try: + # # targets = sl.get_mapped_node_names(self._mapped_workflow, targets) + # # return self._mapped_workflow.get(targets, **kwargs) + # # except ValueError: + # # return self._mapped_workflow.get(targets, **kwargs) # TODO: implement the group() method to group by params in the parameter table diff --git a/tests/reflectometry/workflow_collection_test.py b/tests/reflectometry/workflow_collection_test.py index bb1ff529..a7232a72 100644 --- a/tests/reflectometry/workflow_collection_test.py +++ b/tests/reflectometry/workflow_collection_test.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2025 Scipp contributors (https://github.com/scipp) +import pandas as pd import sciline as sl from ess.reflectometry.tools import WorkflowCollection @@ -14,25 +15,34 @@ def int_float_to_str(x: int, y: float) -> str: return f"{x};{y}" +def make_param_table(params: dict) -> pd.DataFrame: + all_types = {t for v in params.values() for t in v.keys()} + data = {t: [] for t in all_types} + for param in params.values(): + for t in all_types: + data[t].append(param[t]) + return pd.DataFrame(data, index=params.keys()).rename_axis(index='run_id') + + def test_compute() -> None: wf = sl.Pipeline([int_to_float, int_float_to_str]) - wfa = wf.copy() - wfa[int] = 3 - wfb = wf.copy() - wfb[int] = 4 - coll = WorkflowCollection({'a': wfa, 'b': wfb}) - assert coll.compute(float) == {'a': 1.5, 'b': 2.0} - assert coll.compute(str) == {'a': '3;1.5', 'b': '4;2.0'} + coll = WorkflowCollection(wf, make_param_table({'a': {int: 3}, 'b': {int: 4}})) + + assert dict(coll.compute(float)) == {'a': 1.5, 'b': 2.0} + assert dict(coll.compute(str)) == {'a': '3;1.5', 'b': '4;2.0'} def test_compute_multiple() -> None: wf = sl.Pipeline([int_to_float, int_float_to_str]) - wfa = wf.copy() - wfa[int] = 3 - wfb = wf.copy() - wfb[int] = 4 - coll = WorkflowCollection({'a': wfa, 'b': wfb}) + + coll = WorkflowCollection(wf, make_param_table({'a': {int: 3}, 'b': {int: 4}})) + + # wfa = wf.copy() + # wfa[int] = 3 + # wfb = wf.copy() + # wfb[int] = 4 + # coll = WorkflowCollection({'a': wfa, 'b': wfb}) result = coll.compute([float, str]) From c8e43a9e94df4015cfe95bdc4d10dc37ebc86a77 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Tue, 19 Aug 2025 12:25:07 +0200 Subject: [PATCH 29/75] fix compute --- src/ess/reflectometry/tools.py | 83 ++++++---------------------------- 1 file changed, 14 insertions(+), 69 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 247fdd7d..b0808e65 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -132,75 +132,20 @@ def compute(self, keys: type | Sequence[type], **kwargs) -> Mapping[str, Any]: from sciline.pipeline import _is_multiple_keys out = {} - if _is_multiple_keys(keys): - for key in keys: - if sl.is_mapped_node(self._mapped_workflow, key): - targets = [ - n - for x in key - for n in sl.get_mapped_node_names(self._mapped_workflow, x) - ] - results = self._mapped_workflow.compute(targets, **kwargs) - for node, v in results.items(): - key = node.index.values[0] - if key not in out: - out[key] = {node.name: [v]} - else: - out[key][node.name] = v - - # if sl.is_mapped_node(target): - # from sciline.pipeline import _is_multiple_keys - - # targets = [ - # n - # for x in target - # for n in sl.get_mapped_node_names(self._mapped_workflow, x) - # ] - # results = self._mapped_workflow.compute(targets, **kwargs) - # out = {} - # for node, v in results.items(): - # key = node.index.values[0] - # if key not in out: - # out[key] = {node.name: [v]} - # else: - # out[key][node.name] = v - # return out - # else: - # return dict(sl.compute_mapped(self._mapped_workflow, target, **kwargs)) - # else: - # return self._mapped_workflow.compute(target, **kwargs) - - # def get(self, keys, **kwargs): - # if _is_multiple_keys(target): - - # if sl.is_mapped_node(target): - # from sciline.pipeline import _is_multiple_keys - - # if _is_multiple_keys(target): - # targets = [ - # n - # for x in target - # for n in sl.get_mapped_node_names(self._mapped_workflow, x) - # ] - - # results = self._mapped_workflow.compute(targets, **kwargs) - # out = {} - # for node, v in results.items(): - # key = node.index.values[0] - # if key not in out: - # out[key] = {node.name: [v]} - # else: - # out[key][node.name] = v - # return out - # else: - # return dict(sl.compute_mapped(self._mapped_workflow, target, **kwargs)) - # else: - # return self._mapped_workflow.compute(target, **kwargs) - # # try: - # # targets = sl.get_mapped_node_names(self._mapped_workflow, targets) - # # return self._mapped_workflow.get(targets, **kwargs) - # # except ValueError: - # # return self._mapped_workflow.get(targets, **kwargs) + if not _is_multiple_keys(keys): + keys = [keys] + for key in keys: + out[key] = {} + if sl.is_mapped_node(self._mapped_workflow, key): + targets = sl.get_mapped_node_names(self._mapped_workflow, key) + results = self._mapped_workflow.compute(targets, **kwargs) + for node, v in results.items(): + out[key][node.index.values[0]] = v + else: + out[key] = self._mapped_workflow.compute(key, **kwargs) + return next(iter(out.values())) if len(out) == 1 else out + + # TODO: implement get() # TODO: implement the group() method to group by params in the parameter table From 459a40805b131492a51e2a4e4afa97b500df26a5 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Mon, 1 Sep 2025 19:34:20 +0200 Subject: [PATCH 30/75] Revert "fix compute" This reverts commit c8e43a9e94df4015cfe95bdc4d10dc37ebc86a77. --- src/ess/reflectometry/tools.py | 83 ++++++++++++++++++++++++++++------ 1 file changed, 69 insertions(+), 14 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index b0808e65..247fdd7d 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -132,20 +132,75 @@ def compute(self, keys: type | Sequence[type], **kwargs) -> Mapping[str, Any]: from sciline.pipeline import _is_multiple_keys out = {} - if not _is_multiple_keys(keys): - keys = [keys] - for key in keys: - out[key] = {} - if sl.is_mapped_node(self._mapped_workflow, key): - targets = sl.get_mapped_node_names(self._mapped_workflow, key) - results = self._mapped_workflow.compute(targets, **kwargs) - for node, v in results.items(): - out[key][node.index.values[0]] = v - else: - out[key] = self._mapped_workflow.compute(key, **kwargs) - return next(iter(out.values())) if len(out) == 1 else out - - # TODO: implement get() + if _is_multiple_keys(keys): + for key in keys: + if sl.is_mapped_node(self._mapped_workflow, key): + targets = [ + n + for x in key + for n in sl.get_mapped_node_names(self._mapped_workflow, x) + ] + results = self._mapped_workflow.compute(targets, **kwargs) + for node, v in results.items(): + key = node.index.values[0] + if key not in out: + out[key] = {node.name: [v]} + else: + out[key][node.name] = v + + # if sl.is_mapped_node(target): + # from sciline.pipeline import _is_multiple_keys + + # targets = [ + # n + # for x in target + # for n in sl.get_mapped_node_names(self._mapped_workflow, x) + # ] + # results = self._mapped_workflow.compute(targets, **kwargs) + # out = {} + # for node, v in results.items(): + # key = node.index.values[0] + # if key not in out: + # out[key] = {node.name: [v]} + # else: + # out[key][node.name] = v + # return out + # else: + # return dict(sl.compute_mapped(self._mapped_workflow, target, **kwargs)) + # else: + # return self._mapped_workflow.compute(target, **kwargs) + + # def get(self, keys, **kwargs): + # if _is_multiple_keys(target): + + # if sl.is_mapped_node(target): + # from sciline.pipeline import _is_multiple_keys + + # if _is_multiple_keys(target): + # targets = [ + # n + # for x in target + # for n in sl.get_mapped_node_names(self._mapped_workflow, x) + # ] + + # results = self._mapped_workflow.compute(targets, **kwargs) + # out = {} + # for node, v in results.items(): + # key = node.index.values[0] + # if key not in out: + # out[key] = {node.name: [v]} + # else: + # out[key][node.name] = v + # return out + # else: + # return dict(sl.compute_mapped(self._mapped_workflow, target, **kwargs)) + # else: + # return self._mapped_workflow.compute(target, **kwargs) + # # try: + # # targets = sl.get_mapped_node_names(self._mapped_workflow, targets) + # # return self._mapped_workflow.get(targets, **kwargs) + # # except ValueError: + # # return self._mapped_workflow.get(targets, **kwargs) # TODO: implement the group() method to group by params in the parameter table From 99ea9d77b1cc46b8a9b768abe543fa32e356101b Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Mon, 1 Sep 2025 19:34:28 +0200 Subject: [PATCH 31/75] Revert "debugging compute multiple" This reverts commit ccfb1b74f50ebc4de73504eed6b0755bf027c577. --- src/ess/reflectometry/tools.py | 87 +++---------------- .../reflectometry/workflow_collection_test.py | 34 +++----- 2 files changed, 26 insertions(+), 95 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 247fdd7d..a5416484 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -128,79 +128,20 @@ def __setitem__(self, key, value): self._original_workflow[key] = None self._mapped_workflow = self._original_workflow.map(self.param_table) - def compute(self, keys: type | Sequence[type], **kwargs) -> Mapping[str, Any]: - from sciline.pipeline import _is_multiple_keys - - out = {} - if _is_multiple_keys(keys): - for key in keys: - if sl.is_mapped_node(self._mapped_workflow, key): - targets = [ - n - for x in key - for n in sl.get_mapped_node_names(self._mapped_workflow, x) - ] - results = self._mapped_workflow.compute(targets, **kwargs) - for node, v in results.items(): - key = node.index.values[0] - if key not in out: - out[key] = {node.name: [v]} - else: - out[key][node.name] = v - - # if sl.is_mapped_node(target): - # from sciline.pipeline import _is_multiple_keys - - # targets = [ - # n - # for x in target - # for n in sl.get_mapped_node_names(self._mapped_workflow, x) - # ] - # results = self._mapped_workflow.compute(targets, **kwargs) - # out = {} - # for node, v in results.items(): - # key = node.index.values[0] - # if key not in out: - # out[key] = {node.name: [v]} - # else: - # out[key][node.name] = v - # return out - # else: - # return dict(sl.compute_mapped(self._mapped_workflow, target, **kwargs)) - # else: - # return self._mapped_workflow.compute(target, **kwargs) - - # def get(self, keys, **kwargs): - # if _is_multiple_keys(target): - - # if sl.is_mapped_node(target): - # from sciline.pipeline import _is_multiple_keys - - # if _is_multiple_keys(target): - # targets = [ - # n - # for x in target - # for n in sl.get_mapped_node_names(self._mapped_workflow, x) - # ] - - # results = self._mapped_workflow.compute(targets, **kwargs) - # out = {} - # for node, v in results.items(): - # key = node.index.values[0] - # if key not in out: - # out[key] = {node.name: [v]} - # else: - # out[key][node.name] = v - # return out - # else: - # return dict(sl.compute_mapped(self._mapped_workflow, target, **kwargs)) - # else: - # return self._mapped_workflow.compute(target, **kwargs) - # # try: - # # targets = sl.get_mapped_node_names(self._mapped_workflow, targets) - # # return self._mapped_workflow.get(targets, **kwargs) - # # except ValueError: - # # return self._mapped_workflow.get(targets, **kwargs) + def compute(self, target: type | Sequence[type], **kwargs) -> Mapping[str, Any]: + try: + # TODO: Sciline here returns a pandas Series. + # Should we convert it to a dict instead? + return sl.compute_mapped(self._mapped_workflow, target, **kwargs) + except ValueError: + return self._mapped_workflow.compute(target, **kwargs) + + def get(self, targets, **kwargs): + try: + targets = sl.get_mapped_node_names(self._mapped_workflow, targets) + return self._mapped_workflow.get(targets, **kwargs) + except ValueError: + return self._mapped_workflow.get(targets, **kwargs) # TODO: implement the group() method to group by params in the parameter table diff --git a/tests/reflectometry/workflow_collection_test.py b/tests/reflectometry/workflow_collection_test.py index a7232a72..bb1ff529 100644 --- a/tests/reflectometry/workflow_collection_test.py +++ b/tests/reflectometry/workflow_collection_test.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2025 Scipp contributors (https://github.com/scipp) -import pandas as pd import sciline as sl from ess.reflectometry.tools import WorkflowCollection @@ -15,34 +14,25 @@ def int_float_to_str(x: int, y: float) -> str: return f"{x};{y}" -def make_param_table(params: dict) -> pd.DataFrame: - all_types = {t for v in params.values() for t in v.keys()} - data = {t: [] for t in all_types} - for param in params.values(): - for t in all_types: - data[t].append(param[t]) - return pd.DataFrame(data, index=params.keys()).rename_axis(index='run_id') - - def test_compute() -> None: wf = sl.Pipeline([int_to_float, int_float_to_str]) + wfa = wf.copy() + wfa[int] = 3 + wfb = wf.copy() + wfb[int] = 4 + coll = WorkflowCollection({'a': wfa, 'b': wfb}) - coll = WorkflowCollection(wf, make_param_table({'a': {int: 3}, 'b': {int: 4}})) - - assert dict(coll.compute(float)) == {'a': 1.5, 'b': 2.0} - assert dict(coll.compute(str)) == {'a': '3;1.5', 'b': '4;2.0'} + assert coll.compute(float) == {'a': 1.5, 'b': 2.0} + assert coll.compute(str) == {'a': '3;1.5', 'b': '4;2.0'} def test_compute_multiple() -> None: wf = sl.Pipeline([int_to_float, int_float_to_str]) - - coll = WorkflowCollection(wf, make_param_table({'a': {int: 3}, 'b': {int: 4}})) - - # wfa = wf.copy() - # wfa[int] = 3 - # wfb = wf.copy() - # wfb[int] = 4 - # coll = WorkflowCollection({'a': wfa, 'b': wfb}) + wfa = wf.copy() + wfa[int] = 3 + wfb = wf.copy() + wfb[int] = 4 + coll = WorkflowCollection({'a': wfa, 'b': wfb}) result = coll.compute([float, str]) From 745c68d85c8fe1e323b8fa74c7bbf48da10fa888 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Mon, 1 Sep 2025 19:34:35 +0200 Subject: [PATCH 32/75] Revert "update amor notebook" This reverts commit a23c007e51646b0f63bd8a6e61ea6003c168238e. --- docs/user-guide/amor/amor-reduction.ipynb | 27 +++++++++-------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/docs/user-guide/amor/amor-reduction.ipynb b/docs/user-guide/amor/amor-reduction.ipynb index c0c08ec0..18d9224e 100644 --- a/docs/user-guide/amor/amor-reduction.ipynb +++ b/docs/user-guide/amor/amor-reduction.ipynb @@ -184,15 +184,8 @@ "}\n", "\n", "batch = batch_processor(workflow, runs)\n", - "batch.param_table" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ + "display(batch.keys())\n", + "\n", "# Compute R(Q) for all runs\n", "reflectivity = batch.compute(ReflectivityOverQ)\n", "sc.plot(\n", @@ -295,8 +288,8 @@ "from ess.reflectometry.figures import wavelength_theta_figure\n", "\n", "wavelength_theta_figure(\n", - " diagnostics.values,\n", - " theta_bins=scaled_wf.compute(ThetaBins[SampleRun]).values,\n", + " diagnostics.values(),\n", + " theta_bins=scaled_wf.compute(ThetaBins[SampleRun]).values(),\n", " q_edges_to_display=(sc.scalar(0.018, unit='1/angstrom'), sc.scalar(0.113, unit='1/angstrom'))\n", ")" ] @@ -325,8 +318,8 @@ "from ess.reflectometry.figures import q_theta_figure\n", "\n", "q_theta_figure(\n", - " diagnostics.values,\n", - " theta_bins=scaled_wf.compute(ThetaBins[SampleRun]).values,\n", + " diagnostics.values(),\n", + " theta_bins=scaled_wf.compute(ThetaBins[SampleRun]).values(),\n", " q_bins=workflow.compute(QBins)\n", ")" ] @@ -403,7 +396,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We can visualize the workflow for the `OrsoIofQDataset`:" + "We can visualize the workflow for a single run (`'608'`):" ] }, { @@ -412,7 +405,7 @@ "metadata": {}, "outputs": [], "source": [ - "scaled_wf.visualize(orso.OrsoIofQDataset, graph_attr={'rankdir': 'LR'})" + "scaled_wf['608'].visualize(orso.OrsoIofQDataset, graph_attr={'rankdir': 'LR'})" ] }, { @@ -444,7 +437,7 @@ "metadata": {}, "outputs": [], "source": [ - "for ds in iofq_datasets.values:\n", + "for ds in iofq_datasets.values():\n", " ds.info.reduction.script = (\n", " 'https://scipp.github.io/essreflectometry/user-guide/amor/amor-reduction.html'\n", " )" @@ -464,7 +457,7 @@ "metadata": {}, "outputs": [], "source": [ - "fileio.orso.save_orso(datasets=iofq_datasets.values, fname='amor_reduced_iofq.ort')" + "fileio.orso.save_orso(datasets=list(iofq_datasets.values()), fname='amor_reduced_iofq.ort')" ] }, { From 9b9797c72322bdaaf50982a6ac18cb30f739f41d Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Mon, 1 Sep 2025 19:34:37 +0200 Subject: [PATCH 33/75] Revert "formatting" This reverts commit f181b41d42bee5f633c3eec5b8802da70f2c2282. --- tests/reflectometry/tools_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/reflectometry/tools_test.py b/tests/reflectometry/tools_test.py index 70e8838e..256b3d76 100644 --- a/tests/reflectometry/tools_test.py +++ b/tests/reflectometry/tools_test.py @@ -437,8 +437,7 @@ class Reduction: # results = batch.compute(UnscaledReducibleData[SampleRun]) -# assert_almost_equal(results['a'].sum().value, -# 10 + 15 * 0.5 + (10 + 15 * 0.5) * 1.5) +# assert_almost_equal(results['a'].sum().value, 10 + 15 * 0.5 + (10 + 15 * 0.5) * 1.5) # assert_almost_equal(results['b'].sum().value, 10 * 0.8 + 15 * 0.5 * 0.8) # assert_almost_equal( # results['c'].sum().value, (10 + 15 * 0.5) * 0.1 + (10 + 15 * 0.5) * 0.2 From 76aa1cf2c2f624cfb4019a93f5d9db2cc2a9b084 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Mon, 1 Sep 2025 19:34:39 +0200 Subject: [PATCH 34/75] Revert "modify the WorkflowCollection to just use cyclebane mapping under the hood and make its interface more like the Pipeline interface rather than a mix between Pipeline and a dict" This reverts commit 2dea8de9bc081e6a92192c6f95e757e3fc8ebd9f. --- src/ess/reflectometry/tools.py | 168 ++++++++++++++++-------------- tests/reflectometry/tools_test.py | 86 ++++++++------- 2 files changed, 134 insertions(+), 120 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index a5416484..434200aa 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -11,6 +11,7 @@ import scipy.optimize as opt from ess.reflectometry.types import ( + Filename, QBins, ReferenceRun, ReflectivityOverQ, @@ -18,6 +19,7 @@ ScalingFactorForOverlap, UnscaledReducibleData, ) +from ess.reflectometry.workflow import with_filenames _STD_TO_FWHM = sc.scalar(2.0) * sc.sqrt(sc.scalar(2.0) * sc.log(sc.scalar(2.0))) @@ -109,50 +111,55 @@ def linlogspace( class WorkflowCollection: """ A collection of sciline workflows that can be used to compute multiple - targets from mapping a workflow over a parameter table. - It can also be used to set parameters for all mapped nodes in a single shot. + targets from multiple workflows. + It can also be used to set parameters for all workflows in a single shot. """ - def __init__(self, workflow: sl.Pipeline, param_table): - self._original_workflow = workflow.copy() - self.param_table = param_table - self._mapped_workflow = self._original_workflow.map(self.param_table) + def __init__(self, workflows: Mapping[str, sl.Pipeline]): + self._workflows = {name: pl.copy() for name, pl in workflows.items()} - def __setitem__(self, key, value): - if key in self.param_table: - ind = list(self.param_table.keys()).index(key) - self.param_table.iloc[:, ind] = value - self._mapped_workflow = self._original_workflow.map(self.param_table) + def __setitem__(self, key: type, value: Any | Mapping[type, Any]): + if hasattr(value, 'items'): + for name, v in value.items(): + self._workflows[name][key] = v else: - self.param_table.insert(len(self.param_table.columns), key, value) - self._original_workflow[key] = None - self._mapped_workflow = self._original_workflow.map(self.param_table) + for pl in self._workflows.values(): + pl[key] = value + + def __getitem__(self, name: str) -> sl.Pipeline: + """ + Returns a single workflow from the collection given by its name. + """ + return self._workflows[name] def compute(self, target: type | Sequence[type], **kwargs) -> Mapping[str, Any]: - try: - # TODO: Sciline here returns a pandas Series. - # Should we convert it to a dict instead? - return sl.compute_mapped(self._mapped_workflow, target, **kwargs) - except ValueError: - return self._mapped_workflow.compute(target, **kwargs) + return { + name: pl.compute(target, **kwargs) for name, pl in self._workflows.items() + } - def get(self, targets, **kwargs): - try: - targets = sl.get_mapped_node_names(self._mapped_workflow, targets) - return self._mapped_workflow.get(targets, **kwargs) - except ValueError: - return self._mapped_workflow.get(targets, **kwargs) + def copy(self) -> 'WorkflowCollection': + return self.__class__(self._workflows) - # TODO: implement the group() method to group by params in the parameter table + def keys(self) -> Sequence[str]: + return self._workflows.keys() - def visualize(self, targets, **kwargs): - targets = sl.get_mapped_node_names(self._mapped_workflow, targets) - return self._mapped_workflow.visualize(targets, **kwargs) + def values(self) -> Sequence[sl.Pipeline]: + return self._workflows.values() - def copy(self) -> 'WorkflowCollection': - return self.__class__( - workflow=self._original_workflow, param_table=self.param_table - ) + def items(self) -> Sequence[tuple[str, sl.Pipeline]]: + return self._workflows.items() + + def add(self, name: str, workflow: sl.Pipeline): + """ + Adds a new workflow to the collection. + """ + self._workflows[name] = workflow.copy() + + def remove(self, name: str): + """ + Removes a workflow from the collection by its name. + """ + del self._workflows[name] def _sort_by(a, by): @@ -215,7 +222,7 @@ def _interpolate_on_qgrid(curves, grid): def scale_reflectivity_curves_to_overlap( - workflow: WorkflowCollection | sl.Pipeline, + workflows: WorkflowCollection | sl.Pipeline, critical_edge_interval: tuple[sc.Variable, sc.Variable] | None = None, cache_intermediate_results: bool = True, ) -> tuple[list[sc.DataArray], list[sc.Variable]]: @@ -250,9 +257,17 @@ def scale_reflectivity_curves_to_overlap( : A list of scaled reflectivity curves and a list of the scaling factors. ''' - not_collection = isinstance(workflow, sl.Pipeline) + if isinstance(workflows, sl.Pipeline): + # If a single workflow is provided, convert it to a collection + wfc = WorkflowCollection({"": workflows}) + out = scale_reflectivity_curves_to_overlap( + wfc, + critical_edge_interval=critical_edge_interval, + cache_intermediate_results=cache_intermediate_results, + ) + return out[""] - wfc = workflow.copy() + wfc = workflows.copy() if cache_intermediate_results: try: wfc[UnscaledReducibleData[SampleRun]] = wfc.compute( @@ -268,8 +283,6 @@ def scale_reflectivity_curves_to_overlap( pass reflectivities = wfc.compute(ReflectivityOverQ) - if not_collection: - reflectivities = {"": reflectivities} # First sort the dict of reflectivities by the Q min value curves = { @@ -281,21 +294,20 @@ def scale_reflectivity_curves_to_overlap( critical_edge_key = uuid.uuid4().hex if critical_edge_interval is not None: - q = wfc.compute(QBins) - if hasattr(q, "items"): - # If QBins is a mapping, find the one with the lowest Q start - # Note the conversion to a dict, because if pandas is used for the mapping, - # it will return a Series, whose `.values` attribute is not callable. - q = min(dict(q).values(), key=lambda q_: q_.min()) - - # TODO: This is slightly different from before: it extracts the bins from the - # QBins variable that cover the critical edge interval. This means that the - # resulting curve will not necessarily begin and end exactly at the values - # specified, but rather at the closest bin edges. + # Find q bins with the lowest Q start point + q = min( + (wf.compute(QBins) for wf in workflows.values()), + key=lambda q_: q_.min(), + ) + N = ( + ((q >= critical_edge_interval[0]) & (q < critical_edge_interval[1])) + .sum() + .value + ) edge = sc.DataArray( - data=sc.ones(sizes={q.dim: q.sizes[q.dim] - 1}, with_variances=True), - coords={q.dim: q}, - )[q.dim, critical_edge_interval[0] : critical_edge_interval[1]] + data=sc.ones(dims=('Q',), shape=(N,), with_variances=True), + coords={'Q': sc.linspace('Q', *critical_edge_interval, N + 1)}, + ) # Now place the critical edge at the beginning curves = {critical_edge_key: edge} | curves @@ -323,14 +335,11 @@ def cost(scaling_factors): sol = opt.minimize(cost, [1.0] * (len(curves) - 1)) scaling_factors = (1.0, *map(float, sol.x)) - results = { + wfc[ScalingFactorForOverlap[SampleRun]] = { k: v for k, v in zip(curves.keys(), scaling_factors, strict=True) if k != critical_edge_key } - if not_collection: - results = results[""] - wfc[ScalingFactorForOverlap[SampleRun]] = results return wfc @@ -388,10 +397,10 @@ def combine_curves( def batch_processor( - workflow: sl.Pipeline, params: Mapping[Any, Mapping[type, Any]] + workflow: sl.Pipeline, runs: Mapping[Any, Mapping[type, Any]] ) -> WorkflowCollection: """ - Maps the provided workflow over the provided params. + Creates a collection of sciline workflows from the provided runs. Example: @@ -406,7 +415,7 @@ def batch_processor( Filename[SampleRun]: amor.data.amor_run(608), }, '609': { - SampleRotationOffset[SampleRun]: sc.scalar(0.06, unit='deg'), + SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'), Filename[SampleRun]: amor.data.amor_run(609), }, '610': { @@ -414,7 +423,7 @@ def batch_processor( Filename[SampleRun]: amor.data.amor_run(610), }, '611': { - SampleRotationOffset[SampleRun]: sc.scalar(0.07, unit='deg'), + SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'), Filename[SampleRun]: amor.data.amor_run(611), }, } @@ -428,23 +437,30 @@ def batch_processor( ---------- workflow: The sciline workflow used to compute the targets for each of the runs. - params: + runs: The sciline parameters to be used for each run. Should be a mapping where the keys are the names of the runs and the values are mappings of type to value pairs. + In addition, if one of the values for ``Filename[SampleRun]`` + is a list or a tuple, then the events from the files + will be concatenated into a single event list. """ - import pandas as pd - - all_types = {t for v in params.values() for t in v.keys()} - data = {t: [] for t in all_types} - for param in params.values(): - for t in all_types: - if t in param: - data[t].append(param[t]) + workflows = {} + for name, parameters in runs.items(): + wf = workflow.copy() + for tp, value in parameters.items(): + if tp is Filename[SampleRun]: + continue + wf[tp] = value + + if Filename[SampleRun] in parameters: + if isinstance(parameters[Filename[SampleRun]], list | tuple): + wf = with_filenames( + wf, + SampleRun, + parameters[Filename[SampleRun]], + ) else: - # Set the default value - data[t].append(workflow.compute(t)) - - param_table = pd.DataFrame(data, index=params.keys()).rename_axis(index='run_id') - - return WorkflowCollection(workflow, param_table) + wf[Filename[SampleRun]] = parameters[Filename[SampleRun]] + workflows[name] = wf + return WorkflowCollection(workflows) diff --git a/tests/reflectometry/tools_test.py b/tests/reflectometry/tools_test.py index 256b3d76..72e108df 100644 --- a/tests/reflectometry/tools_test.py +++ b/tests/reflectometry/tools_test.py @@ -5,10 +5,12 @@ import pytest import sciline as sl import scipp as sc +from numpy.testing import assert_almost_equal from orsopy.fileio import Orso, OrsoDataset from scipp.testing import assert_allclose from ess.reflectometry.tools import ( + WorkflowCollection, batch_processor, combine_curves, linlogspace, @@ -105,15 +107,14 @@ def test_reflectivity_curve_scaling(): wf[ScalingFactorForOverlap[SampleRun]] = 1.0 wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 params = {'a': (1.0, 0, 0.3), 'b': (0.8, 0.2, 0.7), 'c': (0.1, 0.6, 1.0)} - table = { - k: { - Filename[SampleRun]: "_".join(map(str, v)), - Filename[ReferenceRun]: "_".join(map(str, v[1:])), - QBins: make_reference_events(*v[1:]).coords['Q'], - } - for k, v in params.items() - } - wfc = batch_processor(wf, table) + workflows = {} + for k, v in params.items(): + workflows[k] = wf.copy() + workflows[k][Filename[SampleRun]] = "_".join(map(str, v)) + workflows[k][Filename[ReferenceRun]] = "_".join(map(str, v[1:])) + workflows[k][QBins] = make_reference_events(*v[1:]).coords['Q'] + + wfc = WorkflowCollection(workflows) scaled_wf = scale_reflectivity_curves_to_overlap(wfc) @@ -129,15 +130,14 @@ def test_reflectivity_curve_scaling_with_critical_edge(): wf[ScalingFactorForOverlap[SampleRun]] = 1.0 wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 params = {'a': (2, 0, 0.3), 'b': (0.8, 0.2, 0.7), 'c': (0.1, 0.6, 1.0)} - table = { - k: { - Filename[SampleRun]: "_".join(map(str, v)), - Filename[ReferenceRun]: "_".join(map(str, v[1:])), - QBins: make_reference_events(*v[1:]).coords['Q'], - } - for k, v in params.items() - } - wfc = batch_processor(wf, table) + workflows = {} + for k, v in params.items(): + workflows[k] = wf.copy() + workflows[k][Filename[SampleRun]] = "_".join(map(str, v)) + workflows[k][Filename[ReferenceRun]] = "_".join(map(str, v[1:])) + workflows[k][QBins] = make_reference_events(*v[1:]).coords['Q'] + + wfc = WorkflowCollection(workflows) scaled_wf = scale_reflectivity_curves_to_overlap( wfc, critical_edge_interval=(sc.scalar(0.01), sc.scalar(0.05)) @@ -216,15 +216,14 @@ def reflectivity( wf[ScalingFactorForOverlap[SampleRun]] = 1.0 wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 params = {'a': (1.0, 0, 0.3), 'b': (0.8, 0.2, 0.7), 'c': (0.1, 0.6, 1.0)} - table = { - k: { - Filename[SampleRun]: "_".join(map(str, v)), - Filename[ReferenceRun]: "_".join(map(str, v[1:])), - QBins: make_reference_events(*v[1:]).coords['Q'], - } - for k, v in params.items() - } - wfc = batch_processor(wf, table) + workflows = {} + for k, v in params.items(): + workflows[k] = wf.copy() + workflows[k][Filename[SampleRun]] = "_".join(map(str, v)) + workflows[k][Filename[ReferenceRun]] = "_".join(map(str, v[1:])) + workflows[k][QBins] = make_reference_events(*v[1:]).coords['Q'] + + wfc = WorkflowCollection(workflows) scaled_wf = scale_reflectivity_curves_to_overlap( wfc, cache_intermediate_results=False @@ -422,23 +421,22 @@ class Reduction: assert results['b'].info.name == 'special.orso' -# TODO: need to implement groupby in the mapping -# def test_batch_processor_tool_merges_event_lists(): -# wf = make_workflow() -# wf[ScalingFactorForOverlap[SampleRun]] = 1.0 -# wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 +def test_batch_processor_tool_merges_event_lists(): + wf = make_workflow() + wf[ScalingFactorForOverlap[SampleRun]] = 1.0 + wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 -# runs = { -# 'a': {Filename[SampleRun]: ('1.0_0.0_0.3', '1.5_0.0_0.3')}, -# 'b': {Filename[SampleRun]: '0.8_0.2_0.7'}, -# 'c': {Filename[SampleRun]: ('0.1_0.6_1.0', '0.2_0.6_1.0')}, -# } -# batch = batch_processor(wf, runs) + runs = { + 'a': {Filename[SampleRun]: ('1.0_0.0_0.3', '1.5_0.0_0.3')}, + 'b': {Filename[SampleRun]: '0.8_0.2_0.7'}, + 'c': {Filename[SampleRun]: ('0.1_0.6_1.0', '0.2_0.6_1.0')}, + } + batch = batch_processor(wf, runs) -# results = batch.compute(UnscaledReducibleData[SampleRun]) + results = batch.compute(UnscaledReducibleData[SampleRun]) -# assert_almost_equal(results['a'].sum().value, 10 + 15 * 0.5 + (10 + 15 * 0.5) * 1.5) -# assert_almost_equal(results['b'].sum().value, 10 * 0.8 + 15 * 0.5 * 0.8) -# assert_almost_equal( -# results['c'].sum().value, (10 + 15 * 0.5) * 0.1 + (10 + 15 * 0.5) * 0.2 -# ) + assert_almost_equal(results['a'].sum().value, 10 + 15 * 0.5 + (10 + 15 * 0.5) * 1.5) + assert_almost_equal(results['b'].sum().value, 10 * 0.8 + 15 * 0.5 * 0.8) + assert_almost_equal( + results['c'].sum().value, (10 + 15 * 0.5) * 0.1 + (10 + 15 * 0.5) * 0.2 + ) From ae9b9ac7b3e530cd778a8605473c4bdeeb80bbde Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Mon, 1 Sep 2025 22:22:36 +0200 Subject: [PATCH 35/75] move workflow initialization inside the WorkflowCollection --- src/ess/reflectometry/__init__.py | 3 +- src/ess/reflectometry/tools.py | 93 +++++++++++++++++-------------- 2 files changed, 54 insertions(+), 42 deletions(-) diff --git a/src/ess/reflectometry/__init__.py b/src/ess/reflectometry/__init__.py index 3b6d18b2..bdf2c585 100644 --- a/src/ess/reflectometry/__init__.py +++ b/src/ess/reflectometry/__init__.py @@ -12,7 +12,7 @@ from . import conversions, corrections, figures, normalization, orso from .load import load_reference, save_reference -from .tools import batch_processor +from .tools import WorkflowCollection, batch_processor providers = ( *corrections.providers, @@ -33,6 +33,7 @@ del importlib __all__ = [ + "WorkflowCollection", "__version__", "batch_processor", "conversions", diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 434200aa..52b56e07 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2025 Scipp contributors (https://github.com/scipp) import uuid -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from itertools import chain from typing import Any @@ -115,51 +115,64 @@ class WorkflowCollection: It can also be used to set parameters for all workflows in a single shot. """ - def __init__(self, workflows: Mapping[str, sl.Pipeline]): - self._workflows = {name: pl.copy() for name, pl in workflows.items()} + def __init__(self, workflow: sl.Pipeline, params: Mapping[Any, Mapping[type, Any]]): + # self._original_workflow = workflow + self.workflows = {} + for name, parameters in params.items(): + wf = workflow.copy() + for tp, value in parameters.items(): + # if tp is Filename[SampleRun]: + # continue + wf[tp] = value + self.workflows[name] = wf + # self.workflows = {name: pl.copy() for name, pl in workflows.items()} def __setitem__(self, key: type, value: Any | Mapping[type, Any]): if hasattr(value, 'items'): for name, v in value.items(): - self._workflows[name][key] = v + self.workflows[name][key] = v else: - for pl in self._workflows.values(): - pl[key] = value + for wf in self.workflows.values(): + wf[key] = value def __getitem__(self, name: str) -> sl.Pipeline: - """ - Returns a single workflow from the collection given by its name. - """ - return self._workflows[name] + """ """ + return {key: wf[name] for key, wf in self.workflows.items()} def compute(self, target: type | Sequence[type], **kwargs) -> Mapping[str, Any]: return { - name: pl.compute(target, **kwargs) for name, pl in self._workflows.items() + name: pl.compute(target, **kwargs) for name, pl in self.workflows.items() } def copy(self) -> 'WorkflowCollection': - return self.__class__(self._workflows) + out = self.__class__(sl.Pipeline(), params={}) + for name, wf in self.workflows.items(): + out.workflows[name] = wf.copy() + return out - def keys(self) -> Sequence[str]: - return self._workflows.keys() + def groupby(self, key: type, reduce: Callable) -> 'WorkflowCollection': + results = self.compute(key) - def values(self) -> Sequence[sl.Pipeline]: - return self._workflows.values() + # def keys(self) -> Sequence[str]: + # return self.workflows.keys() - def items(self) -> Sequence[tuple[str, sl.Pipeline]]: - return self._workflows.items() + # def values(self) -> Sequence[sl.Pipeline]: + # return self.workflows.values() - def add(self, name: str, workflow: sl.Pipeline): - """ - Adds a new workflow to the collection. - """ - self._workflows[name] = workflow.copy() + # def items(self) -> Sequence[tuple[str, sl.Pipeline]]: + # return self.workflows.items() - def remove(self, name: str): - """ - Removes a workflow from the collection by its name. - """ - del self._workflows[name] + # def add(self, name: str, workflow: sl.Pipeline): + # """ + # Adds a new workflow to the collection. + # """ + # self.workflows[name] = workflow.copy() + + # def remove(self, name: str): + # """ + # Removes a workflow from the collection by its name. + # """ + # del self.workflows[name] def _sort_by(a, by): @@ -294,20 +307,18 @@ def scale_reflectivity_curves_to_overlap( critical_edge_key = uuid.uuid4().hex if critical_edge_interval is not None: - # Find q bins with the lowest Q start point - q = min( - (wf.compute(QBins) for wf in workflows.values()), - key=lambda q_: q_.min(), - ) - N = ( - ((q >= critical_edge_interval[0]) & (q < critical_edge_interval[1])) - .sum() - .value - ) + q = wfc.compute(QBins) + if hasattr(q, "items"): + # If QBins is a mapping, find the one with the lowest Q start + q = min(q.values(), key=lambda q_: q_.min()) + # TODO: This is slightly different from before: it extracts the bins from the + # QBins variable that cover the critical edge interval. This means that the + # resulting curve will not necessarily begin and end exactly at the values + # specified, but rather at the closest bin edges. edge = sc.DataArray( - data=sc.ones(dims=('Q',), shape=(N,), with_variances=True), - coords={'Q': sc.linspace('Q', *critical_edge_interval, N + 1)}, - ) + data=sc.ones(sizes={q.dim: q.sizes[q.dim] - 1}, with_variances=True), + coords={q.dim: q}, + )[q.dim, critical_edge_interval[0] : critical_edge_interval[1]] # Now place the critical edge at the beginning curves = {critical_edge_key: edge} | curves From 7f914f9ebe023de6f5d6c2ebf9981600dea91654 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Mon, 1 Sep 2025 23:00:02 +0200 Subject: [PATCH 36/75] move toward using pandas dataframes --- src/ess/reflectometry/tools.py | 37 ++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 52b56e07..445d17f9 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -6,6 +6,7 @@ from typing import Any import numpy as np +import pandas as pd import sciline as sl import scipp as sc import scipy.optimize as opt @@ -118,14 +119,20 @@ class WorkflowCollection: def __init__(self, workflow: sl.Pipeline, params: Mapping[Any, Mapping[type, Any]]): # self._original_workflow = workflow self.workflows = {} - for name, parameters in params.items(): - wf = workflow.copy() - for tp, value in parameters.items(): - # if tp is Filename[SampleRun]: - # continue - wf[tp] = value - self.workflows[name] = wf - # self.workflows = {name: pl.copy() for name, pl in workflows.items()} + + for index, row in params.iterrows(): + self.workflows[index] = workflow.copy() + for k, v in row.items(): + self.workflows[index][k] = v + + # for name, parameters in params.items(): + # wf = workflow.copy() + # for tp, value in parameters.items(): + # # if tp is Filename[SampleRun]: + # # continue + # wf[tp] = value + # self.workflows[name] = wf + # # self.workflows = {name: pl.copy() for name, pl in workflows.items()} def __setitem__(self, key: type, value: Any | Mapping[type, Any]): if hasattr(value, 'items'): @@ -140,9 +147,17 @@ def __getitem__(self, name: str) -> sl.Pipeline: return {key: wf[name] for key, wf in self.workflows.items()} def compute(self, target: type | Sequence[type], **kwargs) -> Mapping[str, Any]: - return { - name: pl.compute(target, **kwargs) for name, pl in self.workflows.items() - } + # return { + # name: wf.compute(target, **kwargs) for name, wf in self.workflows.items() + # } + if not isinstance(target, list | tuple): + target = [target] + out = {} + for t in target: + out[t] = { + name: wf.compute(t, **kwargs) for name, wf in self.workflows.items() + } + return pd.DataFrame(out) def copy(self) -> 'WorkflowCollection': out = self.__class__(sl.Pipeline(), params={}) From c44f79b94c2d3c5cf4b3157d6035ee7f813bbce3 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Tue, 2 Sep 2025 10:08:27 +0200 Subject: [PATCH 37/75] make DataTable --- src/ess/reflectometry/__init__.py | 3 +- src/ess/reflectometry/tools.py | 70 +++++++++++++++++++++---------- 2 files changed, 51 insertions(+), 22 deletions(-) diff --git a/src/ess/reflectometry/__init__.py b/src/ess/reflectometry/__init__.py index bdf2c585..c74ec66c 100644 --- a/src/ess/reflectometry/__init__.py +++ b/src/ess/reflectometry/__init__.py @@ -12,7 +12,7 @@ from . import conversions, corrections, figures, normalization, orso from .load import load_reference, save_reference -from .tools import WorkflowCollection, batch_processor +from .tools import DataTable, WorkflowCollection, batch_processor providers = ( *corrections.providers, @@ -33,6 +33,7 @@ del importlib __all__ = [ + "DataTable", "WorkflowCollection", "__version__", "batch_processor", diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 445d17f9..32d8656b 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -109,6 +109,23 @@ def linlogspace( return sc.concat(grids, dim) +class DataTable(dict): + def _repr_html_(self): + clean = { + str(tp.__name__) + if hasattr(tp, '__name__') + else str(tp).split('.')[-1]: value + for tp, value in self.items() + } + try: + import pandas as pd + + df = pd.DataFrame(clean) + return df._repr_html_() + except ImportError: + return clean._repr_html_() + + class WorkflowCollection: """ A collection of sciline workflows that can be used to compute multiple @@ -118,12 +135,16 @@ class WorkflowCollection: def __init__(self, workflow: sl.Pipeline, params: Mapping[Any, Mapping[type, Any]]): # self._original_workflow = workflow - self.workflows = {} + self.workflows = [] + + if not isinstance(params, pd.DataFrame): + params = pd.DataFrame(params) - for index, row in params.iterrows(): - self.workflows[index] = workflow.copy() + for _, row in params.iterrows(): + wf = workflow.copy() for k, v in row.items(): - self.workflows[index][k] = v + wf[k] = v + self.workflows.append(wf) # for name, parameters in params.items(): # wf = workflow.copy() @@ -134,17 +155,19 @@ def __init__(self, workflow: sl.Pipeline, params: Mapping[Any, Mapping[type, Any # self.workflows[name] = wf # # self.workflows = {name: pl.copy() for name, pl in workflows.items()} - def __setitem__(self, key: type, value: Any | Mapping[type, Any]): - if hasattr(value, 'items'): - for name, v in value.items(): - self.workflows[name][key] = v - else: - for wf in self.workflows.values(): - wf[key] = value + def __setitem__(self, key: type, value: Sequence[Any]): + for i, v in enumerate(value): + self.workflows[i][key] = v + # if hasattr(value, 'items'): + # for name, v in value.items(): + # self.workflows[name][key] = v + # else: + # for wf in self.workflows.values(): + # wf[key] = value - def __getitem__(self, name: str) -> sl.Pipeline: - """ """ - return {key: wf[name] for key, wf in self.workflows.items()} + # def __getitem__(self, name: str) -> sl.Pipeline: + # """ """ + # return {key: wf[name] for key, wf in self.workflows.items()} def compute(self, target: type | Sequence[type], **kwargs) -> Mapping[str, Any]: # return { @@ -154,15 +177,20 @@ def compute(self, target: type | Sequence[type], **kwargs) -> Mapping[str, Any]: target = [target] out = {} for t in target: - out[t] = { - name: wf.compute(t, **kwargs) for name, wf in self.workflows.items() - } - return pd.DataFrame(out) + # out[t] = { + # name: wf.compute(t, **kwargs) for name, wf in self.workflows.items() + # } + out[t] = [wf.compute(t, **kwargs) for wf in self.workflows] + # return pd.DataFrame(out), out + return DataTable(out) def copy(self) -> 'WorkflowCollection': - out = self.__class__(sl.Pipeline(), params={}) - for name, wf in self.workflows.items(): - out.workflows[name] = wf.copy() + # out = self.__class__(sl.Pipeline(), params={}) + # for name, wf in self.workflows.items(): + # out.workflows[name] = wf.copy() + # return out + out = WorkflowCollection(None, {}) + out.workflows = [wf.copy() for wf in self.workflows] return out def groupby(self, key: type, reduce: Callable) -> 'WorkflowCollection': From 7a1e05f6272dd5f845fa86e7902924a76d96aebb Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Tue, 16 Sep 2025 20:55:01 +0200 Subject: [PATCH 38/75] go back to making workflowcollection just a collection of workflows --- src/ess/reflectometry/__init__.py | 3 +- src/ess/reflectometry/tools.py | 141 ++++++++++++++++++------------ 2 files changed, 85 insertions(+), 59 deletions(-) diff --git a/src/ess/reflectometry/__init__.py b/src/ess/reflectometry/__init__.py index c74ec66c..bdf2c585 100644 --- a/src/ess/reflectometry/__init__.py +++ b/src/ess/reflectometry/__init__.py @@ -12,7 +12,7 @@ from . import conversions, corrections, figures, normalization, orso from .load import load_reference, save_reference -from .tools import DataTable, WorkflowCollection, batch_processor +from .tools import WorkflowCollection, batch_processor providers = ( *corrections.providers, @@ -33,7 +33,6 @@ del importlib __all__ = [ - "DataTable", "WorkflowCollection", "__version__", "batch_processor", diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 32d8656b..940ddf57 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -1,5 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2025 Scipp contributors (https://github.com/scipp) +from __future__ import annotations + import uuid from collections.abc import Callable, Mapping, Sequence from itertools import chain @@ -109,21 +111,21 @@ def linlogspace( return sc.concat(grids, dim) -class DataTable(dict): - def _repr_html_(self): - clean = { - str(tp.__name__) - if hasattr(tp, '__name__') - else str(tp).split('.')[-1]: value - for tp, value in self.items() - } - try: - import pandas as pd +# class DataTable(dict): +# def _repr_html_(self): +# clean = { +# str(tp.__name__) +# if hasattr(tp, '__name__') +# else str(tp).split('.')[-1]: value +# for tp, value in self.items() +# } +# try: +# import pandas as pd - df = pd.DataFrame(clean) - return df._repr_html_() - except ImportError: - return clean._repr_html_() +# df = pd.DataFrame(clean) +# return df._repr_html_() +# except ImportError: +# return clean._repr_html_() class WorkflowCollection: @@ -133,18 +135,18 @@ class WorkflowCollection: It can also be used to set parameters for all workflows in a single shot. """ - def __init__(self, workflow: sl.Pipeline, params: Mapping[Any, Mapping[type, Any]]): + def __init__(self, workflows: Mapping[str, sl.Pipeline]): # self._original_workflow = workflow - self.workflows = [] + self.workflows = workflows - if not isinstance(params, pd.DataFrame): - params = pd.DataFrame(params) + # if not isinstance(params, pd.DataFrame): + # params = pd.DataFrame(params) - for _, row in params.iterrows(): - wf = workflow.copy() - for k, v in row.items(): - wf[k] = v - self.workflows.append(wf) + # for _, row in params.iterrows(): + # wf = workflow.copy() + # for k, v in row.items(): + # wf[k] = v + # self.workflows.append(wf) # for name, parameters in params.items(): # wf = workflow.copy() @@ -156,45 +158,63 @@ def __init__(self, workflow: sl.Pipeline, params: Mapping[Any, Mapping[type, Any # # self.workflows = {name: pl.copy() for name, pl in workflows.items()} def __setitem__(self, key: type, value: Sequence[Any]): - for i, v in enumerate(value): - self.workflows[i][key] = v - # if hasattr(value, 'items'): - # for name, v in value.items(): - # self.workflows[name][key] = v - # else: - # for wf in self.workflows.values(): - # wf[key] = value - - # def __getitem__(self, name: str) -> sl.Pipeline: - # """ """ - # return {key: wf[name] for key, wf in self.workflows.items()} - - def compute(self, target: type | Sequence[type], **kwargs) -> Mapping[str, Any]: + # for i, v in enumerate(value): + # self.workflows[i][key] = v + if hasattr(value, 'items'): + for name, v in value.items(): + self.workflows[name][key] = v + else: + for wf in self.workflows.values(): + wf[key] = value + + def __getitem__(self, name: str) -> sl.Pipeline: + """ """ + return WorkflowCollection({k: wf[name] for k, wf in self.workflows.items()}) + + def compute(self, targets: type | Sequence[type], **kwargs) -> Mapping[str, Any]: # return { # name: wf.compute(target, **kwargs) for name, wf in self.workflows.items() # } - if not isinstance(target, list | tuple): - target = [target] + if not isinstance(targets, list | tuple): + targets = [targets] out = {} - for t in target: - # out[t] = { - # name: wf.compute(t, **kwargs) for name, wf in self.workflows.items() - # } - out[t] = [wf.compute(t, **kwargs) for wf in self.workflows] + for t in targets: + out[t] = { + name: wf.compute(t, **kwargs) for name, wf in self.workflows.items() + } + # out[t] = [wf.compute(t, **kwargs) for wf in self.workflows] # return pd.DataFrame(out), out - return DataTable(out) + return next(iter(out.values())) if len(out) == 1 else out - def copy(self) -> 'WorkflowCollection': + def copy(self) -> WorkflowCollection: # out = self.__class__(sl.Pipeline(), params={}) # for name, wf in self.workflows.items(): # out.workflows[name] = wf.copy() # return out - out = WorkflowCollection(None, {}) - out.workflows = [wf.copy() for wf in self.workflows] - return out - - def groupby(self, key: type, reduce: Callable) -> 'WorkflowCollection': - results = self.compute(key) + return WorkflowCollection({k: wf.copy() for k, wf in self.workflows.items()}) + + def visualize(self, targets: type | Sequence[type], **kwargs) -> None: + """ + Visualize all workflows in the collection. + + Parameters + ---------- + targets : type | Sequence[type] + The target type(s) to visualize. + **kwargs: + Additional keyword arguments passed to `sciline.Pipeline.visualize`. + """ + # merge all the graphviz Digraphs into a single one + graphs = [wf.visualize(targets, **kwargs) for wf in self.workflows.values()] + from graphviz import Digraph + + combined = Digraph() + for g in graphs: + combined.body.extend(g.body) + return combined + + # def groupby(self, key: type, reduce: Callable) -> 'WorkflowCollection': + # results = self.compute(key) # def keys(self) -> Sequence[str]: # return self.workflows.keys() @@ -509,11 +529,18 @@ def batch_processor( if Filename[SampleRun] in parameters: if isinstance(parameters[Filename[SampleRun]], list | tuple): - wf = with_filenames( - wf, - SampleRun, - parameters[Filename[SampleRun]], - ) + # axis_name = f'{str(runtype).lower()}_runs' + df = pd.DataFrame( + {Filename[SampleRun]: parameters[Filename[SampleRun]]} + ) # .rename_axis() + # wf = workflow.copy() + + wf = wf.map(df) + # wf = with_filenames( + # wf, + # SampleRun, + # parameters[Filename[SampleRun]], + # ) else: wf[Filename[SampleRun]] = parameters[Filename[SampleRun]] workflows[name] = wf From 89facf51e99ed24b2a76768dd8aab7e9f18371ce Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Tue, 16 Sep 2025 23:15:40 +0200 Subject: [PATCH 39/75] fix collection API, also with visualize --- src/ess/reflectometry/tools.py | 70 ++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 12 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 940ddf57..a388445d 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -128,6 +128,19 @@ def linlogspace( # return clean._repr_html_() +class MultiGraphViz: + """ + A dummy class to concatenate multiple graphviz visualizations into a single repr + output for Jupyter notebooks. + """ + + def __init__(self, graphs: Sequence): + self.graphs = graphs + + def _repr_html_(self) -> str: + return "".join(g._repr_image_svg_xml() for g in self.graphs) + + class WorkflowCollection: """ A collection of sciline workflows that can be used to compute multiple @@ -167,7 +180,7 @@ def __setitem__(self, key: type, value: Sequence[Any]): for wf in self.workflows.values(): wf[key] = value - def __getitem__(self, name: str) -> sl.Pipeline: + def __getitem__(self, name: str) -> WorkflowCollection: """ """ return WorkflowCollection({k: wf[name] for k, wf in self.workflows.items()}) @@ -179,9 +192,12 @@ def compute(self, targets: type | Sequence[type], **kwargs) -> Mapping[str, Any] targets = [targets] out = {} for t in targets: - out[t] = { - name: wf.compute(t, **kwargs) for name, wf in self.workflows.items() - } + out[t] = {} + for name, wf in self.workflows.items(): + if sl.is_mapped_node(wf, t): + out[t][name] = sl.compute_mapped(wf, t, **kwargs) + else: + out[t][name] = wf.compute(t, **kwargs) # out[t] = [wf.compute(t, **kwargs) for wf in self.workflows] # return pd.DataFrame(out), out return next(iter(out.values())) if len(out) == 1 else out @@ -193,7 +209,7 @@ def copy(self) -> WorkflowCollection: # return out return WorkflowCollection({k: wf.copy() for k, wf in self.workflows.items()}) - def visualize(self, targets: type | Sequence[type], **kwargs) -> None: + def visualize(self, targets: type | Sequence[type], **kwargs) -> MultiGraphViz: """ Visualize all workflows in the collection. @@ -204,14 +220,34 @@ def visualize(self, targets: type | Sequence[type], **kwargs) -> None: **kwargs: Additional keyword arguments passed to `sciline.Pipeline.visualize`. """ - # merge all the graphviz Digraphs into a single one - graphs = [wf.visualize(targets, **kwargs) for wf in self.workflows.values()] from graphviz import Digraph - combined = Digraph() - for g in graphs: - combined.body.extend(g.body) - return combined + # Place all the graphviz Digraphs side by side into a single one. + if not isinstance(targets, list | tuple): + targets = [targets] + graphs = [] + for key, wf in self.workflows.items(): + v = wf.visualize(targets, **kwargs) + g = Digraph() + with g.subgraph(name=f"cluster_{key}") as c: + c.attr(label=key, style="rounded", color="black") + c.body.extend(v.body) + + graphs.append(g) + + return MultiGraphViz(graphs) + + # master = Digraph(comment="All Graphs Side by Side") + # # master.attr(rankdir="LR") # Left-to-right layout + + # for name, sub in graphs.items(): + # with master.subgraph(name=f"cluster_{name}") as c: + # c.attr(label=name, style="rounded", color="black") + # c.body.extend(sub.body) + + # return master + + # # def groupby(self, key: type, reduce: Callable) -> 'WorkflowCollection': # results = self.compute(key) @@ -470,6 +506,12 @@ def combine_curves( ) +def _concatenate_event_lists(*das): + da = sc.reduce(das).bins.concat() + missing_coords = set(das[0].coords) - set(da.coords) + return da.assign_coords({coord: das[0].coords[coord] for coord in missing_coords}) + + def batch_processor( workflow: sl.Pipeline, runs: Mapping[Any, Mapping[type, Any]] ) -> WorkflowCollection: @@ -535,7 +577,11 @@ def batch_processor( ) # .rename_axis() # wf = workflow.copy() - wf = wf.map(df) + mapped = wf.map(df) + + wf[UnscaledReducibleData[SampleRun]] = mapped[ + UnscaledReducibleData[SampleRun] + ].reduce(func=_concatenate_event_lists) # wf = with_filenames( # wf, # SampleRun, From efd90aa694c6510be86ea7eadf875cb54ff330db Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Tue, 16 Sep 2025 23:22:01 +0200 Subject: [PATCH 40/75] need to fix output format of compute mapped --- src/ess/reflectometry/tools.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index a388445d..71124517 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -196,6 +196,10 @@ def compute(self, targets: type | Sequence[type], **kwargs) -> Mapping[str, Any] for name, wf in self.workflows.items(): if sl.is_mapped_node(wf, t): out[t][name] = sl.compute_mapped(wf, t, **kwargs) + # results = sl.compute_mapped(wf, t, **kwargs) + # # results = self.workflow.compute(targets, **kwargs) + # for node, v in results.items(): + # out[key][node.index.values[0]] = v else: out[t][name] = wf.compute(t, **kwargs) # out[t] = [wf.compute(t, **kwargs) for wf in self.workflows] From 044a4d115d53428c0cd1e6ca34d06b05209655cc Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Wed, 17 Sep 2025 17:18:08 +0200 Subject: [PATCH 41/75] fix batch compute for mapped/reduced nodes --- src/ess/reflectometry/tools.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 71124517..06e339a1 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -194,14 +194,24 @@ def compute(self, targets: type | Sequence[type], **kwargs) -> Mapping[str, Any] for t in targets: out[t] = {} for name, wf in self.workflows.items(): - if sl.is_mapped_node(wf, t): - out[t][name] = sl.compute_mapped(wf, t, **kwargs) - # results = sl.compute_mapped(wf, t, **kwargs) - # # results = self.workflow.compute(targets, **kwargs) - # for node, v in results.items(): - # out[key][node.index.values[0]] = v - else: + try: out[t][name] = wf.compute(t, **kwargs) + except sl.UnsatisfiedRequirement as e: + try: + out[t][name] = sl.compute_mapped( + wf, t, **kwargs + ).values.tolist() + except (sl.UnsatisfiedRequirement, ValueError): + # ValueError is raised when the requested type is not mapped + raise e from e + # if sl.is_mapped_node(wf, t): + # out[t][name] = sl.compute_mapped(wf, t, **kwargs).values.tolist() + # # results = sl.compute_mapped(wf, t, **kwargs) + # # # results = self.workflow.compute(targets, **kwargs) + # # for node, v in results.items(): + # # out[key][node.index.values[0]] = v + # else: + # out[t][name] = wf.compute(t, **kwargs) # out[t] = [wf.compute(t, **kwargs) for wf in self.workflows] # return pd.DataFrame(out), out return next(iter(out.values())) if len(out) == 1 else out @@ -232,7 +242,7 @@ def visualize(self, targets: type | Sequence[type], **kwargs) -> MultiGraphViz: graphs = [] for key, wf in self.workflows.items(): v = wf.visualize(targets, **kwargs) - g = Digraph() + g = Digraph(**kwargs) with g.subgraph(name=f"cluster_{key}") as c: c.attr(label=key, style="rounded", color="black") c.body.extend(v.body) @@ -449,8 +459,10 @@ def cost(scaling_factors): sol = opt.minimize(cost, [1.0] * (len(curves) - 1)) scaling_factors = (1.0, *map(float, sol.x)) + original_factors = wfc.compute(ScalingFactorForOverlap[SampleRun]) + wfc[ScalingFactorForOverlap[SampleRun]] = { - k: v + k: v * original_factors[k] for k, v in zip(curves.keys(), scaling_factors, strict=True) if k != critical_edge_key } From e2ba0307eee1eceeb94bc802043b816a418e274c Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Wed, 17 Sep 2025 17:19:24 +0200 Subject: [PATCH 42/75] static --- docs/user-guide/amor/amor-reduction.ipynb | 80 ++++++++++++++++++++++- src/ess/reflectometry/tools.py | 3 +- 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/docs/user-guide/amor/amor-reduction.ipynb b/docs/user-guide/amor/amor-reduction.ipynb index 18d9224e..44cb620a 100644 --- a/docs/user-guide/amor/amor-reduction.ipynb +++ b/docs/user-guide/amor/amor-reduction.ipynb @@ -172,6 +172,7 @@ " '609': {\n", " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", " Filename[SampleRun]: amor.data.amor_run(609),\n", + " ScalingFactorForOverlap[SampleRun]: 1.0,\n", " },\n", " '610': {\n", " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", @@ -184,7 +185,6 @@ "}\n", "\n", "batch = batch_processor(workflow, runs)\n", - "display(batch.keys())\n", "\n", "# Compute R(Q) for all runs\n", "reflectivity = batch.compute(ReflectivityOverQ)\n", @@ -194,6 +194,15 @@ ")" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "batch.visualize(ReflectivityOverQ, graph_attr={'rankdir': 'LR'})" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -475,6 +484,75 @@ "source": [ "!head amor_reduced_iofq.ort -n50" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Merging multiple runs for the same sample rotation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "runs = {\n", + " '608': {\n", + " # The sample rotation values in the files are slightly off, so we replace\n", + " # them with corrected values.\n", + " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", + " Filename[SampleRun]: amor.data.amor_run(608),\n", + " },\n", + " '609': {\n", + " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", + " Filename[SampleRun]: amor.data.amor_run(609),\n", + " },\n", + " '610': {\n", + " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", + " Filename[SampleRun]: amor.data.amor_run(610),\n", + " },\n", + " '611+612': {\n", + " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", + " Filename[SampleRun]: (amor.data.amor_run(611), amor.data.amor_run(612)), # List of files means their event lists should be merged\n", + " },\n", + "}\n", + "\n", + "batch = batch_processor(workflow, runs)\n", + "\n", + "# Compute R(Q) for all runs\n", + "reflectivity = batch.compute(ReflectivityOverQ)\n", + "sc.plot(\n", + " {key: r.hist() for key, r in reflectivity.items()},\n", + " norm='log', vmin=1e-4\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "batch.visualize(ReflectivityOverQ, graph_attr={'rankdir': 'LR'})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "scaled_wf = scale_reflectivity_curves_to_overlap(\n", + " batch,\n", + " critical_edge_interval=(sc.scalar(0.01, unit='1/angstrom'), sc.scalar(0.014, unit='1/angstrom'))\n", + ")\n", + "\n", + "scaled_r = {key: r.hist() for key, r in scaled_wf.compute(ReflectivityOverQ).items()}\n", + "\n", + "sc.plot(scaled_r, norm='log', vmin=1e-5)" + ] } ], "metadata": { diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 06e339a1..e79144dd 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -3,7 +3,7 @@ from __future__ import annotations import uuid -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Mapping, Sequence from itertools import chain from typing import Any @@ -22,7 +22,6 @@ ScalingFactorForOverlap, UnscaledReducibleData, ) -from ess.reflectometry.workflow import with_filenames _STD_TO_FWHM = sc.scalar(2.0) * sc.sqrt(sc.scalar(2.0) * sc.log(sc.scalar(2.0))) From f0164747bcd6fe9aef7c5a66915c4ba21bd884c8 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Wed, 17 Sep 2025 17:20:43 +0200 Subject: [PATCH 43/75] cleanup --- src/ess/reflectometry/tools.py | 102 +-------------------------------- 1 file changed, 1 insertion(+), 101 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index e79144dd..936dcf8a 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -110,23 +110,6 @@ def linlogspace( return sc.concat(grids, dim) -# class DataTable(dict): -# def _repr_html_(self): -# clean = { -# str(tp.__name__) -# if hasattr(tp, '__name__') -# else str(tp).split('.')[-1]: value -# for tp, value in self.items() -# } -# try: -# import pandas as pd - -# df = pd.DataFrame(clean) -# return df._repr_html_() -# except ImportError: -# return clean._repr_html_() - - class MultiGraphViz: """ A dummy class to concatenate multiple graphviz visualizations into a single repr @@ -148,30 +131,9 @@ class WorkflowCollection: """ def __init__(self, workflows: Mapping[str, sl.Pipeline]): - # self._original_workflow = workflow self.workflows = workflows - # if not isinstance(params, pd.DataFrame): - # params = pd.DataFrame(params) - - # for _, row in params.iterrows(): - # wf = workflow.copy() - # for k, v in row.items(): - # wf[k] = v - # self.workflows.append(wf) - - # for name, parameters in params.items(): - # wf = workflow.copy() - # for tp, value in parameters.items(): - # # if tp is Filename[SampleRun]: - # # continue - # wf[tp] = value - # self.workflows[name] = wf - # # self.workflows = {name: pl.copy() for name, pl in workflows.items()} - def __setitem__(self, key: type, value: Sequence[Any]): - # for i, v in enumerate(value): - # self.workflows[i][key] = v if hasattr(value, 'items'): for name, v in value.items(): self.workflows[name][key] = v @@ -184,9 +146,6 @@ def __getitem__(self, name: str) -> WorkflowCollection: return WorkflowCollection({k: wf[name] for k, wf in self.workflows.items()}) def compute(self, targets: type | Sequence[type], **kwargs) -> Mapping[str, Any]: - # return { - # name: wf.compute(target, **kwargs) for name, wf in self.workflows.items() - # } if not isinstance(targets, list | tuple): targets = [targets] out = {} @@ -203,23 +162,9 @@ def compute(self, targets: type | Sequence[type], **kwargs) -> Mapping[str, Any] except (sl.UnsatisfiedRequirement, ValueError): # ValueError is raised when the requested type is not mapped raise e from e - # if sl.is_mapped_node(wf, t): - # out[t][name] = sl.compute_mapped(wf, t, **kwargs).values.tolist() - # # results = sl.compute_mapped(wf, t, **kwargs) - # # # results = self.workflow.compute(targets, **kwargs) - # # for node, v in results.items(): - # # out[key][node.index.values[0]] = v - # else: - # out[t][name] = wf.compute(t, **kwargs) - # out[t] = [wf.compute(t, **kwargs) for wf in self.workflows] - # return pd.DataFrame(out), out return next(iter(out.values())) if len(out) == 1 else out def copy(self) -> WorkflowCollection: - # out = self.__class__(sl.Pipeline(), params={}) - # for name, wf in self.workflows.items(): - # out.workflows[name] = wf.copy() - # return out return WorkflowCollection({k: wf.copy() for k, wf in self.workflows.items()}) def visualize(self, targets: type | Sequence[type], **kwargs) -> MultiGraphViz: @@ -250,42 +195,6 @@ def visualize(self, targets: type | Sequence[type], **kwargs) -> MultiGraphViz: return MultiGraphViz(graphs) - # master = Digraph(comment="All Graphs Side by Side") - # # master.attr(rankdir="LR") # Left-to-right layout - - # for name, sub in graphs.items(): - # with master.subgraph(name=f"cluster_{name}") as c: - # c.attr(label=name, style="rounded", color="black") - # c.body.extend(sub.body) - - # return master - - # - - # def groupby(self, key: type, reduce: Callable) -> 'WorkflowCollection': - # results = self.compute(key) - - # def keys(self) -> Sequence[str]: - # return self.workflows.keys() - - # def values(self) -> Sequence[sl.Pipeline]: - # return self.workflows.values() - - # def items(self) -> Sequence[tuple[str, sl.Pipeline]]: - # return self.workflows.items() - - # def add(self, name: str, workflow: sl.Pipeline): - # """ - # Adds a new workflow to the collection. - # """ - # self.workflows[name] = workflow.copy() - - # def remove(self, name: str): - # """ - # Removes a workflow from the collection by its name. - # """ - # del self.workflows[name] - def _sort_by(a, by): return [x for x, _ in sorted(zip(a, by, strict=True), key=lambda x: x[1])] @@ -586,22 +495,13 @@ def batch_processor( if Filename[SampleRun] in parameters: if isinstance(parameters[Filename[SampleRun]], list | tuple): - # axis_name = f'{str(runtype).lower()}_runs' df = pd.DataFrame( {Filename[SampleRun]: parameters[Filename[SampleRun]]} - ) # .rename_axis() - # wf = workflow.copy() - + ) mapped = wf.map(df) - wf[UnscaledReducibleData[SampleRun]] = mapped[ UnscaledReducibleData[SampleRun] ].reduce(func=_concatenate_event_lists) - # wf = with_filenames( - # wf, - # SampleRun, - # parameters[Filename[SampleRun]], - # ) else: wf[Filename[SampleRun]] = parameters[Filename[SampleRun]] workflows[name] = wf From 4f28af595918297d582787474e4d87f14c3da838 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Wed, 17 Sep 2025 20:56:49 +0200 Subject: [PATCH 44/75] revert changes in with_filenames --- src/ess/reflectometry/workflow.py | 69 +++++++++---------------------- 1 file changed, 20 insertions(+), 49 deletions(-) diff --git a/src/ess/reflectometry/workflow.py b/src/ess/reflectometry/workflow.py index 8a203eb5..2c0fa9be 100644 --- a/src/ess/reflectometry/workflow.py +++ b/src/ess/reflectometry/workflow.py @@ -6,6 +6,7 @@ import sciline import scipp as sc +from ess.amor.types import RawChopper from ess.reflectometry.orso import ( OrsoExperiment, OrsoOwner, @@ -13,13 +14,11 @@ OrsoSampleFilenames, ) from ess.reflectometry.types import ( - DetectorRotation, Filename, - RawChopper, + ReducibleData, RunType, SampleRotation, SampleRun, - UnscaledReducibleData, ) @@ -63,54 +62,26 @@ def with_filenames( mapped = wf.map(df) - try: - wf[UnscaledReducibleData[runtype]] = mapped[ - UnscaledReducibleData[runtype] - ].reduce(index=axis_name, func=_concatenate_event_lists) - except (ValueError, KeyError): - # UnscaledReducibleData[runtype] is independent of Filename[runtype] or is not - # present in the workflow. - pass - try: - wf[RawChopper[runtype]] = mapped[RawChopper[runtype]].reduce( - index=axis_name, func=_any_value - ) - except (ValueError, KeyError): - # RawChopper[runtype] is independent of Filename[runtype] or is not - # present in the workflow. - pass - try: - wf[SampleRotation[runtype]] = mapped[SampleRotation[runtype]].reduce( - index=axis_name, func=_any_value - ) - except (ValueError, KeyError): - # SampleRotation[runtype] is independent of Filename[runtype] or is not - # present in the workflow. - pass - try: - wf[DetectorRotation[runtype]] = mapped[DetectorRotation[runtype]].reduce( - index=axis_name, func=_any_value - ) - except (ValueError, KeyError): - # DetectorRotation[runtype] is independent of Filename[runtype] or is not - # present in the workflow. - pass + wf[ReducibleData[runtype]] = mapped[ReducibleData[runtype]].reduce( + index=axis_name, func=_concatenate_event_lists + ) + wf[RawChopper[runtype]] = mapped[RawChopper[runtype]].reduce( + index=axis_name, func=_any_value + ) + wf[SampleRotation[runtype]] = mapped[SampleRotation[runtype]].reduce( + index=axis_name, func=_any_value + ) if runtype is SampleRun: - if OrsoSample in wf.underlying_graph: - wf[OrsoSample] = mapped[OrsoSample].reduce(index=axis_name, func=_any_value) - if OrsoExperiment in wf.underlying_graph: - wf[OrsoExperiment] = mapped[OrsoExperiment].reduce( - index=axis_name, func=_any_value - ) - if OrsoOwner in wf.underlying_graph: - wf[OrsoOwner] = mapped[OrsoOwner].reduce( - index=axis_name, func=lambda x, *_: x - ) - if OrsoSampleFilenames in wf.underlying_graph: + wf[OrsoSample] = mapped[OrsoSample].reduce(index=axis_name, func=_any_value) + wf[OrsoExperiment] = mapped[OrsoExperiment].reduce( + index=axis_name, func=_any_value + ) + wf[OrsoOwner] = mapped[OrsoOwner].reduce(index=axis_name, func=lambda x, *_: x) + wf[OrsoSampleFilenames] = mapped[OrsoSampleFilenames].reduce( # When we don't map over filenames # each OrsoSampleFilenames is a list with a single entry. - wf[OrsoSampleFilenames] = mapped[OrsoSampleFilenames].reduce( - index=axis_name, func=_concatenate_lists - ) + index=axis_name, + func=_concatenate_lists, + ) return wf From 4491627ec5871327fbdeb3aa3ba7ac44387c1f96 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Wed, 17 Sep 2025 21:11:13 +0200 Subject: [PATCH 45/75] fix tests --- src/ess/reflectometry/tools.py | 33 ++++++++---- .../reflectometry/workflow_collection_test.py | 52 +------------------ 2 files changed, 25 insertions(+), 60 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 936dcf8a..ea700bca 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -142,10 +142,19 @@ def __setitem__(self, key: type, value: Sequence[Any]): wf[key] = value def __getitem__(self, name: str) -> WorkflowCollection: - """ """ return WorkflowCollection({k: wf[name] for k, wf in self.workflows.items()}) def compute(self, targets: type | Sequence[type], **kwargs) -> Mapping[str, Any]: + """ + Compute the given target(s) for all workflows in the collection. + + Parameters + ---------- + targets: + The target type(s) to compute. + **kwargs: + Additional keyword arguments passed to `sciline.Pipeline.compute`. + """ if not isinstance(targets, list | tuple): targets = [targets] out = {} @@ -165,6 +174,9 @@ def compute(self, targets: type | Sequence[type], **kwargs) -> Mapping[str, Any] return next(iter(out.values())) if len(out) == 1 else out def copy(self) -> WorkflowCollection: + """ + Create a copy of the workflow collection. + """ return WorkflowCollection({k: wf.copy() for k, wf in self.workflows.items()}) def visualize(self, targets: type | Sequence[type], **kwargs) -> MultiGraphViz: @@ -291,15 +303,16 @@ def scale_reflectivity_curves_to_overlap( : A list of scaled reflectivity curves and a list of the scaling factors. ''' - if isinstance(workflows, sl.Pipeline): + is_single_workflow = isinstance(workflows, sl.Pipeline) + if is_single_workflow: # If a single workflow is provided, convert it to a collection - wfc = WorkflowCollection({"": workflows}) - out = scale_reflectivity_curves_to_overlap( - wfc, - critical_edge_interval=critical_edge_interval, - cache_intermediate_results=cache_intermediate_results, - ) - return out[""] + workflows = WorkflowCollection({"": workflows}) + # out = scale_reflectivity_curves_to_overlap( + # wfc, + # critical_edge_interval=critical_edge_interval, + # cache_intermediate_results=cache_intermediate_results, + # ) + # return out[""] wfc = workflows.copy() if cache_intermediate_results: @@ -375,7 +388,7 @@ def cost(scaling_factors): if k != critical_edge_key } - return wfc + return wfc.workflows[""] if is_single_workflow else wfc def combine_curves( diff --git a/tests/reflectometry/workflow_collection_test.py b/tests/reflectometry/workflow_collection_test.py index bb1ff529..42adb79c 100644 --- a/tests/reflectometry/workflow_collection_test.py +++ b/tests/reflectometry/workflow_collection_test.py @@ -36,8 +36,8 @@ def test_compute_multiple() -> None: result = coll.compute([float, str]) - assert result['a'] == {float: 1.5, str: '3;1.5'} - assert result['b'] == {float: 2.0, str: '4;2.0'} + assert result[float] == {'a': 1.5, 'b': 2.0} + assert result[str] == {'a': '3;1.5', 'b': '4;2.0'} def test_setitem_mapping() -> None: @@ -86,51 +86,3 @@ def test_copy() -> None: assert coll.compute(str) == {'a': '3;1.5', 'b': '4;2.0'} assert coll_copy.compute(float) == {'a': 3.5, 'b': 4.0} assert coll_copy.compute(str) == {'a': '7;3.5', 'b': '8;4.0'} - - -def test_add_workflow() -> None: - wf = sl.Pipeline([int_to_float, int_float_to_str]) - wfa = wf.copy() - wfa[int] = 3 - wfb = wf.copy() - wfb[int] = 4 - coll = WorkflowCollection({'a': wfa, 'b': wfb}) - - wfc = wf.copy() - wfc[int] = 5 - coll.add('c', wfc) - - assert coll.compute(float) == {'a': 1.5, 'b': 2.0, 'c': 2.5} - assert coll.compute(str) == {'a': '3;1.5', 'b': '4;2.0', 'c': '5;2.5'} - - -def test_add_workflow_with_existing_key() -> None: - wf = sl.Pipeline([int_to_float, int_float_to_str]) - wfa = wf.copy() - wfa[int] = 3 - wfb = wf.copy() - wfb[int] = 4 - coll = WorkflowCollection({'a': wfa, 'b': wfb}) - - wfc = wf.copy() - wfc[int] = 5 - coll.add('a', wfc) - - assert coll.compute(float) == {'a': 2.5, 'b': 2.0} - assert coll.compute(str) == {'a': '5;2.5', 'b': '4;2.0'} - assert 'c' not in coll.keys() # 'c' should not exist - - -def test_remove_workflow() -> None: - wf = sl.Pipeline([int_to_float, int_float_to_str]) - wfa = wf.copy() - wfa[int] = 3 - wfb = wf.copy() - wfb[int] = 4 - coll = WorkflowCollection({'a': wfa, 'b': wfb}) - - coll.remove('b') - - assert 'b' not in coll.keys() - assert coll.compute(float) == {'a': 1.5} - assert coll.compute(str) == {'a': '3;1.5'} From 536e886ed75a3c144fb5c813c6e187167267806c Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Wed, 17 Sep 2025 21:46:44 +0200 Subject: [PATCH 46/75] fix more tests --- src/ess/reflectometry/tools.py | 4 ++-- src/ess/reflectometry/workflow.py | 14 +++++++------- tests/amor/pipeline_test.py | 19 +++++++------------ 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index ea700bca..e97c6596 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -320,13 +320,13 @@ def scale_reflectivity_curves_to_overlap( wfc[UnscaledReducibleData[SampleRun]] = wfc.compute( UnscaledReducibleData[SampleRun] ) - except sl.UnsatisfiedRequirement: + except (sl.UnsatisfiedRequirement, NotImplementedError): pass try: wfc[UnscaledReducibleData[ReferenceRun]] = wfc.compute( UnscaledReducibleData[ReferenceRun] ) - except sl.UnsatisfiedRequirement: + except (sl.UnsatisfiedRequirement, NotImplementedError): pass reflectivities = wfc.compute(ReflectivityOverQ) diff --git a/src/ess/reflectometry/workflow.py b/src/ess/reflectometry/workflow.py index 2c0fa9be..b9f77891 100644 --- a/src/ess/reflectometry/workflow.py +++ b/src/ess/reflectometry/workflow.py @@ -6,7 +6,6 @@ import sciline import scipp as sc -from ess.amor.types import RawChopper from ess.reflectometry.orso import ( OrsoExperiment, OrsoOwner, @@ -15,6 +14,7 @@ ) from ess.reflectometry.types import ( Filename, + RawChopper, ReducibleData, RunType, SampleRotation, @@ -65,12 +65,12 @@ def with_filenames( wf[ReducibleData[runtype]] = mapped[ReducibleData[runtype]].reduce( index=axis_name, func=_concatenate_event_lists ) - wf[RawChopper[runtype]] = mapped[RawChopper[runtype]].reduce( - index=axis_name, func=_any_value - ) - wf[SampleRotation[runtype]] = mapped[SampleRotation[runtype]].reduce( - index=axis_name, func=_any_value - ) + # wf[RawChopper[runtype]] = mapped[RawChopper[runtype]].reduce( + # index=axis_name, func=_any_value + # ) + # wf[SampleRotation[runtype]] = mapped[SampleRotation[runtype]].reduce( + # index=axis_name, func=_any_value + # ) if runtype is SampleRun: wf[OrsoSample] = mapped[OrsoSample].reduce(index=axis_name, func=_any_value) diff --git a/tests/amor/pipeline_test.py b/tests/amor/pipeline_test.py index 9b5f01b3..50ecb3ba 100644 --- a/tests/amor/pipeline_test.py +++ b/tests/amor/pipeline_test.py @@ -157,10 +157,9 @@ def test_pipeline_can_compute_reflectivity_merging_events_from_multiple_runs( amor.data.amor_run(608), amor.data.amor_run(609), ] - pipeline = with_filenames(amor_pipeline, SampleRun, sample_runs) - pipeline[SampleRotation[SampleRun]] = pipeline.compute( - SampleRotation[SampleRun] - ) + sc.scalar(0.05, unit="deg") + wf = amor_pipeline.copy() + wf[SampleRotationOffset[SampleRun]] = sc.scalar(0.05, unit="deg") + pipeline = with_filenames(wf, SampleRun, sample_runs) result = pipeline.compute(ReflectivityOverQ) assert result.dims == ('Q',) @@ -168,22 +167,18 @@ def test_pipeline_can_compute_reflectivity_merging_events_from_multiple_runs( @pytest.mark.filterwarnings("ignore:Failed to convert .* into a transformation") @pytest.mark.filterwarnings("ignore:Invalid transformation, missing attribute") def test_pipeline_merging_events_result_unchanged(amor_pipeline: sciline.Pipeline): + wf = amor_pipeline.copy() + wf[SampleRotationOffset[SampleRun]] = sc.scalar(0.05, unit="deg") sample_runs = [ amor.data.amor_run(608), ] - pipeline = with_filenames(amor_pipeline, SampleRun, sample_runs) - pipeline[SampleRotation[SampleRun]] = pipeline.compute( - SampleRotation[SampleRun] - ) + sc.scalar(0.05, unit="deg") + pipeline = with_filenames(wf, SampleRun, sample_runs) result = pipeline.compute(ReflectivityOverQ).hist() sample_runs = [ amor.data.amor_run(608), amor.data.amor_run(608), ] - pipeline = with_filenames(amor_pipeline, SampleRun, sample_runs) - pipeline[SampleRotation[SampleRun]] = pipeline.compute( - SampleRotation[SampleRun] - ) + sc.scalar(0.05, unit="deg") + pipeline = with_filenames(wf, SampleRun, sample_runs) result2 = pipeline.compute(ReflectivityOverQ).hist() assert_allclose( 2 * sc.values(result.data), sc.values(result2.data), rtol=sc.scalar(1e-6) From 4b5c6458a563b0f62ace6e36cd38b9b07b3ec5df Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Wed, 17 Sep 2025 22:12:50 +0200 Subject: [PATCH 47/75] fix amor notebooks --- docs/user-guide/amor/amor-reduction.ipynb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/user-guide/amor/amor-reduction.ipynb b/docs/user-guide/amor/amor-reduction.ipynb index 05e8053b..79097528 100644 --- a/docs/user-guide/amor/amor-reduction.ipynb +++ b/docs/user-guide/amor/amor-reduction.ipynb @@ -246,7 +246,9 @@ "outputs": [], "source": [ "from ess.reflectometry.tools import combine_curves\n", - "combined = combine_curves(scaled_r.values(), workflow.compute(QBins))\n", + "\n", + "qbins = sc.geomspace('Q', 0.005, 0.4, 501, unit='1/angstrom')\n", + "combined = combine_curves(scaled_r.values(), qbins)\n", "combined.plot(norm='log')" ] }, @@ -329,7 +331,7 @@ "q_theta_figure(\n", " diagnostics.values(),\n", " theta_bins=scaled_wf.compute(ThetaBins[SampleRun]).values(),\n", - " q_bins=workflow.compute(QBins)\n", + " q_bins=qbins\n", ")" ] }, @@ -414,7 +416,7 @@ "metadata": {}, "outputs": [], "source": [ - "scaled_wf['608'].visualize(orso.OrsoIofQDataset, graph_attr={'rankdir': 'LR'})" + "scaled_wf.workflows['608'].visualize(orso.OrsoIofQDataset, graph_attr={'rankdir': 'LR'})" ] }, { From 09362fdd8f604b6c66f21c18129cedbe87aefb59 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Wed, 17 Sep 2025 22:37:38 +0200 Subject: [PATCH 48/75] remvove commented code --- src/ess/reflectometry/tools.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index e97c6596..12a9e272 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -307,12 +307,6 @@ def scale_reflectivity_curves_to_overlap( if is_single_workflow: # If a single workflow is provided, convert it to a collection workflows = WorkflowCollection({"": workflows}) - # out = scale_reflectivity_curves_to_overlap( - # wfc, - # critical_edge_interval=critical_edge_interval, - # cache_intermediate_results=cache_intermediate_results, - # ) - # return out[""] wfc = workflows.copy() if cache_intermediate_results: From 42aa8cba75bfc7e7ef5e865c81883416f34f6f1d Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Wed, 17 Sep 2025 22:39:59 +0200 Subject: [PATCH 49/75] more cleanup --- src/ess/reflectometry/workflow.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/ess/reflectometry/workflow.py b/src/ess/reflectometry/workflow.py index b9f77891..44ffefb4 100644 --- a/src/ess/reflectometry/workflow.py +++ b/src/ess/reflectometry/workflow.py @@ -65,12 +65,6 @@ def with_filenames( wf[ReducibleData[runtype]] = mapped[ReducibleData[runtype]].reduce( index=axis_name, func=_concatenate_event_lists ) - # wf[RawChopper[runtype]] = mapped[RawChopper[runtype]].reduce( - # index=axis_name, func=_any_value - # ) - # wf[SampleRotation[runtype]] = mapped[SampleRotation[runtype]].reduce( - # index=axis_name, func=_any_value - # ) if runtype is SampleRun: wf[OrsoSample] = mapped[OrsoSample].reduce(index=axis_name, func=_any_value) From 7212ad6e3ed4902ca9dab7bca77d858a453410a0 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Wed, 17 Sep 2025 23:00:05 +0200 Subject: [PATCH 50/75] remove unused imports --- src/ess/reflectometry/workflow.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/ess/reflectometry/workflow.py b/src/ess/reflectometry/workflow.py index 44ffefb4..129a45e3 100644 --- a/src/ess/reflectometry/workflow.py +++ b/src/ess/reflectometry/workflow.py @@ -12,14 +12,7 @@ OrsoSample, OrsoSampleFilenames, ) -from ess.reflectometry.types import ( - Filename, - RawChopper, - ReducibleData, - RunType, - SampleRotation, - SampleRun, -) +from ess.reflectometry.types import Filename, ReducibleData, RunType, SampleRun def _concatenate_event_lists(*das): From 5d17884783788e9f2e7d931c109751a941985502 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Thu, 18 Sep 2025 10:43:31 +0200 Subject: [PATCH 51/75] remove caching of intermediate results in scaling for overlap function and let the user cache manually --- docs/user-guide/amor/amor-reduction.ipynb | 17 +++++++++++++++-- src/ess/reflectometry/tools.py | 19 ------------------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/docs/user-guide/amor/amor-reduction.ipynb b/docs/user-guide/amor/amor-reduction.ipynb index 79097528..b5e84191 100644 --- a/docs/user-guide/amor/amor-reduction.ipynb +++ b/docs/user-guide/amor/amor-reduction.ipynb @@ -36,7 +36,8 @@ "# The files used in this tutorial have some issues that makes scippnexus\n", "# raise warnings when loading them. To avoid noise in the notebook the warnings are silenced.\n", "warnings.filterwarnings('ignore', 'Failed to convert .* into a transformation')\n", - "warnings.filterwarnings('ignore', 'Invalid transformation')" + "warnings.filterwarnings('ignore', 'Invalid transformation')\n", + "warnings.filterwarnings('ignore', 'invalid value encountered')" ] }, { @@ -190,7 +191,7 @@ "reflectivity = batch.compute(ReflectivityOverQ)\n", "sc.plot(\n", " {key: r.hist() for key, r in reflectivity.items()},\n", - " norm='log', vmin=1e-4\n", + " norm='log', vmin=1e-5\n", ")" ] }, @@ -212,6 +213,18 @@ "In case we know the curves are have been scaled by different factors (that are constant in Q) it can be useful to scale them so they overlap:" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Cache intermediate result to avoid re-loading the data\n", + "batch[UnscaledReducibleData[SampleRun]] = batch.compute(\n", + " UnscaledReducibleData[SampleRun]\n", + ")" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 12a9e272..4d720f67 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -16,7 +16,6 @@ from ess.reflectometry.types import ( Filename, QBins, - ReferenceRun, ReflectivityOverQ, SampleRun, ScalingFactorForOverlap, @@ -270,7 +269,6 @@ def _interpolate_on_qgrid(curves, grid): def scale_reflectivity_curves_to_overlap( workflows: WorkflowCollection | sl.Pipeline, critical_edge_interval: tuple[sc.Variable, sc.Variable] | None = None, - cache_intermediate_results: bool = True, ) -> tuple[list[sc.DataArray], list[sc.Variable]]: ''' Set the ``ScalingFactorForOverlap`` parameter on the provided workflows @@ -294,9 +292,6 @@ def scale_reflectivity_curves_to_overlap( A tuple denoting an interval that is known to belong to the critical edge, i.e. where the reflectivity is known to be 1. - cache_intermediate_results: - If ``True`` the intermediate results ``UnscaledReducibleData`` will be cached - (this is the base for all types that are downstream of the scaling factor). Returns --------- @@ -309,20 +304,6 @@ def scale_reflectivity_curves_to_overlap( workflows = WorkflowCollection({"": workflows}) wfc = workflows.copy() - if cache_intermediate_results: - try: - wfc[UnscaledReducibleData[SampleRun]] = wfc.compute( - UnscaledReducibleData[SampleRun] - ) - except (sl.UnsatisfiedRequirement, NotImplementedError): - pass - try: - wfc[UnscaledReducibleData[ReferenceRun]] = wfc.compute( - UnscaledReducibleData[ReferenceRun] - ) - except (sl.UnsatisfiedRequirement, NotImplementedError): - pass - reflectivities = wfc.compute(ReflectivityOverQ) # First sort the dict of reflectivities by the Q min value From 42e132c99f185cdad87a82f44b96db389471d865 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Thu, 18 Sep 2025 14:10:24 +0200 Subject: [PATCH 52/75] improve graph visualization by stacking SVGs --- src/ess/reflectometry/tools.py | 42 +++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 4d720f67..47380463 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -2,6 +2,7 @@ # Copyright (c) 2025 Scipp contributors (https://github.com/scipp) from __future__ import annotations +import re import uuid from collections.abc import Mapping, Sequence from itertools import chain @@ -113,13 +114,46 @@ class MultiGraphViz: """ A dummy class to concatenate multiple graphviz visualizations into a single repr output for Jupyter notebooks. + This combines the SVG representations of multiple graphs vertically with a small gap + in between. """ def __init__(self, graphs: Sequence): self.graphs = graphs - def _repr_html_(self) -> str: - return "".join(g._repr_image_svg_xml() for g in self.graphs) + def _repr_mimebundle_(self, include=None, exclude=None): + gap = 10 + parsed = [] + for svg in [g._repr_image_svg_xml() for g in self.graphs]: + # extract width, height, and inner content + m = re.search(r'width="([\d.]+)pt".*?height="([\d.]+)pt"', svg, re.S) + w, h = float(m.group(1)), float(m.group(2)) + inner = re.search(r']*>(.*)', svg, re.S).group(1) + parsed.append((w, h, inner)) + + # vertical shift + total_width = max(w for w, _, _ in parsed) + total_height = sum(h for _, h, _ in parsed) + gap * (len(parsed) - 1) + + pieces = [] + offset_x = offset_y = 0 + for _, h, inner in parsed: + pieces.append( + f'{inner}' + ) + offset_y += h + gap + + # TODO: for some reason, combining the svgs seems to scale them down. This + # then means that the computed bounding box is too large. For now, we + # apply a fudge factor of 0.75 to the width and height. It is unclear where + # exactly this comes from. + combined = f''' + + {''.join(pieces)} + + ''' + return {"image/svg+xml": combined} class WorkflowCollection: @@ -197,7 +231,9 @@ def visualize(self, targets: type | Sequence[type], **kwargs) -> MultiGraphViz: graphs = [] for key, wf in self.workflows.items(): v = wf.visualize(targets, **kwargs) - g = Digraph(**kwargs) + g = Digraph( + graph_attr=v.graph_attr, node_attr=v.node_attr, edge_attr=v.edge_attr + ) with g.subgraph(name=f"cluster_{key}") as c: c.attr(label=key, style="rounded", color="black") c.body.extend(v.body) From 872deeee37a60ccf7ffebf84bd354c872f2c0e64 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Thu, 18 Sep 2025 14:18:34 +0200 Subject: [PATCH 53/75] rename to BatchProcessor and fix unit tests --- src/ess/reflectometry/__init__.py | 4 +- src/ess/reflectometry/tools.py | 18 ++-- ...ction_test.py => batch_processor_tests.py} | 12 +-- tests/reflectometry/tools_test.py | 85 +------------------ 4 files changed, 20 insertions(+), 99 deletions(-) rename tests/reflectometry/{workflow_collection_test.py => batch_processor_tests.py} (86%) diff --git a/src/ess/reflectometry/__init__.py b/src/ess/reflectometry/__init__.py index bdf2c585..ee144f33 100644 --- a/src/ess/reflectometry/__init__.py +++ b/src/ess/reflectometry/__init__.py @@ -12,7 +12,7 @@ from . import conversions, corrections, figures, normalization, orso from .load import load_reference, save_reference -from .tools import WorkflowCollection, batch_processor +from .tools import BatchProcessor, batch_processor providers = ( *corrections.providers, @@ -33,7 +33,7 @@ del importlib __all__ = [ - "WorkflowCollection", + "BatchProcessor", "__version__", "batch_processor", "conversions", diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 47380463..b459b0b5 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -156,7 +156,7 @@ def _repr_mimebundle_(self, include=None, exclude=None): return {"image/svg+xml": combined} -class WorkflowCollection: +class BatchProcessor: """ A collection of sciline workflows that can be used to compute multiple targets from multiple workflows. @@ -174,8 +174,8 @@ def __setitem__(self, key: type, value: Sequence[Any]): for wf in self.workflows.values(): wf[key] = value - def __getitem__(self, name: str) -> WorkflowCollection: - return WorkflowCollection({k: wf[name] for k, wf in self.workflows.items()}) + def __getitem__(self, name: str) -> BatchProcessor: + return BatchProcessor({k: wf[name] for k, wf in self.workflows.items()}) def compute(self, targets: type | Sequence[type], **kwargs) -> Mapping[str, Any]: """ @@ -206,11 +206,11 @@ def compute(self, targets: type | Sequence[type], **kwargs) -> Mapping[str, Any] raise e from e return next(iter(out.values())) if len(out) == 1 else out - def copy(self) -> WorkflowCollection: + def copy(self) -> BatchProcessor: """ Create a copy of the workflow collection. """ - return WorkflowCollection({k: wf.copy() for k, wf in self.workflows.items()}) + return BatchProcessor({k: wf.copy() for k, wf in self.workflows.items()}) def visualize(self, targets: type | Sequence[type], **kwargs) -> MultiGraphViz: """ @@ -303,7 +303,7 @@ def _interpolate_on_qgrid(curves, grid): def scale_reflectivity_curves_to_overlap( - workflows: WorkflowCollection | sl.Pipeline, + workflows: BatchProcessor | sl.Pipeline, critical_edge_interval: tuple[sc.Variable, sc.Variable] | None = None, ) -> tuple[list[sc.DataArray], list[sc.Variable]]: ''' @@ -337,7 +337,7 @@ def scale_reflectivity_curves_to_overlap( is_single_workflow = isinstance(workflows, sl.Pipeline) if is_single_workflow: # If a single workflow is provided, convert it to a collection - workflows = WorkflowCollection({"": workflows}) + workflows = BatchProcessor({"": workflows}) wfc = workflows.copy() reflectivities = wfc.compute(ReflectivityOverQ) @@ -462,7 +462,7 @@ def _concatenate_event_lists(*das): def batch_processor( workflow: sl.Pipeline, runs: Mapping[Any, Mapping[type, Any]] -) -> WorkflowCollection: +) -> BatchProcessor: """ Creates a collection of sciline workflows from the provided runs. @@ -529,4 +529,4 @@ def batch_processor( else: wf[Filename[SampleRun]] = parameters[Filename[SampleRun]] workflows[name] = wf - return WorkflowCollection(workflows) + return BatchProcessor(workflows) diff --git a/tests/reflectometry/workflow_collection_test.py b/tests/reflectometry/batch_processor_tests.py similarity index 86% rename from tests/reflectometry/workflow_collection_test.py rename to tests/reflectometry/batch_processor_tests.py index 42adb79c..94a868a0 100644 --- a/tests/reflectometry/workflow_collection_test.py +++ b/tests/reflectometry/batch_processor_tests.py @@ -3,7 +3,7 @@ import sciline as sl -from ess.reflectometry.tools import WorkflowCollection +from ess.reflectometry.tools import BatchProcessor def int_to_float(x: int) -> float: @@ -20,7 +20,7 @@ def test_compute() -> None: wfa[int] = 3 wfb = wf.copy() wfb[int] = 4 - coll = WorkflowCollection({'a': wfa, 'b': wfb}) + coll = BatchProcessor({'a': wfa, 'b': wfb}) assert coll.compute(float) == {'a': 1.5, 'b': 2.0} assert coll.compute(str) == {'a': '3;1.5', 'b': '4;2.0'} @@ -32,7 +32,7 @@ def test_compute_multiple() -> None: wfa[int] = 3 wfb = wf.copy() wfb[int] = 4 - coll = WorkflowCollection({'a': wfa, 'b': wfb}) + coll = BatchProcessor({'a': wfa, 'b': wfb}) result = coll.compute([float, str]) @@ -46,7 +46,7 @@ def test_setitem_mapping() -> None: wfa[int] = 3 wfb = wf.copy() wfb[int] = 4 - coll = WorkflowCollection({'a': wfa, 'b': wfb}) + coll = BatchProcessor({'a': wfa, 'b': wfb}) coll[int] = {'a': 7, 'b': 8} @@ -60,7 +60,7 @@ def test_setitem_single_value() -> None: wfa[int] = 3 wfb = wf.copy() wfb[int] = 4 - coll = WorkflowCollection({'a': wfa, 'b': wfb}) + coll = BatchProcessor({'a': wfa, 'b': wfb}) coll[int] = 5 @@ -74,7 +74,7 @@ def test_copy() -> None: wfa[int] = 3 wfb = wf.copy() wfb[int] = 4 - coll = WorkflowCollection({'a': wfa, 'b': wfb}) + coll = BatchProcessor({'a': wfa, 'b': wfb}) coll_copy = coll.copy() diff --git a/tests/reflectometry/tools_test.py b/tests/reflectometry/tools_test.py index 72e108df..7ffac76a 100644 --- a/tests/reflectometry/tools_test.py +++ b/tests/reflectometry/tools_test.py @@ -10,7 +10,7 @@ from scipp.testing import assert_allclose from ess.reflectometry.tools import ( - WorkflowCollection, + BatchProcessor, batch_processor, combine_curves, linlogspace, @@ -114,7 +114,7 @@ def test_reflectivity_curve_scaling(): workflows[k][Filename[ReferenceRun]] = "_".join(map(str, v[1:])) workflows[k][QBins] = make_reference_events(*v[1:]).coords['Q'] - wfc = WorkflowCollection(workflows) + wfc = BatchProcessor(workflows) scaled_wf = scale_reflectivity_curves_to_overlap(wfc) @@ -137,7 +137,7 @@ def test_reflectivity_curve_scaling_with_critical_edge(): workflows[k][Filename[ReferenceRun]] = "_".join(map(str, v[1:])) workflows[k][QBins] = make_reference_events(*v[1:]).coords['Q'] - wfc = WorkflowCollection(workflows) + wfc = BatchProcessor(workflows) scaled_wf = scale_reflectivity_curves_to_overlap( wfc, critical_edge_interval=(sc.scalar(0.01), sc.scalar(0.05)) @@ -167,85 +167,6 @@ def test_reflectivity_curve_scaling_works_with_single_workflow_and_critical_edge assert np.isclose(factor, 0.4) -def test_reflectivity_curve_scaling_caches_intermediate_results(): - sample_count = 0 - reference_count = 0 - - def sample_data_from_filename( - filename: Filename[SampleRun], - ) -> UnscaledReducibleData[SampleRun]: - nonlocal sample_count - sample_count += 1 - return UnscaledReducibleData[SampleRun]( - make_sample_events(*(float(x) for x in filename.split('_'))) - ) - - def reference_data_from_filename( - filename: Filename[ReferenceRun], - ) -> UnscaledReducibleData[ReferenceRun]: - nonlocal reference_count - reference_count += 1 - return UnscaledReducibleData[ReferenceRun]( - make_reference_events(*(float(x) for x in filename.split('_'))) - ) - - def apply_scaling( - da: UnscaledReducibleData[RunType], - scale: ScalingFactorForOverlap[RunType], - ) -> ReducibleData[RunType]: - """ - Scales the raw data by a given factor. - """ - return ReducibleData[RunType](da * scale) - - def reflectivity( - sample: ReducibleData[SampleRun], - reference: ReducibleData[ReferenceRun], - qbins: QBins, - ) -> ReflectivityOverQ: - return ReflectivityOverQ(sample.hist(Q=qbins) / reference.hist(Q=qbins)) - - wf = sl.Pipeline( - [ - sample_data_from_filename, - reference_data_from_filename, - apply_scaling, - reflectivity, - ] - ) - wf[ScalingFactorForOverlap[SampleRun]] = 1.0 - wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 - params = {'a': (1.0, 0, 0.3), 'b': (0.8, 0.2, 0.7), 'c': (0.1, 0.6, 1.0)} - workflows = {} - for k, v in params.items(): - workflows[k] = wf.copy() - workflows[k][Filename[SampleRun]] = "_".join(map(str, v)) - workflows[k][Filename[ReferenceRun]] = "_".join(map(str, v[1:])) - workflows[k][QBins] = make_reference_events(*v[1:]).coords['Q'] - - wfc = WorkflowCollection(workflows) - - scaled_wf = scale_reflectivity_curves_to_overlap( - wfc, cache_intermediate_results=False - ) - scaled_wf.compute(ReflectivityOverQ) - # We expect 6 counts: 3 for each of the 3 runs * 2 for computing ReflectivityOverQ - # inside the scaling function and one more time for the final computation just above - assert sample_count == 6 - assert reference_count == 6 - - sample_count = 0 - reference_count = 0 - - scaled_wf = scale_reflectivity_curves_to_overlap( - wfc, cache_intermediate_results=True - ) - scaled_wf.compute(ReflectivityOverQ) - # We expect 3 counts: 1 for each of the 3 runs * 1 for computing ReflectivityOverQ - assert sample_count == 3 - assert reference_count == 3 - - def test_combined_curves(): qgrid = sc.linspace('Q', 0, 1, 26) curves = ( From 8cb0b2b23738df144414b82c8460a6cdb35ee7e8 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Thu, 18 Sep 2025 14:40:44 +0200 Subject: [PATCH 54/75] comment interactive figs --- docs/user-guide/amor/amor-reduction.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide/amor/amor-reduction.ipynb b/docs/user-guide/amor/amor-reduction.ipynb index b5e84191..2c25471d 100644 --- a/docs/user-guide/amor/amor-reduction.ipynb +++ b/docs/user-guide/amor/amor-reduction.ipynb @@ -24,7 +24,7 @@ "metadata": {}, "outputs": [], "source": [ - "%matplotlib widget\n", + "# %matplotlib widget\n", "import warnings\n", "import scipp as sc\n", "from ess import amor\n", From e35a77da8e4b1cd2884e7e9a8c6f198f8bbab416 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Thu, 25 Sep 2025 12:05:54 +0200 Subject: [PATCH 55/75] remove UnscaledReducibleData and ScalingFactorForOverlap from workflow, and make scaling function return the factors --- src/ess/amor/__init__.py | 2 - src/ess/amor/workflow.py | 18 ++----- src/ess/reflectometry/tools.py | 84 ++++++++++++------------------- src/ess/reflectometry/types.py | 8 --- src/ess/reflectometry/workflow.py | 42 ++++++++++++++-- 5 files changed, 72 insertions(+), 82 deletions(-) diff --git a/src/ess/amor/__init__.py b/src/ess/amor/__init__.py index 5c67959f..4148b94b 100644 --- a/src/ess/amor/__init__.py +++ b/src/ess/amor/__init__.py @@ -16,7 +16,6 @@ Position, RunType, SampleRotationOffset, - ScalingFactorForOverlap, ) from . import ( conversions, @@ -75,7 +74,6 @@ def default_parameters() -> dict: ), GravityToggle: True, SampleRotationOffset[RunType]: sc.scalar(0.0, unit='deg'), - ScalingFactorForOverlap[RunType]: 1.0, } diff --git a/src/ess/amor/workflow.py b/src/ess/amor/workflow.py index 68dc0644..828abe4f 100644 --- a/src/ess/amor/workflow.py +++ b/src/ess/amor/workflow.py @@ -12,8 +12,6 @@ ProtonCurrent, ReducibleData, RunType, - ScalingFactorForOverlap, - UnscaledReducibleData, WavelengthBins, YIndexLimits, ZIndexLimits, @@ -29,7 +27,7 @@ def add_coords_masks_and_apply_corrections( wbins: WavelengthBins, proton_current: ProtonCurrent[RunType], graph: CoordTransformationGraph, -) -> UnscaledReducibleData[RunType]: +) -> ReducibleData[RunType]: """ Computes coordinates, masks and corrections that are the same for the sample measurement and the reference measurement. @@ -45,17 +43,7 @@ def add_coords_masks_and_apply_corrections( da = add_proton_current_mask(da) da = correct_by_proton_current(da) - return UnscaledReducibleData[RunType](da) - - -def scale_raw_reducible_data( - da: UnscaledReducibleData[RunType], - scale: ScalingFactorForOverlap[RunType], -) -> ReducibleData[RunType]: - """ - Scales the raw data by a given factor. - """ - return ReducibleData[RunType](da * scale) + return ReducibleData[RunType](da) -providers = (add_coords_masks_and_apply_corrections, scale_raw_reducible_data) +providers = (add_coords_masks_and_apply_corrections,) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index b459b0b5..3d7edbe0 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -9,19 +9,12 @@ from typing import Any import numpy as np -import pandas as pd import sciline as sl import scipp as sc import scipy.optimize as opt -from ess.reflectometry.types import ( - Filename, - QBins, - ReflectivityOverQ, - SampleRun, - ScalingFactorForOverlap, - UnscaledReducibleData, -) +from ess.reflectometry.types import Filename, SampleRun +from ess.reflectometry.workflow import with_filenames _STD_TO_FWHM = sc.scalar(2.0) * sc.sqrt(sc.scalar(2.0) * sc.log(sc.scalar(2.0))) @@ -192,7 +185,7 @@ def compute(self, targets: type | Sequence[type], **kwargs) -> Mapping[str, Any] targets = [targets] out = {} for t in targets: - out[t] = {} + out[t] = sc.DataGroup() for name, wf in self.workflows.items(): try: out[t][name] = wf.compute(t, **kwargs) @@ -302,16 +295,16 @@ def _interpolate_on_qgrid(curves, grid): ) -def scale_reflectivity_curves_to_overlap( - workflows: BatchProcessor | sl.Pipeline, +def scale_for_reflectivity_overlap( + reflectivities: sc.DataArray | Mapping[str, sc.DataArray] | sc.DataGroup, critical_edge_interval: tuple[sc.Variable, sc.Variable] | None = None, -) -> tuple[list[sc.DataArray], list[sc.Variable]]: +) -> sc.DataArray | sc.DataGroup: ''' - Set the ``ScalingFactorForOverlap`` parameter on the provided workflows - in a way that would makes the 1D reflectivity curves overlap. - One can supply either a collection of workflows or a single workflow. + Compute a scaling for 1D reflectivity curves in a way that would makes the curves + overlap. + One can supply either a single curve or a collection/DataGroup of curves. - If :code:`critical_edge_interval` is not provided, all workflows are scaled except + If :code:`critical_edge_interval` is not provided, all curves are scaled except the data with the lowest Q-range, which is considered to be the reference curve. The scaling factors are determined by a maximum likelihood estimate (assuming the errors are normal distributed). @@ -322,8 +315,8 @@ def scale_reflectivity_curves_to_overlap( Parameters --------- - workflows: - The workflow or collection of workflows that can compute ``ReflectivityOverQ``. + reflectivities: + The reflectivity curves that should be scaled. critical_edge_interval: A tuple denoting an interval that is known to belong to the critical edge, i.e. where the reflectivity is @@ -332,15 +325,12 @@ def scale_reflectivity_curves_to_overlap( Returns --------- : - A list of scaled reflectivity curves and a list of the scaling factors. + A DataGroup with the same keys as the input containing the + scaling factors for each reflectivity curve. ''' - is_single_workflow = isinstance(workflows, sl.Pipeline) - if is_single_workflow: - # If a single workflow is provided, convert it to a collection - workflows = BatchProcessor({"": workflows}) - - wfc = workflows.copy() - reflectivities = wfc.compute(ReflectivityOverQ) + only_one_curve = isinstance(reflectivities, sc.DataArray) + if only_one_curve: + reflectivities = {"": reflectivities} # First sort the dict of reflectivities by the Q min value curves = { @@ -352,10 +342,8 @@ def scale_reflectivity_curves_to_overlap( critical_edge_key = uuid.uuid4().hex if critical_edge_interval is not None: - q = wfc.compute(QBins) - if hasattr(q, "items"): - # If QBins is a mapping, find the one with the lowest Q start - q = min(q.values(), key=lambda q_: q_.min()) + q = {key: c.coords['Q'] for key, c in curves.items()} + q = min(q.values(), key=lambda q_: q_.min()) # TODO: This is slightly different from before: it extracts the bins from the # QBins variable that cover the critical edge interval. This means that the # resulting curve will not necessarily begin and end exactly at the values @@ -391,19 +379,19 @@ def cost(scaling_factors): sol = opt.minimize(cost, [1.0] * (len(curves) - 1)) scaling_factors = (1.0, *map(float, sol.x)) - original_factors = wfc.compute(ScalingFactorForOverlap[SampleRun]) - - wfc[ScalingFactorForOverlap[SampleRun]] = { - k: v * original_factors[k] - for k, v in zip(curves.keys(), scaling_factors, strict=True) - if k != critical_edge_key - } + out = sc.DataGroup( + { + k: v + for k, v in zip(curves.keys(), scaling_factors, strict=True) + if k != critical_edge_key + } + ) - return wfc.workflows[""] if is_single_workflow else wfc + return out[""] if only_one_curve else out def combine_curves( - curves: Sequence[sc.DataArray], + curves: Sequence[sc.DataArray] | sc.DataGroup | Mapping[str, sc.DataArray], q_bin_edges: sc.Variable | None = None, ) -> sc.DataArray: '''Combines the given curves by interpolating them @@ -431,6 +419,8 @@ def combine_curves( : A data array representing the combined reflectivity curve ''' + if hasattr(curves, 'items'): + curves = list(curves.values()) if len({c.data.unit for c in curves}) != 1: raise ValueError('The reflectivity curves must have the same unit') if len({c.coords['Q'].unit for c in curves}) != 1: @@ -454,12 +444,6 @@ def combine_curves( ) -def _concatenate_event_lists(*das): - da = sc.reduce(das).bins.concat() - missing_coords = set(das[0].coords) - set(da.coords) - return da.assign_coords({coord: das[0].coords[coord] for coord in missing_coords}) - - def batch_processor( workflow: sl.Pipeline, runs: Mapping[Any, Mapping[type, Any]] ) -> BatchProcessor: @@ -519,13 +503,7 @@ def batch_processor( if Filename[SampleRun] in parameters: if isinstance(parameters[Filename[SampleRun]], list | tuple): - df = pd.DataFrame( - {Filename[SampleRun]: parameters[Filename[SampleRun]]} - ) - mapped = wf.map(df) - wf[UnscaledReducibleData[SampleRun]] = mapped[ - UnscaledReducibleData[SampleRun] - ].reduce(func=_concatenate_event_lists) + wf = with_filenames(wf, SampleRun, parameters[Filename[SampleRun]]) else: wf[Filename[SampleRun]] = parameters[Filename[SampleRun]] workflows[name] = wf diff --git a/src/ess/reflectometry/types.py b/src/ess/reflectometry/types.py index 6959534b..f1eea80a 100644 --- a/src/ess/reflectometry/types.py +++ b/src/ess/reflectometry/types.py @@ -28,18 +28,10 @@ class RawChopper(sciline.Scope[RunType, sc.DataGroup], sc.DataGroup): """Chopper data loaded from nexus file.""" -class UnscaledReducibleData(sciline.Scope[RunType, sc.DataArray], sc.DataArray): - """""" - - class ReducibleData(sciline.Scope[RunType, sc.DataArray], sc.DataArray): """Event data with common coordinates added""" -class ScalingFactorForOverlap(sciline.Scope[RunType, float], float): - """""" - - ReducedReference = NewType("ReducedReference", sc.DataArray) """Intensity distribution on the detector for a sample with :math`R(Q) = 1`""" diff --git a/src/ess/reflectometry/workflow.py b/src/ess/reflectometry/workflow.py index 129a45e3..abf8deb7 100644 --- a/src/ess/reflectometry/workflow.py +++ b/src/ess/reflectometry/workflow.py @@ -12,7 +12,15 @@ OrsoSample, OrsoSampleFilenames, ) -from ess.reflectometry.types import Filename, ReducibleData, RunType, SampleRun +from ess.reflectometry.types import ( + DetectorRotation, + Filename, + # RawChopper, + ReducibleData, + RunType, + SampleRotation, + SampleRun, +) def _concatenate_event_lists(*das): @@ -55,9 +63,35 @@ def with_filenames( mapped = wf.map(df) - wf[ReducibleData[runtype]] = mapped[ReducibleData[runtype]].reduce( - index=axis_name, func=_concatenate_event_lists - ) + try: + wf[ReducibleData[runtype]] = mapped[ReducibleData[runtype]].reduce( + index=axis_name, func=_concatenate_event_lists + ) + except ValueError: + # ReducibleData[runtype] is independent of Filename[runtype] + pass + # TODO: I didn't understand why we needed the chopper here. + # try: + # wf[RawChopper[runtype]] = mapped[RawChopper[runtype]].reduce( + # index=axis_name, func=_any_value + # ) + # except ValueError: + # # RawChopper[runtype] is independent of Filename[runtype] + # pass + try: + wf[SampleRotation[runtype]] = mapped[SampleRotation[runtype]].reduce( + index=axis_name, func=_any_value + ) + except ValueError: + # SampleRotation[runtype] is independent of Filename[runtype] + pass + try: + wf[DetectorRotation[runtype]] = mapped[DetectorRotation[runtype]].reduce( + index=axis_name, func=_any_value + ) + except ValueError: + # DetectorRotation[runtype] is independent of Filename[runtype] + pass if runtype is SampleRun: wf[OrsoSample] = mapped[OrsoSample].reduce(index=axis_name, func=_any_value) From 2f9f51fcce3638d8efc3cd5a8f2f1c9c325d0d3c Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Thu, 25 Sep 2025 12:11:23 +0200 Subject: [PATCH 56/75] update notebook --- docs/user-guide/amor/amor-reduction.ipynb | 59 ++++++++++++----------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/docs/user-guide/amor/amor-reduction.ipynb b/docs/user-guide/amor/amor-reduction.ipynb index 2c25471d..b7893526 100644 --- a/docs/user-guide/amor/amor-reduction.ipynb +++ b/docs/user-guide/amor/amor-reduction.ipynb @@ -173,7 +173,6 @@ " '609': {\n", " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", " Filename[SampleRun]: amor.data.amor_run(609),\n", - " ScalingFactorForOverlap[SampleRun]: 1.0,\n", " },\n", " '610': {\n", " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", @@ -220,8 +219,8 @@ "outputs": [], "source": [ "# Cache intermediate result to avoid re-loading the data\n", - "batch[UnscaledReducibleData[SampleRun]] = batch.compute(\n", - " UnscaledReducibleData[SampleRun]\n", + "batch[ReducibleData[SampleRun]] = batch.compute(\n", + " ReducibleData[SampleRun]\n", ")" ] }, @@ -231,18 +230,16 @@ "metadata": {}, "outputs": [], "source": [ - "from ess.reflectometry.tools import scale_reflectivity_curves_to_overlap\n", + "from ess.reflectometry.tools import scale_for_reflectivity_overlap\n", "\n", - "# Pass the batch workflow collection and get a new workflow collection as output,\n", - "# with the correct scaling factors applied.\n", - "scaled_wf = scale_reflectivity_curves_to_overlap(\n", - " batch,\n", + "scaling = scale_for_reflectivity_overlap(\n", + " reflectivity,\n", " critical_edge_interval=(sc.scalar(0.01, unit='1/angstrom'), sc.scalar(0.014, unit='1/angstrom'))\n", ")\n", "\n", - "scaled_r = {key: r.hist() for key, r in scaled_wf.compute(ReflectivityOverQ).items()}\n", + "scaled_r = reflectivity * scaling\n", "\n", - "sc.plot(scaled_r, norm='log', vmin=1e-5)" + "sc.plot(scaled_r.hist(), norm='log', vmin=1e-5)" ] }, { @@ -261,7 +258,7 @@ "from ess.reflectometry.tools import combine_curves\n", "\n", "qbins = sc.geomspace('Q', 0.005, 0.4, 501, unit='1/angstrom')\n", - "combined = combine_curves(scaled_r.values(), qbins)\n", + "combined = combine_curves(scaled_r.hist(), qbins)\n", "combined.plot(norm='log')" ] }, @@ -291,7 +288,7 @@ "metadata": {}, "outputs": [], "source": [ - "diagnostics = scaled_wf.compute(ReflectivityOverZW)\n", + "diagnostics = batch.compute(ReflectivityOverZW)\n", "diagnostics['608'].hist().flatten(('blade', 'wire'), to='z').plot(norm='log')" ] }, @@ -313,7 +310,7 @@ "\n", "wavelength_theta_figure(\n", " diagnostics.values(),\n", - " theta_bins=scaled_wf.compute(ThetaBins[SampleRun]).values(),\n", + " theta_bins=batch.compute(ThetaBins[SampleRun]).values(),\n", " q_edges_to_display=(sc.scalar(0.018, unit='1/angstrom'), sc.scalar(0.113, unit='1/angstrom'))\n", ")" ] @@ -343,7 +340,7 @@ "\n", "q_theta_figure(\n", " diagnostics.values(),\n", - " theta_bins=scaled_wf.compute(ThetaBins[SampleRun]).values(),\n", + " theta_bins=batch.compute(ThetaBins[SampleRun]).values(),\n", " q_bins=qbins\n", ")" ] @@ -367,14 +364,14 @@ "source": [ "from ess.reflectometry.figures import wavelength_z_figure\n", "\n", - "workflow[Filename[SampleRun]] = runs['608'][Filename[SampleRun]]\n", - "workflow[SampleRotationOffset[SampleRun]] = runs['608'][SampleRotationOffset[SampleRun]]\n", + "wf_608 = batch.workflows['608']\n", + "\n", "wavelength_z_figure(\n", - " workflow.compute(Sample),\n", - " wavelength_bins=workflow.compute(WavelengthBins),\n", + " wf_608.compute(Sample),\n", + " wavelength_bins=wf_608.compute(WavelengthBins),\n", " grid=False\n", ") + wavelength_z_figure(\n", - " workflow.compute(Reference),\n", + " wf_608.compute(Reference),\n", " grid=False\n", ")" ] @@ -407,7 +404,7 @@ "metadata": {}, "outputs": [], "source": [ - "scaled_wf[orso.OrsoCreator] = orso.OrsoCreator(\n", + "batch[orso.OrsoCreator] = orso.OrsoCreator(\n", " fileio.base.Person(\n", " name='Max Mustermann',\n", " affiliation='European Spallation Source ERIC',\n", @@ -429,7 +426,17 @@ "metadata": {}, "outputs": [], "source": [ - "scaled_wf.workflows['608'].visualize(orso.OrsoIofQDataset, graph_attr={'rankdir': 'LR'})" + "wf_608.visualize(orso.OrsoIofQDataset, graph_attr={'rankdir': 'LR'})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Save scaled reflectivities to file\n", + "batch[ReflectivityOverQ] = scaled_r" ] }, { @@ -445,7 +452,7 @@ "metadata": {}, "outputs": [], "source": [ - "iofq_datasets = scaled_wf.compute(orso.OrsoIofQDataset)" + "iofq_datasets = batch_scaled.compute(orso.OrsoIofQDataset)" ] }, { @@ -559,14 +566,12 @@ "metadata": {}, "outputs": [], "source": [ - "scaled_wf = scale_reflectivity_curves_to_overlap(\n", - " batch,\n", + "scaled_r = reflectivity * scale_for_reflectivity_overlap(\n", + " reflectivity,\n", " critical_edge_interval=(sc.scalar(0.01, unit='1/angstrom'), sc.scalar(0.014, unit='1/angstrom'))\n", ")\n", "\n", - "scaled_r = {key: r.hist() for key, r in scaled_wf.compute(ReflectivityOverQ).items()}\n", - "\n", - "sc.plot(scaled_r, norm='log', vmin=1e-5)" + "sc.plot(scaled_r.hist(), norm='log', vmin=1e-5)" ] } ], From 11627a1a5913db9aef8b91e900701e3b5e39f481 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Thu, 25 Sep 2025 14:24:25 +0200 Subject: [PATCH 57/75] fix type hint --- src/ess/reflectometry/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 3d7edbe0..7636252a 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -159,7 +159,7 @@ class BatchProcessor: def __init__(self, workflows: Mapping[str, sl.Pipeline]): self.workflows = workflows - def __setitem__(self, key: type, value: Sequence[Any]): + def __setitem__(self, key: type, value: Mapping[str, Any] | Any) -> None: if hasattr(value, 'items'): for name, v in value.items(): self.workflows[name][key] = v From d66b0190715c80eef753cf704cd803354ea884c2 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Fri, 26 Sep 2025 08:51:29 +0200 Subject: [PATCH 58/75] disallow setting a single value on BatchProcessor and always require a dict --- docs/user-guide/amor/amor-reduction.ipynb | 73 +++++++++++++---------- src/ess/reflectometry/tools.py | 20 ++++--- 2 files changed, 53 insertions(+), 40 deletions(-) diff --git a/docs/user-guide/amor/amor-reduction.ipynb b/docs/user-guide/amor/amor-reduction.ipynb index b7893526..1ed0f0d4 100644 --- a/docs/user-guide/amor/amor-reduction.ipynb +++ b/docs/user-guide/amor/amor-reduction.ipynb @@ -70,7 +70,10 @@ "# They represent the lower and upper boundaries of a range of pixel indices.\n", "workflow[YIndexLimits] = sc.scalar(11), sc.scalar(41)\n", "workflow[ZIndexLimits] = sc.scalar(80), sc.scalar(370)\n", - "workflow[BeamDivergenceLimits] = sc.scalar(-0.75, unit='deg'), sc.scalar(0.75, unit='deg')" + "workflow[BeamDivergenceLimits] = (\n", + " sc.scalar(-0.75, unit='deg'),\n", + " sc.scalar(0.75, unit='deg'),\n", + ")" ] }, { @@ -188,10 +191,7 @@ "\n", "# Compute R(Q) for all runs\n", "reflectivity = batch.compute(ReflectivityOverQ)\n", - "sc.plot(\n", - " {key: r.hist() for key, r in reflectivity.items()},\n", - " norm='log', vmin=1e-5\n", - ")" + "sc.plot({key: r.hist() for key, r in reflectivity.items()}, norm='log', vmin=1e-5)" ] }, { @@ -219,9 +219,7 @@ "outputs": [], "source": [ "# Cache intermediate result to avoid re-loading the data\n", - "batch[ReducibleData[SampleRun]] = batch.compute(\n", - " ReducibleData[SampleRun]\n", - ")" + "batch[ReducibleData[SampleRun]] = batch.compute(ReducibleData[SampleRun])" ] }, { @@ -234,7 +232,10 @@ "\n", "scaling = scale_for_reflectivity_overlap(\n", " reflectivity,\n", - " critical_edge_interval=(sc.scalar(0.01, unit='1/angstrom'), sc.scalar(0.014, unit='1/angstrom'))\n", + " critical_edge_interval=(\n", + " sc.scalar(0.01, unit='1/angstrom'),\n", + " sc.scalar(0.014, unit='1/angstrom'),\n", + " ),\n", ")\n", "\n", "scaled_r = reflectivity * scaling\n", @@ -311,7 +312,10 @@ "wavelength_theta_figure(\n", " diagnostics.values(),\n", " theta_bins=batch.compute(ThetaBins[SampleRun]).values(),\n", - " q_edges_to_display=(sc.scalar(0.018, unit='1/angstrom'), sc.scalar(0.113, unit='1/angstrom'))\n", + " q_edges_to_display=(\n", + " sc.scalar(0.018, unit='1/angstrom'),\n", + " sc.scalar(0.113, unit='1/angstrom'),\n", + " ),\n", ")" ] }, @@ -341,7 +345,7 @@ "q_theta_figure(\n", " diagnostics.values(),\n", " theta_bins=batch.compute(ThetaBins[SampleRun]).values(),\n", - " q_bins=qbins\n", + " q_bins=qbins,\n", ")" ] }, @@ -367,13 +371,8 @@ "wf_608 = batch.workflows['608']\n", "\n", "wavelength_z_figure(\n", - " wf_608.compute(Sample),\n", - " wavelength_bins=wf_608.compute(WavelengthBins),\n", - " grid=False\n", - ") + wavelength_z_figure(\n", - " wf_608.compute(Reference),\n", - " grid=False\n", - ")" + " wf_608.compute(Sample), wavelength_bins=wf_608.compute(WavelengthBins), grid=False\n", + ") + wavelength_z_figure(wf_608.compute(Reference), grid=False)" ] }, { @@ -404,13 +403,16 @@ "metadata": {}, "outputs": [], "source": [ - "batch[orso.OrsoCreator] = orso.OrsoCreator(\n", - " fileio.base.Person(\n", - " name='Max Mustermann',\n", - " affiliation='European Spallation Source ERIC',\n", - " contact='max.mustermann@ess.eu',\n", + "batch[orso.OrsoCreator] = {\n", + " k: orso.OrsoCreator(\n", + " fileio.base.Person(\n", + " name='Max Mustermann',\n", + " affiliation='European Spallation Source ERIC',\n", + " contact='max.mustermann@ess.eu',\n", + " )\n", " )\n", - ")" + " for k in runs\n", + "}" ] }, { @@ -435,7 +437,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Save scaled reflectivities to file\n", + "# We want to save the scaled results to the file\n", "batch[ReflectivityOverQ] = scaled_r" ] }, @@ -452,7 +454,7 @@ "metadata": {}, "outputs": [], "source": [ - "iofq_datasets = batch_scaled.compute(orso.OrsoIofQDataset)" + "iofq_datasets = batch.compute(orso.OrsoIofQDataset)" ] }, { @@ -488,7 +490,9 @@ "metadata": {}, "outputs": [], "source": [ - "fileio.orso.save_orso(datasets=list(iofq_datasets.values()), fname='amor_reduced_iofq.ort')" + "fileio.orso.save_orso(\n", + " datasets=list(iofq_datasets.values()), fname='amor_reduced_iofq.ort'\n", + ")" ] }, { @@ -537,7 +541,10 @@ " },\n", " '611+612': {\n", " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", - " Filename[SampleRun]: (amor.data.amor_run(611), amor.data.amor_run(612)), # List of files means their event lists should be merged\n", + " Filename[SampleRun]: (\n", + " amor.data.amor_run(611),\n", + " amor.data.amor_run(612),\n", + " ), # List of files means their event lists should be merged\n", " },\n", "}\n", "\n", @@ -545,10 +552,7 @@ "\n", "# Compute R(Q) for all runs\n", "reflectivity = batch.compute(ReflectivityOverQ)\n", - "sc.plot(\n", - " {key: r.hist() for key, r in reflectivity.items()},\n", - " norm='log', vmin=1e-4\n", - ")" + "sc.plot({key: r.hist() for key, r in reflectivity.items()}, norm='log', vmin=1e-4)" ] }, { @@ -568,7 +572,10 @@ "source": [ "scaled_r = reflectivity * scale_for_reflectivity_overlap(\n", " reflectivity,\n", - " critical_edge_interval=(sc.scalar(0.01, unit='1/angstrom'), sc.scalar(0.014, unit='1/angstrom'))\n", + " critical_edge_interval=(\n", + " sc.scalar(0.01, unit='1/angstrom'),\n", + " sc.scalar(0.014, unit='1/angstrom'),\n", + " ),\n", ")\n", "\n", "sc.plot(scaled_r.hist(), norm='log', vmin=1e-5)" diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 7636252a..b834bd01 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -159,15 +159,21 @@ class BatchProcessor: def __init__(self, workflows: Mapping[str, sl.Pipeline]): self.workflows = workflows - def __setitem__(self, key: type, value: Mapping[str, Any] | Any) -> None: - if hasattr(value, 'items'): - for name, v in value.items(): - self.workflows[name][key] = v - else: - for wf in self.workflows.values(): - wf[key] = value + def __setitem__(self, key: type, value: Mapping[str, Any]) -> None: + """ + A mapping (dict or DataGroup) should be supplied as the value. The keys + of the mapping should correspond to the names of the workflows in the + collection. The node matching the key will be set to the corresponding value for + each of the workflows. + """ + for name, v in value.items(): + self.workflows[name][key] = v def __getitem__(self, name: str) -> BatchProcessor: + """ + Get a new BatchProcessor where the workflows are the sub-workflows that lead to + the node with the given name. + """ return BatchProcessor({k: wf[name] for k, wf in self.workflows.items()}) def compute(self, targets: type | Sequence[type], **kwargs) -> Mapping[str, Any]: From ba7b8b0bec3cc7dfa5aabcbe9616d51154756986 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Fri, 26 Sep 2025 09:19:23 +0200 Subject: [PATCH 59/75] fix some tests --- src/ess/reflectometry/workflow.py | 62 ++++++---------- tests/reflectometry/batch_processor_tests.py | 16 +---- tests/reflectometry/tools_test.py | 76 +++++++------------- 3 files changed, 47 insertions(+), 107 deletions(-) diff --git a/src/ess/reflectometry/workflow.py b/src/ess/reflectometry/workflow.py index abf8deb7..1309cb4f 100644 --- a/src/ess/reflectometry/workflow.py +++ b/src/ess/reflectometry/workflow.py @@ -63,46 +63,28 @@ def with_filenames( mapped = wf.map(df) - try: - wf[ReducibleData[runtype]] = mapped[ReducibleData[runtype]].reduce( - index=axis_name, func=_concatenate_event_lists - ) - except ValueError: - # ReducibleData[runtype] is independent of Filename[runtype] - pass - # TODO: I didn't understand why we needed the chopper here. - # try: - # wf[RawChopper[runtype]] = mapped[RawChopper[runtype]].reduce( - # index=axis_name, func=_any_value - # ) - # except ValueError: - # # RawChopper[runtype] is independent of Filename[runtype] - # pass - try: - wf[SampleRotation[runtype]] = mapped[SampleRotation[runtype]].reduce( - index=axis_name, func=_any_value - ) - except ValueError: - # SampleRotation[runtype] is independent of Filename[runtype] - pass - try: - wf[DetectorRotation[runtype]] = mapped[DetectorRotation[runtype]].reduce( - index=axis_name, func=_any_value - ) - except ValueError: - # DetectorRotation[runtype] is independent of Filename[runtype] - pass - + reduce_functions = { + ReducibleData[runtype]: _concatenate_event_lists, + SampleRotation[runtype]: _any_value, + DetectorRotation[runtype]: _any_value, + # RawChopper[runtype]: _any_value, + } if runtype is SampleRun: - wf[OrsoSample] = mapped[OrsoSample].reduce(index=axis_name, func=_any_value) - wf[OrsoExperiment] = mapped[OrsoExperiment].reduce( - index=axis_name, func=_any_value - ) - wf[OrsoOwner] = mapped[OrsoOwner].reduce(index=axis_name, func=lambda x, *_: x) - wf[OrsoSampleFilenames] = mapped[OrsoSampleFilenames].reduce( - # When we don't map over filenames - # each OrsoSampleFilenames is a list with a single entry. - index=axis_name, - func=_concatenate_lists, + reduce_functions.update( + { + OrsoSample: _any_value, + OrsoExperiment: _any_value, + OrsoOwner: _any_value, + OrsoSampleFilenames: _concatenate_lists, + } ) + + for tp, func in reduce_functions.items(): + try: + wf[tp] = mapped[tp].reduce(index=axis_name, func=func) + except (ValueError, KeyError): + # ValueError: tp[runtype] is independent of Filename[runtype] + # KeyError: tp[runtype] not in workflow + pass + return wf diff --git a/tests/reflectometry/batch_processor_tests.py b/tests/reflectometry/batch_processor_tests.py index 94a868a0..547a48de 100644 --- a/tests/reflectometry/batch_processor_tests.py +++ b/tests/reflectometry/batch_processor_tests.py @@ -40,7 +40,7 @@ def test_compute_multiple() -> None: assert result[str] == {'a': '3;1.5', 'b': '4;2.0'} -def test_setitem_mapping() -> None: +def test_setitem() -> None: wf = sl.Pipeline([int_to_float, int_float_to_str]) wfa = wf.copy() wfa[int] = 3 @@ -54,20 +54,6 @@ def test_setitem_mapping() -> None: assert coll.compute(str) == {'a': '7;3.5', 'b': '8;4.0'} -def test_setitem_single_value() -> None: - wf = sl.Pipeline([int_to_float, int_float_to_str]) - wfa = wf.copy() - wfa[int] = 3 - wfb = wf.copy() - wfb[int] = 4 - coll = BatchProcessor({'a': wfa, 'b': wfb}) - - coll[int] = 5 - - assert coll.compute(float) == {'a': 2.5, 'b': 2.5} - assert coll.compute(str) == {'a': '5;2.5', 'b': '5;2.5'} - - def test_copy() -> None: wf = sl.Pipeline([int_to_float, int_float_to_str]) wfa = wf.copy() diff --git a/tests/reflectometry/tools_test.py b/tests/reflectometry/tools_test.py index 7ffac76a..319df99e 100644 --- a/tests/reflectometry/tools_test.py +++ b/tests/reflectometry/tools_test.py @@ -14,7 +14,7 @@ batch_processor, combine_curves, linlogspace, - scale_reflectivity_curves_to_overlap, + scale_for_reflectivity_overlap, ) from ess.reflectometry.types import ( Filename, @@ -24,8 +24,6 @@ ReflectivityOverQ, RunType, SampleRun, - ScalingFactorForOverlap, - UnscaledReducibleData, ) @@ -64,27 +62,18 @@ def make_reference_events(qmin, qmax): def make_workflow(): def sample_data_from_filename( filename: Filename[SampleRun], - ) -> UnscaledReducibleData[SampleRun]: - return UnscaledReducibleData[SampleRun]( + ) -> ReducibleData[SampleRun]: + return ReducibleData[SampleRun]( make_sample_events(*(float(x) for x in filename.split('_'))) ) def reference_data_from_filename( filename: Filename[ReferenceRun], - ) -> UnscaledReducibleData[ReferenceRun]: - return UnscaledReducibleData[ReferenceRun]( + ) -> ReducibleData[ReferenceRun]: + return ReducibleData[ReferenceRun]( make_reference_events(*(float(x) for x in filename.split('_'))) ) - def apply_scaling( - da: UnscaledReducibleData[RunType], - scale: ScalingFactorForOverlap[RunType], - ) -> ReducibleData[RunType]: - """ - Scales the raw data by a given factor. - """ - return ReducibleData[RunType](da * scale) - def reflectivity( sample: ReducibleData[SampleRun], reference: ReducibleData[ReferenceRun], @@ -93,19 +82,12 @@ def reflectivity( return ReflectivityOverQ(sample.hist(Q=qbins) / reference.hist(Q=qbins)) return sl.Pipeline( - [ - sample_data_from_filename, - reference_data_from_filename, - apply_scaling, - reflectivity, - ] + [sample_data_from_filename, reference_data_from_filename, reflectivity] ) def test_reflectivity_curve_scaling(): wf = make_workflow() - wf[ScalingFactorForOverlap[SampleRun]] = 1.0 - wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 params = {'a': (1.0, 0, 0.3), 'b': (0.8, 0.2, 0.7), 'c': (0.1, 0.6, 1.0)} workflows = {} for k, v in params.items(): @@ -114,21 +96,17 @@ def test_reflectivity_curve_scaling(): workflows[k][Filename[ReferenceRun]] = "_".join(map(str, v[1:])) workflows[k][QBins] = make_reference_events(*v[1:]).coords['Q'] - wfc = BatchProcessor(workflows) - - scaled_wf = scale_reflectivity_curves_to_overlap(wfc) + batch = BatchProcessor(workflows) - factors = scaled_wf.compute(ScalingFactorForOverlap[SampleRun]) + scaling_factors = scale_for_reflectivity_overlap(batch.compute(ReflectivityOverQ)) - assert np.isclose(factors['a'], 1.0) - assert np.isclose(factors['b'], 0.5 / 0.8) - assert np.isclose(factors['c'], 0.25 / 0.1) + assert np.isclose(scaling_factors['a'], 1.0) + assert np.isclose(scaling_factors['b'], 0.5 / 0.8) + assert np.isclose(scaling_factors['c'], 0.25 / 0.1) def test_reflectivity_curve_scaling_with_critical_edge(): wf = make_workflow() - wf[ScalingFactorForOverlap[SampleRun]] = 1.0 - wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 params = {'a': (2, 0, 0.3), 'b': (0.8, 0.2, 0.7), 'c': (0.1, 0.6, 1.0)} workflows = {} for k, v in params.items(): @@ -137,34 +115,30 @@ def test_reflectivity_curve_scaling_with_critical_edge(): workflows[k][Filename[ReferenceRun]] = "_".join(map(str, v[1:])) workflows[k][QBins] = make_reference_events(*v[1:]).coords['Q'] - wfc = BatchProcessor(workflows) + batch = BatchProcessor(workflows) - scaled_wf = scale_reflectivity_curves_to_overlap( - wfc, critical_edge_interval=(sc.scalar(0.01), sc.scalar(0.05)) + scaling_factors = scale_for_reflectivity_overlap( + batch.compute(ReflectivityOverQ), + critical_edge_interval=(sc.scalar(0.01), sc.scalar(0.05)), ) - factors = scaled_wf.compute(ScalingFactorForOverlap[SampleRun]) + assert np.isclose(scaling_factors['a'], 0.5) + assert np.isclose(scaling_factors['b'], 0.5 / 0.8) + assert np.isclose(scaling_factors['c'], 0.25 / 0.1) - assert np.isclose(factors['a'], 0.5) - assert np.isclose(factors['b'], 0.5 / 0.8) - assert np.isclose(factors['c'], 0.25 / 0.1) - -def test_reflectivity_curve_scaling_works_with_single_workflow_and_critical_edge(): +def test_reflectivity_curve_scaling_works_with_single_curve_and_critical_edge(): wf = make_workflow() - wf[ScalingFactorForOverlap[SampleRun]] = 1.0 - wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 wf[Filename[SampleRun]] = '2.5_0.4_0.8' wf[Filename[ReferenceRun]] = '0.4_0.8' wf[QBins] = make_reference_events(0.4, 0.8).coords['Q'] - scaled_wf = scale_reflectivity_curves_to_overlap( - wf, critical_edge_interval=(sc.scalar(0.0), sc.scalar(0.5)) + scaling_factor = scale_for_reflectivity_overlap( + wf.compute(ReflectivityOverQ), + critical_edge_interval=(sc.scalar(0.0), sc.scalar(0.5)), ) - factor = scaled_wf.compute(ScalingFactorForOverlap[SampleRun]) - - assert np.isclose(factor, 0.4) + assert np.isclose(scaling_factor, 0.4) def test_combined_curves(): @@ -344,8 +318,6 @@ class Reduction: def test_batch_processor_tool_merges_event_lists(): wf = make_workflow() - wf[ScalingFactorForOverlap[SampleRun]] = 1.0 - wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 runs = { 'a': {Filename[SampleRun]: ('1.0_0.0_0.3', '1.5_0.0_0.3')}, @@ -354,7 +326,7 @@ def test_batch_processor_tool_merges_event_lists(): } batch = batch_processor(wf, runs) - results = batch.compute(UnscaledReducibleData[SampleRun]) + results = batch.compute(ReducibleData[SampleRun]) assert_almost_equal(results['a'].sum().value, 10 + 15 * 0.5 + (10 + 15 * 0.5) * 1.5) assert_almost_equal(results['b'].sum().value, 10 * 0.8 + 15 * 0.5 * 0.8) From 68c45302fa549c525e3c3b9f8c8bdf617c1a6750 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Fri, 26 Sep 2025 09:21:10 +0200 Subject: [PATCH 60/75] remove unused import --- tests/reflectometry/tools_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/reflectometry/tools_test.py b/tests/reflectometry/tools_test.py index 319df99e..9c3c0c13 100644 --- a/tests/reflectometry/tools_test.py +++ b/tests/reflectometry/tools_test.py @@ -22,7 +22,6 @@ ReducibleData, ReferenceRun, ReflectivityOverQ, - RunType, SampleRun, ) From 28f9fbe46c5b49b80e4d3a6934221ee7895d53ab Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Fri, 26 Sep 2025 09:36:30 +0200 Subject: [PATCH 61/75] fix remaining tests --- tests/amor/pipeline_test.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/amor/pipeline_test.py b/tests/amor/pipeline_test.py index 50ecb3ba..91f45845 100644 --- a/tests/amor/pipeline_test.py +++ b/tests/amor/pipeline_test.py @@ -12,7 +12,7 @@ from ess import amor from ess.amor import data from ess.reflectometry import orso -from ess.reflectometry.tools import scale_reflectivity_curves_to_overlap +from ess.reflectometry.tools import scale_for_reflectivity_overlap from ess.reflectometry.types import ( BeamDivergenceLimits, Filename, @@ -128,14 +128,15 @@ def test_save_reduced_orso_file(output_folder: Path): wf[Filename[ReferenceRun]] = data.amor_run(4152) wf[QBins] = sc.geomspace(dim="Q", start=0.01, stop=0.06, num=201, unit="1/angstrom") - scaled_wf = scale_reflectivity_curves_to_overlap( - wf, + r_of_q = wf.compute(ReflectivityOverQ) + wf[ReflectivityOverQ] = r_of_q * scale_for_reflectivity_overlap( + r_of_q, critical_edge_interval=( sc.scalar(0.01, unit='1/angstrom'), sc.scalar(0.014, unit='1/angstrom'), ), ) - scaled_wf[orso.OrsoCreator] = orso.OrsoCreator( + wf[orso.OrsoCreator] = orso.OrsoCreator( fileio.base.Person( name="Max Mustermann", affiliation="European Spallation Source ERIC", @@ -143,7 +144,7 @@ def test_save_reduced_orso_file(output_folder: Path): ) ) fileio.orso.save_orso( - datasets=[scaled_wf.compute(orso.OrsoIofQDataset)], + datasets=[wf.compute(orso.OrsoIofQDataset)], fname=output_folder / 'amor_reduced_iofq.ort', ) From 19a3f1bad6c2764ed11c4c1c367f2ca0389aafad Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Fri, 26 Sep 2025 10:08:42 +0200 Subject: [PATCH 62/75] use the fact that we are returning results as DataGroup for plotting --- docs/user-guide/amor/amor-reduction.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user-guide/amor/amor-reduction.ipynb b/docs/user-guide/amor/amor-reduction.ipynb index 1ed0f0d4..87136458 100644 --- a/docs/user-guide/amor/amor-reduction.ipynb +++ b/docs/user-guide/amor/amor-reduction.ipynb @@ -191,7 +191,7 @@ "\n", "# Compute R(Q) for all runs\n", "reflectivity = batch.compute(ReflectivityOverQ)\n", - "sc.plot({key: r.hist() for key, r in reflectivity.items()}, norm='log', vmin=1e-5)" + "sc.plot(reflectivity.hist(), norm='log', vmin=1e-5)" ] }, { @@ -552,7 +552,7 @@ "\n", "# Compute R(Q) for all runs\n", "reflectivity = batch.compute(ReflectivityOverQ)\n", - "sc.plot({key: r.hist() for key, r in reflectivity.items()}, norm='log', vmin=1e-4)" + "sc.plot(reflectivity.hist(), norm='log', vmin=1e-4)" ] }, { From 34198cf67a7e53c2394a74039ebb5fe483660f65 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Fri, 26 Sep 2025 15:11:51 +0200 Subject: [PATCH 63/75] add batch_compute helper and simple notebook --- .../amor/amor-reduction-simple.ipynb | 321 ++++++++++++++++++ src/ess/reflectometry/__init__.py | 9 +- src/ess/reflectometry/tools.py | 85 ++++- tests/reflectometry/tools_test.py | 96 +++++- 4 files changed, 507 insertions(+), 4 deletions(-) create mode 100644 docs/user-guide/amor/amor-reduction-simple.ipynb diff --git a/docs/user-guide/amor/amor-reduction-simple.ipynb b/docs/user-guide/amor/amor-reduction-simple.ipynb new file mode 100644 index 00000000..0b3e0aa3 --- /dev/null +++ b/docs/user-guide/amor/amor-reduction-simple.ipynb @@ -0,0 +1,321 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Divergent data reduction for Amor - Simple\n", + "\n", + "In this notebook, we will look at how to use the `essreflectometry` package with Sciline,\n", + "for reflectometry data collected from the PSI instrument [Amor](https://www.psi.ch/en/sinq/amor) in [divergent beam mode](https://www.psi.ch/en/sinq/amor/selene).\n", + "\n", + "This notebook shows the basics of computing reduced results for a set of runs and saving them to Orso file format." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# %matplotlib widget\n", + "import warnings\n", + "import scipp as sc\n", + "from ess import amor\n", + "from ess.amor import data # noqa: F401\n", + "from ess.reflectometry.types import *\n", + "from ess.amor.types import *\n", + "from ess.reflectometry import batch_processor, batch_compute\n", + "\n", + "# The files used in this tutorial have some issues that makes scippnexus\n", + "# raise warnings when loading them. To avoid noise in the notebook the warnings are silenced.\n", + "warnings.filterwarnings('ignore', 'Failed to convert .* into a transformation')\n", + "warnings.filterwarnings('ignore', 'Invalid transformation')\n", + "warnings.filterwarnings('ignore', 'invalid value encountered')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create and configure the workflow\n", + "\n", + "We begin by creating the Amor workflow object which is a skeleton for reducing Amor data,\n", + "with pre-configured steps, and then set the missing parameters which are specific to each experiment:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "workflow = amor.AmorWorkflow()\n", + "workflow[SampleSize[SampleRun]] = sc.scalar(10.0, unit='mm')\n", + "workflow[SampleSize[ReferenceRun]] = sc.scalar(10.0, unit='mm')\n", + "\n", + "workflow[ChopperPhase[ReferenceRun]] = sc.scalar(-7.5, unit='deg')\n", + "workflow[ChopperPhase[SampleRun]] = sc.scalar(-7.5, unit='deg')\n", + "\n", + "workflow[WavelengthBins] = sc.geomspace('wavelength', 2.8, 12.5, 2001, unit='angstrom')\n", + "\n", + "# The YIndexLimits and ZIndexLimits define ranges on the detector where\n", + "# data is considered to be valid signal.\n", + "# They represent the lower and upper boundaries of a range of pixel indices.\n", + "workflow[YIndexLimits] = sc.scalar(11), sc.scalar(41)\n", + "workflow[ZIndexLimits] = sc.scalar(80), sc.scalar(370)\n", + "workflow[BeamDivergenceLimits] = (\n", + " sc.scalar(-0.75, unit='deg'),\n", + " sc.scalar(0.75, unit='deg'),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setting the reference run\n", + "\n", + "The reference represents the intensity reflected by the super-mirror.\n", + "The same run is used for normalizing all sample runs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "workflow[Filename[ReferenceRun]] = amor.data.amor_run(614)\n", + "\n", + "# The sample rotation value in the file is slightly off, so we set it manually\n", + "workflow[SampleRotationOffset[ReferenceRun]] = sc.scalar(0.05, unit='deg')" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Computing sample reflectivity from batch reduction\n", + "\n", + "We now compute the sample reflectivity from 4 runs that used different sample rotation angles.\n", + "The measurements at different rotation angles cover different ranges of $Q$.\n", + "\n", + "We use the `batch_compute` function which makes it easy to process multiple runs at once." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "runs = {\n", + " '608': {\n", + " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", + " Filename[SampleRun]: amor.data.amor_run(608),\n", + " },\n", + " '609': {\n", + " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", + " Filename[SampleRun]: amor.data.amor_run(609),\n", + " },\n", + " '610': {\n", + " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", + " Filename[SampleRun]: amor.data.amor_run(610),\n", + " },\n", + " '611': {\n", + " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", + " Filename[SampleRun]: amor.data.amor_run(611),\n", + " },\n", + "}\n", + "\n", + "# Compute R(Q) for all runs\n", + "r_of_q = batch_compute(workflow, runs, target=ReflectivityOverQ)\n", + "r_of_q" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sc.plot(r_of_q.hist(), norm='log', vmin=1e-5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Scaling the reflectivity curves to overlap\n", + "\n", + "In case we know the curves are have been scaled by different factors (that are constant in Q) it can be useful to scale them so they overlap:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "scaled_r = batch_compute(\n", + " workflow,\n", + " runs,\n", + " target=ReflectivityOverQ,\n", + " scale_to_overlap=True,\n", + " critical_edge_interval=(\n", + " sc.scalar(0.01, unit='1/angstrom'),\n", + " sc.scalar(0.014, unit='1/angstrom'),\n", + " ),\n", + ")\n", + "\n", + "sc.plot(scaled_r.hist(), norm='log', vmin=1e-5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Save data\n", + "\n", + "We can save the computed $I(Q)$ to an [ORSO](https://www.reflectometry.org) [.ort](https://github.com/reflectivity/file_format/blob/master/specification.md) file using the [orsopy](https://orsopy.readthedocs.io/en/latest/index.html) package.\n", + "\n", + "First, we need to collect the metadata for that file.\n", + "To this end, we insert a parameter to indicate the creator of the processed data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ess.reflectometry import orso\n", + "from orsopy import fileio" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "workflow[orso.OrsoCreator] = orso.OrsoCreator(\n", + " fileio.base.Person(\n", + " name='Max Mustermann',\n", + " affiliation='European Spallation Source ERIC',\n", + " contact='max.mustermann@ess.eu',\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We build our ORSO dataset from the computed $I(Q)$ and the ORSO metadata:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "iofq_datasets = batch_compute(\n", + " workflow,\n", + " runs,\n", + " target=orso.OrsoIofQDataset,\n", + " scale_to_overlap=True,\n", + " critical_edge_interval=(\n", + " sc.scalar(0.01, unit='1/angstrom'),\n", + " sc.scalar(0.014, unit='1/angstrom'),\n", + " ),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We also add the URL of this notebook to make it easier to reproduce the data:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for ds in iofq_datasets.values():\n", + " ds.info.reduction.script = (\n", + " 'https://scipp.github.io/essreflectometry/user-guide/amor/amor-reduction-simple.html'\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, we can save the data to a file.\n", + "Note that `iofq_datasets` contains [orsopy.fileio.orso.OrsoDataset](https://orsopy.readthedocs.io/en/latest/orsopy.fileio.orso.html#orsopy.fileio.orso.OrsoDataset)s." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fileio.orso.save_orso(\n", + " datasets=list(iofq_datasets.values()), fname='amor_reduced_iofq.ort'\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Look at the first 50 lines of the file to inspect the metadata:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!head amor_reduced_iofq.ort -n50" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/src/ess/reflectometry/__init__.py b/src/ess/reflectometry/__init__.py index ee144f33..000cfaaf 100644 --- a/src/ess/reflectometry/__init__.py +++ b/src/ess/reflectometry/__init__.py @@ -12,7 +12,12 @@ from . import conversions, corrections, figures, normalization, orso from .load import load_reference, save_reference -from .tools import BatchProcessor, batch_processor +from .tools import ( + BatchProcessor, + batch_compute, + batch_processor, + scale_for_reflectivity_overlap, +) providers = ( *corrections.providers, @@ -35,6 +40,7 @@ __all__ = [ "BatchProcessor", "__version__", + "batch_compute", "batch_processor", "conversions", "corrections", @@ -44,4 +50,5 @@ "orso", "providers", "save_reference", + "scale_for_reflectivity_overlap", ] diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index b834bd01..2aeca17b 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -13,7 +13,15 @@ import scipp as sc import scipy.optimize as opt -from ess.reflectometry.types import Filename, SampleRun +from ess.reflectometry import orso +from ess.reflectometry.types import ( + Filename, + ReducedReference, + ReducibleData, + ReferenceRun, + ReflectivityOverQ, + SampleRun, +) from ess.reflectometry.workflow import with_filenames _STD_TO_FWHM = sc.scalar(2.0) * sc.sqrt(sc.scalar(2.0) * sc.log(sc.scalar(2.0))) @@ -514,3 +522,78 @@ def batch_processor( wf[Filename[SampleRun]] = parameters[Filename[SampleRun]] workflows[name] = wf return BatchProcessor(workflows) + + +def batch_compute( + workflow: sl.Pipeline, + runs: Sequence[Mapping[type, Any]] | Mapping[Any, Mapping[type, Any]], + target: type | Sequence[type] = orso.OrsoIofQDataset, + *, + scale_to_overlap: bool = False, + critical_edge_interval: tuple[sc.Variable, sc.Variable] | None = None, +) -> list | Mapping: + ''' + Computes requested target(s) from a supplied workflow for a number of runs. + Each entry of :code:`runs` is a mapping of parameters and + values needed to produce the targets. + + This is an alternative to using :func:`batch_processor`: instead of returning a + BatchProcessor object which can operate on multiple workflows at once, + this function directly computes the requested targets, reducing the risk of + accidentally compromizing the workflows in the collection. + + It also provides the option to scale the reflectivity curves so that they overlap + in the regions where they have the same Q-value. + + Beginners should prefer this function over :func:`batch_processor` unless + they need the extra flexibility of the latter (caching intermediate results, + quickly exploring results, etc). + + Parameters + ----------- + workflow: + The sciline workflow used to compute `ReflectivityOverQ` for each of the runs. + + runs: + The sciline parameters to be used for each run. + + target: + The domain type(s) to compute for each run. + + scale_to_overlap: + If ``True`` the loaded data will be scaled so that the computed reflectivity + curves to overlap. + critical_edge_interval: + A tuple denoting an interval that is known to belong to the critical edge, + i.e. where the reflectivity is known to be 1. + ''' + batch = batch_processor(workflow=workflow, runs=runs) + + # Cache the Reference results as it is often the case that the same reference is + # used for multiple sample runs. + try: + reference_filenames = batch.compute(Filename[ReferenceRun]) + reference_results = {} + wf = workflow.copy() + for fname in set(reference_filenames.values()): + wf[Filename[ReferenceRun]] = fname + reference_results[fname] = wf.compute(ReducedReference) + batch[ReducedReference] = sc.DataGroup( + {k: reference_results[v] for k, v in reference_filenames.items()} + ) + except sl.UnsatisfiedRequirement: + # No reference run found in pipeline + pass + + if scale_to_overlap: + results = batch.compute((ReflectivityOverQ, ReducibleData[SampleRun])) + scale_factors = scale_for_reflectivity_overlap( + results[ReflectivityOverQ].hist(), + critical_edge_interval=critical_edge_interval, + ) + batch[ReducibleData[SampleRun]] = ( + scale_factors * results[ReducibleData[SampleRun]] + ) + batch[ReflectivityOverQ] = scale_factors * results[ReflectivityOverQ] + + return batch.compute(target) diff --git a/tests/reflectometry/tools_test.py b/tests/reflectometry/tools_test.py index 9c3c0c13..520a7e1e 100644 --- a/tests/reflectometry/tools_test.py +++ b/tests/reflectometry/tools_test.py @@ -11,6 +11,7 @@ from ess.reflectometry.tools import ( BatchProcessor, + batch_compute, batch_processor, combine_curves, linlogspace, @@ -291,7 +292,7 @@ def test_linlogspace_bad_input(): @pytest.mark.filterwarnings("ignore:No suitable") -def test_batch_processor_tool_uses_expected_parameters_from_each_run(): +def test_batch_processor_uses_expected_parameters_from_each_run(): def normalized_ioq(filename: Filename[SampleRun]) -> ReflectivityOverQ: return filename @@ -315,7 +316,7 @@ class Reduction: assert results['b'].info.name == 'special.orso' -def test_batch_processor_tool_merges_event_lists(): +def test_batch_processor_merges_event_lists(): wf = make_workflow() runs = { @@ -332,3 +333,94 @@ def test_batch_processor_tool_merges_event_lists(): assert_almost_equal( results['c'].sum().value, (10 + 15 * 0.5) * 0.1 + (10 + 15 * 0.5) * 0.2 ) + + +def test_batch_compute_single_target(): + def A(x: str) -> int: + return int(x) + + params = {'a': {str: '1'}, 'b': {str: '2'}} + workflow = sl.Pipeline([A]) + results = batch_compute(workflow, params, target=int) + assert results == {'a': 1, 'b': 2} + + +def test_batch_compute_multiple_targets(): + def A(x: str) -> float: + return float(x) + + def B(x: str) -> int: + return int(x) + + params = {'a': {str: '1'}, 'b': {str: '2'}} + workflow = sl.Pipeline([A, B]) + results = batch_compute(workflow, params, target=(float, int)) + assert results[float] == {'a': 1.0, 'b': 2.0} + assert results[int] == {'a': 1, 'b': 2} + + +def test_batch_compute_does_not_recompute_reflectivity(): + R = sc.DataArray( + sc.ones(dims=['Q'], shape=(50,), with_variances=True), + coords={'Q': sc.linspace('Q', 0.1, 1, 50)}, + ).bin(Q=10) + + times_evaluated = 0 + + def reflectivity() -> ReflectivityOverQ: + nonlocal times_evaluated + times_evaluated += 1 + return ReflectivityOverQ(R) + + def reducible_data() -> ReducibleData[SampleRun]: + return ReducibleData[SampleRun](1.5) + + pl = sl.Pipeline([reflectivity, reducible_data]) + + batch_compute( + pl, {'a': {}, 'b': {}}, target=(ReflectivityOverQ,), scale_to_overlap=True + ) + assert times_evaluated == 2 + + +def test_batch_compute_applies_scaling_to_reflectivityoverq(): + R1 = sc.DataArray( + sc.ones(dims=['Q'], shape=(50,), with_variances=True), + coords={'Q': sc.linspace('Q', 0.1, 1, 50)}, + ).bin(Q=10) + R2 = 0.5 * R1 + + def reducible_data() -> ReducibleData[SampleRun]: + return 1.5 + + pl = sl.Pipeline([reducible_data]) + + results = batch_compute( + pl, + {'a': {ReflectivityOverQ: R1}, 'b': {ReflectivityOverQ: R2}}, + target=ReflectivityOverQ, + scale_to_overlap=True, + critical_edge_interval=(sc.scalar(0.0), sc.scalar(1.0)), + ) + assert_allclose(results['a'], results['b']) + + +def test_batch_compute_applies_scaling_to_reducibledata(): + R1 = sc.DataArray( + sc.ones(dims=['Q'], shape=(50,), with_variances=True), + coords={'Q': sc.linspace('Q', 0.1, 1, 50)}, + ).bin(Q=10) + R2 = 0.5 * R1 + + def reducible_data() -> ReducibleData[SampleRun]: + return sc.scalar(1) + + pl = sl.Pipeline([reducible_data]) + + results = batch_compute( + pl, + {'a': {ReflectivityOverQ: R1}, 'b': {ReflectivityOverQ: R2}}, + target=ReducibleData[SampleRun], + scale_to_overlap=(sc.scalar(0.0), sc.scalar(1.0)), + ) + assert_allclose(results['a'], 0.5 * results['b']) From f9ffa255255718200c529f6d768b71f8f7baf646 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Fri, 26 Sep 2025 15:18:28 +0200 Subject: [PATCH 64/75] add test on amor workflow with batch_compute that concatenates event lists --- tests/amor/pipeline_test.py | 46 ++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/tests/amor/pipeline_test.py b/tests/amor/pipeline_test.py index 91f45845..75f912d9 100644 --- a/tests/amor/pipeline_test.py +++ b/tests/amor/pipeline_test.py @@ -11,14 +11,17 @@ from ess import amor from ess.amor import data +from ess.amor.types import ChopperPhase from ess.reflectometry import orso -from ess.reflectometry.tools import scale_for_reflectivity_overlap +from ess.reflectometry.tools import batch_compute, scale_for_reflectivity_overlap from ess.reflectometry.types import ( BeamDivergenceLimits, + DetectorRotation, Filename, ProtonCurrent, QBins, RawSampleRotation, + ReducedReference, ReducibleData, ReferenceRun, ReflectivityOverQ, @@ -243,3 +246,44 @@ def test_sample_rotation_offset(amor_pipeline: sciline.Pipeline): ) ).values() assert mu == muoffset.to(unit=muraw.unit) + muraw + + +@pytest.fixture +def pipeline_with_1632_reference(amor_pipeline): # noqa: F811 + amor_pipeline[ChopperPhase[ReferenceRun]] = sc.scalar(7.5, unit='deg') + amor_pipeline[ChopperPhase[SampleRun]] = sc.scalar(7.5, unit='deg') + amor_pipeline[Filename[ReferenceRun]] = data.amor_run('1632') + amor_pipeline[ReducedReference] = amor_pipeline.compute(ReducedReference) + return amor_pipeline + + +def test_batch_compute_concatenates_event_lists( + pipeline_with_1632_reference: sciline.Pipeline, +): + pl = pipeline_with_1632_reference + + run = { + Filename[SampleRun]: list(map(data.amor_run, (1636, 1639, 1641))), + QBins: sc.geomspace( + dim='Q', start=0.062, stop=0.18, num=391, unit='1/angstrom' + ), + DetectorRotation[SampleRun]: sc.scalar(0.140167, unit='rad'), + SampleRotation[SampleRun]: sc.scalar(0.0680678, unit='rad'), + } + result = batch_compute( + pl, + {"": run}, + target=ReflectivityOverQ, + scale_to_overlap=False, + )[""] + + result2 = [] + for fname in run[Filename[SampleRun]]: + pl.copy() + pl[Filename[SampleRun]] = fname + pl[QBins] = run[QBins] + pl[DetectorRotation[SampleRun]] = run[DetectorRotation[SampleRun]] + pl[SampleRotation[SampleRun]] = run[SampleRotation[SampleRun]] + result2.append(pl.compute(ReflectivityOverQ).hist().data) + + assert_allclose(sum(result2), result.hist().data) From 66a4632f7e64abea1fa2bc62d6c10ff3c687e02e Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Fri, 26 Sep 2025 15:19:23 +0200 Subject: [PATCH 65/75] add simple notebook to docs TOC --- docs/user-guide/amor/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/user-guide/amor/index.md b/docs/user-guide/amor/index.md index 2c0d7023..03a09fe1 100644 --- a/docs/user-guide/amor/index.md +++ b/docs/user-guide/amor/index.md @@ -4,6 +4,7 @@ --- maxdepth: 1 --- +amor-reduction-simple amor-reduction compare-to-eos workflow-widget From a74f54b84b91140f014fdc0aefc7ea56f322d1e7 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Fri, 26 Sep 2025 15:20:47 +0200 Subject: [PATCH 66/75] static analysis --- docs/user-guide/amor/amor-reduction-simple.ipynb | 2 +- tests/amor/pipeline_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user-guide/amor/amor-reduction-simple.ipynb b/docs/user-guide/amor/amor-reduction-simple.ipynb index 0b3e0aa3..b799b4f5 100644 --- a/docs/user-guide/amor/amor-reduction-simple.ipynb +++ b/docs/user-guide/amor/amor-reduction-simple.ipynb @@ -32,7 +32,7 @@ "from ess.amor import data # noqa: F401\n", "from ess.reflectometry.types import *\n", "from ess.amor.types import *\n", - "from ess.reflectometry import batch_processor, batch_compute\n", + "from ess.reflectometry import batch_compute\n", "\n", "# The files used in this tutorial have some issues that makes scippnexus\n", "# raise warnings when loading them. To avoid noise in the notebook the warnings are silenced.\n", diff --git a/tests/amor/pipeline_test.py b/tests/amor/pipeline_test.py index 75f912d9..501e89d0 100644 --- a/tests/amor/pipeline_test.py +++ b/tests/amor/pipeline_test.py @@ -249,7 +249,7 @@ def test_sample_rotation_offset(amor_pipeline: sciline.Pipeline): @pytest.fixture -def pipeline_with_1632_reference(amor_pipeline): # noqa: F811 +def pipeline_with_1632_reference(amor_pipeline): amor_pipeline[ChopperPhase[ReferenceRun]] = sc.scalar(7.5, unit='deg') amor_pipeline[ChopperPhase[SampleRun]] = sc.scalar(7.5, unit='deg') amor_pipeline[Filename[ReferenceRun]] = data.amor_run('1632') From 26834529ca7c02fb925fdb64968dbf618e28148d Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Fri, 26 Sep 2025 17:19:28 +0200 Subject: [PATCH 67/75] return to single arg api for batch_compute for scaling_to_overlap --- src/ess/reflectometry/tools.py | 14 ++++++++------ tests/reflectometry/tools_test.py | 3 +-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 2aeca17b..48edc718 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -529,8 +529,9 @@ def batch_compute( runs: Sequence[Mapping[type, Any]] | Mapping[Any, Mapping[type, Any]], target: type | Sequence[type] = orso.OrsoIofQDataset, *, - scale_to_overlap: bool = False, - critical_edge_interval: tuple[sc.Variable, sc.Variable] | None = None, + scale_to_overlap: bool + | tuple[sc.Variable, sc.Variable] + | list[sc.Variable, sc.Variable] = False, ) -> list | Mapping: ''' Computes requested target(s) from a supplied workflow for a number of runs. @@ -563,9 +564,8 @@ def batch_compute( scale_to_overlap: If ``True`` the loaded data will be scaled so that the computed reflectivity curves to overlap. - critical_edge_interval: - A tuple denoting an interval that is known to belong to the critical edge, - i.e. where the reflectivity is known to be 1. + If a tuple is provided, it is interpreted as a critical edge interval where + the reflectivity is known to be 1. ''' batch = batch_processor(workflow=workflow, runs=runs) @@ -589,7 +589,9 @@ def batch_compute( results = batch.compute((ReflectivityOverQ, ReducibleData[SampleRun])) scale_factors = scale_for_reflectivity_overlap( results[ReflectivityOverQ].hist(), - critical_edge_interval=critical_edge_interval, + critical_edge_interval=scale_to_overlap + if isinstance(scale_to_overlap, tuple | list) + else None, ) batch[ReducibleData[SampleRun]] = ( scale_factors * results[ReducibleData[SampleRun]] diff --git a/tests/reflectometry/tools_test.py b/tests/reflectometry/tools_test.py index 520a7e1e..3d4e52be 100644 --- a/tests/reflectometry/tools_test.py +++ b/tests/reflectometry/tools_test.py @@ -399,8 +399,7 @@ def reducible_data() -> ReducibleData[SampleRun]: pl, {'a': {ReflectivityOverQ: R1}, 'b': {ReflectivityOverQ: R2}}, target=ReflectivityOverQ, - scale_to_overlap=True, - critical_edge_interval=(sc.scalar(0.0), sc.scalar(1.0)), + scale_to_overlap=(sc.scalar(0.0), sc.scalar(1.0)), ) assert_allclose(results['a'], results['b']) From ad61d6905456432f78b6983ffc0914acd447b4d5 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Fri, 26 Sep 2025 17:24:44 +0200 Subject: [PATCH 68/75] expand docstrings --- src/ess/reflectometry/tools.py | 71 ++++++++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 4 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 48edc718..fa3c3a42 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -474,19 +474,19 @@ def batch_processor( runs = { '608': { SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'), - Filename[SampleRun]: amor.data.amor_run(608), + Filename[SampleRun]: "file_608.hdf", }, '609': { SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'), - Filename[SampleRun]: amor.data.amor_run(609), + Filename[SampleRun]: "file_609.hdf", }, '610': { SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'), - Filename[SampleRun]: amor.data.amor_run(610), + Filename[SampleRun]: "file_610.hdf", }, '611': { SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'), - Filename[SampleRun]: amor.data.amor_run(611), + Filename[SampleRun]: "file_611.hdf", }, } @@ -495,6 +495,23 @@ def batch_processor( results = batch.compute(ReflectivityOverQ) ``` + Additionally, if a list of filenames is provided for + ``Filename[SampleRun]``, the events from the files will be concatenated + into a single event list before processing. + + Example: + + ``` + runs = { + '608': { + Filename[SampleRun]: "file_608.hdf", + }, + '609+610': { + Filename[SampleRun]: ["file_609.hdf", "file_610.hdf"], + }, + } + ``` + Parameters ---------- workflow: @@ -550,6 +567,52 @@ def batch_compute( they need the extra flexibility of the latter (caching intermediate results, quickly exploring results, etc). + Example: + + ``` + from ess.reflectometry import amor, tools + + workflow = amor.AmorWorkflow() + + runs = { + '608': { + SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'), + Filename[SampleRun]: "file_608.hdf", + }, + '609': { + SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'), + Filename[SampleRun]: "file_609.hdf", + }, + '610': { + SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'), + Filename[SampleRun]: "file_610.hdf", + }, + '611': { + SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'), + Filename[SampleRun]: "file_611.hdf", + }, + } + + r_of_q = tools.batch_compute(workflow, runs, target=ReflectivityOverQ) + ``` + + Additionally, if a list of filenames is provided for + ``Filename[SampleRun]``, the events from the files will be concatenated + into a single event list before processing. + + Example: + + ``` + runs = { + '608': { + Filename[SampleRun]: "file_608.hdf", + }, + '609+610': { + Filename[SampleRun]: ["file_609.hdf", "file_610.hdf"], + }, + } + ``` + Parameters ----------- workflow: From a1a2b98f00ed075dfb6b06be43620b9390bf4bd3 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Fri, 26 Sep 2025 17:27:18 +0200 Subject: [PATCH 69/75] remove automatic caching of reference run, as it is not always guaranteed that the same filename yields the same reference result --- src/ess/reflectometry/tools.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index fa3c3a42..7aa1d52d 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -632,22 +632,6 @@ def batch_compute( ''' batch = batch_processor(workflow=workflow, runs=runs) - # Cache the Reference results as it is often the case that the same reference is - # used for multiple sample runs. - try: - reference_filenames = batch.compute(Filename[ReferenceRun]) - reference_results = {} - wf = workflow.copy() - for fname in set(reference_filenames.values()): - wf[Filename[ReferenceRun]] = fname - reference_results[fname] = wf.compute(ReducedReference) - batch[ReducedReference] = sc.DataGroup( - {k: reference_results[v] for k, v in reference_filenames.items()} - ) - except sl.UnsatisfiedRequirement: - # No reference run found in pipeline - pass - if scale_to_overlap: results = batch.compute((ReflectivityOverQ, ReducibleData[SampleRun])) scale_factors = scale_for_reflectivity_overlap( From 7607214ad7481828f198fffff36c3e56fe13696a Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Fri, 26 Sep 2025 17:30:35 +0200 Subject: [PATCH 70/75] rename notebooks to normal and advanced --- .../amor/amor-reduction-advanced.ipynb | 606 ++++++++++++++++++ .../amor/amor-reduction-simple.ipynb | 321 ---------- docs/user-guide/amor/amor-reduction.ipynb | 355 +--------- docs/user-guide/amor/index.md | 2 +- 4 files changed, 642 insertions(+), 642 deletions(-) create mode 100644 docs/user-guide/amor/amor-reduction-advanced.ipynb delete mode 100644 docs/user-guide/amor/amor-reduction-simple.ipynb diff --git a/docs/user-guide/amor/amor-reduction-advanced.ipynb b/docs/user-guide/amor/amor-reduction-advanced.ipynb new file mode 100644 index 00000000..147200f6 --- /dev/null +++ b/docs/user-guide/amor/amor-reduction-advanced.ipynb @@ -0,0 +1,606 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Divergent data reduction for Amor - advanced\n", + "\n", + "In this notebook, we will look at how to use the `essreflectometry` package with Sciline, for reflectometry data collected from the PSI instrument [Amor](https://www.psi.ch/en/sinq/amor) in [divergent beam mode](https://www.psi.ch/en/sinq/amor/selene).\n", + "\n", + "We will begin by importing the modules that are necessary for this notebook." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# %matplotlib widget\n", + "import warnings\n", + "import scipp as sc\n", + "from ess import amor\n", + "from ess.amor import data # noqa: F401\n", + "from ess.reflectometry.types import *\n", + "from ess.amor.types import *\n", + "from ess.reflectometry import batch_processor\n", + "\n", + "# The files used in this tutorial have some issues that makes scippnexus\n", + "# raise warnings when loading them. To avoid noise in the notebook the warnings are silenced.\n", + "warnings.filterwarnings('ignore', 'Failed to convert .* into a transformation')\n", + "warnings.filterwarnings('ignore', 'Invalid transformation')\n", + "warnings.filterwarnings('ignore', 'invalid value encountered')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create and configure the workflow\n", + "\n", + "We begin by creating the Amor workflow object which is a skeleton for reducing Amor data,\n", + "with pre-configured steps, and then set the missing parameters which are specific to each experiment:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "workflow = amor.AmorWorkflow()\n", + "workflow[SampleSize[SampleRun]] = sc.scalar(10.0, unit='mm')\n", + "workflow[SampleSize[ReferenceRun]] = sc.scalar(10.0, unit='mm')\n", + "\n", + "workflow[ChopperPhase[ReferenceRun]] = sc.scalar(-7.5, unit='deg')\n", + "workflow[ChopperPhase[SampleRun]] = sc.scalar(-7.5, unit='deg')\n", + "\n", + "workflow[WavelengthBins] = sc.geomspace('wavelength', 2.8, 12.5, 2001, unit='angstrom')\n", + "\n", + "# The YIndexLimits and ZIndexLimits define ranges on the detector where\n", + "# data is considered to be valid signal.\n", + "# They represent the lower and upper boundaries of a range of pixel indices.\n", + "workflow[YIndexLimits] = sc.scalar(11), sc.scalar(41)\n", + "workflow[ZIndexLimits] = sc.scalar(80), sc.scalar(370)\n", + "workflow[BeamDivergenceLimits] = (\n", + " sc.scalar(-0.75, unit='deg'),\n", + " sc.scalar(0.75, unit='deg'),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "workflow.visualize(ReflectivityOverQ, graph_attr={'rankdir': 'LR'})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Caching the reference result\n", + "\n", + "The reference result (used for normalizing the sample data) only needs to be computed once.\n", + "It represents the intensity reflected by the super-mirror.\n", + "\n", + "We compute it using the pipeline and thereafter set the result back on the original pipeline." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "workflow[Filename[ReferenceRun]] = amor.data.amor_run(614)\n", + "# The sample rotation value in the file is slightly off, so we set it manually\n", + "workflow[SampleRotationOffset[ReferenceRun]] = sc.scalar(0.05, unit='deg')\n", + "\n", + "reference_result = workflow.compute(ReducedReference)\n", + "# Set the result back onto the pipeline to cache it\n", + "workflow[ReducedReference] = reference_result" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If we now visualize the pipeline again, we can see that the reference is not re-computed:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "workflow.visualize(ReflectivityOverQ, graph_attr={'rankdir': 'LR'})" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Computing sample reflectivity from batch reduction\n", + "\n", + "We now compute the sample reflectivity from 4 runs that used different sample rotation angles.\n", + "The measurements at different rotation angles cover different ranges of $Q$.\n", + "\n", + "We set up a batch reduction helper (using the `batch_processor` function) which makes it easy to process multiple runs at once.\n", + "\n", + "In this tutorial we use some Amor data files we have received.\n", + "The file paths to the tutorial files are obtained by calling:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "amor.data.amor_run(608)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When you encounter `amor.data.amor_run` you should imagining replacing that with a path to your own dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "runs = {\n", + " '608': {\n", + " # The sample rotation values in the files are slightly off, so we replace\n", + " # them with corrected values.\n", + " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", + " Filename[SampleRun]: amor.data.amor_run(608),\n", + " },\n", + " '609': {\n", + " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", + " Filename[SampleRun]: amor.data.amor_run(609),\n", + " },\n", + " '610': {\n", + " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", + " Filename[SampleRun]: amor.data.amor_run(610),\n", + " },\n", + " '611': {\n", + " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", + " Filename[SampleRun]: amor.data.amor_run(611),\n", + " },\n", + "}\n", + "\n", + "batch = batch_processor(workflow, runs)\n", + "\n", + "# Compute R(Q) for all runs\n", + "reflectivity = batch.compute(ReflectivityOverQ)\n", + "sc.plot(reflectivity.hist(), norm='log', vmin=1e-5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "batch.visualize(ReflectivityOverQ, graph_attr={'rankdir': 'LR'})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Scaling the reflectivity curves to overlap\n", + "\n", + "In case we know the curves are have been scaled by different factors (that are constant in Q) it can be useful to scale them so they overlap:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Cache intermediate result to avoid re-loading the data\n", + "batch[ReducibleData[SampleRun]] = batch.compute(ReducibleData[SampleRun])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ess.reflectometry.tools import scale_for_reflectivity_overlap\n", + "\n", + "scaling = scale_for_reflectivity_overlap(\n", + " reflectivity,\n", + " critical_edge_interval=(\n", + " sc.scalar(0.01, unit='1/angstrom'),\n", + " sc.scalar(0.014, unit='1/angstrom'),\n", + " ),\n", + ")\n", + "\n", + "scaled_r = reflectivity * scaling\n", + "\n", + "sc.plot(scaled_r.hist(), norm='log', vmin=1e-5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Curves obtained from measurements at different angles can be combined to one common reflectivity curve:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ess.reflectometry.tools import combine_curves\n", + "\n", + "qbins = sc.geomspace('Q', 0.005, 0.4, 501, unit='1/angstrom')\n", + "combined = combine_curves(scaled_r.hist(), qbins)\n", + "combined.plot(norm='log')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Diagnostic figures" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "There are some useful visualizations that can be used to troubleshoot the instrument.\n", + "They typically operate on the `ReflectivityData`.\n", + "\n", + "The difference between `ReflectivityData` and `ReflectivityOverQ` is that `ReflectivityData` is not binned in $Q$, but instead has the same shape as the reference.\n", + "\n", + "Essentially it represents a \"reflectivity\" computed in every wavelength-detector coordinate (`z_index`) bin.\n", + "This makes it easier to spot inhomogeneities and diagnose problems." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "diagnostics = batch.compute(ReflectivityOverZW)\n", + "diagnostics['608'].hist().flatten(('blade', 'wire'), to='z').plot(norm='log')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Make a $(\\lambda, \\theta)$ map\n", + "A good sanity check is to create a two-dimensional map of the counts in $\\lambda$ and $\\theta$ bins and make sure the triangles converge at the origin." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ess.reflectometry.figures import wavelength_theta_figure\n", + "\n", + "wavelength_theta_figure(\n", + " diagnostics.values(),\n", + " theta_bins=batch.compute(ThetaBins[SampleRun]).values(),\n", + " q_edges_to_display=(\n", + " sc.scalar(0.018, unit='1/angstrom'),\n", + " sc.scalar(0.113, unit='1/angstrom'),\n", + " ),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This plot can be used to check if the value of the sample rotation angle $\\omega$ is correct. The bright triangles should be pointing back to the origin $\\lambda = \\theta = 0$. In the figure above the black lines are all passing through the origin." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Make a $(Q, \\theta)$ map\n", + "Another good sanity check is to create a two-dimensional map of the counts in $\\lambda$ and $Q$ and make sure the stripes are vertical. If they are not that could indicate that the `ChopperPhase` setting is incorrect." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ess.reflectometry.figures import q_theta_figure\n", + "\n", + "q_theta_figure(\n", + " diagnostics.values(),\n", + " theta_bins=batch.compute(ThetaBins[SampleRun]).values(),\n", + " q_bins=qbins,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Compare the sample measurement to the reference on the detector\n", + "\n", + "Here we compare the raw number of counts on the detector for the sample measurement and the reference respectively.\n", + "\n", + "`z_index` is the $z-$detector coordinate, that is, the detector coordinate in the direction of the scattering angle $\\theta$." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ess.reflectometry.figures import wavelength_z_figure\n", + "\n", + "wf_608 = batch.workflows['608']\n", + "\n", + "wavelength_z_figure(\n", + " wf_608.compute(Sample), wavelength_bins=wf_608.compute(WavelengthBins), grid=False\n", + ") + wavelength_z_figure(wf_608.compute(Reference), grid=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Save data\n", + "\n", + "We can save the computed $I(Q)$ to an [ORSO](https://www.reflectometry.org) [.ort](https://github.com/reflectivity/file_format/blob/master/specification.md) file using the [orsopy](https://orsopy.readthedocs.io/en/latest/index.html) package.\n", + "\n", + "First, we need to collect the metadata for that file.\n", + "To this end, we insert a parameter to indicate the creator of the processed data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ess.reflectometry import orso\n", + "from orsopy import fileio" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "batch[orso.OrsoCreator] = {\n", + " k: orso.OrsoCreator(\n", + " fileio.base.Person(\n", + " name='Max Mustermann',\n", + " affiliation='European Spallation Source ERIC',\n", + " contact='max.mustermann@ess.eu',\n", + " )\n", + " )\n", + " for k in runs\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can visualize the workflow for a single run (`'608'`):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wf_608.visualize(orso.OrsoIofQDataset, graph_attr={'rankdir': 'LR'})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# We want to save the scaled results to the file\n", + "batch[ReflectivityOverQ] = scaled_r" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We build our ORSO dataset from the computed $I(Q)$ and the ORSO metadata:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "iofq_datasets = batch.compute(orso.OrsoIofQDataset)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We also add the URL of this notebook to make it easier to reproduce the data:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for ds in iofq_datasets.values():\n", + " ds.info.reduction.script = (\n", + " 'https://scipp.github.io/essreflectometry/user-guide/amor/amor-reduction.html'\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, we can save the data to a file.\n", + "Note that `iofq_datasets` contains [orsopy.fileio.orso.OrsoDataset](https://orsopy.readthedocs.io/en/latest/orsopy.fileio.orso.html#orsopy.fileio.orso.OrsoDataset)s." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fileio.orso.save_orso(\n", + " datasets=list(iofq_datasets.values()), fname='amor_reduced_iofq.ort'\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Look at the first 50 lines of the file to inspect the metadata:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!head amor_reduced_iofq.ort -n50" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Merging multiple runs for the same sample rotation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "runs = {\n", + " '608': {\n", + " # The sample rotation values in the files are slightly off, so we replace\n", + " # them with corrected values.\n", + " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", + " Filename[SampleRun]: amor.data.amor_run(608),\n", + " },\n", + " '609': {\n", + " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", + " Filename[SampleRun]: amor.data.amor_run(609),\n", + " },\n", + " '610': {\n", + " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", + " Filename[SampleRun]: amor.data.amor_run(610),\n", + " },\n", + " '611+612': {\n", + " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", + " Filename[SampleRun]: (\n", + " amor.data.amor_run(611),\n", + " amor.data.amor_run(612),\n", + " ), # List of files means their event lists should be merged\n", + " },\n", + "}\n", + "\n", + "batch = batch_processor(workflow, runs)\n", + "\n", + "# Compute R(Q) for all runs\n", + "reflectivity = batch.compute(ReflectivityOverQ)\n", + "sc.plot(reflectivity.hist(), norm='log', vmin=1e-4)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "batch.visualize(ReflectivityOverQ, graph_attr={'rankdir': 'LR'})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "scaled_r = reflectivity * scale_for_reflectivity_overlap(\n", + " reflectivity,\n", + " critical_edge_interval=(\n", + " sc.scalar(0.01, unit='1/angstrom'),\n", + " sc.scalar(0.014, unit='1/angstrom'),\n", + " ),\n", + ")\n", + "\n", + "sc.plot(scaled_r.hist(), norm='log', vmin=1e-5)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/user-guide/amor/amor-reduction-simple.ipynb b/docs/user-guide/amor/amor-reduction-simple.ipynb deleted file mode 100644 index b799b4f5..00000000 --- a/docs/user-guide/amor/amor-reduction-simple.ipynb +++ /dev/null @@ -1,321 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Divergent data reduction for Amor - Simple\n", - "\n", - "In this notebook, we will look at how to use the `essreflectometry` package with Sciline,\n", - "for reflectometry data collected from the PSI instrument [Amor](https://www.psi.ch/en/sinq/amor) in [divergent beam mode](https://www.psi.ch/en/sinq/amor/selene).\n", - "\n", - "This notebook shows the basics of computing reduced results for a set of runs and saving them to Orso file format." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Setup" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# %matplotlib widget\n", - "import warnings\n", - "import scipp as sc\n", - "from ess import amor\n", - "from ess.amor import data # noqa: F401\n", - "from ess.reflectometry.types import *\n", - "from ess.amor.types import *\n", - "from ess.reflectometry import batch_compute\n", - "\n", - "# The files used in this tutorial have some issues that makes scippnexus\n", - "# raise warnings when loading them. To avoid noise in the notebook the warnings are silenced.\n", - "warnings.filterwarnings('ignore', 'Failed to convert .* into a transformation')\n", - "warnings.filterwarnings('ignore', 'Invalid transformation')\n", - "warnings.filterwarnings('ignore', 'invalid value encountered')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create and configure the workflow\n", - "\n", - "We begin by creating the Amor workflow object which is a skeleton for reducing Amor data,\n", - "with pre-configured steps, and then set the missing parameters which are specific to each experiment:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "workflow = amor.AmorWorkflow()\n", - "workflow[SampleSize[SampleRun]] = sc.scalar(10.0, unit='mm')\n", - "workflow[SampleSize[ReferenceRun]] = sc.scalar(10.0, unit='mm')\n", - "\n", - "workflow[ChopperPhase[ReferenceRun]] = sc.scalar(-7.5, unit='deg')\n", - "workflow[ChopperPhase[SampleRun]] = sc.scalar(-7.5, unit='deg')\n", - "\n", - "workflow[WavelengthBins] = sc.geomspace('wavelength', 2.8, 12.5, 2001, unit='angstrom')\n", - "\n", - "# The YIndexLimits and ZIndexLimits define ranges on the detector where\n", - "# data is considered to be valid signal.\n", - "# They represent the lower and upper boundaries of a range of pixel indices.\n", - "workflow[YIndexLimits] = sc.scalar(11), sc.scalar(41)\n", - "workflow[ZIndexLimits] = sc.scalar(80), sc.scalar(370)\n", - "workflow[BeamDivergenceLimits] = (\n", - " sc.scalar(-0.75, unit='deg'),\n", - " sc.scalar(0.75, unit='deg'),\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Setting the reference run\n", - "\n", - "The reference represents the intensity reflected by the super-mirror.\n", - "The same run is used for normalizing all sample runs." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "workflow[Filename[ReferenceRun]] = amor.data.amor_run(614)\n", - "\n", - "# The sample rotation value in the file is slightly off, so we set it manually\n", - "workflow[SampleRotationOffset[ReferenceRun]] = sc.scalar(0.05, unit='deg')" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Computing sample reflectivity from batch reduction\n", - "\n", - "We now compute the sample reflectivity from 4 runs that used different sample rotation angles.\n", - "The measurements at different rotation angles cover different ranges of $Q$.\n", - "\n", - "We use the `batch_compute` function which makes it easy to process multiple runs at once." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "runs = {\n", - " '608': {\n", - " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", - " Filename[SampleRun]: amor.data.amor_run(608),\n", - " },\n", - " '609': {\n", - " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", - " Filename[SampleRun]: amor.data.amor_run(609),\n", - " },\n", - " '610': {\n", - " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", - " Filename[SampleRun]: amor.data.amor_run(610),\n", - " },\n", - " '611': {\n", - " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", - " Filename[SampleRun]: amor.data.amor_run(611),\n", - " },\n", - "}\n", - "\n", - "# Compute R(Q) for all runs\n", - "r_of_q = batch_compute(workflow, runs, target=ReflectivityOverQ)\n", - "r_of_q" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "sc.plot(r_of_q.hist(), norm='log', vmin=1e-5)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Scaling the reflectivity curves to overlap\n", - "\n", - "In case we know the curves are have been scaled by different factors (that are constant in Q) it can be useful to scale them so they overlap:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "scaled_r = batch_compute(\n", - " workflow,\n", - " runs,\n", - " target=ReflectivityOverQ,\n", - " scale_to_overlap=True,\n", - " critical_edge_interval=(\n", - " sc.scalar(0.01, unit='1/angstrom'),\n", - " sc.scalar(0.014, unit='1/angstrom'),\n", - " ),\n", - ")\n", - "\n", - "sc.plot(scaled_r.hist(), norm='log', vmin=1e-5)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Save data\n", - "\n", - "We can save the computed $I(Q)$ to an [ORSO](https://www.reflectometry.org) [.ort](https://github.com/reflectivity/file_format/blob/master/specification.md) file using the [orsopy](https://orsopy.readthedocs.io/en/latest/index.html) package.\n", - "\n", - "First, we need to collect the metadata for that file.\n", - "To this end, we insert a parameter to indicate the creator of the processed data." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from ess.reflectometry import orso\n", - "from orsopy import fileio" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "workflow[orso.OrsoCreator] = orso.OrsoCreator(\n", - " fileio.base.Person(\n", - " name='Max Mustermann',\n", - " affiliation='European Spallation Source ERIC',\n", - " contact='max.mustermann@ess.eu',\n", - " )\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We build our ORSO dataset from the computed $I(Q)$ and the ORSO metadata:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "iofq_datasets = batch_compute(\n", - " workflow,\n", - " runs,\n", - " target=orso.OrsoIofQDataset,\n", - " scale_to_overlap=True,\n", - " critical_edge_interval=(\n", - " sc.scalar(0.01, unit='1/angstrom'),\n", - " sc.scalar(0.014, unit='1/angstrom'),\n", - " ),\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We also add the URL of this notebook to make it easier to reproduce the data:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for ds in iofq_datasets.values():\n", - " ds.info.reduction.script = (\n", - " 'https://scipp.github.io/essreflectometry/user-guide/amor/amor-reduction-simple.html'\n", - " )" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Finally, we can save the data to a file.\n", - "Note that `iofq_datasets` contains [orsopy.fileio.orso.OrsoDataset](https://orsopy.readthedocs.io/en/latest/orsopy.fileio.orso.html#orsopy.fileio.orso.OrsoDataset)s." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "fileio.orso.save_orso(\n", - " datasets=list(iofq_datasets.values()), fname='amor_reduced_iofq.ort'\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Look at the first 50 lines of the file to inspect the metadata:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!head amor_reduced_iofq.ort -n50" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.7" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/docs/user-guide/amor/amor-reduction.ipynb b/docs/user-guide/amor/amor-reduction.ipynb index 87136458..c39e8a0e 100644 --- a/docs/user-guide/amor/amor-reduction.ipynb +++ b/docs/user-guide/amor/amor-reduction.ipynb @@ -6,9 +6,10 @@ "source": [ "# Divergent data reduction for Amor\n", "\n", - "In this notebook, we will look at how to use the `essreflectometry` package with Sciline, for reflectometry data collected from the PSI instrument [Amor](https://www.psi.ch/en/sinq/amor) in [divergent beam mode](https://www.psi.ch/en/sinq/amor/selene).\n", + "In this notebook, we will look at how to use the `essreflectometry` package with Sciline,\n", + "for reflectometry data collected from the PSI instrument [Amor](https://www.psi.ch/en/sinq/amor) in [divergent beam mode](https://www.psi.ch/en/sinq/amor/selene).\n", "\n", - "We will begin by importing the modules that are necessary for this notebook." + "This notebook shows the basics of computing reduced results for a set of runs and saving them to Orso file format." ] }, { @@ -31,7 +32,7 @@ "from ess.amor import data # noqa: F401\n", "from ess.reflectometry.types import *\n", "from ess.amor.types import *\n", - "from ess.reflectometry import batch_processor\n", + "from ess.reflectometry import batch_compute\n", "\n", "# The files used in this tutorial have some issues that makes scippnexus\n", "# raise warnings when loading them. To avoid noise in the notebook the warnings are silenced.\n", @@ -76,25 +77,14 @@ ")" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "workflow.visualize(ReflectivityOverQ, graph_attr={'rankdir': 'LR'})" - ] - }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Caching the reference result\n", + "## Setting the reference run\n", "\n", - "The reference result (used for normalizing the sample data) only needs to be computed once.\n", - "It represents the intensity reflected by the super-mirror.\n", - "\n", - "We compute it using the pipeline and thereafter set the result back on the original pipeline." + "The reference represents the intensity reflected by the super-mirror.\n", + "The same run is used for normalizing all sample runs." ] }, { @@ -104,28 +94,9 @@ "outputs": [], "source": [ "workflow[Filename[ReferenceRun]] = amor.data.amor_run(614)\n", - "# The sample rotation value in the file is slightly off, so we set it manually\n", - "workflow[SampleRotationOffset[ReferenceRun]] = sc.scalar(0.05, unit='deg')\n", "\n", - "reference_result = workflow.compute(ReducedReference)\n", - "# Set the result back onto the pipeline to cache it\n", - "workflow[ReducedReference] = reference_result" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If we now visualize the pipeline again, we can see that the reference is not re-computed:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "workflow.visualize(ReflectivityOverQ, graph_attr={'rankdir': 'LR'})" + "# The sample rotation value in the file is slightly off, so we set it manually\n", + "workflow[SampleRotationOffset[ReferenceRun]] = sc.scalar(0.05, unit='deg')" ] }, { @@ -138,26 +109,7 @@ "We now compute the sample reflectivity from 4 runs that used different sample rotation angles.\n", "The measurements at different rotation angles cover different ranges of $Q$.\n", "\n", - "We set up a batch reduction helper (using the `batch_processor` function) which makes it easy to process multiple runs at once.\n", - "\n", - "In this tutorial we use some Amor data files we have received.\n", - "The file paths to the tutorial files are obtained by calling:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "amor.data.amor_run(608)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "When you encounter `amor.data.amor_run` you should imagining replacing that with a path to your own dataset." + "We use the `batch_compute` function which makes it easy to process multiple runs at once." ] }, { @@ -168,8 +120,6 @@ "source": [ "runs = {\n", " '608': {\n", - " # The sample rotation values in the files are slightly off, so we replace\n", - " # them with corrected values.\n", " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", " Filename[SampleRun]: amor.data.amor_run(608),\n", " },\n", @@ -187,11 +137,9 @@ " },\n", "}\n", "\n", - "batch = batch_processor(workflow, runs)\n", - "\n", "# Compute R(Q) for all runs\n", - "reflectivity = batch.compute(ReflectivityOverQ)\n", - "sc.plot(reflectivity.hist(), norm='log', vmin=1e-5)" + "r_of_q = batch_compute(workflow, runs, target=ReflectivityOverQ)\n", + "r_of_q" ] }, { @@ -200,7 +148,7 @@ "metadata": {}, "outputs": [], "source": [ - "batch.visualize(ReflectivityOverQ, graph_attr={'rankdir': 'LR'})" + "sc.plot(r_of_q.hist(), norm='log', vmin=1e-5)" ] }, { @@ -218,163 +166,20 @@ "metadata": {}, "outputs": [], "source": [ - "# Cache intermediate result to avoid re-loading the data\n", - "batch[ReducibleData[SampleRun]] = batch.compute(ReducibleData[SampleRun])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from ess.reflectometry.tools import scale_for_reflectivity_overlap\n", - "\n", - "scaling = scale_for_reflectivity_overlap(\n", - " reflectivity,\n", + "scaled_r = batch_compute(\n", + " workflow,\n", + " runs,\n", + " target=ReflectivityOverQ,\n", + " scale_to_overlap=True,\n", " critical_edge_interval=(\n", " sc.scalar(0.01, unit='1/angstrom'),\n", " sc.scalar(0.014, unit='1/angstrom'),\n", " ),\n", ")\n", "\n", - "scaled_r = reflectivity * scaling\n", - "\n", "sc.plot(scaled_r.hist(), norm='log', vmin=1e-5)" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Curves obtained from measurements at different angles can be combined to one common reflectivity curve:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from ess.reflectometry.tools import combine_curves\n", - "\n", - "qbins = sc.geomspace('Q', 0.005, 0.4, 501, unit='1/angstrom')\n", - "combined = combine_curves(scaled_r.hist(), qbins)\n", - "combined.plot(norm='log')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Diagnostic figures" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "There are some useful visualizations that can be used to troubleshoot the instrument.\n", - "They typically operate on the `ReflectivityData`.\n", - "\n", - "The difference between `ReflectivityData` and `ReflectivityOverQ` is that `ReflectivityData` is not binned in $Q$, but instead has the same shape as the reference.\n", - "\n", - "Essentially it represents a \"reflectivity\" computed in every wavelength-detector coordinate (`z_index`) bin.\n", - "This makes it easier to spot inhomogeneities and diagnose problems." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "diagnostics = batch.compute(ReflectivityOverZW)\n", - "diagnostics['608'].hist().flatten(('blade', 'wire'), to='z').plot(norm='log')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Make a $(\\lambda, \\theta)$ map\n", - "A good sanity check is to create a two-dimensional map of the counts in $\\lambda$ and $\\theta$ bins and make sure the triangles converge at the origin." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from ess.reflectometry.figures import wavelength_theta_figure\n", - "\n", - "wavelength_theta_figure(\n", - " diagnostics.values(),\n", - " theta_bins=batch.compute(ThetaBins[SampleRun]).values(),\n", - " q_edges_to_display=(\n", - " sc.scalar(0.018, unit='1/angstrom'),\n", - " sc.scalar(0.113, unit='1/angstrom'),\n", - " ),\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This plot can be used to check if the value of the sample rotation angle $\\omega$ is correct. The bright triangles should be pointing back to the origin $\\lambda = \\theta = 0$. In the figure above the black lines are all passing through the origin." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Make a $(Q, \\theta)$ map\n", - "Another good sanity check is to create a two-dimensional map of the counts in $\\lambda$ and $Q$ and make sure the stripes are vertical. If they are not that could indicate that the `ChopperPhase` setting is incorrect." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from ess.reflectometry.figures import q_theta_figure\n", - "\n", - "q_theta_figure(\n", - " diagnostics.values(),\n", - " theta_bins=batch.compute(ThetaBins[SampleRun]).values(),\n", - " q_bins=qbins,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Compare the sample measurement to the reference on the detector\n", - "\n", - "Here we compare the raw number of counts on the detector for the sample measurement and the reference respectively.\n", - "\n", - "`z_index` is the $z-$detector coordinate, that is, the detector coordinate in the direction of the scattering angle $\\theta$." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from ess.reflectometry.figures import wavelength_z_figure\n", - "\n", - "wf_608 = batch.workflows['608']\n", - "\n", - "wavelength_z_figure(\n", - " wf_608.compute(Sample), wavelength_bins=wf_608.compute(WavelengthBins), grid=False\n", - ") + wavelength_z_figure(wf_608.compute(Reference), grid=False)" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -403,42 +208,13 @@ "metadata": {}, "outputs": [], "source": [ - "batch[orso.OrsoCreator] = {\n", - " k: orso.OrsoCreator(\n", - " fileio.base.Person(\n", - " name='Max Mustermann',\n", - " affiliation='European Spallation Source ERIC',\n", - " contact='max.mustermann@ess.eu',\n", - " )\n", + "workflow[orso.OrsoCreator] = orso.OrsoCreator(\n", + " fileio.base.Person(\n", + " name='Max Mustermann',\n", + " affiliation='European Spallation Source ERIC',\n", + " contact='max.mustermann@ess.eu',\n", " )\n", - " for k in runs\n", - "}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can visualize the workflow for a single run (`'608'`):" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "wf_608.visualize(orso.OrsoIofQDataset, graph_attr={'rankdir': 'LR'})" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# We want to save the scaled results to the file\n", - "batch[ReflectivityOverQ] = scaled_r" + ")" ] }, { @@ -454,7 +230,16 @@ "metadata": {}, "outputs": [], "source": [ - "iofq_datasets = batch.compute(orso.OrsoIofQDataset)" + "iofq_datasets = batch_compute(\n", + " workflow,\n", + " runs,\n", + " target=orso.OrsoIofQDataset,\n", + " scale_to_overlap=True,\n", + " critical_edge_interval=(\n", + " sc.scalar(0.01, unit='1/angstrom'),\n", + " sc.scalar(0.014, unit='1/angstrom'),\n", + " ),\n", + ")" ] }, { @@ -472,7 +257,7 @@ "source": [ "for ds in iofq_datasets.values():\n", " ds.info.reduction.script = (\n", - " 'https://scipp.github.io/essreflectometry/user-guide/amor/amor-reduction.html'\n", + " 'https://scipp.github.io/essreflectometry/user-guide/amor/amor-reduction-simple.html'\n", " )" ] }, @@ -510,76 +295,6 @@ "source": [ "!head amor_reduced_iofq.ort -n50" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Merging multiple runs for the same sample rotation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "runs = {\n", - " '608': {\n", - " # The sample rotation values in the files are slightly off, so we replace\n", - " # them with corrected values.\n", - " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", - " Filename[SampleRun]: amor.data.amor_run(608),\n", - " },\n", - " '609': {\n", - " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", - " Filename[SampleRun]: amor.data.amor_run(609),\n", - " },\n", - " '610': {\n", - " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", - " Filename[SampleRun]: amor.data.amor_run(610),\n", - " },\n", - " '611+612': {\n", - " SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'),\n", - " Filename[SampleRun]: (\n", - " amor.data.amor_run(611),\n", - " amor.data.amor_run(612),\n", - " ), # List of files means their event lists should be merged\n", - " },\n", - "}\n", - "\n", - "batch = batch_processor(workflow, runs)\n", - "\n", - "# Compute R(Q) for all runs\n", - "reflectivity = batch.compute(ReflectivityOverQ)\n", - "sc.plot(reflectivity.hist(), norm='log', vmin=1e-4)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "batch.visualize(ReflectivityOverQ, graph_attr={'rankdir': 'LR'})" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "scaled_r = reflectivity * scale_for_reflectivity_overlap(\n", - " reflectivity,\n", - " critical_edge_interval=(\n", - " sc.scalar(0.01, unit='1/angstrom'),\n", - " sc.scalar(0.014, unit='1/angstrom'),\n", - " ),\n", - ")\n", - "\n", - "sc.plot(scaled_r.hist(), norm='log', vmin=1e-5)" - ] } ], "metadata": { diff --git a/docs/user-guide/amor/index.md b/docs/user-guide/amor/index.md index 03a09fe1..4a50a14b 100644 --- a/docs/user-guide/amor/index.md +++ b/docs/user-guide/amor/index.md @@ -4,8 +4,8 @@ --- maxdepth: 1 --- -amor-reduction-simple amor-reduction +amor-reduction-advanced compare-to-eos workflow-widget ``` From 027bfa7103dc66d27705885640eb5b0e79b19054 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Fri, 26 Sep 2025 17:40:03 +0200 Subject: [PATCH 71/75] remove unused imports --- src/ess/reflectometry/tools.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 7aa1d52d..3a2cb3a3 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -16,9 +16,7 @@ from ess.reflectometry import orso from ess.reflectometry.types import ( Filename, - ReducedReference, ReducibleData, - ReferenceRun, ReflectivityOverQ, SampleRun, ) From 0cc001aa6f89e5b181f4c223ceec6bc24da39385 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Fri, 26 Sep 2025 17:56:38 +0200 Subject: [PATCH 72/75] fix notebook --- docs/user-guide/amor/amor-reduction.ipynb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/user-guide/amor/amor-reduction.ipynb b/docs/user-guide/amor/amor-reduction.ipynb index c39e8a0e..3ffe854b 100644 --- a/docs/user-guide/amor/amor-reduction.ipynb +++ b/docs/user-guide/amor/amor-reduction.ipynb @@ -170,8 +170,7 @@ " workflow,\n", " runs,\n", " target=ReflectivityOverQ,\n", - " scale_to_overlap=True,\n", - " critical_edge_interval=(\n", + " scale_to_overlap=(\n", " sc.scalar(0.01, unit='1/angstrom'),\n", " sc.scalar(0.014, unit='1/angstrom'),\n", " ),\n", @@ -234,8 +233,7 @@ " workflow,\n", " runs,\n", " target=orso.OrsoIofQDataset,\n", - " scale_to_overlap=True,\n", - " critical_edge_interval=(\n", + " scale_to_overlap=(\n", " sc.scalar(0.01, unit='1/angstrom'),\n", " sc.scalar(0.014, unit='1/angstrom'),\n", " ),\n", From f0578c461929da15ec3e2f8284a72762589395cb Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Mon, 29 Sep 2025 13:51:43 +0200 Subject: [PATCH 73/75] fix type hint --- src/ess/reflectometry/tools.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 3a2cb3a3..f77f823b 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -309,7 +309,9 @@ def _interpolate_on_qgrid(curves, grid): def scale_for_reflectivity_overlap( reflectivities: sc.DataArray | Mapping[str, sc.DataArray] | sc.DataGroup, - critical_edge_interval: tuple[sc.Variable, sc.Variable] | None = None, + critical_edge_interval: tuple[sc.Variable, sc.Variable] + | list[sc.Variable] + | None = None, ) -> sc.DataArray | sc.DataGroup: ''' Compute a scaling for 1D reflectivity curves in a way that would makes the curves @@ -546,7 +548,7 @@ def batch_compute( *, scale_to_overlap: bool | tuple[sc.Variable, sc.Variable] - | list[sc.Variable, sc.Variable] = False, + | list[sc.Variable] = False, ) -> list | Mapping: ''' Computes requested target(s) from a supplied workflow for a number of runs. From c5237b4f2d24e6ab693bbb17c945dc3160fc9c26 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Mon, 29 Sep 2025 14:41:20 +0200 Subject: [PATCH 74/75] remove warning filtering which does not seem to be needed, and add a couple of tests on batch processor when working with a mapped workflow --- tests/amor/pipeline_test.py | 16 ----- tests/reflectometry/batch_processor_tests.py | 63 ++++++++++++++------ 2 files changed, 45 insertions(+), 34 deletions(-) diff --git a/tests/amor/pipeline_test.py b/tests/amor/pipeline_test.py index 501e89d0..b56427ce 100644 --- a/tests/amor/pipeline_test.py +++ b/tests/amor/pipeline_test.py @@ -67,8 +67,6 @@ def amor_pipeline() -> sciline.Pipeline: return pl -@pytest.mark.filterwarnings("ignore:Failed to convert .* into a transformation") -@pytest.mark.filterwarnings("ignore:Invalid transformation, missing attribute") def test_has_expected_coordinates(amor_pipeline: sciline.Pipeline): # The sample rotation value in the file is slightly off, so we set it manually amor_pipeline[SampleRotation[SampleRun]] = sc.scalar(0.85, unit="deg") @@ -78,8 +76,6 @@ def test_has_expected_coordinates(amor_pipeline: sciline.Pipeline): assert "Q_resolution" in reflectivity_over_q.coords -@pytest.mark.filterwarnings("ignore:Failed to convert .* into a transformation") -@pytest.mark.filterwarnings("ignore:Invalid transformation, missing attribute") def test_pipeline_no_gravity_correction(amor_pipeline: sciline.Pipeline): # The sample rotation value in the file is slightly off, so we set it manually amor_pipeline[SampleRotation[SampleRun]] = sc.scalar(0.85, unit="deg") @@ -90,8 +86,6 @@ def test_pipeline_no_gravity_correction(amor_pipeline: sciline.Pipeline): assert "Q_resolution" in reflectivity_over_q.coords -@pytest.mark.filterwarnings("ignore:Failed to convert .* into a transformation") -@pytest.mark.filterwarnings("ignore:Invalid transformation, missing attribute") def test_orso_pipeline(amor_pipeline: sciline.Pipeline): # The sample rotation value in the file is slightly off, so we set it manually amor_pipeline[SampleRotation[SampleRun]] = sc.scalar(0.85, unit="deg") @@ -110,8 +104,6 @@ def test_orso_pipeline(amor_pipeline: sciline.Pipeline): assert np.isfinite(res.data).all() -@pytest.mark.filterwarnings("ignore:Failed to convert .* into a transformation") -@pytest.mark.filterwarnings("ignore:Invalid transformation, missing attribute") def test_save_reduced_orso_file(output_folder: Path): from orsopy import fileio @@ -152,8 +144,6 @@ def test_save_reduced_orso_file(output_folder: Path): ) -@pytest.mark.filterwarnings("ignore:Failed to convert .* into a transformation") -@pytest.mark.filterwarnings("ignore:Invalid transformation, missing attribute") def test_pipeline_can_compute_reflectivity_merging_events_from_multiple_runs( amor_pipeline: sciline.Pipeline, ): @@ -168,8 +158,6 @@ def test_pipeline_can_compute_reflectivity_merging_events_from_multiple_runs( assert result.dims == ('Q',) -@pytest.mark.filterwarnings("ignore:Failed to convert .* into a transformation") -@pytest.mark.filterwarnings("ignore:Invalid transformation, missing attribute") def test_pipeline_merging_events_result_unchanged(amor_pipeline: sciline.Pipeline): wf = amor_pipeline.copy() wf[SampleRotationOffset[SampleRun]] = sc.scalar(0.05, unit="deg") @@ -192,8 +180,6 @@ def test_pipeline_merging_events_result_unchanged(amor_pipeline: sciline.Pipelin ) -@pytest.mark.filterwarnings("ignore:Failed to convert .* into a transformation") -@pytest.mark.filterwarnings("ignore:Invalid transformation, missing attribute") def test_proton_current(amor_pipeline: sciline.Pipeline): amor_pipeline[Filename[SampleRun]] = amor.data.amor_run(611) da_without_proton_current = amor_pipeline.compute(ReducibleData[SampleRun]) @@ -233,8 +219,6 @@ def test_proton_current(amor_pipeline: sciline.Pipeline): ) -@pytest.mark.filterwarnings("ignore:Failed to convert .* into a transformation") -@pytest.mark.filterwarnings("ignore:Invalid transformation, missing attribute") def test_sample_rotation_offset(amor_pipeline: sciline.Pipeline): amor_pipeline[Filename[SampleRun]] = amor.data.amor_run(608) amor_pipeline[SampleRotationOffset[SampleRun]] = sc.scalar(1.0, unit='deg') diff --git a/tests/reflectometry/batch_processor_tests.py b/tests/reflectometry/batch_processor_tests.py index 547a48de..c11863e4 100644 --- a/tests/reflectometry/batch_processor_tests.py +++ b/tests/reflectometry/batch_processor_tests.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2025 Scipp contributors (https://github.com/scipp) +import pandas as pd import sciline as sl from ess.reflectometry.tools import BatchProcessor @@ -20,10 +21,10 @@ def test_compute() -> None: wfa[int] = 3 wfb = wf.copy() wfb[int] = 4 - coll = BatchProcessor({'a': wfa, 'b': wfb}) + batch = BatchProcessor({'a': wfa, 'b': wfb}) - assert coll.compute(float) == {'a': 1.5, 'b': 2.0} - assert coll.compute(str) == {'a': '3;1.5', 'b': '4;2.0'} + assert batch.compute(float) == {'a': 1.5, 'b': 2.0} + assert batch.compute(str) == {'a': '3;1.5', 'b': '4;2.0'} def test_compute_multiple() -> None: @@ -32,26 +33,52 @@ def test_compute_multiple() -> None: wfa[int] = 3 wfb = wf.copy() wfb[int] = 4 - coll = BatchProcessor({'a': wfa, 'b': wfb}) + batch = BatchProcessor({'a': wfa, 'b': wfb}) - result = coll.compute([float, str]) + result = batch.compute([float, str]) assert result[float] == {'a': 1.5, 'b': 2.0} assert result[str] == {'a': '3;1.5', 'b': '4;2.0'} +def test_compute_mapped() -> None: + ints = [1, 2, 3] + df = pd.DataFrame({int: ints}) + wf = sl.Pipeline([int_to_float, int_float_to_str]).map(df) + batch = BatchProcessor({"": wf}) + res_float = batch.compute(float) + assert res_float[""] == [0.5, 1.0, 1.5] + res_str = batch.compute(str) + assert res_str[""] == ["1;0.5", "2;1.0", "3;1.5"] + + +def test_compute_mixed_mapped_unmapped() -> None: + ints = [1, 2, 3] + df = pd.DataFrame({int: ints}) + unmapped = sl.Pipeline([int_to_float, int_float_to_str]) + mapped = unmapped.map(df) + unmapped[int] = 5 + batch = BatchProcessor({"unmapped": unmapped, "mapped": mapped}) + res_float = batch.compute(float) + assert res_float['unmapped'] == 2.5 + assert res_float['mapped'] == [0.5, 1.0, 1.5] + res_str = batch.compute(str) + assert res_str['unmapped'] == '5;2.5' + assert res_str['mapped'] == ['1;0.5', '2;1.0', '3;1.5'] + + def test_setitem() -> None: wf = sl.Pipeline([int_to_float, int_float_to_str]) wfa = wf.copy() wfa[int] = 3 wfb = wf.copy() wfb[int] = 4 - coll = BatchProcessor({'a': wfa, 'b': wfb}) + batch = BatchProcessor({'a': wfa, 'b': wfb}) - coll[int] = {'a': 7, 'b': 8} + batch[int] = {'a': 7, 'b': 8} - assert coll.compute(float) == {'a': 3.5, 'b': 4.0} - assert coll.compute(str) == {'a': '7;3.5', 'b': '8;4.0'} + assert batch.compute(float) == {'a': 3.5, 'b': 4.0} + assert batch.compute(str) == {'a': '7;3.5', 'b': '8;4.0'} def test_copy() -> None: @@ -60,15 +87,15 @@ def test_copy() -> None: wfa[int] = 3 wfb = wf.copy() wfb[int] = 4 - coll = BatchProcessor({'a': wfa, 'b': wfb}) + batch = BatchProcessor({'a': wfa, 'b': wfb}) - coll_copy = coll.copy() + batch_copy = batch.copy() - assert coll_copy.compute(float) == {'a': 1.5, 'b': 2.0} - assert coll_copy.compute(str) == {'a': '3;1.5', 'b': '4;2.0'} + assert batch_copy.compute(float) == {'a': 1.5, 'b': 2.0} + assert batch_copy.compute(str) == {'a': '3;1.5', 'b': '4;2.0'} - coll_copy[int] = {'a': 7, 'b': 8} - assert coll.compute(float) == {'a': 1.5, 'b': 2.0} - assert coll.compute(str) == {'a': '3;1.5', 'b': '4;2.0'} - assert coll_copy.compute(float) == {'a': 3.5, 'b': 4.0} - assert coll_copy.compute(str) == {'a': '7;3.5', 'b': '8;4.0'} + batch_copy[int] = {'a': 7, 'b': 8} + assert batch.compute(float) == {'a': 1.5, 'b': 2.0} + assert batch.compute(str) == {'a': '3;1.5', 'b': '4;2.0'} + assert batch_copy.compute(float) == {'a': 3.5, 'b': 4.0} + assert batch_copy.compute(str) == {'a': '7;3.5', 'b': '8;4.0'} From 0297493830d9c22f8bd91bbff4a2facb2333a3b1 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Mon, 29 Sep 2025 14:45:48 +0200 Subject: [PATCH 75/75] cache reference result in Amor simple notebook to avoid computing it many times --- docs/user-guide/amor/amor-reduction.ipynb | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/user-guide/amor/amor-reduction.ipynb b/docs/user-guide/amor/amor-reduction.ipynb index 3ffe854b..4b13e699 100644 --- a/docs/user-guide/amor/amor-reduction.ipynb +++ b/docs/user-guide/amor/amor-reduction.ipynb @@ -84,7 +84,10 @@ "## Setting the reference run\n", "\n", "The reference represents the intensity reflected by the super-mirror.\n", - "The same run is used for normalizing all sample runs." + "The same run is used for normalizing all sample runs,\n", + "and it thus only needs to be reduced once.\n", + "\n", + "In this subsection, we reduce the reference run and cache it onto the workflow to speed-up subsequent processing." ] }, { @@ -96,7 +99,10 @@ "workflow[Filename[ReferenceRun]] = amor.data.amor_run(614)\n", "\n", "# The sample rotation value in the file is slightly off, so we set it manually\n", - "workflow[SampleRotationOffset[ReferenceRun]] = sc.scalar(0.05, unit='deg')" + "workflow[SampleRotationOffset[ReferenceRun]] = sc.scalar(0.05, unit='deg')\n", + "\n", + "# Set the result back onto the pipeline to cache it\n", + "workflow[ReducedReference] = workflow.compute(ReducedReference)" ] }, {