Made it work with scanpointgenerator2

malcolm/controllers/runnablecontroller.py

@@ -267,11 +267,15 @@ def do_configure(self, params):
         self.run_hook(self.Load, self.part_tasks, self.load_structure)
         # Store the params for use in seek()
         self.configure_params = params
+        # This will calculate what we need from the generator, possibly a long
+        # call
+        params.generator.prepare()
         # Set the steps attributes that we will do across many run() calls
-        self.total_steps.set_value(params.generator.num)
+        self.total_steps.set_value(params.generator.size)
         self.completed_steps.set_value(0)
         self.configured_steps.set_value(0)
-        # TODO: this should come from tne generator
+        # TODO: We can be cleverer about this and support a different number
+        # of steps per run for each run by examining the generator structure
         self.steps_per_run = self._get_steps_per_run(
             params.generator, params.axesToMove)
         # Get any status from all parts
@@ -293,18 +297,17 @@ def do_configure(self, params):
     def _get_steps_per_run(self, generator, axes_to_move):
         steps = 1
         axes_set = set(axes_to_move)
-        for g in reversed(generator.generators):
+        for dim in reversed(generator.dimensions):
             # If the axes_set is empty then we are done
             if not axes_set:
                 break
             # Consume the axes that this generator scans
-            for axis in g.position_units:
+            for axis in dim.axes:
                 assert axis in axes_set, \
                     "Axis %s is not in %s" % (axis, axes_to_move)
                 axes_set.remove(axis)
             # Now multiply by the dimensions to get the number of steps
-            for dim in g.index_dims:
-                steps *= dim
+            steps *= dim.size
         return steps
 
     @method_writeable_in(sm.READY)

malcolm/parts/ADCore/detectordriverpart.py

@@ -5,9 +5,6 @@
 from malcolm.parts.ADCore.hdfwriterpart import NDArrayDatasetInfo
 
 
-# Maximum number of points to check for fixed duration
-MAX_CHECK = 100
-
 # Args for configure() and validate
 configure_args = [
     "generator", PointGeneratorMeta("Generator instance"), REQUIRED]
@@ -47,16 +44,10 @@ def report_configuration(self, _):
     @RunnableController.Validate
     @method_takes(*configure_args)
     def validate(self, task, part_info, params):
-        durations = set()
-        max_points = min(MAX_CHECK, params.generator.num)
-        for i in range(max_points):
-            point = params.generator.get_point(i)
-            durations.add(point.duration)
-        assert len(durations) == 1, \
-            "Expected a fixed duration time, got %s" % list(durations)
-        exposure = durations.pop()
-        assert exposure is not None, \
-            "Expected duration to be specified, got None"
+        exposure = params.generator.duration
+        assert exposure > 0, \
+            "Duration %s for generator must be >0 to signify constant exposure"\
+            % exposure
         # TODO: should really get this from an Info from pmac trajectory part...
         exposure -= self.readout_time.value
         assert exposure > 0.0, \
@@ -69,8 +60,7 @@ def validate(self, task, part_info, params):
     @method_takes(*configure_args)
     def configure(self, task, completed_steps, steps_to_do, part_info, params):
         task.unsubscribe_all()
-        exposure = params.generator.get_point(0).duration
-        exposure -= self.readout_time.value
+        exposure = params.generator.duration - self.readout_time.value
         task.put_many(self.child, dict(
             exposure=exposure,
             imageMode="Multiple",

malcolm/parts/ADCore/hdfwriterpart.py

@@ -1,6 +1,5 @@
 import os
 from xml.etree import cElementTree as ET
-from scanpointgenerator import FixedDurationMutator
 
 from malcolm.compat import et_to_string
 from malcolm.core import method_takes, REQUIRED, Info
@@ -51,7 +50,7 @@ class HDFWriterPart(ChildPart):
     def _create_dataset_infos(self, part_info, generator, filename):
         # Update the dataset table
         uniqueid = "/entry/NDAttributes/NDArrayUniqueId"
-        generator_rank = len(generator.index_dims)
+        generator_rank = len(generator.dimensions)
 
         # Get the detector name from the primary source
         ndarray_infos = NDArrayDatasetInfo.filter_values(part_info)
@@ -149,23 +148,15 @@ def configure(self, task, completed_steps, steps_to_do, part_info, params):
         task.wait_all(futures)
         # Reset numCapture back to 0
         task.put(self.child["numCapture"], 0)
-
         # We want the HDF writer to flush this often:
         flush_time = 1  # seconds
         # (In particular this means that HDF files can be read cleanly by
         # SciSoft at the start of a scan.)
-        # To achieve this we'll tell the HDF writer how many frames it should
-        # write between each flush. Thus we need to know the exposure time. Get
-        # it from the last FDM. (There's probably only one, and we don't care
-        # about other cases.)
-        # Choose a default exposure time in case there is no FDM.
-        exposure_time = 0.1  # seconds
-        for mutator in params.generator.mutators:
-            if isinstance(mutator, FixedDurationMutator):
-                exposure_time = mutator.duration
-        # Now do some maths and set the relevant PV. (Xspress3 does not seem to
-        # support flushing more often than once per 2 frames.)
-        n_frames_between_flushes = max(2, round(flush_time/exposure_time))
+        assert params.generator.duration > 0, \
+            "Duration %s for generator must be >0 to signify constant exposure"\
+            % params.generator.duration
+        n_frames_between_flushes = max(2, round(
+            flush_time/params.generator.duration))
         task.put(self.child["flushDataPerNFrames"], n_frames_between_flushes)
         task.put(self.child["flushAttrPerNFrames"], n_frames_between_flushes)
 
@@ -208,16 +199,19 @@ def abort(self, task):
         task.post(self.child["stop"])
 
     def _set_dimensions(self, task, generator):
-        num_dims = len(generator.index_dims)
+        num_dims = len(generator.dimensions)
         assert num_dims <= 10, \
             "Can only do 10 dims, you gave me %s" % num_dims
         attr_dict = dict(numExtraDims=num_dims-1)
         # Fill in dim name and size
+        # NOTE: HDF writer has these filled with fastest moving first
+        # while dimensions is slowest moving first
         for i in range(10):
             suffix = SUFFIXES[i]
-            if i < len(generator.index_names):
-                index_name = generator.index_names[-i - 1]
-                index_size = generator.index_dims[-i - 1]
+            if i < num_dims:
+                forward_i = num_dims - i - 1
+                index_name = "d%d" % forward_i
+                index_size = generator.dimensions[forward_i].size
             else:
                 index_name = ""
                 index_size = 1
@@ -226,53 +220,46 @@ def _set_dimensions(self, task, generator):
         futures = task.put_many_async(self.child, attr_dict)
         return futures
 
-    def _find_generator_index(self, generator, dim):
-        ndims = 0
-        for g in generator.generators:
-            if dim in g.position_units:
-                return ndims, g
-            else:
-                ndims += len(g.index_dims)
-        raise ValueError("Can't find generator for %s" % dim)
-
     def _make_nxdata(self, name, rank, entry_el, generator, link=False):
         # Make a dataset for the data
         data_el = ET.SubElement(entry_el, "group", name=name)
         ET.SubElement(data_el, "attribute", name="signal", source="constant",
                       value=name, type="string")
         pad_dims = []
-        for n in generator.index_names:
-            if n in generator.position_units:
-                pad_dims.append("%s_set" % n)
+        for d in generator.dimensions:
+            if len(d.axes) == 1:
+                pad_dims.append("%s_set" % d.axes[0])
             else:
                 pad_dims.append(".")
+
         pad_dims += ["."] * rank
         ET.SubElement(data_el, "attribute", name="axes", source="constant",
                       value=",".join(pad_dims), type="string")
         ET.SubElement(data_el, "attribute", name="NX_class", source="constant",
                       value="NXdata", type="string")
         # Add in the indices into the dimensions array that our axes refer to
-        for dim, units in sorted(generator.position_units.items()):
-            # Find the generator for this dimension
-            ndims, g = self._find_generator_index(generator, dim)
-            ET.SubElement(data_el, "attribute",
-                          name="%s_set_indices" % dim,
-                          source="constant", value=str(ndims), type="string")
-            if link:
-                ET.SubElement(data_el, "hardlink",
-                              name="%s_set" % dim,
-                              target="/entry/detector/%s_set" % dim)
-            else:
-                axes_vals = []
-                for point in g.iterator():
-                    axes_vals.append("%.12g" % point.positions[dim])
-                axis_el = ET.SubElement(
-                    data_el, "dataset", name="%s_set" % dim,
-                    source="constant", type="float", value=",".join(axes_vals))
-                ET.SubElement(axis_el, "attribute", name="units",
-                              source="constant", value=units, type="string")
+        for i, d in enumerate(generator.dimensions):
+            for axis in d.axes:
+                ET.SubElement(data_el, "attribute",
+                              name="%s_set_indices" % axis,
+                              source="constant", value=str(i), type="string")
+                if link:
+                    ET.SubElement(data_el, "hardlink",
+                                  name="%s_set" % axis,
+                                  target="/entry/detector/%s_set" % axis)
+                else:
+                    self._make_set_points(
+                        d, axis, data_el, generator.units[axis])
         return data_el
 
+    def _make_set_points(self, dimension, axis, data_el, units):
+        axis_vals = ["%.12g" % p for p in dimension.get_positions(axis)]
+        axis_el = ET.SubElement(
+            data_el, "dataset", name="%s_set" % axis, source="constant",
+            type="float", value=",".join(axis_vals))
+        ET.SubElement(axis_el, "attribute", name="units", source="constant",
+                      value=units, type="string")
+
     def _make_layout_xml(self, generator, part_info):
         # Make a root element with an NXEntry
         root_el = ET.Element("hdf5_layout")

malcolm/parts/ADCore/positionlabellerpart.py

@@ -35,8 +35,8 @@ def _make_xml(self, start_index):
         dimensions_el = ET.SubElement(root_el, "dimensions")
 
         # Make an index for every hdf index
-        for index_name in sorted(self.generator.index_names):
-            ET.SubElement(dimensions_el, "dimension", name=index_name)
+        for i in range(len(self.generator.dimensions)):
+            ET.SubElement(dimensions_el, "dimension", name="d%d" % i)
 
         # Add the a file close command for the HDF writer
         ET.SubElement(dimensions_el, "dimension", name="FilePluginClose")
@@ -50,13 +50,13 @@ def _make_xml(self, start_index):
 
         for i in range(start_index, end_index):
             point = self.generator.get_point(i)
-            if i == self.generator.num - 1:
+            if i == self.generator.size - 1:
                 do_close = True
             else:
                 do_close = False
             positions = dict(FilePluginClose="%d" % do_close)
-            for name, value in zip(self.generator.index_names, point.indexes):
-                positions[name] = str(value)
+            for j, value in enumerate(point.indexes):
+                positions["d%d" % j] = str(value)
             position_el = ET.Element("position", **positions)
             positions_el.append(position_el)
 

malcolm/parts/pmac/pmactrajectorypart.py

@@ -3,7 +3,7 @@
 from collections import Counter
 
 import numpy as np
-from scanpointgenerator import FixedDurationMutator, CompoundGenerator
+from scanpointgenerator import CompoundGenerator
 
 from malcolm.controllers.runnablecontroller import RunnableController, \
     ParameterTweakInfo
@@ -282,27 +282,23 @@ def reset(self, task):
     @method_takes(*configure_args)
     def validate(self, task, part_info, params):
         self._make_axis_mapping(part_info, params.axesToMove)
-        # Find the last FixedDurationMutator
-        mutators = []
-        fdm = None
-        for mutator in params.generator.mutators:
-            if isinstance(mutator, FixedDurationMutator):
-                fdm = mutator
-            else:
-                mutators.append(mutator)
+        # Find the duration
+        assert params.generator.duration > 0, \
+            "Can only do fixed duration at the moment"
         servo_freq = 8388608000. / self.child.i10
         # convert half an exposure to multiple of servo ticks, rounding down
         # + 0.002 for some observed jitter in the servo frequency (I18)
-        ticks = np.floor(servo_freq * 0.5 * fdm.duration) + 0.002
+        ticks = np.floor(servo_freq * 0.5 * params.generator.duration) + 0.002
         # convert to integer number of microseconds, rounding up
         micros = np.ceil(ticks / servo_freq * 1e6)
         # back to duration
         duration = 2 * float(micros) / 1e6
-        if duration != fdm.duration:
+        if duration != params.generator.duration:
             new_generator = CompoundGenerator(
                 generators=params.generator.generators,
                 excluders=params.generator.excluders,
-                mutators=mutators + [FixedDurationMutator(duration)])
+                mutators=params.generator.mutators,
+                duration=duration)
             return [ParameterTweakInfo("generator", new_generator)]
 
     def _make_axis_mapping(self, part_info, axes_to_move):

malcolm/parts/xspress3/xspress3driverpart.py

@@ -14,8 +14,8 @@ class Xspress3DriverPart(DetectorDriverPart):
     @method_takes(*configure_args)
     def configure(self, task, completed_steps, steps_to_do, part_info, params):
         if steps_to_do > XSPRESS3_BUFFER:
-            # Set the PointsPerRow from the innermost generator
-            gen_num = params.generator.generators[-1].num
+            # Set the PointsPerRow from the innermost dimension
+            gen_num = params.generator.dimensions[-1].size
             steps_per_row = XSPRESS3_BUFFER // gen_num * gen_num
         else:
             steps_per_row = steps_to_do

requirements/test.txt

@@ -2,6 +2,6 @@ mock>=2.0.0
 nose>=1.3.0
 coverage>=3.7.1
 tornado>=4.1
-scanpointgenerator>=1.6
+scanpointgenerator>=2.0.0
 cothread
 ruamel.yaml

tests/test_controllers/test_runnablecontroller.py

@@ -15,8 +15,7 @@
 from malcolm.core import Process, Part, Task, Map, AbortedError, ResponseError
 from malcolm.core.syncfactory import SyncFactory
 from malcolm.controllers.runnablecontroller import RunnableController
-from scanpointgenerator import LineGenerator, CompoundGenerator, \
-    FixedDurationMutator
+from scanpointgenerator import LineGenerator, CompoundGenerator
 from malcolm.parts.builtin.runnablechildpart import RunnableChildPart
 from malcolm.blocks.demo import Ticker
 
@@ -143,8 +142,7 @@ def test_validate(self):
     def prepare_half_run(self, duration=0.01, exception=0):
         line1 = LineGenerator('y', 'mm', 0, 2, 3)
         line2 = LineGenerator('x', 'mm', 0, 2, 2)
-        duration = FixedDurationMutator(duration)
-        compound = CompoundGenerator([line1, line2], [], [duration])
+        compound = CompoundGenerator([line1, line2], [], [], duration)
         self.b.configure(
             generator=compound, axesToMove=['x'], exceptionStep=exception)
 

tests/test_parts/test_ADCore/test_detectordriverpart.py

@@ -6,8 +6,7 @@
 import unittest
 from mock import Mock, MagicMock, ANY, call
 
-from scanpointgenerator import LineGenerator, CompoundGenerator, \
-    FixedDurationMutator
+from scanpointgenerator import LineGenerator, CompoundGenerator
 from malcolm.parts.ADCore.detectordriverpart import DetectorDriverPart
 
 
@@ -31,10 +30,9 @@ def getitem(name):
     def test_configure(self):
         task = MagicMock()
         params = MagicMock()
-        xs = LineGenerator("x", "mm", 0.0, 0.5, 3, alternate_direction=True)
+        xs = LineGenerator("x", "mm", 0.0, 0.5, 3, alternate=True)
         ys = LineGenerator("y", "mm", 0.0, 0.1, 2)
-        duration = FixedDurationMutator(0.1)
-        params.generator = CompoundGenerator([ys, xs], [], [duration])
+        params.generator = CompoundGenerator([ys, xs], [], [], 0.1)
         completed_steps = 0
         steps_to_do = 6
         part_info = ANY

tests/test_parts/test_ADCore/test_hdfwriterpart.py

@@ -34,9 +34,11 @@ def test_configure(self):
         task = MagicMock()
         params = MagicMock()
         energy = LineGenerator("energy", "kEv", 13.0, 15.2, 2)
-        spiral = SpiralGenerator(["x", "y"], "mm", [0., 0.], 5., scale=2.0)
-        params.generator = CompoundGenerator([energy, spiral], [], [])
+        spiral = SpiralGenerator(
+            ["x", "y"], ["mm", "mm"], [0., 0.], 5., scale=2.0)
+        params.generator = CompoundGenerator([energy, spiral], [], [], 0.1)
         params.filePath = "/tmp/file.h5"
+        params.generator.prepare()
         completed_steps = 0
         steps_to_do = 38
         part_info = {
@@ -99,9 +101,9 @@ def test_configure(self):
         self.assertEqual(task.put_many_async.call_args_list[1],
                          call(self.child, dict(
                              numExtraDims=1,
-                             posNameDimN="x_y_Spiral",
-                             extraDimSizeN=19,
-                             posNameDimX="energy",
+                             posNameDimN="d1",
+                             extraDimSizeN=20,
+                             posNameDimX="d0",
                              extraDimSizeX=2,
                              posNameDimY="",
                              extraDimSizeY=1,