Named regions

Use names to make working with regions and application pointer tables
easier.

nengo_spinnaker/regions/utils.py

@@ -1,8 +1,62 @@
 """Region utilities.
 """
+import collections
+from six import iteritems, iterkeys, itervalues
 import struct
 
 
+def create_app_ptr_and_region_files_named(fp, regions, region_indices,
+                                          region_args):
+    """Split up a file-like view of memory into smaller views, one per region,
+    and write into the first region of memory the offsets to these later
+    regions.
+
+    Parameters
+    ----------
+    regions : {name: Region, ...}
+        Map from names to region objects.
+    region_indices : {name: int, ...}
+        Map from region names to their index in the application pointer table.
+    region_args : {name: (*args, **kwargs)}
+        Map from region names to the arguments and keyword-arguments that
+        should be used when determining the size of a region.
+
+    Returns
+    -------
+    {name: file-like}
+        Map from region name to file-like view of memory.
+    """
+    # Determine the number of entries needed in the application pointer table
+    ptr_len = max(index for index in itervalues(region_indices)) + 1
+
+    # Construct an empty pointer table of the correct length
+    ptrs = [0] * ptr_len
+
+    # Update the offset and then begin to allocate memory
+    region_memory = dict()
+    offset = ptr_len * 4  # In bytes
+    for k, region in iteritems(regions):
+        # Get the size of this region
+        args, kwargs = region_args[k]
+        region_size = region.sizeof_padded(*args, **kwargs)
+
+        # Store the current offset as the pointer for this region
+        ptrs[region_indices[k]] = offset
+
+        # Get the memory region and update the offset
+        next_offset = offset + region_size
+        region_memory[k] = fp[offset:next_offset]
+        offset = next_offset
+
+    # Write the pointer table into memory
+    fp.seek(0)
+    fp.write(struct.pack("<{}I".format(ptr_len), *ptrs))
+    fp.seek(0)
+
+    # Return the region memories
+    return region_memory
+
+
 def create_app_ptr_and_region_files(fp, regions, vertex_slice):
     """Split up a file-like view of memory into smaller views, one per region,
     and write into the first region of memory the offsets to these later
@@ -13,34 +67,74 @@ def create_app_ptr_and_region_files(fp, regions, vertex_slice):
     [file-like view, ...]
         A file-like view of memory for each region.
     """
-    # First we split off the application pointer region
-    ptrs = [0 for n in range(len(regions) + 1)]
-    offset = len(ptrs)*4  # 1 word per region
-
-    # Then we go through and assign each region in turn
-    region_memory = list()
-    for i, r in enumerate(regions, start=1):
-        if r is None:
-            region_memory.append(None)
-        else:
-            ptrs[i] = offset
-            next_offset = offset + r.sizeof_padded(vertex_slice)
-            region_memory.append(fp[offset:next_offset])
-            offset = next_offset
-
-    # Write in the pointer table
-    fp.seek(0)
-    fp.write(struct.pack("<{}I".format(len(ptrs)), *ptrs))
+    # Construct an ordered dictionary of regions and additional dictionaries of
+    # index and arguments.
+    regions, region_indices, region_args = _name_regions(regions, vertex_slice)
 
-    # Return the file views
-    return region_memory
+    # Allocate memory as before
+    region_mem = create_app_ptr_and_region_files_named(
+        fp, regions, region_indices, region_args
+    )
+
+    # Serialise the dictionary in the correct order and return
+    filelikes = [None] * max(iterkeys(region_mem))
+    for i, mem in iteritems(region_mem):
+        filelikes[i - 1] = mem
+    return filelikes
+
+
+def sizeof_regions_named(regions, region_indices, region_args,
+                         include_app_ptr=True):
+    """Return the total amount of memory required to represent all regions when
+    padded to a whole number of words each.
+
+    Parameters
+    ----------
+    regions : {name: Region, ...}
+        Map from names to region objects.
+    region_indices : {name: int, ...}
+        Map from region names to their index in the application pointer table.
+    region_args : {name: (*args, **kwargs)}
+        Map from region names to the arguments and keyword-arguments that
+        should be used when determining the size of a region.
+    """
+    if include_app_ptr:
+        # Get the size of the application pointer
+        size = (max(itervalues(region_indices)) + 1) * 4
+    else:
+        # Don't include the application pointer
+        size = 0
+
+    # Get the size of all the regions
+    for key, region in iteritems(regions):
+        # Get the arguments for the region
+        args, kwargs = region_args[key]
+
+        # Add the size of the region
+        size += region.sizeof_padded(*args, **kwargs)
+
+    return size
 
 
 def sizeof_regions(regions, vertex_slice, include_app_ptr=True):
     """Return the total amount of memory required to represent all the regions
     when they are padded to take a whole number of words each.
     """
-    size = sum(r.sizeof_padded(vertex_slice) for r in regions if r is not None)
-    if include_app_ptr:
-        size += len(regions) * 4 + 4
-    return size
+    # Get the size as before
+    return sizeof_regions_named(*_name_regions(regions, vertex_slice),
+                                include_app_ptr=include_app_ptr)
+
+
+def _name_regions(regions, vertex_slice):
+    """Convert a list of regions into the correct form for a method expecting
+    three dictionaries describing the regions.
+    """
+    regions = collections.OrderedDict({
+        i: r for i, r in enumerate(regions, start=1) if r is not None
+    })
+    region_indices = {i: i for i in iterkeys(regions)}
+    region_args = collections.defaultdict(
+        lambda: ((vertex_slice, ), {})
+    )
+
+    return regions, region_indices, region_args

tests/regions/test_utils.py

@@ -1,12 +1,94 @@
 import mock
 import pytest
+from six import iteritems, itervalues
 import struct
 import tempfile
 
 from nengo_spinnaker.regions.region import Region
 from nengo_spinnaker.regions import utils
 
 
+def test_create_app_ptr_and_filelike_named():
+    """Test creating an application pointer table and file-like views of memory
+    from named regions.
+    """
+    # Create a file-like object which can be the initial file that we pass in,
+    # and wrap it so that creating slices of it creates objects that we can
+    # interpret.
+    actual_fp = tempfile.TemporaryFile()
+
+    class Subfilelike(object):
+        def __init__(self, sl):
+            self.slice = sl
+
+        def __repr__(self):
+            return "Subfilelike({!r})".format(self.slice)
+
+        def __eq__(self, other):
+            return self.slice == other.slice
+
+    def _getitem_(self, sl):
+        return Subfilelike(sl)
+
+    fp = mock.Mock(wraps=actual_fp)
+    fp.__getitem__ = _getitem_
+
+    # Now create a series of regions with different sizes.
+    class MyRegion(Region):
+        def __init__(self, size, expected_arg):
+            self.size = size
+            self.called = False
+            self.expected_arg = expected_arg
+
+        def sizeof(self, arg):
+            assert arg is self.expected_arg
+            self.called = True
+            return self.size
+
+    # Create all the regions, region indices and region slices
+    regions = {
+        "A": MyRegion(4, mock.Mock()),  # 1 word
+        "B": MyRegion(3, mock.Mock()),  # < 1 word
+        "C": MyRegion(5, mock.Mock()),  # < 2 words
+        "D": MyRegion(100, mock.Mock()),  # 25 words
+        "E": MyRegion(32, mock.Mock()),  # 8 words
+    }
+    region_indices = {"A": 1, "B": 2, "C": 3, "D": 4, "E": 6}
+    region_args = {k: ((v.expected_arg, ), {}) for k, v in iteritems(regions)}
+
+    # Now create the application pointer table and all the sub-filelikes
+    fps = utils.create_app_ptr_and_region_files_named(
+        fp, regions, region_indices, region_args
+    )
+
+    # Check that the size was called in all cases
+    assert all(r.called for r in itervalues(regions))
+
+    # Read back the application pointer table
+    actual_fp.seek(0)
+    pointer_table = struct.unpack("<7I", actual_fp.read())
+
+    # Check that the regions are the correct size and do not overlap
+    allocated_bytes = set()
+    for k, filelike in iteritems(fps):
+        # Check that the size is correct
+        exp_size = regions[k].size
+        if exp_size % 4:
+            exp_size += 4 - (exp_size % 4)
+
+        size = filelike.slice.stop - filelike.slice.start
+
+        assert exp_size == size
+
+        # Check that the offset in the pointer table is correct
+        assert pointer_table[region_indices[k]] == filelike.slice.start
+
+        # Finally, ensure that this region of memory wasn't previously assigned
+        used = set(range(filelike.slice.start, filelike.slice.stop))
+        assert not used & allocated_bytes
+        allocated_bytes |= used
+
+
 @pytest.mark.parametrize("vertex_slice", [slice(0, 1), slice(100, 150)])
 def test_create_app_ptr_and_filelikes(vertex_slice):
     """Test creation of an application pointer table and a series of smaller
@@ -51,6 +133,7 @@ def sizeof(self, sl):
         MyRegion(5),  # < 2 words
         MyRegion(100),  # 25 words
         None,  # No region
+        None,
         MyRegion(32),  # 8 words
     ]
 
@@ -59,12 +142,13 @@ def sizeof(self, sl):
     assert all(r.called for r in regions if r is not None)
 
     expected_slices = [
-        slice(28, 32),  # 7 words for the pointer table : +1 word
-        slice(32, 36),  # : +1 word
-        slice(36, 44),  # : +2 words
-        slice(44, 144),  # : +25 words
+        slice(32, 36),  # 8 words for the pointer table : +1 word
+        slice(36, 40),  # : +1 word
+        slice(40, 48),  # : +2 words
+        slice(48, 148),  # : +25 words
         None,
-        slice(144, 176),  # : +8 words
+        None,
+        slice(148, 180),  # : +8 words
     ]
     expected_filelikes = [None if sl is None else Subfilelike(sl) for sl in
                           expected_slices]
@@ -74,11 +158,44 @@ def sizeof(self, sl):
     # correct.
     actual_fp.seek(0)
     assert actual_fp.read() == struct.pack(
-        "<7I",
+        "<8I",
         0, *[0 if s is None else s.slice.start for s in expected_filelikes]
     )
 
 
+@pytest.mark.parametrize("include_app_ptr", (True, False))
+def test_sizeof_regions_named(include_app_ptr):
+    """Test getting the total memory usage of some regions."""
+    class MyRegion(Region):
+        def __init__(self, size, expected_arg):
+            self.size = size
+            self.called = False
+            self.expected_arg = expected_arg
+
+        def sizeof(self, *args, **kwargs):
+            assert args == (self.expected_arg, )
+            self.called = True
+            return self.size
+
+    # Create all the regions
+    regions = {
+        "A": MyRegion(4, mock.Mock()),  # 1 word
+        "B": MyRegion(3, mock.Mock()),  # < 1 word
+        "C": MyRegion(5, mock.Mock()),  # < 2 words
+        "D": MyRegion(100, mock.Mock()),  # 25 words
+        "E": MyRegion(32, mock.Mock()),  # 8 words
+    }
+    region_indices = {"A": 1, "B": 2, "C": 3, "D": 4, "E": 6}
+    region_args = {k: ((v.expected_arg, ), {}) for k, v in iteritems(regions)}
+
+    # Now query their size
+    expected_size = 37*4 + (7*4 if include_app_ptr else 0)
+    size = utils.sizeof_regions_named(regions, region_indices, region_args,
+                                      include_app_ptr)
+    assert all(r.called for r in itervalues(regions))
+    assert size == expected_size
+
+
 @pytest.mark.parametrize(
     "vertex_slice, include_app_ptr",
     [(slice(0, 1), True), (slice(100, 150), False)])