Merge pull request #121 from SpiNNakerManchester/reduce_overhead

Reduce overhead

data_specification/constants.py

@@ -47,10 +47,21 @@
 #: Maximum number of random distributions in DSG virtual machine.
 MAX_RANDOM_DISTS = 16
 
+# conversion from words to bytes
+BYTES_PER_WORD = 4
+
 #: Size of header of data spec pointer table produced by DSE, in bytes.
-APP_PTR_TABLE_HEADER_BYTE_SIZE = 8
+#: Note that the header consists of 2 uint32_t variables
+#: (magic_number, version)
+APP_PTR_TABLE_HEADER_BYTE_SIZE = 2 * BYTES_PER_WORD
+#: Size of a region description in the pointer table.
+#: Note that the description consists of a pointer and 2 uint32_t variables:
+#: (pointer, checksum, n_words)
+APP_PTR_TABLE_REGION_BYTE_SIZE = 3 * BYTES_PER_WORD
 #: Size of data spec pointer table produced by DSE, in bytes.
-APP_PTR_TABLE_BYTE_SIZE = APP_PTR_TABLE_HEADER_BYTE_SIZE + MAX_MEM_REGIONS * 4
+APP_PTR_TABLE_BYTE_SIZE = (
+    APP_PTR_TABLE_HEADER_BYTE_SIZE +
+    (MAX_MEM_REGIONS * APP_PTR_TABLE_REGION_BYTE_SIZE))
 
 # Constants used by DSG command encoding; not relevant outside
 LEN1 = 0
@@ -68,6 +79,3 @@
 
 # return values from functions of the data spec executor
 END_SPEC_EXECUTOR = -1
-
-# conversion from words to bytes
-BYTES_PER_WORD = 4

data_specification/data_specification_executor.py

@@ -17,11 +17,12 @@
 import struct
 import functools
 import numpy
+import math
 from spinn_utilities.log import FormatAdapter
 from .data_specification_executor_functions import (
     DataSpecificationExecutorFunctions)
 from .constants import (
-    APPDATA_MAGIC_NUM, DSE_VERSION, END_SPEC_EXECUTOR,
+    APPDATA_MAGIC_NUM, DSE_VERSION, END_SPEC_EXECUTOR, BYTES_PER_WORD,
     MAX_MEM_REGIONS, APP_PTR_TABLE_BYTE_SIZE, APP_PTR_TABLE_HEADER_BYTE_SIZE)
 from .enums import Commands
 from .exceptions import DataSpecificationException
@@ -44,6 +45,9 @@ class DataSpecificationExecutor(object):
         "dsef"
     ]
 
+    TABLE_TYPE = numpy.dtype(
+        [("pointer", "<u4"), ("checksum", "<u4"), ("n_words", "<u4")])
+
     def __init__(self, spec_reader, memory_space):
         """
         :param ~io.RawIOBase spec_reader:
@@ -127,16 +131,27 @@ def get_pointer_table(self, start_address):
         :param int start_address: The base address of the data to be written
         :rtype: numpy.ndarray
         """
-        pointer_table = numpy.zeros(MAX_MEM_REGIONS, dtype="<u4")
-        pointer_table_size = MAX_MEM_REGIONS * 4
+        pointer_table = numpy.zeros(MAX_MEM_REGIONS, dtype=self.TABLE_TYPE)
+        pointer_table_size = MAX_MEM_REGIONS * self.TABLE_TYPE.itemsize
         next_free_offset = pointer_table_size + APP_PTR_TABLE_HEADER_BYTE_SIZE
 
         for i, region in self.mem_regions:
             if isinstance(region, MemoryRegionReal):
-                pointer_table[i] = next_free_offset + start_address
+                data = numpy.array(region.region_data, dtype="uint8")
+                if data.size % BYTES_PER_WORD != 0:
+                    data = numpy.concatenate((
+                        data, numpy.zeros(
+                            BYTES_PER_WORD - (data.size % BYTES_PER_WORD))))
+                pointer_table[i]["pointer"] = next_free_offset + start_address
+                pointer_table[i]["n_words"] = int(
+                    math.ceil(region.max_write_pointer / BYTES_PER_WORD))
+                pointer_table[i]["checksum"] = int(numpy.sum(
+                    data.view("uint32"))) & 0xFFFFFFFF
                 next_free_offset += region.allocated_size
             else:
-                pointer_table[i] = 0
+                pointer_table[i]["pointer"] = 0
+                pointer_table[i]["n_words"] = 0
+                pointer_table[i]["checksum"] = 0
         return pointer_table
 
     @property

unittests/test_data_spec_executor.py

@@ -60,7 +60,7 @@ def test_simple_spec(self):
         executor.execute()
 
         # Test the size
-        header_and_table_size = (constants.MAX_MEM_REGIONS + 2) * 4
+        header_and_table_size = ((constants.MAX_MEM_REGIONS * 3) + 2) * 4
         self.assertEqual(
             executor.get_constructed_data_size(),
             header_and_table_size + 100 + 200 + 4 + 12)
@@ -109,13 +109,13 @@ def test_simple_spec(self):
         # Test the pointer table
         table = executor.get_pointer_table(0)
         self.assertEqual(len(table), constants.MAX_MEM_REGIONS)
-        self.assertEqual(table[0], header_and_table_size)
-        self.assertEqual(table[1], header_and_table_size + 100)
-        self.assertEqual(table[2], header_and_table_size + 300)
-        self.assertEqual(table[3], header_and_table_size + 304)
+        self.assertEqual(table[0]["pointer"], header_and_table_size)
+        self.assertEqual(table[1]["pointer"], header_and_table_size + 100)
+        self.assertEqual(table[2]["pointer"], header_and_table_size + 300)
+        self.assertEqual(table[3]["pointer"], header_and_table_size + 304)
         # 4 is also 0 because it is a reference
         for region in range(4, constants.MAX_MEM_REGIONS):
-            self.assertEqual(table[region], 0)
+            self.assertEqual(table[region]["pointer"], 0)
 
         # Test the header
         header = executor.get_header()