Merge 1461209 into b36193b

pacman/operations/chip_id_allocator_algorithms/malloc_based_chip_id_allocator.py

@@ -52,7 +52,7 @@ class MallocBasedChipIdAllocator(ElementAllocatorAlgorithm):
     ]
 
     def __init__(self):
-        super(MallocBasedChipIdAllocator, self).__init__(0, 2 ** 32)
+        super(MallocBasedChipIdAllocator, self).__init__([(0, 2 ** 32)])
 
         # we only want one virtual chip per 'link'
         self._virtual_chips = dict()
@@ -72,7 +72,7 @@ def allocate_chip_ids(self, machine, graph):
         # allocate standard IDs for real chips
         for x, y in progress.over(machine.chip_coordinates, False):
             expected_chip_id = (x << 8) + y
-            self._allocate_elements(expected_chip_id, 1)
+            self.allocate_elements(expected_chip_id, 1)
 
         # allocate IDs for virtual chips
         for vertex in progress.over(graph.vertices):
@@ -120,5 +120,5 @@ def _allocate_id(self):
         # otherwise it will have already been deleted already.
         free_space_chunk = self._free_space_tracker[0]
         chip_id = free_space_chunk.start_address
-        self._allocate_elements(chip_id, 1)
+        self.allocate_elements(chip_id, 1)
         return (chip_id >> 8), (chip_id & _LOWER_16_BITS)

pacman/operations/routing_info_allocator_algorithms/malloc_based_routing_allocator/compressible_malloc_based_routing_info_allocator.py

@@ -18,8 +18,6 @@
 except ImportError:
     from collections import defaultdict, OrderedDict
 import logging
-import numpy
-from past.builtins import xrange
 from six import iteritems, itervalues
 from spinn_utilities.log import FormatAdapter
 from spinn_utilities.ordered_set import OrderedSet
@@ -33,12 +31,12 @@
 from pacman.model.routing_info import (
     RoutingInfo, BaseKeyAndMask, PartitionRoutingInfo)
 from pacman.utilities.utility_calls import (
-    check_algorithm_can_support_constraints, locate_constraints_of_type,
-    compress_from_bit_array, expand_to_bit_array)
+    check_algorithm_can_support_constraints, locate_constraints_of_type)
 from pacman.utilities.algorithm_utilities import ElementAllocatorAlgorithm
 from pacman.utilities.algorithm_utilities.routing_info_allocator_utilities \
     import (
-        check_types_of_edge_constraint, get_edge_groups)
+        check_types_of_edge_constraint, get_edge_groups,
+        generate_key_ranges_from_mask)
 from pacman.exceptions import (
     PacmanConfigurationException, PacmanRouteInfoAllocationException)
 from .utils import get_possible_masks
@@ -56,7 +54,7 @@ class CompressibleMallocBasedRoutingInfoAllocator(ElementAllocatorAlgorithm):
 
     def __init__(self):
         super(CompressibleMallocBasedRoutingInfoAllocator, self).__init__(
-            0, 2 ** 32)
+            [(0, 2 ** 32)])
 
     def __call__(self, machine_graph, n_keys_map, routing_tables):
         # check that this algorithm supports the constraints
@@ -76,15 +74,15 @@ def __call__(self, machine_graph, n_keys_map, routing_tables):
 
         # Get the edges grouped by those that require the same key
         (fixed_keys, _shared_keys, fixed_masks, fixed_fields, flexi_fields,
-         continuous, noncontinuous) = \
-            get_edge_groups(machine_graph, EdgeTrafficType.MULTICAST)
+         continuous, noncontinuous) = get_edge_groups(
+             machine_graph, EdgeTrafficType.MULTICAST)
         if flexi_fields:
             raise PacmanConfigurationException(
                 "MallocBasedRoutingInfoAllocator does not support FlexiField")
 
         # Even non-continuous keys will be continuous
         for group in noncontinuous:
-            continuous.add(group)
+            continuous.append(group)
 
         # Go through the groups and allocate keys
         progress = ProgressBar(
@@ -95,13 +93,12 @@ def __call__(self, machine_graph, n_keys_map, routing_tables):
         for group in progress.over(fixed_keys, False):
             # Get any fixed keys and masks from the group and attempt to
             # allocate them
-            fixed_mask = None
             fixed_key_and_mask_constraint = locate_constraints_of_type(
                 group.constraints, FixedKeyAndMaskConstraint)[0]
 
             # attempt to allocate them
             self._allocate_fixed_keys_and_masks(
-                fixed_key_and_mask_constraint.keys_and_masks, fixed_mask)
+                fixed_key_and_mask_constraint.keys_and_masks)
 
             # update the pacman data objects
             self._update_routing_objects(
@@ -155,18 +152,16 @@ def __call__(self, machine_graph, n_keys_map, routing_tables):
             for partition, entry in iteritems(routing_table):
                 if partition in continuous:
                     entry_hash = sum(
-                        1 << i
-                        for i in entry.link_ids)
+                        1 << i for i in entry.link_ids)
                     entry_hash += sum(
-                        1 << (i + 6)
-                        for i in entry.processor_ids)
+                        1 << (i + 6) for i in entry.processor_ids)
                     partitions_by_route[entry_hash].add(partition)
 
             for entry_hash, partitions in iteritems(partitions_by_route):
-                found_groups = list()
-                for partition in partitions:
-                    if partition in partition_groups:
-                        found_groups.append(partition_groups[partition])
+                found_groups = [
+                    partition_groups[partition]
+                    for partition in partitions
+                    if partition in partition_groups]
 
                 if not found_groups:
                     # If no group was found, create a new one
@@ -205,69 +200,23 @@ def __call__(self, machine_graph, n_keys_map, routing_tables):
         return routing_infos
 
     @staticmethod
-    def _update_routing_objects(
-            keys_and_masks, routing_infos, group):
+    def _update_routing_objects(keys_and_masks, routing_infos, group):
         # Allocate the routing information
-        partition_info = PartitionRoutingInfo(keys_and_masks, group)
-        routing_infos.add_partition_info(partition_info)
+        routing_infos.add_partition_info(PartitionRoutingInfo(
+            keys_and_masks, group))
 
-    @staticmethod
-    def _get_key_ranges(key, mask):
-        """ Get a generator of base_key, n_keys pairs that represent ranges
-            allowed by the mask
-
-        :param key: The base key
-        :param mask: The mask
-        """
-        unwrapped_mask = expand_to_bit_array(mask)
-        first_zeros = list()
-        remaining_zeros = list()
-        pos = len(unwrapped_mask) - 1
-
-        # Keep the indices of the first set of zeros
-        while pos >= 0 and unwrapped_mask[pos] == 0:
-            first_zeros.append(pos)
-            pos -= 1
-
-        # Find all the remaining zeros
-        while pos >= 0:
-            if unwrapped_mask[pos] == 0:
-                remaining_zeros.append(pos)
-            pos -= 1
-
-        # Loop over 2^len(remaining_zeros) to produce the base key,
-        # with n_keys being 2^len(first_zeros)
-        n_sets = 2 ** len(remaining_zeros)
-        n_keys = 2 ** len(first_zeros)
-        if not remaining_zeros:
-            yield key, n_keys
-            return
-        unwrapped_key = expand_to_bit_array(key)
-        for value in xrange(n_sets):
-            generated_key = numpy.copy(unwrapped_key)
-            generated_key[remaining_zeros] = \
-                expand_to_bit_array(value)[-len(remaining_zeros):]
-            yield compress_from_bit_array(generated_key), n_keys
-
-    def _allocate_fixed_keys_and_masks(self, keys_and_masks, fixed_mask):
+    def _allocate_fixed_keys_and_masks(self, keys_and_masks):
         """ Allocate fixed keys and masks
 
-        :param keys_and_masks: the fixed keys and masks combos
-        :param fixed_mask: fixed mask
-        :type fixed_mask: None or FixedMask object
+        :param keys_and_masks: set of keys and masks
         :rtype: None
         """
         # If there are fixed keys and masks, allocate them
         for key_and_mask in keys_and_masks:
-            # If there is a fixed mask, check it doesn't clash
-            if fixed_mask is not None and fixed_mask != key_and_mask.mask:
-                raise PacmanRouteInfoAllocationException(
-                    "Cannot meet conflicting constraints")
-
             # Go through the mask sets and allocate
-            for key, n_keys in self._get_key_ranges(
+            for key, n_keys in generate_key_ranges_from_mask(
                     key_and_mask.key, key_and_mask.mask):
-                self._allocate_elements(key, n_keys)
+                self.allocate_elements(key, n_keys)
 
     def _allocate_keys_and_masks(self, fixed_mask, fields, partition_n_keys,
                                  contiguous_keys=True):
@@ -294,23 +243,7 @@ def _allocate_keys_and_masks(self, fixed_mask, fields, partition_n_keys,
                 logger.debug("Trying key {}", hex(key))
 
                 # Check if all the key ranges can be allocated
-                matched_all = True
-                index = 0
-                for (base_key, n_keys) in self._get_key_ranges(key, mask):
-                    logger.debug("Finding slot for {}, n_keys={}",
-                                 hex(base_key), n_keys)
-                    index = self._find_slot(base_key, lo=index)
-                    logger.debug("Slot for {} is {}", hex(base_key), index)
-                    if index is None:
-                        matched_all = False
-                        break
-                    space = self._check_allocation(index, base_key, n_keys)
-                    logger.debug("Space for {} is {}", hex(base_key), space)
-                    if space is None:
-                        matched_all = False
-                        break
-
-                if matched_all:
+                if self.__check_match(key, mask):
                     logger.debug("Matched key {}", hex(key))
                     key_found = key
                     break
@@ -324,11 +257,26 @@ def _allocate_keys_and_masks(self, fixed_mask, fields, partition_n_keys,
         # If we found a working key and mask that can be assigned,
         # Allocate them
         if key_found is not None and mask_found is not None:
-            for (base_key, n_keys) in self._get_key_ranges(key_found, mask):
-                self._allocate_elements(base_key, n_keys)
-
+            for (base_key, n_keys) in generate_key_ranges_from_mask(
+                    key_found, mask):
+                self.allocate_elements(base_key, n_keys)
             # If we get here, we can assign the keys to the edges
             return [BaseKeyAndMask(base_key=key_found, mask=mask)]
 
         raise PacmanRouteInfoAllocationException(
             "Could not find space to allocate keys")
+
+    def __check_match(self, key, mask):
+        index = 0
+        for (base_key, n_keys) in generate_key_ranges_from_mask(key, mask):
+            logger.debug("Finding slot for {}, n_keys={}",
+                         hex(base_key), n_keys)
+            index = self._find_slot(base_key, lo=index)
+            logger.debug("Slot for {} is {}", hex(base_key), index)
+            if index is None:
+                return False
+            space = self._check_allocation(index, base_key, n_keys)
+            logger.debug("Space for {} is {}", hex(base_key), space)
+            if space is None:
+                return False
+        return True

pacman/operations/routing_info_allocator_algorithms/malloc_based_routing_allocator/malloc_based_routing_info_allocator.py

@@ -14,8 +14,6 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import logging
-import numpy
-from past.builtins import xrange
 from spinn_utilities.progress_bar import ProgressBar
 from spinn_utilities.log import FormatAdapter
 from pacman.model.graphs.common import EdgeTrafficType
@@ -26,11 +24,11 @@
 from pacman.model.routing_info import (
     RoutingInfo, BaseKeyAndMask, PartitionRoutingInfo)
 from pacman.utilities.utility_calls import (
-    check_algorithm_can_support_constraints,
-    compress_from_bit_array, expand_to_bit_array)
+    check_algorithm_can_support_constraints)
 from pacman.utilities.algorithm_utilities import ElementAllocatorAlgorithm
 from pacman.utilities.algorithm_utilities.routing_info_allocator_utilities \
-    import (check_types_of_edge_constraint, get_edge_groups)
+    import (check_types_of_edge_constraint, get_edge_groups,
+            generate_key_ranges_from_mask)
 from pacman.exceptions import (
     PacmanConfigurationException, PacmanRouteInfoAllocationException)
 from .utils import get_possible_masks
@@ -46,7 +44,8 @@ class MallocBasedRoutingInfoAllocator(ElementAllocatorAlgorithm):
     __slots__ = []
 
     def __init__(self):
-        super(MallocBasedRoutingInfoAllocator, self).__init__(0, 2 ** 32)
+        super(MallocBasedRoutingInfoAllocator, self).__init__(
+            [(0, 2 ** 32)])
 
     def __call__(self, machine_graph, n_keys_map, graph_mapper=None):
         # check that this algorithm supports the constraints
@@ -173,44 +172,6 @@ def _update_routing_objects(
         partition_info = PartitionRoutingInfo(keys_and_masks, group)
         routing_infos.add_partition_info(partition_info)
 
-    @staticmethod
-    def _get_key_ranges(key, mask):
-        """ Get a generator of base_key, n_keys pairs that represent ranges
-            allowed by the mask
-
-        :param key: The base key
-        :param mask: The mask
-        """
-        unwrapped_mask = expand_to_bit_array(mask)
-        first_zeros = list()
-        remaining_zeros = list()
-        pos = len(unwrapped_mask) - 1
-
-        # Keep the indices of the first set of zeros
-        while pos >= 0 and unwrapped_mask[pos] == 0:
-            first_zeros.append(pos)
-            pos -= 1
-
-        # Find all the remaining zeros
-        while pos >= 0:
-            if unwrapped_mask[pos] == 0:
-                remaining_zeros.append(pos)
-            pos -= 1
-
-        # Loop over 2^len(remaining_zeros) to produce the base key,
-        # with n_keys being 2^len(first_zeros)
-        n_sets = 2 ** len(remaining_zeros)
-        n_keys = 2 ** len(first_zeros)
-        if not remaining_zeros:
-            yield key, n_keys
-            return
-        unwrapped_key = expand_to_bit_array(key)
-        for value in xrange(n_sets):
-            generated_key = numpy.copy(unwrapped_key)
-            generated_key[remaining_zeros] = \
-                expand_to_bit_array(value)[-len(remaining_zeros):]
-            yield compress_from_bit_array(generated_key), n_keys
-
     def _allocate_fixed_keys_and_masks(self, keys_and_masks, fixed_mask):
         """ Allocate fixed keys and masks
 
@@ -227,9 +188,9 @@ def _allocate_fixed_keys_and_masks(self, keys_and_masks, fixed_mask):
                     "Cannot meet conflicting constraints")
 
             # Go through the mask sets and allocate
-            for key, n_keys in self._get_key_ranges(
+            for key, n_keys in generate_key_ranges_from_mask(
                     key_and_mask.key, key_and_mask.mask):
-                self._allocate_elements(key, n_keys)
+                self.allocate_elements(key, n_keys)
 
     def _allocate_keys_and_masks(self, fixed_mask, fields, partition_n_keys,
                                  contiguous_keys=True):
@@ -258,7 +219,8 @@ def _allocate_keys_and_masks(self, fixed_mask, fields, partition_n_keys,
                 # Check if all the key ranges can be allocated
                 matched_all = True
                 index = 0
-                for (base_key, n_keys) in self._get_key_ranges(key, mask):
+                for (base_key, n_keys) in generate_key_ranges_from_mask(
+                        key, mask):
                     logger.debug("Finding slot for {}, n_keys={}",
                                  hex(base_key), n_keys)
                     index = self._find_slot(base_key, lo=index)
@@ -286,8 +248,9 @@ def _allocate_keys_and_masks(self, fixed_mask, fields, partition_n_keys,
         # If we found a working key and mask that can be assigned,
         # Allocate them
         if key_found is not None and mask_found is not None:
-            for (base_key, n_keys) in self._get_key_ranges(key_found, mask):
-                self._allocate_elements(base_key, n_keys)
+            for (base_key, n_keys) in generate_key_ranges_from_mask(
+                    key_found, mask):
+                self.allocate_elements(base_key, n_keys)
 
             # If we get here, we can assign the keys to the edges
             return [BaseKeyAndMask(base_key=key_found, mask=mask)]