Merge pull request #419 from SpiNNakerManchester/range_compress

Range compress

pacman/model/routing_tables/multicast_routing_tables.py

@@ -30,7 +30,9 @@ class MulticastRoutingTables(object):
         # set that holds routing tables
         "_routing_tables",
         # dict of (x,y) -> routing table
-        "_routing_tables_by_chip"
+        "_routing_tables_by_chip",
+        # maximum value for number_of_entries in all tables
+        "_max_number_of_entries"
     ]
 
     def __init__(self, routing_tables=None):
@@ -42,6 +44,7 @@ def __init__(self, routing_tables=None):
         """
         self._routing_tables = set()
         self._routing_tables_by_chip = dict()
+        self._max_number_of_entries = 0
 
         if routing_tables is not None:
             for routing_table in routing_tables:
@@ -69,6 +72,8 @@ def add_routing_table(self, routing_table):
         self._routing_tables_by_chip[(routing_table.x, routing_table.y)] = \
             routing_table
         self._routing_tables.add(routing_table)
+        self._max_number_of_entries = max(
+            self._max_number_of_entries, routing_table.number_of_entries)
 
     @property
     def routing_tables(self):
@@ -80,6 +85,18 @@ def routing_tables(self):
         """
         return self._routing_tables
 
+    @property
+    def max_number_of_entries(self):
+        """
+        The maximumn number of multi-cast routing entries there are in any\
+            multicast routing table
+
+        Will return zero if there are no routing tables
+
+        :rtype: int
+        """
+        return self._max_number_of_entries
+
     def get_routing_table_for_chip(self, x, y):
         """ Get a routing table for a particular chip
 

pacman/model/routing_tables/uncompressed_multicast_routing_table.py

@@ -14,11 +14,18 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from collections import OrderedDict
+import csv
+import gzip
+import logging
+from spinn_utilities.log import FormatAdapter
+from spinn_machine import MulticastRoutingEntry
 from pacman.exceptions import (
     PacmanAlreadyExistsException, PacmanRoutingException)
 from pacman.model.routing_tables import AbstractMulticastRoutingTable
 from spinn_utilities.overrides import overrides
 
+logger = FormatAdapter(logging.getLogger(__name__))
+
 
 class UnCompressedMulticastRoutingTable(AbstractMulticastRoutingTable):
     """ Represents a uncompressed routing table for a chip.
@@ -205,3 +212,34 @@ def __repr__(self):
     @overrides(AbstractMulticastRoutingTable.__hash__)
     def __hash__(self):
         return id(self)
+
+
+def _from_csv_file(csvfile):
+    table_reader = csv.reader(csvfile)
+    table = UnCompressedMulticastRoutingTable(0, 0)
+    for row in table_reader:
+        try:
+            if len(row) == 3:
+                key = int(row[0], base=16)
+                mask = int(row[1], base=16)
+                route = int(row[2], base=16)
+                table.add_multicast_routing_entry(
+                    MulticastRoutingEntry(key, mask, spinnaker_route=route))
+            elif len(row) == 6:
+                key = int(row[1], base=16)
+                mask = int(row[2], base=16)
+                route = int(row[3], base=16)
+                table.add_multicast_routing_entry(
+                    MulticastRoutingEntry(key, mask, spinnaker_route=route))
+        except ValueError as ex:
+            logger.warning(f"csv read error {ex}")
+    return table
+
+
+def from_csv(file_name):
+    if file_name.endswith(".gz"):
+        with gzip.open(file_name, mode="rt", newline='') as csvfile:
+            return _from_csv_file(csvfile)
+    else:
+        with open(file_name, newline='') as csvfile:
+            return _from_csv_file(csvfile)

pacman/operations/router_compressors/__init__.py

@@ -16,5 +16,7 @@
 from .abstract_compressor import AbstractCompressor
 from .entry import Entry
 from .pair_compressor import pair_compressor
+from .ranged_compressor import range_compressor, RangeCompressor
 
-__all__ = ['AbstractCompressor', 'Entry', 'pair_compressor']
+__all__ = ['AbstractCompressor', 'Entry', 'pair_compressor',
+           'RangeCompressor', 'range_compressor']

pacman/operations/router_compressors/ranged_compressor.py

@@ -0,0 +1,211 @@
+# Copyright (c) 2017-2019 The University of Manchester
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import sys
+from spinn_utilities.config_holder import get_config_bool
+from spinn_utilities.log import FormatAdapter
+from spinn_utilities.progress_bar import ProgressBar
+from spinn_machine import Machine, MulticastRoutingEntry
+from pacman.model.routing_tables import (
+    CompressedMulticastRoutingTable, MulticastRoutingTables)
+from pacman.exceptions import MinimisationFailedError
+
+logger = FormatAdapter(logging.getLogger(__name__))
+
+
+def range_compressor(router_tables, accept_overflow=True):
+    """
+
+    :param MulticastRoutingTables router_tables:
+    :param bool accept_overflow:
+        A flag which should only be used in testing to stop raising an
+        exception if result is too big
+    :rtype: MulticastRoutingTables
+    """
+    if accept_overflow:
+        message = "Precompressing tables using Range Compressor"
+    else:
+        message = "Compressing tables using Range Compressor"
+    progress = ProgressBar(len(router_tables.routing_tables), message)
+    compressor = RangeCompressor()
+    compressed_tables = MulticastRoutingTables()
+    for table in progress.over(router_tables.routing_tables):
+        new_table = compressor.compress_table(table)
+        if (new_table.number_of_entries > Machine.ROUTER_ENTRIES and
+                not accept_overflow):
+            raise MinimisationFailedError(
+                f"The routing table {table.x} {table.y} with "
+                f"{table.number_of_entries} entries after compression "
+                f"still has {new_table.number_of_entries} so will not fit")
+        compressed_tables.add_routing_table(new_table)
+    logger.info(f"Ranged compressor resulted with the largest table of size "
+                f"{compressed_tables.max_number_of_entries}")
+    return compressed_tables
+
+
+class RangeCompressor(object):
+    """
+
+    """
+
+    __slots__ = [
+        # Router table to add merged rutes into
+        "_compressed",
+        # List of entries to be merged
+        "_entries"
+    ]
+
+    def compress_table(self, uncompressed):
+        """ Compresses all the entries for a single table.
+
+        Compressed the entries for this unordered table
+        returning a new table with possibly fewer entries
+
+        :param UnCompressedMulticastRoutingTable uncompressed:
+            Original Routing table for a single chip
+        :return: Compressed routing table for the same chip
+        :rtype: list(Entry)
+        """
+        # Check you need to compress
+        if not get_config_bool(
+                "Mapping", "router_table_compress_as_far_as_possible"):
+            if uncompressed.number_of_entries < Machine.ROUTER_ENTRIES:
+                return uncompressed
+
+        # Step 1 get the entries and make sure they are sorted by key
+        self._entries = uncompressed.multicast_routing_entries
+        self._entries.sort(key=lambda x: x.routing_entry_key)
+        if not self._validate():
+            return uncompressed
+
+        # Step 2 Create the results Table
+        self._compressed = CompressedMulticastRoutingTable(
+            uncompressed.x, uncompressed.y)
+
+        # Step 3 Find ranges of entries with the same route and merge them
+        # Start the first range
+        route = self._entries[0].spinnaker_route
+        first = 0
+        for i, entry in enumerate(self._entries):
+            # Keep building the range until the route changes
+            if entry.spinnaker_route != route:
+                # Merge all the entries in the range
+                self._merge_range(first, i-1)
+                # Start the next range with this entry
+                first = i
+                route = entry.spinnaker_route
+        # Merge the last range
+        self._merge_range(first, i)
+
+        # return the results as a list
+        return self._compressed
+
+    def _validate(self):
+        for i in range(len(self._entries)):
+            if self._get_endpoint(i) >= self._get_key(i+1):
+                logger.warning(
+                    "Unable to run range compressor because entries overlap")
+                return False
+        return True
+
+    def _get_key(self, index):
+        """
+        Gets routing_entry_key for entry index with support for index overflow
+
+        if index == len(self._entries):
+            return sys.maxsize
+        :param int index:
+        """
+        if index == len(self._entries):
+            return sys.maxsize
+        entry = self._entries[index]
+        return entry.routing_entry_key & entry.mask
+
+    def _get_endpoint(self, index):
+        """
+        Get the end of the range covered by entry index's key and mask
+
+        With support for index underflow
+
+        :param index:
+        :return:
+        """
+        if index < 0:
+            return 0
+        entry = self._entries[index]
+        # return the key plus the mask flipping ones and zeros
+        return (entry.routing_entry_key | ~entry.mask) & 0xFFFFFFFF
+
+    def _merge_range(self, first, last):
+        # With a range of 1 just use the existing
+        if first == last:
+            self._compressed.add_multicast_routing_entry(self._entries[first])
+            return
+
+        # Find the points the range must cover
+        first_point = self._get_key(first)
+        last_point = self._get_endpoint(last)
+        # Find the points the range may NOT go into
+        pre_point = self._get_endpoint(first-1)
+        post_point = self._get_key(last+1)
+
+        # find the power big enough to include the first and last enrty
+        dif = last_point - first_point
+        power = self.next_power(dif)
+
+        # Find the start range cutoffs
+        low_cut = first_point // power * power
+        high_cut = low_cut + power
+
+        # If that does not cover all try one power higher
+        if high_cut < last_point:
+            power <<= 1
+            low_cut = first_point // power * power
+            high_cut = low_cut + power
+
+        # The power is too big if it touches the entry before or after
+        while power > 1 and (low_cut < pre_point or high_cut > post_point):
+            power >>= 1
+            low_cut = first_point // power * power
+            high_cut = low_cut + power
+
+        # The range may now not cover all the index so reduce the indexes
+        full_last = last
+        while high_cut <= last_point:
+            last -= 1
+            last_point = self._get_endpoint(last)
+
+        # make the new router entry
+        new_mask = 2 ** 32 - power
+        route = self._entries[first].spinnaker_route
+        new_entry = MulticastRoutingEntry(
+            low_cut, new_mask,  spinnaker_route=route)
+        self._compressed.add_multicast_routing_entry(new_entry)
+
+        # Do any indexes skip from before
+        if full_last != last:
+            self._merge_range(last + 1, full_last)
+
+    @staticmethod
+    def next_power(number):
+        power = 1
+        while power < number:
+            power *= 2
+        return power
+
+    @staticmethod
+    def cut_off(key, power):
+        return key // power * power

unittests/operations_tests/router_compressor_tests/test_compressors.py

@@ -21,8 +21,8 @@
     UnCompressedMulticastRoutingTable, MulticastRoutingTables)
 from pacman.operations.router_compressors.routing_compression_checker import (
     compare_tables)
-from pacman.operations.router_compressors.pair_compressor import (
-    pair_compressor)
+from pacman.operations.router_compressors import (
+    pair_compressor, range_compressor)
 from pacman.operations.router_compressors.ordered_covering_router_compressor \
     import ordered_covering_compressor
 
@@ -64,6 +64,13 @@ def test_pair_compressor(self):
         compressed_tables = pair_compressor(self.original_tables)
         self.check_compression(compressed_tables)
 
+    def test_range_compressor_skipped(self):
+        compressed_tables = range_compressor(self.original_tables)
+        for original in self.original_tables:
+            compressed = compressed_tables.get_routing_table_for_chip(
+                original.x, original.y)
+            self.assertEqual(original, compressed)
+
     def test_checked_unordered_pair_compressor(self):
         compressed_tables = pair_compressor(
             self.original_tables, ordered=False, accept_overflow=False)