Discover and use multiple Ethernet connections

This commit adds support for discovering and using multiple Ethernet
connections (when available) but does not (yet) feature any way to use these
connections in parallel to improve performance. A limited performance
improvement due to reduced average latency within the machine may be plausible,
however.

This commit lifts board geometry functions from commit b666e80 (part of the now
defunct non-blocking-io branch) along with the basic principles for probing the
machine for Ethernet connections.

rig/geometry.py

@@ -5,6 +5,8 @@
 
 from math import sqrt
 
+import numpy as np
+
 
 def to_xyz(xy):
     """Convert a two-tuple (x, y) coordinate into an (x, y, 0) coordinate."""
@@ -214,3 +216,84 @@ def standard_system_dimensions(num_boards):
     # Convert the number of triads into numbers of chips (each triad of boards
     # contributes as 12x12 block of chips).
     return (w * 12, h * 12)
+
+
+def spinn5_eth_coords(width, height):
+    """Generate a list of board coordinates with Ethernet connectivity in a
+    SpiNNaker machine.
+
+    Specifically, generates the coordinates for the Ethernet connected chips of
+    SpiNN-5 boards arranged in a standard torus topology.
+
+    Parameters
+    ----------
+    width : int
+        Width of the system in chips.
+    height : int
+        Height of the system in chips.
+    """
+    # Internally, work with the width and height rounded up to the next
+    # multiple of 12
+    w = ((width + 11) // 12) * 12
+    h = ((height + 11) // 12) * 12
+
+    for x in range(0, w, 12):
+        for y in range(0, h, 12):
+            for dx, dy in ((0, 0), (4, 8), (8, 4)):
+                nx = (x + dx) % w
+                ny = (y + dy) % h
+                # Skip points which are outside the range available
+                if nx < width and ny < height:
+                    yield (nx, ny)
+
+
+def spinn5_local_eth_coord(x, y, w, h):
+    """Get the coordinates of a chip's local ethernet connected chip.
+
+    .. note::
+        This function assumes the system is constructed from SpiNN-5 boards
+        returns the coordinates of the ethernet connected chip on the current
+        board.
+
+    Parameters
+    ----------
+    x : int
+    y : int
+    w : int
+        Width of the system in chips.
+    h : int
+        Height of the system in chips.
+    """
+    dx, dy = SPINN5_ETH_OFFSET[y % 12][x % 12]
+    return ((x + dx) % w), ((y + dy) % h)
+
+
+SPINN5_ETH_OFFSET = np.array([
+    [(vx - x, vy - y) for x, (vx, vy) in enumerate(row)]
+    for y, row in enumerate([
+        # Below is an enumeration of the absolute coordinates of the nearest
+        # ethernet connected chip. Note that the above list comprehension
+        # changes these into offsets to the nearest chip.
+        # X:   0         1         2         3         4         5         6         7         8         9        10        11     # noqa Y:
+        [(+0, +0), (+0, +0), (+0, +0), (+0, +0), (+0, +0), (+4, -4), (+4, -4), (+4, -4), (+4, -4), (+4, -4), (+4, -4), (+4, -4)],  # noqa  0
+        [(+0, +0), (+0, +0), (+0, +0), (+0, +0), (+0, +0), (+0, +0), (+4, -4), (+4, -4), (+4, -4), (+4, -4), (+4, -4), (+4, -4)],  # noqa  1
+        [(+0, +0), (+0, +0), (+0, +0), (+0, +0), (+0, +0), (+0, +0), (+0, +0), (+4, -4), (+4, -4), (+4, -4), (+4, -4), (+4, -4)],  # noqa  2
+        [(+0, +0), (+0, +0), (+0, +0), (+0, +0), (+0, +0), (+0, +0), (+0, +0), (+0, +0), (+4, -4), (+4, -4), (+4, -4), (+4, -4)],  # noqa  3
+        [(-4, +4), (+0, +0), (+0, +0), (+0, +0), (+0, +0), (+0, +0), (+0, +0), (+0, +0), (+8, +4), (+8, +4), (+8, +4), (+8, +4)],  # noqa  4
+        [(-4, +4), (-4, +4), (+0, +0), (+0, +0), (+0, +0), (+0, +0), (+0, +0), (+0, +0), (+8, +4), (+8, +4), (+8, +4), (+8, +4)],  # noqa  5
+        [(-4, +4), (-4, +4), (-4, +4), (+0, +0), (+0, +0), (+0, +0), (+0, +0), (+0, +0), (+8, +4), (+8, +4), (+8, +4), (+8, +4)],  # noqa  6
+        [(-4, +4), (-4, +4), (-4, +4), (-4, +4), (+0, +0), (+0, +0), (+0, +0), (+0, +0), (+8, +4), (+8, +4), (+8, +4), (+8, +4)],  # noqa  7
+        [(-4, +4), (-4, +4), (-4, +4), (-4, +4), (+4, +8), (+4, +8), (+4, +8), (+4, +8), (+4, +8), (+8, +4), (+8, +4), (+8, +4)],  # noqa  8
+        [(-4, +4), (-4, +4), (-4, +4), (-4, +4), (+4, +8), (+4, +8), (+4, +8), (+4, +8), (+4, +8), (+4, +8), (+8, +4), (+8, +4)],  # noqa  9
+        [(-4, +4), (-4, +4), (-4, +4), (-4, +4), (+4, +8), (+4, +8), (+4, +8), (+4, +8), (+4, +8), (+4, +8), (+4, +8), (+8, +4)],  # noqa 10
+        [(-4, +4), (-4, +4), (-4, +4), (-4, +4), (+4, +8), (+4, +8), (+4, +8), (+4, +8), (+4, +8), (+4, +8), (+4, +8), (+4, +8)]   # noqa 11
+    ])
+], dtype=int)
+"""SpiNN-5 ethernet connected chip lookup.
+
+Used by :py:func:`.spinn5_local_eth_coord`. Given an x and y chip position
+modulo 12, return the offset of the board's bottom-left chip from the chip's
+position.
+
+Note: the order of indexes: ``SPINN5_ETH_OFFSET[y][x]``!
+"""

rig/machine_control/machine_controller.py

@@ -19,6 +19,8 @@
 from rig import routing_table
 from rig.machine import Cores, SDRAM, SRAM, Links, Machine
 
+from rig.geometry import spinn5_eth_coords, spinn5_local_eth_coord
+
 from rig.utils.contexts import ContextMixin, Required
 from rig.utils.docstrings import add_signature_to_docstring
 
@@ -109,10 +111,20 @@ def __init__(self, initial_host, scp_port=consts.SCP_PORT,
                                                         "boot/sark.struct")
             self.structs = struct_file.read_struct_file(struct_data)
 
-        # Create the initial connection
-        self.connections = [
-            SCPConnection(initial_host, scp_port, n_tries, timeout)
-        ]
+        # This dictionary contains a lookup from chip (x, y) to the
+        # SCPConnection associated with that chip. The special entry with the
+        # key None is reserved for the connection initially made to the
+        # machine and is special since it is always known to exist but its
+        # actual position in the network is unknown.
+        self.connections = {
+            None: SCPConnection(initial_host, scp_port, n_tries, timeout)
+        }
+
+        # The dimensions of the system. This is set by discover_connections()
+        # and is used by _get_connection to determine which of the above
+        # connections to use.
+        self._width = None
+        self._height = None
 
     def __call__(self, **context_args):
         """For use with `with`: set default argument values.
@@ -154,9 +166,9 @@ def send_scp(self, *args, **kwargs):
         This function is a thin wrapper around
         :py:meth:`rig.machine_control.scp_connection.SCPConnection.send_scp`.
 
-        Future versions of this command will automatically choose the most
-        appropriate connection to use for machines with more than one Ethernet
-        connection.
+        This function will attempt to use the SCP connection nearest the
+        destination of the SCP command if multiple connections have been
+        discovered using :py:meth:`.discover_connections`.
 
         Parameters
         ----------
@@ -175,7 +187,20 @@ def send_scp(self, *args, **kwargs):
 
     def _get_connection(self, x, y):
         """Get the appropriate connection for a chip."""
-        return self.connections[0]
+        if self._width is None or self._height is None:
+            return self.connections[None]
+        else:
+            # If possible, use the local Ethernet connected chip
+            eth_chip = spinn5_local_eth_coord(x, y, self._width, self._height)
+            conn = self.connections.get(eth_chip)
+            if conn is not None:
+                return conn
+            else:
+                # If no connection was available to the local board, chose
+                # another arbitrarily.
+                # XXX: This choice will cause lots of contention in systems
+                # with many missing Ethernet connections.
+                return self.connections[None]
 
     def _send_scp(self, x, y, p, *args, **kwargs):
         """Determine the best connection to use to send an SCP packet and use
@@ -251,6 +276,63 @@ def boot(self, width, height, **boot_kwargs):
                                  **boot_kwargs)
         assert len(self.structs) > 0
 
+    @ContextMixin.use_contextual_arguments()
+    def discover_connections(self, x=0, y=0):
+        """Attempt to discover all available Ethernet connections to a machine.
+
+        After calling this method, :py:class:`.MachineController` will attempt
+        to communicate via the Ethernet connection on the same board as the
+        destination chip for all commands.
+
+        If called multiple times, existing connections will be retained in
+        preference to new ones.
+
+        .. note::
+            The system must be booted for this command to succeed.
+
+        .. note::
+            Currently, only systems comprised of multiple Ethernet-connected
+            SpiNN-5 boards are supported.
+
+        Parameters
+        ----------
+        x : int
+        y : int
+            (Optional) The coordinates of the chip to initially use to query
+            the system for the set of live chips.
+
+        Returns
+        -------
+        int
+            The number of new connections established.
+        """
+        working_chips = set(
+            (x, y)
+            for (x, y), route in iteritems(self.get_p2p_routing_table(x, y))
+            if route != consts.P2PTableEntry.none)
+        self._width = max(x for x, y in working_chips) + 1
+        self._height = max(y for x, y in working_chips) + 1
+
+        num_new_connections = 0
+
+        for x, y in spinn5_eth_coords(self._width, self._height):
+            if (x, y) in working_chips and (x, y) not in self.connections:
+                ip = self.get_ip_address(x, y)
+                if ip is not None:
+                    # Create a connection to the IP
+                    self.connections[(x, y)] = \
+                        SCPConnection(ip, self.scp_port,
+                                      self.n_tries, self.timeout)
+                    # Attempt to use the connection (and remove it if it
+                    # doesn't work)
+                    try:
+                        self.get_software_version(x, y)
+                        num_new_connections += 1
+                    except SCPError:
+                        self.connections.pop((x, y)).close()
+
+        return num_new_connections
+
     @ContextMixin.use_contextual_arguments()
     def application(self, app_id):
         """Update the context to use the given application ID and stop the
@@ -294,6 +376,24 @@ def get_software_version(self, x, y, processor=0):
         return CoreInfo(p2p_address, pcpu, vcpu, version, buffer_size,
                         sver.arg3, sver.data.decode("utf-8"))
 
+    @ContextMixin.use_contextual_arguments()
+    def get_ip_address(self, x, y):
+        """Get the IP address of a particular SpiNNaker chip's Ethernet link.
+
+        Returns
+        -------
+        str or None
+            The IPv4 address (as a string) of the chip's Ethernet link or None
+            if the chip does not have an Ethernet connection or the link is
+            currently down.
+        """
+        if self.read_struct_field("sv", "eth_up", x=x, y=y):
+            ip = self.read_struct_field("sv", "ip_addr", x=x, y=y)
+            # Convert the IP address to the standard decimal string format
+            return ".".join(str((ip >> i) & 0xFF) for i in range(0, 32, 8))
+        else:
+            return None
+
     @ContextMixin.use_contextual_arguments()
     def write(self, address, data, x, y, p=0):
         """Write a bytestring to an address in memory.

rig/machine_control/scp_connection.py

@@ -72,7 +72,6 @@ def __init__(self, spinnaker_host, port=consts.SCP_PORT,
 
         # Create a socket to communicate with the SpiNNaker machine
         self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-        self.sock.settimeout(self.default_timeout)
         self.sock.connect((spinnaker_host, port))
 
         # Store the number of tries that will be allowed
@@ -443,6 +442,10 @@ def packets(address, data):
         # Run the event loop and then return the retrieved data
         self.send_scp_burst(buffer_size, window_size, packets(address, data))
 
+    def close(self):
+        """Close the SCP connection."""
+        self.sock.close()
+
 
 def seqs(mask=0xffff):
     i = 0