Merge branch 'master' into auto-publish

.travis.yml

@@ -14,18 +14,23 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 language: python
+os: linux
 jobs:
   include:
     - name: Py3.6 (and doc deploy)
       python: 3.6
+      dist: bionic
       env:
         - DEPLOY_DIR=$PWD/doc/source/_build/html
     - name: Py3.7
       python: 3.7
+      dist: focal
+    - name: Py3.8
+      python: 3.8
+      dist: focal
     - name: Py2.7 (legacy)
       python: 2.7
       dist: xenial
-dist: focal
 cache: pip
 addons:
   apt:

pacman/model/graphs/graph.py

@@ -175,6 +175,8 @@ def add_edge(self, edge, outgoing_edge_partition_name):
         self._incoming_edges_by_partition_name[
             (edge.post_vertex, outgoing_edge_partition_name)].append(edge)
         self._incoming_edges[edge.post_vertex].add(edge)
+        if edge in self._outgoing_edge_partition_by_edge:
+            raise PacmanAlreadyExistsException("edge", edge)
         self._outgoing_edge_partition_by_edge[edge] = partition
 
     def _new_edge_partition(self, name):

pacman/model/graphs/machine/machine_vertex.py

@@ -88,6 +88,16 @@ def vertex_slice(self):
         """
         return self._vertex_slice
 
+    def get_n_keys_for_partition(self, _partition):
+        """ Get the number of keys required by the given partition of edges.
+
+        :param ~pacman.model.graphs.OutgoingEdgePartition _partition:
+            An partition that comes out of this vertex
+        :return: The number of keys required
+        :rtype: int
+        """
+        return self._vertex_slice.n_atoms
+
     @property
     def index(self):
         """ The index into the collection of machine vertices for an\

pacman/model/partitioner_interfaces/__init__.py

@@ -0,0 +1,18 @@
+# Copyright (c) 2017-2019 The University of Manchester
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from .abstract_slices_connect import AbstractSlicesConnect
+
+__all__ = ["AbstractSlicesConnect"]

pacman/model/partitioner_interfaces/abstract_slices_connect.py

@@ -0,0 +1,45 @@
+# Copyright (c) 2017-2019 The University of Manchester
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from six import add_metaclass
+from spinn_utilities.abstract_base import AbstractBase, abstractmethod
+
+
+@add_metaclass(AbstractBase)
+class AbstractSlicesConnect(object):
+    """ An object that can check if a pre slice and a post slice could
+        connect.
+
+        Typically used to determine if a Machine Edge should be created by
+        checking that at least one of the indexes in the source slice could
+        connect to at least one of the indexes in the destination slice.
+    """
+
+    __slots__ = ()
+
+    @abstractmethod
+    def could_connect(self, pre_slice, post_slice):
+        """ Determine if there is a chance that one of the indexes in the
+        pre slice could connect to at least one of the indexes in the
+        post slice.
+
+        note: This method should never return a false negative,
+        but may return a false positives
+
+        :param ~pacman.model.graphs.common.Slice pre_slice:
+        :param ~pacman.model.graphs.common.Slice post_slice:
+        :return: True if a connection could be possible
+        :rtype: bool
+        """

pacman/operations/algorithm_reports/reports.py

@@ -270,7 +270,8 @@ def partitioner_report(report_folder, hostname, graph):
 
     :param str report_folder: the folder to which the reports are being written
     :param str hostname: the machine's hostname to which the placer worked on
-    :param ApplicationGraph graph:
+    :param ApplicationGraph graph: the app graph
+
     """
 
     # Cycle through all vertices, and for each cycle through its vertices.
@@ -282,7 +283,7 @@ def partitioner_report(report_folder, hostname, graph):
             progress = ProgressBar(graph.n_vertices,
                                    "Generating partitioner report")
 
-            f.write("        Placement Information by Vertex\n")
+            f.write("        Partitioning Information by Vertex\n")
             f.write("        ===============================\n\n")
             f.write("Generated: {} for target machine '{}'\n\n".format(
                 time_date_string, hostname))

pacman/operations/algorithm_reports/reports_metadata.xml

@@ -358,6 +358,7 @@
             <param_name>report_folder</param_name>
             <param_name>hostname</param_name>
             <param_name>graph</param_name>
+            <token>PartitioningDone</token>
         </required_inputs>
     </algorithm>
     <algorithm name="TagReport">

pacman/operations/algorithms_metadata.xml

@@ -73,6 +73,7 @@
         <outputs>
             <param_type>MemoryMachineGraph</param_type>
             <param_type>NChipsRequired</param_type>
+            <token>PartitioningDone</token>
         </outputs>
     </algorithm>
     <algorithm name="PartitionAndPlacePartitioner">
@@ -109,6 +110,7 @@
         <outputs>
             <param_type>MemoryMachineGraph</param_type>
             <param_type>NChipsRequired</param_type>
+            <token>PartitioningDone</token>
         </outputs>
     </algorithm>
     <algorithm name="RadialPlacer">

pacman/operations/placer_algorithms/__init__.py

@@ -16,5 +16,10 @@
 from .radial_placer import RadialPlacer
 from .one_to_one_placer import OneToOnePlacer
 from .spreader_placer import SpreaderPlacer
+from .connective_based_placer import ConnectiveBasedPlacer
 
-__all__ = ['OneToOnePlacer', 'RadialPlacer', "SpreaderPlacer"]
+__all__ = [
+    'ConnectiveBasedPlacer',
+    'OneToOnePlacer',
+    'RadialPlacer',
+    "SpreaderPlacer"]

pacman/operations/placer_algorithms/connective_based_placer.py

@@ -0,0 +1,143 @@
+# Copyright (c) 2017-2019 The University of Manchester
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+from spinn_utilities.progress_bar import ProgressBar
+from pacman.model.constraints.placer_constraints import (
+    AbstractPlacerConstraint)
+from pacman.model.placements import Placements
+from pacman.operations.placer_algorithms import RadialPlacer
+from pacman.utilities.algorithm_utilities.placer_algorithm_utilities import (
+    sort_vertices_by_known_constraints, get_same_chip_vertex_groups)
+from pacman.utilities.utility_calls import locate_constraints_of_type
+from pacman.utilities.utility_objs import ResourceTracker
+
+logger = logging.getLogger(__name__)
+
+
+class ConnectiveBasedPlacer(RadialPlacer):
+    """ A radial algorithm that can place a machine graph onto a\
+        machine using a circle out behaviour from a Ethernet at a given point\
+        and which will place things that are most connected closest to each\
+        other
+    """
+
+    __slots__ = []
+
+    def __call__(self, machine_graph, machine, plan_n_timesteps):
+        """
+        :param machine_graph: The machine_graph to place
+        :type machine_graph:\
+            :py:class:`pacman.model.graphs.machine.MachineGraph`
+        :param machine:\
+            The machine with respect to which to partition the application\
+            graph
+        :type machine: :py:class:`spinn_machine.Machine`
+        :param plan_n_timesteps: number of timesteps to plan for
+        :type  plan_n_timesteps: int
+        :return: A set of placements
+        :rtype: :py:class:`pacman.model.placements.Placements`
+        :raise pacman.exceptions.PacmanPlaceException: \
+            If something goes wrong with the placement
+        """
+        # check that the algorithm can handle the constraints
+        self._check_constraints(machine_graph.vertices)
+
+        # Sort the vertices into those with and those without
+        # placement constraints
+        placements = Placements()
+        constrained = list()
+        unconstrained = set()
+        for vertex in machine_graph.vertices:
+            if locate_constraints_of_type(
+                    vertex.constraints, AbstractPlacerConstraint):
+                constrained.append(vertex)
+            else:
+                unconstrained.add(vertex)
+
+        # Iterate over constrained vertices and generate placements
+        progress = ProgressBar(
+            machine_graph.n_vertices, "Placing graph vertices")
+        resource_tracker = ResourceTracker(
+            machine, plan_n_timesteps, self._generate_radial_chips(machine))
+        constrained = sort_vertices_by_known_constraints(constrained)
+        vertices_on_same_chip = get_same_chip_vertex_groups(machine_graph)
+        for vertex in progress.over(constrained, False):
+            self._place_vertex(
+                vertex, resource_tracker, machine, placements,
+                vertices_on_same_chip)
+
+        while unconstrained:
+            # Place the subgraph with the overall most connected vertex
+            max_connected_vertex = self._find_max_connected_vertex(
+                unconstrained, machine_graph)
+            self._place_unconstrained_subgraph(
+                max_connected_vertex, machine_graph, unconstrained,
+                machine, placements, resource_tracker, progress,
+                vertices_on_same_chip)
+
+        # finished, so stop progress bar and return placements
+        progress.end()
+        return placements
+
+    def _place_unconstrained_subgraph(
+            self, starting_vertex, machine_graph, unplaced_vertices,
+            machine, placements, resource_tracker, progress,
+            vertices_on_same_chip):
+        # pylint: disable=too-many-arguments
+        # Keep track of all unplaced_vertices connected to the currently
+        # placed ones
+        to_do = set()
+        to_do.add(starting_vertex)
+
+        while to_do:
+            # Find the vertex most connected of the currently-to-be-placed ones
+            vertex = self._find_max_connected_vertex(to_do, machine_graph)
+
+            # Place the vertex
+            self._place_vertex(
+                vertex, resource_tracker, machine, placements,
+                vertices_on_same_chip)
+            progress.update()
+
+            # Remove from collections of unplaced_vertices to work on
+            unplaced_vertices.remove(vertex)
+            to_do.remove(vertex)
+
+            # Add all unplaced_vertices connected to this one to the set
+            for edge in machine_graph.get_edges_ending_at_vertex(vertex):
+                if edge.pre_vertex in unplaced_vertices:
+                    to_do.add(edge.pre_vertex)
+            for edge in machine_graph.get_edges_starting_at_vertex(vertex):
+                if edge.post_vertex in unplaced_vertices:
+                    to_do.add(edge.post_vertex)
+
+    @staticmethod
+    def _find_max_connected_vertex(vertices, graph):
+        max_connected_vertex = None
+        max_weight = 0
+        for vertex in vertices:
+            in_weight = sum(
+                edge.pre_vertex.vertex_slice.n_atoms
+                for edge in graph.get_edges_starting_at_vertex(vertex))
+            out_weight = sum(
+                edge.pre_vertex.vertex_slice.n_atoms
+                for edge in graph.get_edges_ending_at_vertex(vertex))
+            weight = in_weight + out_weight
+
+            if max_connected_vertex is None or weight > max_weight:
+                max_connected_vertex = vertex
+                max_weight = weight
+        return max_connected_vertex

pacman/utilities/algorithm_utilities/partition_algorithm_utilities.py

@@ -18,6 +18,7 @@
 from collections import OrderedDict
 from spinn_utilities.progress_bar import ProgressBar
 from spinn_utilities.ordered_set import OrderedSet
+from pacman.model.partitioner_interfaces import AbstractSlicesConnect
 from pacman.utilities import utility_calls as utils
 from pacman.exceptions import PacmanPartitionException
 from pacman.model.constraints.partitioner_constraints import (
@@ -72,18 +73,23 @@ def generate_machine_edges(machine_graph, application_graph):
             for source_vertex in vertex.machine_vertices:
                 # create new partitions
                 for dest_vertex in edge.post_vertex.machine_vertices:
-                    machine_edge = edge.create_machine_edge(
-                        source_vertex, dest_vertex,
-                        "machine_edge_for{}".format(edge.label))
-                    machine_graph.add_edge(
-                        machine_edge, application_partition.identifier)
-
-                    # add constraints from the application partition
-                    machine_partition = machine_graph.\
-                        get_outgoing_edge_partition_starting_at_vertex(
-                            source_vertex, application_partition.identifier)
-                    machine_partition.add_constraints(
-                        application_partition.constraints)
+                    if (not isinstance(edge, AbstractSlicesConnect) or
+                            edge.could_connect(
+                                source_vertex.vertex_slice,
+                                dest_vertex.vertex_slice)):
+                        machine_edge = edge.create_machine_edge(
+                            source_vertex, dest_vertex,
+                            "machine_edge_for{}".format(edge.label))
+                        machine_graph.add_edge(
+                            machine_edge, application_partition.identifier)
+
+                        # add constraints from the application partition
+                        machine_partition = machine_graph.\
+                            get_outgoing_edge_partition_starting_at_vertex(
+                                source_vertex,
+                                application_partition.identifier)
+                        machine_partition.add_constraints(
+                            application_partition.constraints)
 
 
 def get_remaining_constraints(vertex):

setup.py

@@ -56,7 +56,7 @@
         "Intended Audience :: Developers",
         "Intended Audience :: Science/Research",
 
-        "License :: OSI Approved :: GNU General Public License v2 (GPLv2)",
+        "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
 
         "Natural Language :: English",
 
@@ -68,6 +68,7 @@
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
     ],
     packages=packages,
     package_data=package_data,

uinit_test_objects/placer_test_support.py

@@ -64,7 +64,8 @@ class MachineVertex(SimpleMachineVertex):
     def __init__(self, lo_atom, hi_atom, resources_required, label=None,
                  constraints=None):
         super(MachineVertex, self).__init__(
-            resources_required, label=label, constraints=constraints)
+            resources_required, label=label, constraints=constraints,
+            vertex_slice=Slice(lo_atom, hi_atom))
         self.lo_atom = lo_atom
         self.hi_atom = hi_atom
         self._model_based_max_atoms_per_core = 256

unittests/model_tests/machine_graph_tests/test_machine_graph_model.py

@@ -110,7 +110,8 @@ def test_add_duplicate_edge(self):
         edges.append(edge)
         graph = MachineGraph("foo")
         graph.add_vertices(vertices)
-        graph.add_edges(edges, "bar")
+        with self.assertRaises(PacmanAlreadyExistsException):
+            graph.add_edges(edges, "bar")
 
     def test_add_edge_with_no_existing_pre_vertex_in_graph(self):
         """