Merge pull request #1 from dbrnz/mahyar-network-routing

Network routing fixes and tidy up

README.rst

@@ -24,7 +24,7 @@ It is recommended to install and run ``mapmaker`` in its own Python virtual envi
 
 * Create and activate a Python virtual environment in which to install ``mapmaker``.
 
-* Within this environment, install the latest ``mapmaker`` wheel from https://github.com/dbrnz/flatmap-maker/releases/latest (currently ``mapmaker-1.3.0b5-py3-none-any.whl``).
+* Within this environment, install the latest ``mapmaker`` wheel from https://github.com/dbrnz/flatmap-maker/releases/latest (currently ``mapmaker-1.3.0b6-py3-none-any.whl``).
 
 Using pipenv
 ~~~~~~~~~~~~
@@ -33,7 +33,7 @@ Using pipenv
 
 * Install ``mapmaker`` directly from GitHub with::
 
-    $ pipenv install --python 3.8 https://github.com/dbrnz/flatmap-maker/releases/download/v1.3.0b5/mapmaker-1.3.0b5-py3-none-any.whl
+    $ pipenv install --python 3.8 https://github.com/dbrnz/flatmap-maker/releases/download/v1.3.0b6/mapmaker-1.3.0b6-py3-none-any.whl
 
 
 Development
@@ -123,7 +123,7 @@ An example run
 
 .. code-block:: text
 
-    Mapmaker 1.3.0b5
+    Mapmaker 1.3.0b6
     100%|█████████████████████████▉| 678/679
      98%|███████████████████████████▌| 65/66
     Adding details...

mapmaker/__init__.py

@@ -18,7 +18,7 @@
 #
 #===============================================================================
 
-__version__ = '1.3.0b5routing'
+__version__ = '1.3.0b6routing'
 
 #===============================================================================
 

mapmaker/flatmap/__init__.py

@@ -176,6 +176,8 @@ def close(self):
         self.__setup_feature_search()
         # Generate metadata with connection information
         self.__resolve_paths()
+        # Save all we know about the map
+        self.__map_properties.save_knowledge()
         # Set creation time
         self.__created = datetime.datetime.utcnow()
         self.__metadata['created'] = self.__created.isoformat()

mapmaker/knowledgebase/__init__.py

@@ -120,19 +120,21 @@ def entity_knowledge(self, entity):
             knowledge = self.__scicrunch.get_knowledge(entity)
             if 'label' in knowledge:
                 self.db.execute('replace into labels values (?, ?)', (entity, knowledge['label']))
-            # We don't return the list of publications to thenmap maker but just save them
-            # in the knowledge base
-            publications = knowledge.pop('publications', [])
-            with self.db:
-                self.db.execute('delete from publications where entity = ?', (entity, ))
-                self.db.executemany('insert into publications(entity, publication) values (?, ?)',
-                    ((entity, publication) for publication in publications))
+            # Save the list of publications in the knowledge base
+            self.update_publications(entity, knowledge.pop('publications', []))
         # Use the entity's value as its label if none is defined
         if 'label' not in knowledge:
             knowledge['label'] = entity
         # Cache local knowledge
         self.__entity_knowledge[entity] = knowledge
-
         return knowledge
 
+    #---------------------------------------------------------------------------
+
+    def update_publications(self, entity, publications):
+        with self.db:
+            self.db.execute('delete from publications where entity = ?', (entity, ))
+            self.db.executemany('insert into publications(entity, publication) values (?, ?)',
+                ((entity, publication) for publication in publications))
+
 #===============================================================================

mapmaker/maker.py

@@ -178,6 +178,9 @@ def __init__(self, options):
         self.__geojson_files = []
         self.__tippe_inputs = []
 
+        # Raster tile layers
+        self.__raster_layers = []
+
         # Our source of knowledge, updated with information
         # about maps we've made
         self.__knowledgebase = KnowledgeStore(map_base)
@@ -211,9 +214,8 @@ def make(self):
             self.__output_geojson()
             # Generate vector tiles from GeoJSON
             self.__make_vector_tiles()
-            # Generate image tiles
-            if settings.get('backgroundTiles', False):
-                self.__make_raster_tiles()
+            # Generate image tiles as needed
+            self.__check_raster_tiles()
             # Save the flatmap's metadata
             self.__save_metadata()
 
@@ -242,7 +244,6 @@ def __finish_make(self):
 
     def __process_sources(self):
     #===========================
-        tile_background = settings.get('backgroundTiles', False)
         # Make sure ``base`` and ``slides`` source kinds are processed first
         def kind_order(source):
             kind = source.get('kind', '')
@@ -252,8 +253,7 @@ def kind_order(source):
             kind = source.get('kind')
             href = source['href']
             if kind == 'slides':
-                source_layer = PowerpointSource(self.__flatmap, id, href,
-                                                get_background=tile_background)
+                source_layer = PowerpointSource(self.__flatmap, id, href)
             elif kind == 'image':
                 if layer_number > 0 and 'boundary' not in source:
                     raise ValueError('An image source must specify a boundary')
@@ -274,13 +274,16 @@ def kind_order(source):
         if len(self.__flatmap) == 0:
             raise ValueError('No map layers in sources...')
 
-    def __make_raster_tiles(self):
+    def __check_raster_tiles(self):
     #============================
-        log('Generating background tiles (may take a while...)')
+        log('Checking and making background tiles (may take a while...)')
         for layer in self.__flatmap.layers:
             for raster_layer in layer.raster_layers:
                 tilemaker = RasterTileMaker(raster_layer, self.__map_dir, self.__zoom[1])
-                raster_tile_file = tilemaker.make_tiles()
+                if settings.get('backgroundTiles', False):
+                    tilemaker.make_tiles()
+                if tilemaker.have_tiles():
+                    self.__raster_layers.append(raster_layer)
 
     def __make_vector_tiles(self, compressed=True):
     #==============================================
@@ -364,19 +367,14 @@ def __save_metadata(self):
 #*        ## TODO: set ``layer.properties`` for annotations...
 #*        ##update_RDF(options['map_base'], options['map_id'], source, annotations)
 
-        # Get list of all image sources from all layers
-        raster_layers = []
-        for layer in self.__flatmap.layers:
-            raster_layers.extend(layer.raster_layers)
-
         map_index = {
             'id': self.__id,
             'source': self.__manifest.url,
             'min-zoom': self.__zoom[0],
             'max-zoom': self.__zoom[1],
             'bounds': self.__flatmap.extent,
             'version': FLATMAP_VERSION,
-            'image_layer': len(raster_layers) > 0  ## For compatibility
+            'image_layer': len(self.__raster_layers) > 0
         }
         if self.__flatmap.models is not None:
             map_index['describes'] = self.__flatmap.models
@@ -387,7 +385,7 @@ def __save_metadata(self):
 
         # Create style file
         metadata = tile_db.metadata()
-        style_dict = MapStyle.style(raster_layers, metadata, self.__zoom)
+        style_dict = MapStyle.style(self.__raster_layers, metadata, self.__zoom)
         with open(os.path.join(self.__map_dir, 'style.json'), 'w') as output_file:
             json.dump(style_dict, output_file)
 

mapmaker/output/tilemaker.py

@@ -433,16 +433,15 @@ class RasterTileMaker(object):
     """
     def __init__(self, raster_layer, output_dir, max_zoom=MAX_ZOOM):
         self.__raster_layer = raster_layer
-        self.__output_dir = output_dir
         self.__max_zoom = max_zoom
         self.__id = raster_layer.id
+        self.__database_path = os.path.join(output_dir, '{}.mbtiles'.format(raster_layer.id))
         self.__min_zoom = raster_layer.min_zoom
         self.__tile_set = TileSet(raster_layer.extent, max_zoom)
 
     def __make_zoomed_tiles(self, tile_extractor):
     #=============================================
-        raster_database_name = '{}.mbtiles'.format(self.__id)
-        mbtiles = MBTiles(os.path.join(self.__output_dir, raster_database_name), True, True)
+        mbtiles = MBTiles(self.__database_path, True, True)
         mbtiles.add_metadata(id=self.__id)
         zoom = self.__max_zoom
         log('Tiling zoom level {} for {}'.format(zoom, self.__id))
@@ -459,7 +458,6 @@ def __make_zoomed_tiles(self, tile_extractor):
         self.__make_overview_tiles(mbtiles, zoom, self.__tile_set.start_coords,
                                                   self.__tile_set.end_coords)
         mbtiles.close(compress=True)
-        return raster_database_name
 
     def __make_overview_tiles(self, mbtiles, zoom, start_coords, end_coords):
     #========================================================================
@@ -490,6 +488,10 @@ def __make_overview_tiles(self, mbtiles, zoom, start_coords, end_coords):
             progress_bar.close()
             self.__make_overview_tiles(mbtiles, zoom, half_start, half_end)
 
+    def have_tiles(self):
+    #====================
+        return os.path.exists(self.__database_path)
+
     def make_tiles(self):
     #====================
         log('Tiling {}...'.format(self.__id))
@@ -505,7 +507,7 @@ def make_tiles(self):
                 tile_extractor = SVGImageTiler(self.__raster_layer, self.__tile_set)
         else:
             raise TypeError('Unsupported kind of background tile source: {}'.format(source_kind))
-        return self.__make_zoomed_tiles(tile_extractor)
+        self.__make_zoomed_tiles(tile_extractor)
 
 #===============================================================================
 

mapmaker/properties/__init__.py

@@ -83,6 +83,11 @@ def generate_networks(self, feature_map):
                 network.create_geometry(feature_map)
                 self.__pathways.resolve_pathways(network, feature_map, self.__model_to_features)
 
+    def save_knowledge(self):
+    #========================
+        if self.__pathways is not None:
+            self.__pathways.save_knowledge(self.__knowledgebase)
+
     def update_properties(self, properties):
     #=======================================
         cls = properties.get('class')

mapmaker/properties/pathways.py

@@ -160,6 +160,7 @@ def __resolve_nodes_for_path(self, path_id, nodes):
             for feature in self.__feature_map.features(id):
                 if not feature.get_property('exclude'):
                     node_id = feature.feature_id
+                    feature.set_property('nodeId', node_id)
                     self.__node_paths[node_id].append(path_id)
                     node_ids.append(node_id)
                     node_count += 1
@@ -397,13 +398,16 @@ def get_point_for_anatomy(anatomical_id, error_list):
                 for path_id, routed_path in network.layout(connectivity_model.connections).items():
                     properties = { 'tile-layer': 'autopaths' }
                     properties.update(self.__line_properties(path_id))
-                    feature = self.__flatmap.new_feature(routed_path.geometry(), properties)
-                    layer.add_feature(feature)
-                    self.__resolved_pathways.add_line_feature(path_id, feature)
-                    self.__resolved_pathways.add_nerves(path_id, self.__nerves_by_path_id.get(path_id, []))
-                    self.__resolved_pathways.add_nodes(path_id, routed_path.node_set)
-                    self.__resolved_pathways.add_path_type(path_id, properties.get('type'))
-                    self.__resolved_pathways.set_model_id(path_id, self.__path_models.get(path_id))
+                    for n, geometric_shape in enumerate(routed_path.geometry()):
+                        properties.update(geometric_shape.properties)
+                        feature = self.__flatmap.new_feature(geometric_shape.geometry, properties)
+                        layer.add_feature(feature)
+                        id = f'{path_id}__F_{n}'
+                        self.__resolved_pathways.add_line_feature(path_id, feature)
+                        self.__resolved_pathways.add_nerves(path_id, self.__nerves_by_path_id.get(path_id, []))
+                        self.__resolved_pathways.add_nodes(path_id, routed_path.node_set)
+                        self.__resolved_pathways.add_path_type(id, properties.get('type'))
+                        self.__resolved_pathways.set_model_id(id, self.__path_models.get(path_id))
 
     def resolve_pathways(self, network, feature_map, model_to_features):
     #===================================================================
@@ -428,4 +432,10 @@ def resolve_pathways(self, network, feature_map, model_to_features):
         if errors:
             raise ValueError('Errors in mapping paths and routes')
 
+    def save_knowledge(self, knowledgebase):
+    #=======================================
+        for model in self.__connectivity_models:
+            if model.source is not None:
+                knowledgebase.update_publications(model.source, model.publications)
+
 #===============================================================================

mapmaker/routing/__init__.py

@@ -33,6 +33,7 @@
 
 #===============================================================================
 
+from mapmaker.settings import settings
 from mapmaker.utils import log
 
 from .network import RoutedPath
@@ -95,6 +96,7 @@ def __init__(self, network):
         self.__terminal_nodes = { n for n, d in self.__graph.degree() if d == 1 }
         # Used to lookup features but only known when `maker` has processed sources
         self.__feature_map = None
+        self.__centreline_scaffold = None
 
     @property
     def id(self):
@@ -111,28 +113,21 @@ def __find_feature(self, id):
             log.warn('Multiple network features for: {}'.format(id))
         return None
 
-    def __set_node_properties(self, node_id, graph):
-    #===============================================
-        feature = self.__find_feature(node_id)
+    def __set_node_properties(self, node, id):
+    #=========================================
+        feature = self.__find_feature(id)
         if feature is not None:
-            node = graph.nodes[node_id]
             if 'geometry' not in node:
                 for key, value in feature.properties.items():
                     node[key] = value
                 node['geometry'] = feature.geometry
 
-    def __set_node_type(self, node_id, node_type, graph):
-    #===============================================
-        node = graph.nodes[node_id]
-        if 'type' not in node:
-            node['type'] = node_type[node_id]
-
     def create_geometry(self, feature_map):
     #======================================
         self.__feature_map = feature_map
         for edge in self.__graph.edges(data='id'):  # Returns triples: (node, node, id)
             for node_id in edge[0:2]:
-                self.__set_node_properties(node_id, self.__graph)
+                self.__set_node_properties(self.__graph.nodes[node_id], node_id)
             feature = self.__find_feature(edge[2])
             if feature is not None:
                 bezier_path = feature.get_property('bezier-path')
@@ -144,37 +139,45 @@ def create_geometry(self, feature_map):
                     end_node_0 = self.__graph.nodes[edge[0]].get('geometry')
                     end_node_1 = self.__graph.nodes[edge[1]].get('geometry')
                     if end_node_0 is not None and end_node_1 is not None:
-                        if start_point.distance(end_node_0) > start_point.distance(end_node_1):
-                            bezier_path = bezier_path.reverse()
                         self.__graph.edges[edge[0:2]]['geometry'] = bezier_path
+                        if start_point.distance(end_node_0) < start_point.distance(end_node_1):
+                            self.__graph.edges[edge[0:2]]['start-node'] = edge[0]
+                        else:
+                            self.__graph.edges[edge[0:2]]['start-node'] = edge[1]
+        if settings.get('pathLayout', 'automatic'):
+            # Construct the centreline scaffold for the network
+            ##self.__centreline_scaffold = Sheath(self.__id, self.__graph)
+            pass
 
     def layout(self, connections: dict) -> dict:
     #===========================================
         routed_paths = {}
         for path_id, connects in connections.items():
             end_nodes = []
             terminals = {}
-            node_type = {}
+            node_types = {}
             for node in connects:
                 if isinstance(node, dict):
                     # Check that dict has 'node' and 'terminals'...
                     end_node = node['node']
                     end_nodes.append(end_node)
-                    terminals[end_node] = node['terminals']
-                    if 'type' in node:
-                        for terminal in terminals[end_node]:
-                            node_type[terminal] = node['type']
+                    terminals[end_node] = node.get('terminals', [])
+                    node_types[end_node] = node['type']
                 else:
                     end_nodes.append(node)
             # Find our route as a subgraph of the centreline network
             route_graph = nx.Graph(get_connected_subgraph(self.__graph, end_nodes))
+            for (node_id, node_type) in node_types.items():
+                node = route_graph.nodes[node_id]
+                node['type'] = node_type
+
             # Add edges to terminal nodes that aren't part of the centreline network
             for end_node, terminal_nodes in terminals.items():
-                for terminal in terminal_nodes:
-                    route_graph.add_edge(end_node, terminal)
-                    self.__set_node_properties(terminal, route_graph)
-                    if any(node_type):
-                        self.__set_node_type(terminal, node_type, route_graph)
+                for terminal_id in terminal_nodes:
+                    route_graph.add_edge(end_node, terminal_id)
+                    node = route_graph.nodes[terminal_id]
+                    self.__set_node_properties(node, terminal_id)
+                    route_graph.edges[end_node, terminal_id]['type'] = 'terminal'
             # Save the geometry of any intermediate points on an edge
             for edge in route_graph.edges(data='intermediates'):
                 if edge[2] is not None:
@@ -185,7 +188,10 @@ def layout(self, connections: dict) -> dict:
                             way_point_geometry.append(feature.geometry)
                     del(route_graph.edges[edge[0:2]]['intermediates'])
                     route_graph.edges[edge[0:2]]['way-points'] = way_point_geometry
-            routed_paths[path_id] = RoutedPath(path_id, route_graph)
+
+            # Route the connection's path through the centreline scaffold
+            routed_paths[path_id] = RoutedPath(path_id, route_graph, self.__centreline_scaffold)
+
         return routed_paths
 
     def has_node(self, id):