Merge pull request #47 from oemof/features/cleanup-osm-import

Clean up OSM import

dhnx/dhn_from_osm.py

@@ -11,11 +11,20 @@
 SPDX-License-Identifier: MIT
 """
 
-import geopandas as gpd
+try:
+    import geopandas as gpd
+
+except ImportError:
+    print("Need to install geopandas to process osm data.")
+
 import pandas as pd
 
-from shapely.ops import nearest_points
-from shapely.geometry import LineString
+try:
+    from shapely.ops import nearest_points
+    from shapely.geometry import LineString
+
+except ImportError:
+    print("Need to install shapely to download from osm.")
 
 
 def connect_points_to_network(points, nodes, edges):
@@ -62,19 +71,19 @@ def connect_points_to_network(points, nodes, edges):
 
         nearest_point = nearest_points(edges_united, point)[0]
 
-        n_points.append([id_point, point.x, point.y, point])
+        n_points.append([id_point, point])
 
-        n_nearest_points.append([id_nearest_point, nearest_point.x, nearest_point.y, nearest_point])
+        n_nearest_points.append([id_nearest_point, nearest_point])
 
         n_edges.append([id_point, id_nearest_point, LineString([point, nearest_point])])
 
     n_points = gpd.GeoDataFrame(
         n_points,
-        columns=['index', 'x', 'y', 'geometry']).set_index('index')
+        columns=['index', 'geometry']).set_index('index')
 
     n_nearest_points = gpd.GeoDataFrame(
         n_nearest_points,
-        columns=['index', 'x', 'y', 'geometry']).set_index('index')
+        columns=['index', 'geometry']).set_index('index')
 
     n_edges = gpd.GeoDataFrame(n_edges, columns=['u', 'v', 'geometry'])
 

dhnx/graph.py

@@ -56,7 +56,7 @@ def thermal_network_to_nx_graph(thermal_network):
     for k, v in nodes.items():
         v.index = [k + '-' + str(id) for id in v.index]
 
-    nodes = pd.concat(nodes.values())
+    nodes = pd.concat(nodes.values(), sort=True)
 
     node_attrs = {node_id: dict(data) for node_id, data in nodes.iterrows()}
 

dhnx/input_output.py

@@ -15,6 +15,29 @@
 
 from addict import Dict
 import pandas as pd
+import networkx as nx
+import numpy as np
+
+try:
+    from shapely.geometry import Point
+    from shapely.geometry import LineString
+
+except ImportError:
+    print("Need to install shapely to download from osm.")
+
+try:
+    import geopandas as gpd
+
+except ImportError:
+    print("Need to install geopandas to download from osm.")
+
+try:
+    import osmnx as ox
+
+except ImportError:
+    print("Need to install osmnx to download from osm.")
+
+from dhnx.dhn_from_osm import connect_points_to_network
 
 
 logger = logging.getLogger()
@@ -158,6 +181,231 @@ class OSMNetworkImporter(NetworkImporter):
 
     Not yet implemented.
     """
+    def __init__(self, thermal_network, place, distance):
+        super().__init__(thermal_network, None)
+
+        self.place = place
+
+        self.distance = distance
+
+    def download_street_network(self):
+
+        print('Downloading street network...')
+
+        graph = ox.graph_from_point(
+            center_point=self.place, distance=self.distance
+        )
+
+        graph = ox.project_graph(graph)
+
+        return graph
+
+    def download_footprints(self):
+
+        print('Downloading footprints...')
+
+        footprints = ox.footprints.footprints_from_point(
+            point=self.place, distance=self.distance
+        )
+
+        footprints = footprints.drop(labels='nodes', axis=1)
+
+        footprints = ox.project_gdf(footprints)
+
+        return footprints
+
+    @staticmethod
+    def remove_duplicate_edges(graph):
+
+        graph = nx.DiGraph(graph)
+        return graph
+
+    @staticmethod
+    def remove_self_loops(graph):
+
+        self_loop_edges = list(nx.selfloop_edges(graph))
+
+        graph.remove_edges_from(self_loop_edges)
+
+        return graph
+
+    @staticmethod
+    def graph_to_gdfs(G, nodes=True, edges=True, node_geometry=True, fill_edge_geometry=True):
+        """
+        Convert a graph into node and/or edge GeoDataFrames. Adapted from osmnx for DiGraph.
+
+        Parameters
+        ----------
+        G : networkx digraph
+        nodes : bool
+            if True, convert graph nodes to a GeoDataFrame and return it
+        edges : bool
+            if True, convert graph edges to a GeoDataFrame and return it
+        node_geometry : bool
+            if True, create a geometry column from node x and y data
+        fill_edge_geometry : bool
+            if True, fill in missing edge geometry fields using origin and
+            destination nodes
+
+        Returns
+        -------
+        GeoDataFrame or tuple
+            gdf_nodes or gdf_edges or both as a tuple
+        """
+        if not (nodes or edges):
+            raise ValueError('You must request nodes or edges, or both.')
+
+        to_return = []
+
+        if nodes:
+
+            nodes, data = zip(*G.nodes(data=True))
+            gdf_nodes = gpd.GeoDataFrame(list(data), index=nodes)
+            if node_geometry:
+                gdf_nodes['geometry'] = gdf_nodes.apply(lambda row: Point(row['x'], row['y']),
+                                                        axis=1)
+                gdf_nodes.set_geometry('geometry', inplace=True)
+            gdf_nodes.crs = G.graph['crs']
+            gdf_nodes.gdf_name = '{}_nodes'.format(G.graph['name'])
+
+            to_return.append(gdf_nodes)
+
+        if edges:
+
+            # create a list to hold our edges, then loop through each edge in the
+            # graph
+            edges = []
+            for u, v, data in G.edges(data=True):
+
+                # for each edge, add key and all attributes in data dict to the
+                # edge_details
+                edge_details = {'u': u, 'v': v}
+                for attr_key in data:
+                    edge_details[attr_key] = data[attr_key]
+
+                # if edge doesn't already have a geometry attribute, create one now
+                # if fill_edge_geometry==True
+                if 'geometry' not in data:
+                    if fill_edge_geometry:
+                        point_u = Point((G.nodes[u]['x'], G.nodes[u]['y']))
+                        point_v = Point((G.nodes[v]['x'], G.nodes[v]['y']))
+                        edge_details['geometry'] = LineString([point_u, point_v])
+                    else:
+                        edge_details['geometry'] = np.nan
+
+                edges.append(edge_details)
+
+            # create a GeoDataFrame from the list of edges and set the CRS
+            gdf_edges = gpd.GeoDataFrame(edges)
+            gdf_edges.crs = G.graph['crs']
+            gdf_edges.gdf_name = '{}_edges'.format(G.graph['name'])
+
+            to_return.append(gdf_edges)
+
+        if len(to_return) > 1:
+            return tuple(to_return)
+
+        return to_return[0]
+
+    @staticmethod
+    def get_building_midpoints(footprints):
+
+        building_midpoints = gpd.GeoDataFrame(
+            footprints.geometry.centroid,
+            columns=['geometry'],
+            crs=footprints.geometry.centroid.crs
+        )
+
+        return building_midpoints
+
+    def graph_to_component_dfs(self, graph, building_midpoints):
+
+        # get nodes and edges from graph
+        nodes, edges = self.graph_to_gdfs(graph)
+
+        nodes = nodes.loc[:, ['geometry']]
+
+        edges = edges.loc[:, ['u', 'v', 'geometry']]
+
+        nodes = nodes.to_crs("epsg:4326")
+
+        edges = edges.to_crs("epsg:4326")
+
+        building_midpoints = building_midpoints.to_crs("epsg:4326")
+
+        endpoints, forks, pipes = connect_points_to_network(
+            building_midpoints, nodes, edges)
+
+        pipes = pipes.rename(columns={'u': 'from_node', 'v': 'to_node'})
+
+        forks['node_type'] = 'Fork'
+
+        consumers = endpoints
+
+        consumers['node_type'] = 'Consumer'
+
+        # Update names of nodes in pipe's from_node/to_node
+        rename_nodes = {i: 'forks-' + str(i) for i in forks.index}
+
+        rename_nodes.update({i: 'consumers-' + str(i) for i in consumers.index})
+
+        pipes['from_node'].replace(rename_nodes, inplace=True)
+
+        pipes['to_node'].replace(rename_nodes, inplace=True)
+
+        pipes['length_m'] = pipes['geometry'].length
+
+        for node in [consumers, forks]:
+            node['lat'] = node.geometry.y
+            node['lon'] = node.geometry.x
+
+        component_dfs = {
+            'consumers': consumers,
+            'forks': forks,
+            'pipes': pipes
+        }
+
+        for df in component_dfs.values():
+            df.index.name = 'id'
+
+        return component_dfs
+
+    def add_component_data_to_network(self, components):
+
+        for k, v in components.items():
+            self.thermal_network.components[k] = v
+
+        self.thermal_network.is_consistent()
+
+    def process(self, graph, footprints):
+
+        print('Processing...')
+
+        graph = self.remove_self_loops(graph)
+
+        graph = self.remove_duplicate_edges(graph)
+
+        graph = nx.relabel.convert_node_labels_to_integers(graph)
+
+        building_midpoints = self.get_building_midpoints(footprints)
+
+        component_dfs = self.graph_to_component_dfs(graph, building_midpoints)
+
+        return component_dfs
+
+    def load(self):
+
+        graph = self.download_street_network()
+
+        footprints = self.download_footprints()
+
+        footprints = ox.project_gdf(footprints)
+
+        component_dfs = self.process(graph, footprints)
+
+        self.add_component_data_to_network(component_dfs)
+
+        return self.thermal_network
 
 
 class GDFNetworkExporter(NetworkExporter):

dhnx/network.py

@@ -186,7 +186,7 @@ def is_consistent(self):
         for k, v in nodes.items():
             v.index = [k + '-' + str(id) for id in v.index]
 
-        nodes = pd.concat(nodes.values())
+        nodes = pd.concat(nodes.values(), sort=True)
 
         node_indices = nodes.index