Merge pull request #860 from openego/fixes/#856-avoid-conflicting-sub…

…station-ids

Fixes/#856 avoid conflicting substation ids

CHANGELOG.rst

@@ -403,6 +403,7 @@ Changed
   `#921 <https://github.com/openego/eGon-data/issues/921>`_
 * Update creation of heat demand timeseries
   `#857 <https://github.com/openego/eGon-data/issues/857>`_
+  `#856 <https://github.com/openego/eGon-data/issues/856>`_
 
 Bug Fixes
 ---------

src/egon/data/airflow/dags/pipeline.py

@@ -180,7 +180,7 @@
 
     # Create Voronoi polygons
     substation_voronoi = SubstationVoronoi(
-        dependencies=[tasks["osmtgmod_substation"], vg250]
+        dependencies=[tasks["osmtgmod.substation.extract"], vg250]
     )
 
     # MV (medium voltage) grid districts

src/egon/data/datasets.yml

@@ -211,10 +211,10 @@ substation_extraction:
   targets:
     hvmv_substation:
       schema: 'grid'
-      table: 'egon_hvmv_substation'
+      table: 'egon_hvmv_transfer_buses'
     ehv_substation:
       schema: 'grid'
-      table: 'egon_ehv_substation'
+      table: 'egon_ehv_transfer_buses'
     transfer_busses:
       table: 'transfer_busses_complete'
 

src/egon/data/datasets/mv_grid_districts.py

@@ -78,7 +78,7 @@
 
 from egon.data import db
 from egon.data.datasets import Dataset
-from egon.data.datasets.substation import EgonHvmvSubstation
+from egon.data.datasets.osmtgmod.substation import EgonHvmvSubstation
 from egon.data.datasets.substation_voronoi import EgonHvmvSubstationVoronoi
 from egon.data.db import session_scope
 

src/egon/data/datasets/osmtgmod/__init__.py

@@ -8,13 +8,12 @@
 import shutil
 import sys
 
-from airflow.operators.postgres_operator import PostgresOperator
-import importlib_resources as resources
 import psycopg2
 
 from egon.data import db
 from egon.data.config import settings
 from egon.data.datasets import Dataset
+from egon.data.datasets.osmtgmod.substation import extract
 from egon.data.datasets.scenario_parameters import get_sector_parameters
 import egon.data.config
 import egon.data.subprocess as subproc
@@ -776,20 +775,13 @@ class Osmtgmod(Dataset):
     def __init__(self, dependencies):
         super().__init__(
             name="Osmtgmod",
-            version="0.0.4",
+            version="0.0.5",
             dependencies=dependencies,
             tasks=(
                 import_osm_data,
                 run,
                 {
-                    PostgresOperator(
-                        task_id="osmtgmod_substation",
-                        sql=resources.read_text(
-                            __name__, "substation_otg.sql"
-                        ),
-                        postgres_conn_id="egon_data",
-                        autocommit=True,
-                    ),
+                    extract,
                     to_pypsa,
                 },
             ),

src/egon/data/datasets/osmtgmod/substation.py

@@ -0,0 +1,155 @@
+"""The central module containing code to create substation tables
+
+"""
+from geoalchemy2.types import Geometry
+from sqlalchemy import Column, Float, Integer, Text
+from sqlalchemy.ext.declarative import declarative_base
+
+from egon.data import db
+
+Base = declarative_base()
+
+
+class EgonEhvSubstation(Base):
+    __tablename__ = "egon_ehv_substation"
+    __table_args__ = {"schema": "grid"}
+    bus_id = Column(
+        Integer,
+        primary_key=True,
+    )
+    lon = Column(Float(53))
+    lat = Column(Float(53))
+    point = Column(Geometry("POINT", 4326), index=True)
+    polygon = Column(Geometry)
+    voltage = Column(Text)
+    power_type = Column(Text)
+    substation = Column(Text)
+    osm_id = Column(Text)
+    osm_www = Column(Text)
+    frequency = Column(Text)
+    subst_name = Column(Text)
+    ref = Column(Text)
+    operator = Column(Text)
+    dbahn = Column(Text)
+    status = Column(Integer)
+
+
+class EgonHvmvSubstation(Base):
+    __tablename__ = "egon_hvmv_substation"
+    __table_args__ = {"schema": "grid"}
+    bus_id = Column(
+        Integer,
+        primary_key=True,
+    )
+    lon = Column(Float(53))
+    lat = Column(Float(53))
+    point = Column(Geometry("POINT", 4326), index=True)
+    polygon = Column(Geometry)
+    voltage = Column(Text)
+    power_type = Column(Text)
+    substation = Column(Text)
+    osm_id = Column(Text)
+    osm_www = Column(Text)
+    frequency = Column(Text)
+    subst_name = Column(Text)
+    ref = Column(Text)
+    operator = Column(Text)
+    dbahn = Column(Text)
+    status = Column(Integer)
+
+
+def create_tables():
+    """Create tables for substation data
+    Returns
+    -------
+    None.
+    """
+
+    db.execute_sql(
+        f"CREATE SCHEMA IF NOT EXISTS {EgonHvmvSubstation.__table__.schema};"
+    )
+
+    # Drop tables
+    db.execute_sql(
+        f"""DROP TABLE IF EXISTS
+            {EgonEhvSubstation.__table__.schema}.
+            {EgonEhvSubstation.__table__.name} CASCADE;"""
+    )
+
+    db.execute_sql(
+        f"""DROP TABLE IF EXISTS
+            {EgonHvmvSubstation.__table__.schema}.
+            {EgonHvmvSubstation.__table__.name} CASCADE;"""
+    )
+
+    engine = db.engine()
+    EgonEhvSubstation.__table__.create(bind=engine, checkfirst=True)
+    EgonHvmvSubstation.__table__.create(bind=engine, checkfirst=True)
+
+
+def extract():
+    """
+    Extract ehv and hvmv substation from transfer buses and results from osmtgmod
+
+    Returns
+    -------
+    None.
+
+    """
+    # Create tables for substations
+    create_tables()
+
+    # Extract eHV substations
+    db.execute_sql(
+        f"""
+        INSERT INTO {EgonEhvSubstation.__table__.schema}.{EgonEhvSubstation.__table__.name}
+        
+        SELECT * FROM grid.egon_ehv_transfer_buses;
+        
+        
+        -- update ehv_substation table with new column of respective osmtgmod bus_i
+        ALTER TABLE {EgonEhvSubstation.__table__.schema}.{EgonEhvSubstation.__table__.name}
+        	ADD COLUMN otg_id bigint;
+
+        -- fill table with bus_i from osmtgmod
+        UPDATE {EgonEhvSubstation.__table__.schema}.{EgonEhvSubstation.__table__.name}
+        	SET otg_id = osmtgmod_results.bus_data.bus_i
+        	FROM osmtgmod_results.bus_data
+        	WHERE osmtgmod_results.bus_data.base_kv > 110 AND(SELECT TRIM(leading 'n' FROM TRIM(leading 'w' FROM TRIM(leading 'r' FROM grid.egon_ehv_substation.osm_id)))::BIGINT)=osmtgmod_results.bus_data.osm_substation_id; 
+
+        DELETE FROM {EgonEhvSubstation.__table__.schema}.{EgonEhvSubstation.__table__.name} WHERE otg_id IS NULL;
+
+        UPDATE {EgonEhvSubstation.__table__.schema}.{EgonEhvSubstation.__table__.name}
+        	SET 	bus_id = otg_id;
+
+        ALTER TABLE {EgonEhvSubstation.__table__.schema}.{EgonEhvSubstation.__table__.name}
+        	DROP COLUMN otg_id;
+        """
+    )
+
+    # Extract HVMV substations
+    db.execute_sql(
+        f"""
+        INSERT INTO {EgonHvmvSubstation.__table__.schema}.{EgonHvmvSubstation.__table__.name}
+        
+        SELECT * FROM grid.egon_hvmv_transfer_buses;
+        
+        
+        ALTER TABLE {EgonHvmvSubstation.__table__.schema}.{EgonHvmvSubstation.__table__.name}
+        	ADD COLUMN otg_id bigint;
+        
+        -- fill table with bus_i from osmtgmod
+        UPDATE {EgonHvmvSubstation.__table__.schema}.{EgonHvmvSubstation.__table__.name}
+        	SET 	otg_id = osmtgmod_results.bus_data.bus_i
+        	FROM 	osmtgmod_results.bus_data
+        	WHERE 	osmtgmod_results.bus_data.base_kv <= 110 AND (SELECT TRIM(leading 'n' FROM TRIM(leading 'w' FROM grid.egon_hvmv_substation.osm_id))::BIGINT)=osmtgmod_results.bus_data.osm_substation_id; 
+        
+        DELETE FROM {EgonHvmvSubstation.__table__.schema}.{EgonHvmvSubstation.__table__.name} WHERE otg_id IS NULL;
+        
+        UPDATE {EgonHvmvSubstation.__table__.schema}.{EgonHvmvSubstation.__table__.name}
+        	SET 	bus_id = otg_id;
+        
+        ALTER TABLE {EgonHvmvSubstation.__table__.schema}.{EgonHvmvSubstation.__table__.name}
+        	DROP COLUMN otg_id;
+        """
+    )

src/egon/data/datasets/substation/__init__.py

@@ -14,14 +14,14 @@
 Base = declarative_base()
 
 
-class EgonEhvSubstation(Base):
-    __tablename__ = "egon_ehv_substation"
+class EgonEhvTransferBuses(Base):
+    __tablename__ = "egon_ehv_transfer_buses"
     __table_args__ = {"schema": "grid"}
     bus_id = Column(
         Integer,
-        Sequence("egon_ehv_substation_bus_id_seq", schema="grid"),
+        Sequence("egon_ehv_transfer_buses_bus_id_seq", schema="grid"),
         server_default=Sequence(
-            "egon_ehv_substation_bus_id_seq", schema="grid"
+            "egon_ehv_transfer_buses_bus_id_seq", schema="grid"
         ).next_value(),
         primary_key=True,
     )
@@ -42,14 +42,14 @@ class EgonEhvSubstation(Base):
     status = Column(Integer)
 
 
-class EgonHvmvSubstation(Base):
-    __tablename__ = "egon_hvmv_substation"
+class EgonHvmvTransferBuses(Base):
+    __tablename__ = "egon_hvmv_transfer_buses"
     __table_args__ = {"schema": "grid"}
     bus_id = Column(
         Integer,
-        Sequence("egon_hvmv_substation_bus_id_seq", schema="grid"),
+        Sequence("egon_hvmv_transfer_buses_bus_id_seq", schema="grid"),
         server_default=Sequence(
-            "egon_hvmv_substation_bus_id_seq", schema="grid"
+            "egon_hvmv_transfer_buses_bus_id_seq", schema="grid"
         ).next_value(),
         primary_key=True,
     )
@@ -74,7 +74,7 @@ class SubstationExtraction(Dataset):
     def __init__(self, dependencies):
         super().__init__(
             name="substation_extraction",
-            version="0.0.1",
+            version="0.0.2",
             dependencies=dependencies,
             tasks=(
                 create_tables,
@@ -142,8 +142,8 @@ def create_tables():
     )
 
     engine = db.engine()
-    EgonEhvSubstation.__table__.create(bind=engine, checkfirst=True)
-    EgonHvmvSubstation.__table__.create(bind=engine, checkfirst=True)
+    EgonEhvTransferBuses.__table__.create(bind=engine, checkfirst=True)
+    EgonHvmvTransferBuses.__table__.create(bind=engine, checkfirst=True)
 
 
 def create_sql_functions():

src/egon/data/datasets/substation/ehv_substation.sql

@@ -154,7 +154,7 @@ CREATE VIEW 		grid.egon_final_result_hoes AS
 	WHERE 	grid.egon_summary_de_hoes.osm_id NOT IN ( SELECT grid.egon_substations_to_drop_hoes.osm_id FROM grid.egon_substations_to_drop_hoes);
 
 -- insert results
-INSERT INTO grid.egon_ehv_substation (lon, lat, point, polygon, voltage, power_type, substation, osm_id, osm_www, frequency, subst_name, ref, operator, dbahn, status)
+INSERT INTO grid.egon_ehv_transfer_buses (lon, lat, point, polygon, voltage, power_type, substation, osm_id, osm_www, frequency, subst_name, ref, operator, dbahn, status)
 	SELECT 	lon, lat, point, polygon, voltage, power_type, substation, osm_id, osm_www, frequency, subst_name, ref, operator, dbahn, status
 	FROM 	grid.egon_final_result_hoes;
 

src/egon/data/datasets/substation/hvmv_substation.sql

@@ -178,16 +178,16 @@ CREATE VIEW 		grid.egon_final_result AS
 
 
 -- insert results
-INSERT INTO grid.egon_hvmv_substation (lon, lat, point, polygon, voltage, power_type, substation, osm_id, osm_www, frequency, subst_name, ref, operator, dbahn, status)
+INSERT INTO grid.egon_hvmv_transfer_buses (lon, lat, point, polygon, voltage, power_type, substation, osm_id, osm_www, frequency, subst_name, ref, operator, dbahn, status)
 	SELECT lon, lat, point, polygon, voltage, power_type, substation, osm_id, osm_www, frequency, subst_name, ref, operator, dbahn, status
 	FROM grid.egon_final_result;
 
 -- update voltage level if split by '/' instead of ';' or contains '--'
-UPDATE grid.egon_hvmv_substation
+UPDATE grid.egon_hvmv_transfer_buses
 SET voltage = (SELECT REPLACE (voltage, '/', ';'))
 WHERE voltage LIKE '%/%';
 
-UPDATE grid.egon_hvmv_substation
+UPDATE grid.egon_hvmv_transfer_buses
 SET voltage = (SELECT REPLACE (voltage, '-', ''))
 WHERE voltage LIKE '%-%';