From 2d1a2891f6fa55b0099979c7dcb45555c9965b4a Mon Sep 17 00:00:00 2001
From: ulfmueller <ulf.p.mueller@hs-flensburg.de>
Date: Thu, 14 Jan 2021 15:45:14 +0100
Subject: [PATCH 01/12] proposal for etrago sqlalchemy table structure

---
 src/egon/data/importing/etrago/etrago_grid.py | 312 ++++++++++++++++++
 1 file changed, 312 insertions(+)
 create mode 100644 src/egon/data/importing/etrago/etrago_grid.py

diff --git a/src/egon/data/importing/etrago/etrago_grid.py b/src/egon/data/importing/etrago/etrago_grid.py
new file mode 100644
index 000000000..f0e4500e4
--- /dev/null
+++ b/src/egon/data/importing/etrago/etrago_grid.py
@@ -0,0 +1,312 @@
+# coding: utf-8
+from sqlalchemy import ARRAY, BigInteger, Boolean, CHAR, Column, Date, DateTime, Float, ForeignKey, Integer, JSON, Numeric, SmallInteger, String, Table, Text, text
+from geoalchemy2.types import Geometry
+from sqlalchemy.orm import relationship
+from sqlalchemy.ext.declarative import declarative_base
+
+Base = declarative_base()
+metadata = Base.metadata
+
+
+
+class EgonPfHvBus(Base):
+    __tablename__ = 'egon_pf_hv_bus'
+    __table_args__ = {'schema': 'grid'}
+
+    version = Column(Text, primary_key=True, nullable=False)
+    scn_name = Column(String, primary_key=True, nullable=False)
+    bus_id = Column(BigInteger, primary_key=True, nullable=False)
+    v_nom = Column(Float(53))
+    carrier = Column(Text)
+    v_mag_pu_min = Column(Float(53))
+    v_mag_pu_max = Column(Float(53))
+    geom = Column(Geometry('POINT', 4326), index=True)
+
+
+class EgonPfHvGenerator(Base):
+    __tablename__ = 'egon_pf_hv_generator'
+    __table_args__ = {'schema': 'grid'}
+
+    version = Column(Text, primary_key=True, nullable=False)
+    scn_name = Column(String, primary_key=True, nullable=False)
+    generator_id = Column(BigInteger, primary_key=True, nullable=False)
+    bus = Column(BigInteger)
+    control = Column(Text)
+    carrier = Column(Text)
+    p_nom = Column(Float(53))
+    p_nom_extendable = Column(Boolean)
+    p_nom_min = Column(Float(53))
+    p_nom_max = Column(Float(53))
+    p_min_pu_fixed = Column(Float(53))
+    p_max_pu_fixed = Column(Float(53))
+    sign = Column(Float(53))
+    marginal_cost_fixed = Column(Float(53))
+    capital_cost = Column(Float(53))
+    efficiency = Column(Float(53))
+    committable = Column(Boolean)
+    start_up_cost = Column(Float(53))
+    shut_down_cost = Column(Float(53))
+    min_up_time = Column(BigInteger)
+    min_down_time = Column(BigInteger)
+    up_time_before = Column(BigInteger)    
+    down_time_before = Column(BigInteger)    
+    ramp_limit_up = Column(Float(53))
+    ramp_limit_down = Column(Float(53))
+    ramp_limit_start_up = Column(Float(53))
+    ramp_limit_shut_down = Column(Float(53))
+
+
+class EgonPfHvGeneratorTimeseries(Base):
+    __tablename__ = 'egon_pf_hv_generator_timeseries'
+    __table_args__ = {'schema': 'grid'}
+
+    version = Column(Text, primary_key=True, nullable=False)
+    scn_name = Column(String, primary_key=True, nullable=False)
+    generator_id = Column(Integer, primary_key=True, nullable=False)
+    temp_id = Column(Integer, nullable=False)
+    p_set = Column(ARRAY(Float(precision=53)))
+    q_set = Column(ARRAY(Float(precision=53)))
+    p_min_pu = Column(ARRAY(Float(precision=53)))
+    p_max_pu = Column(ARRAY(Float(precision=53)))
+    marginal_cost = Column(ARRAY(Float(precision=53)))
+
+
+class EgonPfHvLine(Base):
+    __tablename__ = 'egon_pf_hv_line'
+    __table_args__ = {'schema': 'grid'}
+
+    version = Column(Text, primary_key=True, nullable=False)
+    scn_name = Column(String, primary_key=True, nullable=False)
+    line_id = Column(BigInteger, primary_key=True, nullable=False)
+    bus0 = Column(BigInteger)
+    bus1 = Column(BigInteger)
+    x = Column(Numeric)
+    r = Column(Numeric)
+    g = Column(Numeric)
+    b = Column(Numeric)
+    s_nom = Column(Numeric)
+    s_nom_extendable = Column(Boolean)
+    s_nom_min = Column(Float(53))
+    s_nom_max = Column(Float(53))
+    s_max_pu_fixed = Column(ARRAY(Float(precision=53)))
+    capital_cost = Column(Float(53))
+    length = Column(Float(53))
+    cables = Column(Integer)
+    terrain_factor = Column(Float(53))
+    geom = Column(Geometry('MULTILINESTRING', 4326))
+    topo = Column(Geometry('LINESTRING', 4326))
+
+
+class EgonPfHvLineTimeseries(Base):
+    __tablename__ = 'egon_pf_hv_line_timeseries'
+    __table_args__ = {'schema': 'grid'}
+
+    version = version = Column(Text, primary_key=True, nullable=False)
+    scn_name = Column(String, primary_key=True, nullable=False)
+    line_id = Column(BigInteger, primary_key=True, nullable=False)
+    s_max_pu = Column(ARRAY(Float(precision=53)))
+
+
+class EgonPfHvLink(Base):
+    __tablename__ = 'egon_pf_hv_link'
+    __table_args__ = {'schema': 'grid'}
+
+    version = version = Column(Text, primary_key=True, nullable=False)
+    scn_name = Column(String, primary_key=True, nullable=False)
+    link_id = Column(BigInteger, primary_key=True, nullable=False)
+    bus0 = Column(BigInteger)
+    bus1 = Column(BigInteger)
+    carrier = Column(Text)
+    efficiency_fixed = Column(Float(53))
+    p_nom = Column(Numeric)
+    p_nom_extendable = Column(Boolean)
+    p_nom_min = Column(Float(53))
+    p_nom_max = Column(Float(53))
+    p_min_pu_fixed = Column(Float(53))
+    p_max_pu_fixed = Column(Float(53))
+    p_set_fixed = Column(Float(53))
+    capital_cost = Column(Float(53))
+    marginal_cost_fixed = Column(Float(53))
+    length = Column(Float(53))
+    terrain_factor = Column(Float(53))
+    geom = Column(Geometry('MULTILINESTRING', 4326))
+    topo = Column(Geometry('LINESTRING', 4326))
+
+class EgonPfHvLinkTimeseries(Base):
+    __tablename__ = 'egon_pf_hv_link_timeseries'
+    __table_args__ = {'schema': 'grid'}
+
+    version = version = Column(Text, primary_key=True, nullable=False)
+    scn_name = Column(String, primary_key=True, nullable=False)
+    link_id = Column(BigInteger, primary_key=True, nullable=False)
+    p_set = Column(ARRAY(Float(precision=53)))
+    p_min_pu = Column(ARRAY(Float(precision=53)))
+    p_max_pu = Column(ARRAY(Float(precision=53)))
+    efficiency = Column(ARRAY(Float(precision=53)))
+    marginal_cost = Column(ARRAY(Float(precision=53)))
+
+
+class EgonPfHvLoad(Base):
+    __tablename__ = 'egon_pf_hv_load'
+    __table_args__ = {'schema': 'grid'}
+
+    version = Column(Text, primary_key=True, nullable=False)
+    scn_name = Column(String, primary_key=True, nullable=False)
+    load_id = Column(BigInteger, primary_key=True, nullable=False)
+    bus = Column(BigInteger)
+    carrier = Column(Text)
+    sign = Column(Float(53))
+
+
+class EgonPfHvLoadTimeseries(Base):
+    __tablename__ = 'egon_pf_hv_load_timeseries'
+    __table_args__ = {'schema': 'grid'}
+
+    version = Column(Text, primary_key=True, nullable=False)
+    scn_name = Column(String, primary_key=True, nullable=False)
+    load_id = Column(BigInteger, primary_key=True, nullable=False)
+    temp_id = Column(Integer, nullable=False)
+    p_set = Column(ARRAY(Float(precision=53)))
+    q_set = Column(ARRAY(Float(precision=53)))
+
+
+class EgonPfHvCarrier(Base):
+    __tablename__ = 'egon_pf_hv_carrier'
+    __table_args__ = {'schema': 'grid'}
+
+    version = Column(Text, primary_key=True, nullable=False)
+    name = Column(Text)
+    co2_emissions = Column(Float(53))
+    commentary = Column(Text)
+
+
+class EgonPfHvStorage(Base):
+    __tablename__ = 'egon_pf_hv_storage'
+    __table_args__ = {'schema': 'grid'}
+
+    version = Column(Text, primary_key=True, nullable=False)
+    scn_name = Column(String, primary_key=True, nullable=False)
+    storage_id = Column(BigInteger, primary_key=True, nullable=False)
+    bus = Column(BigInteger)
+    control = Column(Text)
+    carrier = Column(Text)
+    p_nom = Column(Float(53))
+    p_nom_extendable = Column(Boolean)
+    p_nom_min = Column(Float(53))
+    p_nom_max = Column(Float(53))
+    p_min_pu_fixed = Column(Float(53))
+    p_max_pu_fixed = Column(Float(53))
+    sign = Column(Float(53))
+    marginal_cost_fixed = Column(Float(53))
+    capital_cost = Column(Float(53))
+    state_of_charge_initial = Column(Float(53))
+    cyclic_state_of_charge = Column(Boolean)
+    max_hours = Column(Float(53))
+    efficiency_store = Column(Float(53))
+    efficiency_dispatch = Column(Float(53))
+    standing_loss = Column(Float(53))
+
+
+class EgonPfHvStorageTimeseries(Base):
+    __tablename__ = 'egon_pf_hv_storage_timeseries'
+    __table_args__ = {'schema': 'grid'}
+
+    version = Column(Text, primary_key=True, nullable=False)
+    scn_name = Column(String, primary_key=True, nullable=False)
+    storage_id = Column(BigInteger, primary_key=True, nullable=False)
+    temp_id = Column(Integer, primary_key=True, nullable=False)
+    p_set = Column(ARRAY(Float(precision=53)))
+    q_set = Column(ARRAY(Float(precision=53)))
+    p_min_pu = Column(ARRAY(Float(precision=53)))
+    p_max_pu = Column(ARRAY(Float(precision=53)))
+    state_of_charge_set = Column(ARRAY(Float(precision=53)))
+    inflow = Column(ARRAY(Float(precision=53)))
+    marginal_cost = Column(ARRAY(Float(precision=53)))
+
+class EgonPfHvStore(Base):
+    __tablename__ = 'egon_pf_hv_store'
+    __table_args__ = {'schema': 'grid'}
+
+    version = Column(Text, primary_key=True, nullable=False)
+    scn_name = Column(String, primary_key=True, nullable=False)
+    store_id = Column(BigInteger, primary_key=True, nullable=False)
+    bus = Column(BigInteger)
+    carrier = Column(Text)
+    e_nom = Column(Float(53))
+    e_nom_extendable = Column(Boolean)
+    e_nom_min = Column(Float(53))
+    e_nom_max = Column(Float(53))
+    e_min_pu_fixed = Column(Float(53))
+    e_max_pu_fixed = Column(Float(53))
+    e_initial = Column(Float(53))
+    e_cyclic = Column(Boolean)
+    sign = Column(Float(53))
+    marginal_cost_fixed = Column(Float(53))
+    capital_cost = Column(Float(53))
+    standing_loss = Column(Float(53))
+
+class EgonPfHvStoreTimeseries(Base):
+    __tablename__ = 'egon_pf_hv_store_timeseries'
+    __table_args__ = {'schema': 'grid'}
+
+    version = Column(Text, primary_key=True, nullable=False)
+    scn_name = Column(String, primary_key=True, nullable=False)
+    store_id = Column(BigInteger, primary_key=True, nullable=False)
+    temp_id = Column(Integer, primary_key=True, nullable=False)
+    p_set = Column(ARRAY(Float(precision=53)))
+    q_set = Column(ARRAY(Float(precision=53)))
+    e_min_pu = Column(ARRAY(Float(precision=53)))
+    e_max_pu = Column(ARRAY(Float(precision=53)))
+    marginal_cost = Column(ARRAY(Float(precision=53)))
+
+class EgonPfHvTempResolution(Base):
+    __tablename__ = 'egon_pf_hv_temp_resolution'
+    __table_args__ = {'schema': 'grid'}
+
+    version = Column(Text, primary_key=True, nullable=False)
+    temp_id = Column(BigInteger, primary_key=True, nullable=False)
+    timesteps = Column(BigInteger, nullable=False)
+    resolution = Column(Text)
+    start_time = Column(DateTime)
+
+
+class EgonPfHvTransformer(Base):
+    __tablename__ = 'egon_pf_hv_transformer'
+    __table_args__ = {'schema': 'grid'}
+
+    version = Column(Text, primary_key=True, nullable=False)
+    scn_name = Column(String, primary_key=True, nullable=False)
+    trafo_id = Column(BigInteger, primary_key=True, nullable=False)
+    bus0 = Column(BigInteger)
+    bus1 = Column(BigInteger)
+    model = Column(Text)
+    x = Column(Numeric)
+    r = Column(Numeric)
+    g = Column(Numeric)
+    b = Column(Numeric)
+    s_nom = Column(Float(53))
+    s_nom_extendable = Column(Boolean)
+    s_nom_min = Column(Float(53))
+    s_nom_max = Column(Float(53))
+    s_max_pu_fixed = Column(Float(53))
+    tap_ratio = Column(Float(53))
+    tap_side = Column(Float(53))
+    tap_position = Column(BigInteger)
+    phase_shift = Column(Float(53))
+    v_ang_min = Column(Float(53))
+    v_ang_max = Column(Float(53))
+    capital_cost = Column(Float(53))
+    num_parallel = Column(Float(53))
+    geom = Column(Geometry('MULTILINESTRING', 4326))
+    topo = Column(Geometry('LINESTRING', 4326))
+
+
+class EgonPfHvTransformerTimeseries(Base):
+    __tablename__ = 'egon_pf_hv_transformer_timeseries'
+    __table_args__ = {'schema': 'grid'}
+
+    version = version = Column(Text, primary_key=True, nullable=False)
+    scn_name = Column(String, primary_key=True, nullable=False)
+    trafo_id = Column(BigInteger, primary_key=True, nullable=False)
+    s_max_pu = Column(ARRAY(Float(precision=53)))
+

From 7fd59e45cfb6aa8e9ff5d9340ce5741028f4cfec Mon Sep 17 00:00:00 2001
From: ulfmueller <ulf.p.mueller@hs-flensburg.de>
Date: Mon, 18 Jan 2021 10:38:56 +0100
Subject: [PATCH 02/12] add carrier column to lines table

---
 src/egon/data/importing/etrago/etrago_grid.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/egon/data/importing/etrago/etrago_grid.py b/src/egon/data/importing/etrago/etrago_grid.py
index f0e4500e4..ed0b362da 100644
--- a/src/egon/data/importing/etrago/etrago_grid.py
+++ b/src/egon/data/importing/etrago/etrago_grid.py
@@ -80,6 +80,7 @@ class EgonPfHvLine(Base):
     line_id = Column(BigInteger, primary_key=True, nullable=False)
     bus0 = Column(BigInteger)
     bus1 = Column(BigInteger)
+    carrier = Column(Text)    
     x = Column(Numeric)
     r = Column(Numeric)
     g = Column(Numeric)

From c52988400005f33630c824606d93838cc6b213bc Mon Sep 17 00:00:00 2001
From: ulfmueller <ulf.p.mueller@hs-flensburg.de>
Date: Mon, 18 Jan 2021 10:48:31 +0100
Subject: [PATCH 03/12] correct erroneous column definition

---
 src/egon/data/importing/etrago/etrago_grid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/egon/data/importing/etrago/etrago_grid.py b/src/egon/data/importing/etrago/etrago_grid.py
index ed0b362da..d59506dae 100644
--- a/src/egon/data/importing/etrago/etrago_grid.py
+++ b/src/egon/data/importing/etrago/etrago_grid.py
@@ -89,7 +89,7 @@ class EgonPfHvLine(Base):
     s_nom_extendable = Column(Boolean)
     s_nom_min = Column(Float(53))
     s_nom_max = Column(Float(53))
-    s_max_pu_fixed = Column(ARRAY(Float(precision=53)))
+    s_max_pu_fixed = Column(Float(53))
     capital_cost = Column(Float(53))
     length = Column(Float(53))
     cables = Column(Integer)

From 81a0d2d76d5aebb41a0836d983a5a65f5d27d623 Mon Sep 17 00:00:00 2001
From: ulfmueller <ulf.p.mueller@hs-flensburg.de>
Date: Mon, 18 Jan 2021 10:53:11 +0100
Subject: [PATCH 04/12] correct erroneous column definition

---
 src/egon/data/importing/etrago/etrago_grid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/egon/data/importing/etrago/etrago_grid.py b/src/egon/data/importing/etrago/etrago_grid.py
index d59506dae..8893dc5b8 100644
--- a/src/egon/data/importing/etrago/etrago_grid.py
+++ b/src/egon/data/importing/etrago/etrago_grid.py
@@ -291,7 +291,7 @@ class EgonPfHvTransformer(Base):
     s_nom_max = Column(Float(53))
     s_max_pu_fixed = Column(Float(53))
     tap_ratio = Column(Float(53))
-    tap_side = Column(Float(53))
+    tap_side = Column(BigInteger)
     tap_position = Column(BigInteger)
     phase_shift = Column(Float(53))
     v_ang_min = Column(Float(53))

From 62b9e708baf8018b67a4f6e1e4d7b5c176c402c8 Mon Sep 17 00:00:00 2001
From: ulfmueller <ulf.p.mueller@hs-flensburg.de>
Date: Mon, 18 Jan 2021 15:43:06 +0100
Subject: [PATCH 05/12] add v_mag_pu_set / _fixed for the buses

---
 src/egon/data/importing/etrago/etrago_grid.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/egon/data/importing/etrago/etrago_grid.py b/src/egon/data/importing/etrago/etrago_grid.py
index 8893dc5b8..84532a154 100644
--- a/src/egon/data/importing/etrago/etrago_grid.py
+++ b/src/egon/data/importing/etrago/etrago_grid.py
@@ -18,10 +18,20 @@ class EgonPfHvBus(Base):
     bus_id = Column(BigInteger, primary_key=True, nullable=False)
     v_nom = Column(Float(53))
     carrier = Column(Text)
+    v_mag_pu_set_fixed = Column(Float(53))
     v_mag_pu_min = Column(Float(53))
     v_mag_pu_max = Column(Float(53))
     geom = Column(Geometry('POINT', 4326), index=True)
 
+    
+class EgonPfHvBusTimeseries(Base):
+    __tablename__ = 'egon_pf_hv_bus_timeseries'
+    __table_args__ = {'schema': 'grid'}
+
+    version = Column(Text, primary_key=True, nullable=False)
+    scn_name = Column(String, primary_key=True, nullable=False)
+    bus_id = Column(BigInteger, primary_key=True, nullable=False)
+    v_mag_pu_set = Column(ARRAY(Float(precision=53)))
 
 class EgonPfHvGenerator(Base):
     __tablename__ = 'egon_pf_hv_generator'

From bc6865d879c3a5fac42e1edd8224d6394adedfbb Mon Sep 17 00:00:00 2001
From: ulfmueller <ulf.p.mueller@hs-flensburg.de>
Date: Mon, 18 Jan 2021 15:54:18 +0100
Subject: [PATCH 06/12] Update etrago_grid.py

---
 src/egon/data/importing/etrago/etrago_grid.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/egon/data/importing/etrago/etrago_grid.py b/src/egon/data/importing/etrago/etrago_grid.py
index 84532a154..aa02546fe 100644
--- a/src/egon/data/importing/etrago/etrago_grid.py
+++ b/src/egon/data/importing/etrago/etrago_grid.py
@@ -104,6 +104,8 @@ class EgonPfHvLine(Base):
     length = Column(Float(53))
     cables = Column(Integer)
     terrain_factor = Column(Float(53))
+    v_ang_min = Column(Float(53))
+    v_ang_max = Column(Float(53))
     geom = Column(Geometry('MULTILINESTRING', 4326))
     topo = Column(Geometry('LINESTRING', 4326))
 
@@ -212,10 +214,13 @@ class EgonPfHvStorage(Base):
     capital_cost = Column(Float(53))
     state_of_charge_initial = Column(Float(53))
     cyclic_state_of_charge = Column(Boolean)
+    state_of_charge_set_fixed = Column(Float(53))
     max_hours = Column(Float(53))
     efficiency_store = Column(Float(53))
     efficiency_dispatch = Column(Float(53))
     standing_loss = Column(Float(53))
+    inflow_fixed = Column(Float(53))
+    
 
 
 class EgonPfHvStorageTimeseries(Base):

From 5b2218b5816599da7820efcb656713b3098e44ba Mon Sep 17 00:00:00 2001
From: ulfmueller <ulf.p.mueller@hs-flensburg.de>
Date: Wed, 3 Mar 2021 11:05:57 +0100
Subject: [PATCH 07/12] add less important columns for more pypsa consistency

---
 src/egon/data/importing/etrago/etrago_grid.py | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/src/egon/data/importing/etrago/etrago_grid.py b/src/egon/data/importing/etrago/etrago_grid.py
index aa02546fe..ff9d54575 100644
--- a/src/egon/data/importing/etrago/etrago_grid.py
+++ b/src/egon/data/importing/etrago/etrago_grid.py
@@ -17,10 +17,13 @@ class EgonPfHvBus(Base):
     scn_name = Column(String, primary_key=True, nullable=False)
     bus_id = Column(BigInteger, primary_key=True, nullable=False)
     v_nom = Column(Float(53))
+    type = Column(Text)
     carrier = Column(Text)
     v_mag_pu_set_fixed = Column(Float(53))
     v_mag_pu_min = Column(Float(53))
     v_mag_pu_max = Column(Float(53))
+    x = Column(Float(53))
+    y = Column(Float(53))
     geom = Column(Geometry('POINT', 4326), index=True)
 
     
@@ -42,6 +45,7 @@ class EgonPfHvGenerator(Base):
     generator_id = Column(BigInteger, primary_key=True, nullable=False)
     bus = Column(BigInteger)
     control = Column(Text)
+    type = Column(Text)
     carrier = Column(Text)
     p_nom = Column(Float(53))
     p_nom_extendable = Column(Boolean)
@@ -49,6 +53,8 @@ class EgonPfHvGenerator(Base):
     p_nom_max = Column(Float(53))
     p_min_pu_fixed = Column(Float(53))
     p_max_pu_fixed = Column(Float(53))
+    p_set_fixed = Column(Float(53))
+    q_set_fixed = Column(Float(53))
     sign = Column(Float(53))
     marginal_cost_fixed = Column(Float(53))
     capital_cost = Column(Float(53))
@@ -90,6 +96,7 @@ class EgonPfHvLine(Base):
     line_id = Column(BigInteger, primary_key=True, nullable=False)
     bus0 = Column(BigInteger)
     bus1 = Column(BigInteger)
+    type = Column(Text)
     carrier = Column(Text)    
     x = Column(Numeric)
     r = Column(Numeric)
@@ -104,6 +111,7 @@ class EgonPfHvLine(Base):
     length = Column(Float(53))
     cables = Column(Integer)
     terrain_factor = Column(Float(53))
+    num_parallel = Column(Float(53))
     v_ang_min = Column(Float(53))
     v_ang_max = Column(Float(53))
     geom = Column(Geometry('MULTILINESTRING', 4326))
@@ -129,6 +137,7 @@ class EgonPfHvLink(Base):
     link_id = Column(BigInteger, primary_key=True, nullable=False)
     bus0 = Column(BigInteger)
     bus1 = Column(BigInteger)
+    type = Column(Text)
     carrier = Column(Text)
     efficiency_fixed = Column(Float(53))
     p_nom = Column(Numeric)
@@ -167,7 +176,10 @@ class EgonPfHvLoad(Base):
     scn_name = Column(String, primary_key=True, nullable=False)
     load_id = Column(BigInteger, primary_key=True, nullable=False)
     bus = Column(BigInteger)
+    type = Column(Text)
     carrier = Column(Text)
+    p_set_fixed = Column(Float(53))
+    q_set_fixed = Column(Float(53))
     sign = Column(Float(53))
 
 
@@ -190,6 +202,8 @@ class EgonPfHvCarrier(Base):
     version = Column(Text, primary_key=True, nullable=False)
     name = Column(Text)
     co2_emissions = Column(Float(53))
+    color = Column(Text)
+    nice_name = Column(Text)
     commentary = Column(Text)
 
 
@@ -202,6 +216,7 @@ class EgonPfHvStorage(Base):
     storage_id = Column(BigInteger, primary_key=True, nullable=False)
     bus = Column(BigInteger)
     control = Column(Text)
+    type = Column(Text)
     carrier = Column(Text)
     p_nom = Column(Float(53))
     p_nom_extendable = Column(Boolean)
@@ -209,6 +224,8 @@ class EgonPfHvStorage(Base):
     p_nom_max = Column(Float(53))
     p_min_pu_fixed = Column(Float(53))
     p_max_pu_fixed = Column(Float(53))
+    p_set_fixed = Column(Float(53))
+    q_set_fixed = Column(Float(53))
     sign = Column(Float(53))
     marginal_cost_fixed = Column(Float(53))
     capital_cost = Column(Float(53))
@@ -247,6 +264,7 @@ class EgonPfHvStore(Base):
     scn_name = Column(String, primary_key=True, nullable=False)
     store_id = Column(BigInteger, primary_key=True, nullable=False)
     bus = Column(BigInteger)
+    type = Column(Text)
     carrier = Column(Text)
     e_nom = Column(Float(53))
     e_nom_extendable = Column(Boolean)
@@ -254,6 +272,8 @@ class EgonPfHvStore(Base):
     e_nom_max = Column(Float(53))
     e_min_pu_fixed = Column(Float(53))
     e_max_pu_fixed = Column(Float(53))
+    p_set_fixed = Column(Float(53))
+    q_set_fixed = Column(Float(53))
     e_initial = Column(Float(53))
     e_cyclic = Column(Boolean)
     sign = Column(Float(53))
@@ -295,6 +315,7 @@ class EgonPfHvTransformer(Base):
     trafo_id = Column(BigInteger, primary_key=True, nullable=False)
     bus0 = Column(BigInteger)
     bus1 = Column(BigInteger)
+    type = Column(Text)
     model = Column(Text)
     x = Column(Numeric)
     r = Column(Numeric)

From e2ef4963fd485e2fc6e6be4f9ca67a8bf6ad9341 Mon Sep 17 00:00:00 2001
From: ulfmueller <ulf.p.mueller@hs-flensburg.de>
Date: Wed, 3 Mar 2021 11:28:22 +0100
Subject: [PATCH 08/12] add table creation function

---
 src/egon/data/importing/etrago/etrago_grid.py | 29 +++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/src/egon/data/importing/etrago/etrago_grid.py b/src/egon/data/importing/etrago/etrago_grid.py
index ff9d54575..14fdeea21 100644
--- a/src/egon/data/importing/etrago/etrago_grid.py
+++ b/src/egon/data/importing/etrago/etrago_grid.py
@@ -3,6 +3,7 @@
 from geoalchemy2.types import Geometry
 from sqlalchemy.orm import relationship
 from sqlalchemy.ext.declarative import declarative_base
+from egon.data import db
 
 Base = declarative_base()
 metadata = Base.metadata
@@ -347,3 +348,31 @@ class EgonPfHvTransformerTimeseries(Base):
     trafo_id = Column(BigInteger, primary_key=True, nullable=False)
     s_max_pu = Column(ARRAY(Float(precision=53)))
 
+
+def create_tables():
+    """Create tables for eTraGo input data.
+    Returns
+    -------
+    None.
+    """
+    db.execute_sql(
+        f"CREATE SCHEMA IF NOT EXISTS grid;")
+    engine = db.engine()
+    EgonPfHvBus.__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvBusTimeseries.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvGenerator.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvGeneratorTimeseries.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvLine.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvLineTimeseries.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvLink.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvLinkTimeseries.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvLoad.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvLoadTimeseries.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvCarrier.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvStorage.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvStorageTimeseries.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvStore.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvStoreTimeseries.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvTempResolution.__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvTransformer.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvTransformerTimeseries.__table__.create(bind=engine, checkfirst=True)

From 8d8a345055ecd4497b44e1c0d96da66306e02593 Mon Sep 17 00:00:00 2001
From: ulfmueller <ulf.p.mueller@hs-flensburg.de>
Date: Wed, 3 Mar 2021 11:36:43 +0100
Subject: [PATCH 09/12] Rename etrago_grid.py to __init__.py

---
 src/egon/data/importing/etrago/{etrago_grid.py => __init__.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/egon/data/importing/etrago/{etrago_grid.py => __init__.py} (100%)

diff --git a/src/egon/data/importing/etrago/etrago_grid.py b/src/egon/data/importing/etrago/__init__.py
similarity index 100%
rename from src/egon/data/importing/etrago/etrago_grid.py
rename to src/egon/data/importing/etrago/__init__.py

From 6204931f5ac93a160cf5e306a4b0972744e74091 Mon Sep 17 00:00:00 2001
From: ulfmueller <ulf.p.mueller@hs-flensburg.de>
Date: Wed, 3 Mar 2021 11:38:29 +0100
Subject: [PATCH 10/12] adding etrago table creation to pipeline

---
 src/egon/data/airflow/dags/pipeline.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/egon/data/airflow/dags/pipeline.py b/src/egon/data/airflow/dags/pipeline.py
index 206246f8e..340beae1c 100644
--- a/src/egon/data/airflow/dags/pipeline.py
+++ b/src/egon/data/airflow/dags/pipeline.py
@@ -10,6 +10,7 @@
 import egon.data.importing.vg250 as import_vg250
 import egon.data.processing.openstreetmap as process_osm
 import egon.data.importing.zensus as import_zs
+import egon.data.importing.etrago as etrago
 
 # Prepare connection to db for operators
 airflow_db_connection()
@@ -81,3 +82,10 @@
         python_callable=import_zs.population_to_postgres
     )
     setup >> zs_pop_download >> zs_pop_import
+    
+    # setting etrago input tables
+    etrago_input_data = PythonOperator(
+        task_id = "setting-etrago-input-tables",
+        python_callable = etrago.create_tables
+    )
+    setup >> etrago_input_data

From 1a5adb65e006c9728142e6d7e132763b52f851f3 Mon Sep 17 00:00:00 2001
From: ulfmueller <ulf.p.mueller@hs-flensburg.de>
Date: Wed, 3 Mar 2021 12:13:29 +0100
Subject: [PATCH 11/12] Optimize codacity quality

---
 src/egon/data/importing/etrago/__init__.py | 34 +++++++++++-----------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/src/egon/data/importing/etrago/__init__.py b/src/egon/data/importing/etrago/__init__.py
index 14fdeea21..3ee08aae4 100644
--- a/src/egon/data/importing/etrago/__init__.py
+++ b/src/egon/data/importing/etrago/__init__.py
@@ -1,7 +1,6 @@
 # coding: utf-8
-from sqlalchemy import ARRAY, BigInteger, Boolean, CHAR, Column, Date, DateTime, Float, ForeignKey, Integer, JSON, Numeric, SmallInteger, String, Table, Text, text
+from sqlalchemy import ARRAY, BigInteger, Boolean, Column, Date, DateTime, Float, ForeignKey, Integer, JSON, Numeric, SmallInteger, String, Table, Text, text
 from geoalchemy2.types import Geometry
-from sqlalchemy.orm import relationship
 from sqlalchemy.ext.declarative import declarative_base
 from egon.data import db
 
@@ -359,20 +358,21 @@ def create_tables():
         f"CREATE SCHEMA IF NOT EXISTS grid;")
     engine = db.engine()
     EgonPfHvBus.__table__.create(bind=engine, checkfirst=True)
-    EgonPfHvBusTimeseries.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
-    EgonPfHvGenerator.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
-    EgonPfHvGeneratorTimeseries.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
-    EgonPfHvLine.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
-    EgonPfHvLineTimeseries.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
-    EgonPfHvLink.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
-    EgonPfHvLinkTimeseries.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
-    EgonPfHvLoad.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
-    EgonPfHvLoadTimeseries.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
-    EgonPfHvCarrier.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
-    EgonPfHvStorage.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
-    EgonPfHvStorageTimeseries.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
-    EgonPfHvStore.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
-    EgonPfHvStoreTimeseries.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvBusTimeseries.__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvGenerator.__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvGeneratorTimeseries.__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvLine.__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvLineTimeseries.__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvLink.__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvLinkTimeseries.__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvLoad.__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvLoadTimeseries.__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvCarrier.__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvStorage.__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvStorageTimeseries.__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvStore.__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvStoreTimeseries.__table__.create(bind=engine, checkfirst=True)
     EgonPfHvTempResolution.__table__.create(bind=engine, checkfirst=True)
-    EgonPfHvTransformer.__table__.create(bind=engine, checkfirst=True).__table__.create(bind=engine, checkfirst=True)
+    EgonPfHvTransformer.__table__.create(bind=engine, checkfirst=True)
     EgonPfHvTransformerTimeseries.__table__.create(bind=engine, checkfirst=True)
+

From c9b52c672a048ba3fc358713e1537ebb38c8c248 Mon Sep 17 00:00:00 2001
From: ulfmueller <ulf.p.mueller@hs-flensburg.de>
Date: Wed, 3 Mar 2021 12:22:40 +0100
Subject: [PATCH 12/12] optimize codacity quality 2.0

---
 src/egon/data/importing/etrago/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/egon/data/importing/etrago/__init__.py b/src/egon/data/importing/etrago/__init__.py
index 3ee08aae4..583c67753 100644
--- a/src/egon/data/importing/etrago/__init__.py
+++ b/src/egon/data/importing/etrago/__init__.py
@@ -1,5 +1,5 @@
 # coding: utf-8
-from sqlalchemy import ARRAY, BigInteger, Boolean, Column, Date, DateTime, Float, ForeignKey, Integer, JSON, Numeric, SmallInteger, String, Table, Text, text
+from sqlalchemy import ARRAY, BigInteger, Boolean, Column, DateTime, Float, Integer, Numeric, String, Text
 from geoalchemy2.types import Geometry
 from sqlalchemy.ext.declarative import declarative_base
 from egon.data import db