Merge dev into features/#474

CHANGELOG.rst

@@ -137,6 +137,8 @@ Added
 * Extend zensus by a combined table with all cells where
   there's either building, apartment or population data
   `#359 <https://github.com/openego/eGon-data/issues/359>`_
+* Include allocation of pumped hydro units
+  `#332 <https://github.com/openego/eGon-data/issues/332>`_
 * Add example metadata for OSM, VG250 and Zensus VG250.
   Add metadata templates for licences, context and some helper
   functions. Extend docs on how to create metadata for tables.
@@ -305,4 +307,7 @@ Bug fixes
   `#511 <https://github.com/openego/eGon-data/issues/511>`_
 * Add `data_bundle` to `industrial_sites` task dependencies
   `#468 <https://github.com/openego/eGon-data/issues/468>`_
-
+* Lift `geopandas` minimum requirement to `0.10.0`
+  `#504 <https://github.com/openego/eGon-data/issues/504>`_
+* Use inbuilt `datetime` package instead of `pandas.datetime`
+  `#516 <https://github.com/openego/eGon-data/issues/516>`_

setup.py

@@ -86,7 +86,8 @@ def read(*names, **kwargs):
         "atlite==0.2.5",
         "cdsapi",
         "click",
-        "geopandas>=0.9.0",
+        "geopandas>=0.10.0",
+        "geopy",
         "importlib-resources",
         "loguru",
         "matplotlib",

src/egon/data/airflow/dags/pipeline.py

@@ -10,6 +10,7 @@
 from egon.data.datasets import database
 from egon.data.datasets.saltcavern import SaltcavernData
 from egon.data.datasets.calculate_dlr import Calculate_dlr
+from egon.data.datasets.ch4_storages import CH4Storages
 from egon.data.datasets.chp import Chp
 from egon.data.datasets.chp_etrago import ChpEtrago
 from egon.data.datasets.data_bundle import DataBundle
@@ -27,6 +28,7 @@
 from egon.data.datasets.gas_areas import GasAreas
 from egon.data.datasets.ch4_storages import CH4Storages
 from egon.data.datasets.gas_grid import GasNodesandPipes
+from egon.data.datasets.gas_prod import CH4Production
 from egon.data.datasets.heat_demand import HeatDemandImport
 from egon.data.datasets.heat_demand_timeseries.HTS import HeatTimeSeries
 from egon.data.datasets.heat_etrago import HeatEtrago
@@ -54,6 +56,7 @@
 from egon.data.datasets.scenario_capacities import ScenarioCapacities
 from egon.data.datasets.scenario_parameters import ScenarioParameters
 from egon.data.datasets.society_prognosis import SocietyPrognosis
+from egon.data.datasets.storages import PumpedHydro
 from egon.data.datasets.vg250 import Vg250
 from egon.data.datasets.vg250_mv_grid_districts import Vg250MvGridDistricts
 from egon.data.datasets.zensus_mv_grid_districts import ZensusMvGridDistricts
@@ -494,11 +497,22 @@
     chp_etrago = ChpEtrago(dependencies=[chp, heat_etrago])
 
     # DSM
-    components_dsm = dsm_Potential(
+    components_dsm =  dsm_Potential(
+        dependencies = [cts_electricity_demand_annual,
+                        demand_curves_industry,
+                        osmtgmod_pypsa])
+
+    # Pumped hydro units
+
+    pumped_hydro = PumpedHydro(
         dependencies=[
-            cts_electricity_demand_annual,
-            demand_curves_industry,
-            osmtgmod_pypsa,
+            setup,
+            mv_grid_districts,
+            mastr_data,
+            scenario_parameters,
+            scenario_capacities,
+            Vg250MvGridDistricts,
+            power_plants,
         ]
     )
 

src/egon/data/datasets.yml

@@ -327,6 +327,7 @@ power_plants:
       mastr_hydro: "bnetza_mastr_hydro_cleaned.csv"
       mastr_location: "location_elec_generation_raw.csv"
       mastr_combustion_without_chp: "supply.egon_mastr_conventional_without_chp"
+      mastr_storage: "bnetza_mastr_storage_cleaned.csv"
       capacities: "supply.egon_scenario_capacities"
       geom_germany: "boundaries.vg250_sta_union"
       geom_federal_states: "boundaries.vg250_lan"
@@ -340,6 +341,19 @@ power_plants:
       table: 'egon_power_plants'
       schema: 'supply'
 
+storages:
+  sources:
+      mastr_storage: "bnetza_mastr_storage_cleaned.csv"
+      capacities: "supply.egon_scenario_capacities"
+      geom_germany: "boundaries.vg250_sta_union"
+      geom_federal_states: "boundaries.vg250_lan"
+      egon_mv_grid_district: "grid.egon_mv_grid_district"
+      ehv_voronoi: "grid.egon_ehv_substation_voronoi"
+      nep_conv: "supply.egon_nep_2021_conventional_powerplants"
+  target:
+      table: 'egon_storages'
+      schema: 'supply'
+
 landuse:
   sources:
     osm_polygons:

src/egon/data/datasets/heat_demand_timeseries/HTS.py

@@ -1,36 +1,35 @@
-import pandas as pd
-import numpy as np
-import psycopg2
-import pandas.io.sql as sqlio
+from datetime import datetime
+import glob
+import os
+
+from sqlalchemy import ARRAY, Column, Float, ForeignKey, Integer, String
+from sqlalchemy.ext.declarative import declarative_base
 import geopandas as gpd
 import matplotlib.pyplot as plt
-import os
-import glob
+import numpy as np
+import pandas as pd
+import pandas.io.sql as sqlio
+import psycopg2
+import xarray as xr
 
 from egon.data import db, subprocess
-
-import xarray as xr
-from sqlalchemy import Column, String, Float, Integer, ForeignKey, ARRAY
 import egon.data.datasets.era5 as era
 
-from sqlalchemy.ext.declarative import declarative_base
-
 try:
-    from disaggregator import config, data, spatial, temporal, plot
+    from disaggregator import config, data, plot, spatial, temporal
 except ImportError as e:
     pass
 
 from math import ceil
 
-
 from egon.data.datasets import Dataset
 import egon
 
 
 class IdpProfiles:
     def __init__(self, df_index, **kwargs):
         index = pd.date_range(
-            pd.datetime(2011, 1, 1, 0), periods=8760, freq="H"
+            datetime(2011, 1, 1, 0), periods=8760, freq="H"
         )
 
         self.df = pd.DataFrame(index=df_index)
@@ -39,7 +38,7 @@ def __init__(self, df_index, **kwargs):
 
     def get_temperature_interval(self, how="geometric_series"):
         index = pd.date_range(
-            pd.datetime(2011, 1, 1, 0), periods=8760, freq="H"
+            datetime(2011, 1, 1, 0), periods=8760, freq="H"
         )
         """Appoints the corresponding temperature interval to each temperature
         in the temperature vector.
@@ -202,7 +201,7 @@ def temp_interval():
         Hourly temperature intrerval of all 15 TRY Climate station#s temperature profile
 
     """
-    index = pd.date_range(pd.datetime(2011, 1, 1, 0), periods=8760, freq="H")
+    index = pd.date_range(datetime(2011, 1, 1, 0), periods=8760, freq="H")
     temperature_interval = pd.DataFrame()
     temp_profile = temperature_profile_extract()
 
@@ -235,7 +234,7 @@ def idp_pool_generator():
         "household_heat_demand_profiles",
         "household_heat_demand_profiles.hdf5",
     )
-    index = pd.date_range(pd.datetime(2011, 1, 1, 0), periods=8760, freq="H")
+    index = pd.date_range(datetime(2011, 1, 1, 0), periods=8760, freq="H")
 
     sfh = pd.read_hdf(path, key="SFH")
     mfh = pd.read_hdf(path, key="MFH")
@@ -920,7 +919,7 @@ def h_value():
         Extracted from demandlib.
 
     """
-    index = pd.date_range(pd.datetime(2011, 1, 1, 0), periods=8760, freq="H")
+    index = pd.date_range(datetime(2011, 1, 1, 0), periods=8760, freq="H")
 
     a = 3.0469695
 
@@ -1696,7 +1695,7 @@ class HeatTimeSeries(Dataset):
     def __init__(self, dependencies):
         super().__init__(
             name="HeatTimeSeries",
-            version="0.0.0",
+            version="0.0.1",
             dependencies=dependencies,
             tasks=(demand_profile_generator),
         )

src/egon/data/datasets/scenario_capacities.py

@@ -60,7 +60,7 @@ class ScenarioCapacities(Dataset):
     def __init__(self, dependencies):
         super().__init__(
             name="ScenarioCapacities",
-            version="0.0.4",
+            version="0.0.5",
             dependencies=dependencies,
             tasks=(create_table, insert_data_nep),
         )