Merge branch 'dev' into features/#265-limit-number-of-processes-per-task

CHANGELOG.rst

@@ -94,6 +94,8 @@ Added
   `#109 <https://github.com/openego/eGon-data/issues/109>`_
 * Assign voltage level and bus_id to power plants
   `#15 <https://github.com/openego/eGon-data/issues/15>`_
+* Integrate solar rooftop for etrago tables
+  `#255 <https://github.com/openego/eGon-data/issues/255>`_
 
 .. _PR #159: https://github.com/openego/eGon-data/pull/159
 
@@ -149,8 +151,9 @@ Bug fixes
   `#204 <https://github.com/openego/eGon-data/issues/204>`_
 * Replace wrong table name in SQL function used in substation extraction
   `#236 <https://github.com/openego/eGon-data/issues/236>`_
-* Fix osmtgmod for osm data from 2021 by updating substation in Garenfeld 
+* Fix osmtgmod for osm data from 2021 by updating substation in Garenfeld and set srid
   `#241 <https://github.com/openego/eGon-data/issues/241>`_
+  `#258 <https://github.com/openego/eGon-data/issues/258>`_
 * Adjust format of voltage levels in hvmv substation
   `#248 <https://github.com/openego/eGon-data/issues/248>`_
 * Change order of osmtgmod tasks

src/egon/data/airflow/dags/pipeline.py

@@ -5,7 +5,8 @@
 from airflow.utils.dates import days_ago
 import importlib_resources as resources
 
-from egon.data.airflow.tasks import initdb
+from egon.data.datasets import database
+from egon.data.datasets.osm import OpenStreetMap
 from egon.data.processing.zensus_vg250 import (
     zensus_population_inside_germany as zensus_vg250,
 )
@@ -18,15 +19,14 @@
 import egon.data.importing.industrial_sites as industrial_sites
 import egon.data.importing.mastr as mastr
 import egon.data.importing.nep_input_data as nep_input
-import egon.data.importing.openstreetmap as import_osm
 import egon.data.importing.re_potential_areas as re_potential_areas
 import egon.data.importing.scenarios as import_scenarios
 import egon.data.importing.vg250 as import_vg250
 import egon.data.importing.zensus as import_zs
+import egon.data.processing.boundaries_grid_districts as boundaries_grid_districts
 import egon.data.processing.demandregio as process_dr
 import egon.data.processing.district_heating_areas as district_heating_areas
 import egon.data.processing.loadarea as loadarea
-import egon.data.processing.openstreetmap as process_osm
 import egon.data.processing.osmtgmod as osmtgmod
 import egon.data.processing.power_plants as power_plants
 import egon.data.processing.renewable_feedin as import_feedin
@@ -56,26 +56,17 @@
     is_paused_upon_creation=False,
     schedule_interval=None,
 ) as pipeline:
-    setup = PythonOperator(task_id="initdb", python_callable=initdb)
 
-    # Openstreetmap data import
-    osm_download = PythonOperator(
-        task_id="download-osm",
-        python_callable=import_osm.download_pbf_file,
-    )
-    osm_import = PythonOperator(
-        task_id="import-osm",
-        python_callable=import_osm.to_postgres,
-    )
-    osm_migrate = PythonOperator(
-        task_id="migrate-osm",
-        python_callable=process_osm.modify_tables,
-    )
-    osm_add_metadata = PythonOperator(
-        task_id="add-osm-metadata",
-        python_callable=import_osm.add_metadata,
-    )
-    setup >> osm_download >> osm_import >> osm_migrate >> osm_add_metadata
+    tasks = pipeline.task_dict
+
+    database_setup = database.Setup()
+    database_setup.insert_into(pipeline)
+    setup = tasks["database.setup"]
+
+    osm = OpenStreetMap(dependencies=[setup])
+    osm.insert_into(pipeline)
+    osm_add_metadata = tasks["osm.add-metadata"]
+    osm_download = tasks["osm.download"]
 
     # VG250 (Verwaltungsgebiete 250) data import
     vg250_download = PythonOperator(
@@ -535,3 +526,24 @@
     demandregio_demand_cts_ind >> electrical_load_curves_cts
     map_zensus_vg250 >> electrical_load_curves_cts
     etrago_input_data >> electrical_load_curves_cts
+
+    # Map federal states to mv_grid_districts
+    map_boundaries_grid_districts = PythonOperator(
+        task_id="map_vg250_grid_districts",
+        python_callable=boundaries_grid_districts.map_mvgriddistricts_vg250,
+    )
+    define_mv_grid_districts >> map_boundaries_grid_districts
+    vg250_clean_and_prepare >> map_boundaries_grid_districts
+
+    # Solar rooftop per mv grid district
+    solar_rooftop_etrago = PythonOperator(
+        task_id="etrago_solar_rooftop",
+        python_callable=power_plants.pv_rooftop_per_mv_grid,
+    )
+    map_boundaries_grid_districts >> solar_rooftop_etrago
+    feedin_pv >> solar_rooftop_etrago
+    elec_cts_demands_zensus >> solar_rooftop_etrago
+    elec_household_demands_zensus >> solar_rooftop_etrago
+    nep_insert_data >> solar_rooftop_etrago
+    etrago_input_data >> solar_rooftop_etrago
+    map_zensus_grid_districts >> solar_rooftop_etrago

src/egon/data/cli.py

@@ -342,6 +342,12 @@ def render(template, target, update=True, inserts={}, **more_inserts):
     airflow.add(connection)
     airflow.commit()
 
+    # TODO: This should probably rather be done during the database
+    #       initialization workflow task.
+    from egon.data.datasets import setup
+
+    setup()
+
 
 @egon_data.command(
     add_help_option=False,

src/egon/data/config.py

@@ -53,6 +53,8 @@ def settings() -> dict[str, dict[str, str]]:
     """
     files = paths(pid="*") + paths()
     if not files[0].exists():
+        # TODO: Fatal errors should be raised as exceptions, so one can figure
+        #       out where they are coming from without having to debug.
         logger.error(
             f"Unable to determine settings.\nConfiguration file:"
             f"\n\n{files[0]}\n\nnot found.\nExiting."

src/egon/data/datasets.yml

@@ -435,3 +435,49 @@ electrical_load_curves_cts:
       schema: 'grid'
       table: 'egon_pf_hv_load_timeseries'
 
+map_mvgrid_vg250:
+  sources:
+    mv_grid_districts:
+      schema: 'grid'
+      table: 'mv_grid_districts'
+    federal_states:
+      schema: 'boundaries'
+      table: 'vg250_lan_union'
+  targets:
+    map:
+      schema: 'boundaries'
+      table: 'egon_map_mvgriddistrict_vg250'
+
+solar_rooftop:
+  sources:
+    solar_feedin:
+      schema: 'supply'
+      table: 'egon_era5_renewable_feedin'
+    mv_grid_districts:
+      schema: 'grid'
+      table: 'mv_grid_districts'
+    weather_cells:
+      schema: 'supply'
+      table: 'egon_era5_weather_cells'
+    electricity_demand:
+      schema: 'demand'
+      table: 'egon_demandregio_zensus_electricity'
+    map_zensus_grid_districts:
+      schema: 'boundaries'
+      table: 'egon_map_zensus_grid_districts'
+    map_grid_boundaries:
+      schema: 'boundaries'
+      table: 'egon_map_mvgriddistrict_vg250'
+    scenario_capacities:
+      schema: 'supply'
+      table: 'egon_scenario_capacities'
+    federal_states:
+      schema: 'boundaries'
+      table: 'vg250_lan'
+  targets:
+    generators:
+      schema: 'grid'
+      table: 'egon_pf_hv_generator'
+    generator_timeseries:
+      schema: 'grid'
+      table: 'egon_pf_hv_generator_timeseries'