diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4b9ad36b5..8bb9d2738 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: rev: v2.2.5 hooks: - id: codespell - args: ['--ignore-regex="(\b[A-Z]+\b)"', '--ignore-words-list=fom,appartment,bage,ore,setis,tabacco,berfore'] # Ignore capital case words, e.g. country codes + args: ['--ignore-regex="(\b[A-Z]+\b)"', '--ignore-words-list=fom,appartment,bage,ore,setis,tabacco,berfore,fo,FO'] types_or: [python, rst, markdown] files: ^(scripts|doc)/ diff --git a/README.md b/README.md index c4d9284b8..3eab8b0c8 100644 --- a/README.md +++ b/README.md @@ -376,6 +376,13 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe + + + kma33 +
+ Katherine M. Antonio + + pitmonticone diff --git a/Snakefile b/Snakefile index ae4bc6c8d..ea3b690f0 100644 --- a/Snakefile +++ b/Snakefile @@ -318,7 +318,27 @@ rule build_bus_regions: "scripts/build_bus_regions.py" +def terminate_if_cutout_exists(config=config): + """ + Check if any of the requested cutout files exist. + If that's the case, terminate execution to avoid data loss. + """ + config_cutouts = [ + d_value["cutout"] for tc, d_value in config["renewable"].items() + ] + list(config["atlite"]["cutouts"].keys()) + + for ct in set(config_cutouts): + cutout_fl = "cutouts/" + CDIR + ct + ".nc" + if os.path.exists(cutout_fl): + raise Exception( + "An option `build_cutout` is enabled, while a cutout file '" + + cutout_fl + + "' still exists and risks to be overwritten. If this is an intended behavior, please move or delete this file and re-run the rule. Otherwise, just disable the `build_cutout` rule in the config file." + ) + + if config["enable"].get("build_cutout", False): + terminate_if_cutout_exists(config) rule build_cutout: params: diff --git a/config.default.yaml b/config.default.yaml index 40a00662d..c2d62bafc 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -293,6 +293,7 @@ renewable: carriers: [ror, PHS, hydro] PHS_max_hours: 6 hydro_max_hours: "energy_capacity_totals_by_country" # not active + hydro_max_hours_default: 6.0 # (optional, default 6) Default value of max_hours for hydro when NaN values are found clip_min_inflow: 1.0 extendable: true normalization: diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 2115c00df..b31691be9 100644 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -291,6 +291,7 @@ renewable: carriers: [ror, PHS, hydro] PHS_max_hours: 6 hydro_max_hours: "energy_capacity_totals_by_country" # one of energy_capacity_totals_by_country, estimate_by_large_installations or a float + hydro_max_hours_default: 6.0 # (optional, default 6) Default value of max_hours for hydro when NaN values are found clip_min_inflow: 1.0 normalization: method: hydro_capacities # 'hydro_capacities' to rescale country hydro production by using hydro_capacities, 'eia' to rescale by eia data, false for no rescaling diff --git a/configs/bundle_config.yaml b/configs/bundle_config.yaml index 978824b80..2a5ebb30d 100644 --- a/configs/bundle_config.yaml +++ b/configs/bundle_config.yaml @@ -147,6 +147,14 @@ databundles: - data/gebco/GEBCO_2021_TID.nc - data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif - data/ssp2-2.6/2030/era5_2013/Africa.nc + - data/ssp2-2.6/2030/era5_2013/Asia.nc + - data/ssp2-2.6/2030/era5_2013/Europe.nc + - data/ssp2-2.6/2030/era5_2013/NorthAmerica.nc + - data/ssp2-2.6/2030/era5_2013/SouthAmerica.nc + - data/ssp2-2.6/2030/era5_2013/Oceania.nc + - data/hydrobasins/hybas_world_lev04_v1c.shp + - data/hydrobasins/hybas_world_lev05_v1c.shp + - data/hydrobasins/hybas_world_lev06_v1c.shp # resources bundle containing the resources folder for Africa only bundle_natura_earth: @@ -336,3 +344,15 @@ databundles: # output: [cutouts/cutout-2013-era5.nc] # disable_by_opt: # build_cutout: [all] + + # Cutout for South America, approx 18 GB + bundle_cutouts_southamerica: + countries: [SouthAmerica] + category: cutouts + destination: "cutouts" + urls: + # zenodo: + gdrive: https://drive.google.com/file/d/1Jeu2Vzoq4mNDUKSvIviN8HqSFx5gl61b/view?usp=sharing + output: [cutouts/cutout-2013-era5.nc] + disable_by_opt: + build_cutout: [all] diff --git a/configs/osm_config.yaml b/configs/osm_config.yaml new file mode 100644 index 000000000..d4d70f33b --- /dev/null +++ b/configs/osm_config.yaml @@ -0,0 +1,367 @@ +# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC0-1.0 + + +# Columns of CLEAN OSM DATA names and dtypes +osm_clean_columns: + substation: + bus_id: object + station_id: float + voltage: float + dc: bool + symbol: object + under_construction: bool + tag_substation: str + tag_area: str + lon: float + lat: float + country: str + geometry: object + generator: + id: object + tags.power: object + Type: object + Country: str + Area: object + name: object + tags.generator:type: object + tags.generator:method: object + tags.generator:source: object + power_output_MW: object + geometry: object + line: + line_id: object + bus0: object + bus1: object + voltage: float + circuits: float + length: float + underground: bool + under_construction: bool + tag_type: str + tag_frequency: float + dc: bool + country: object + geometry: object +# Python dictionary of ISO 3166-1-alpha-2 codes, as per publicly +# available data on official ISO site in July 2015. +# +# Available under MIT license +# Dimitris Karagkasidis, https://github.com/pageflt +continents: + LA: NorthAmerica + SA: SouthAmerica + AS: Asia + OC: Oceania + AF: Africa + EU: Europe + +world_iso: + Africa: + DZ: "algeria" # Algeria + AO: "angola" + BJ: "benin" + BW: "botswana" + BF: "burkina-faso" + BI: "burundi" + CM: "cameroon" + CV: "cape-verde" + CF: "central-african-republic" + TD: "chad" + KM: "comoros" + CG: "congo-brazzaville" + CD: "congo-democratic-republic" + DJ: "djibouti" + EG: "egypt" + GQ: "equatorial-guinea" + ER: "eritrea" + ET: "ethiopia" + GA: "gabon" + GH: "ghana" + GW: "guinea-bissau" # No Data + GN: "guinea" + CI: "ivory-coast" + KE: "kenya" + LS: "lesotho" + LR: "liberia" + LY: "libya" + MG: "madagascar" + MW: "malawi" + ML: "mali" + MR: "mauritania" + MU: "mauritius" + MZ: "mozambique" + NA: "namibia" + NE: "niger" + NG: "nigeria" + RW: "rwanda" + ST: "sao-tome-and-principe" + SN: "senegal" + GM: "gambia" + SC: "seychelles" + SO: "somalia" + ZA: "south-africa" + SS: "south-sudan" + SD: "sudan" + SZ: "swaziland" + TZ: "tanzania" + TG: "togo" + TN: "tunisia" + UG: "uganda" + ZM: "zambia" + ZW: "zimbabwe" + EH: "western-sahara" + Asia: + AF: "afghanistan" + AM: "armenia" + AZ: "azerbaijan" + BH: "bahrain" + BD: "bangladesh" + BT: "bhutan" + KH: "cambodia" + CN: "china" + CY: "cyprus" + GE: "georgia" + IN: "india" + ID: "indonesia" + IR: "iran" + IQ: "iraq" + IL: "israel" + JP: "japan" + JO: "jordan" + KZ: "kazakhstan" + KP: "north-korea" + KR: "south-korea" + KW: "kuwait" + KG: "kyrgyzstan" + LA: "lao-peoples-democratic-republic" + LB: "lebanon" + MY: "malaysia" + BN: "brunei" + MV: "maldives" + MN: "mongolia" + MM: "myanmar" + NP: "nepal" + OM: "oman" + PK: "pakistan" + PS: "palestine" + PH: "philippines" + QA: "qatar" + RU: "russian-federation" + SA: "saudi-arabia" + SG: "singapore" # merged with MY + LK: "sri-lanka" + SY: "syria" + TW: "taiwan" + TJ: "tajikistan" + TH: "thailand" + TL: "timor-leste" + TR: "turkey" + TM: "turkmenistan" + AE: "united-arab-emirates" + UZ: "uzbekistan" + VN: "vietnam" + YE: "yemen" + Oceania: + AS: "american-samoa" # Island + AU: "australia" + FJ: "fiji" + KI: "kiribati" # Island + MH: "marshall-islands" + FM: "micronesia" + NR: "nauru" + NC: "new-caledonia" # Island + NZ: "new-zealand" + NU: "niue" # Island + PG: "papua-new-guinea" + PW: "palau" + WS: "samoa" + SB: "solomon-islands" + TO: "tonga" + TV: "tuvalu" + VU: "vanuatu" # Island + Europe: + AL: "albania" + AD: "andorra" + AM: "armenia" + AT: "austria" + AZ: "Azerbaijan" + BY: "belarus" + BE: "belgium" + BA: "bosnia-herzegovina" + BG: "bulgaria" + HR: "croatia" + CY: "cyprus" + CZ: "czech-republic" + DK: "denmark" + EE: "estonia" + FI: "finland" + FR: "france" + GE: "georgia" + DE: "germany" + GR: "greece" + HU: "hungary" + IS: "iceland" + IE: "ireland-and-northern-ireland" + IT: "italy" + KZ: "kazakhstan" + XK: "kosovo" + LV: "latvia" + LI: "liechtenstein" + LT: "lithuania" + LU: "luxembourg" + MK: "macedonia" + MT: "malta" + MD: "moldova" + MC: "monaco" + ME: "montenegro" + NL: "netherlands" + NO: "norway" + PL: "poland" + PT: "portugal" + RO: "romania" + RU: "russia" + SM: "san-marino" + RS: "serbia" + SK: "slovakia" + SI: "slovenia" + ES: "spain" + SE: "sweden" + CH: "switzerland" + UA: "ukraine" + GB: "great-britain" + TR: "turkey" + VA: "vatican" + NorthAmerica: + AG: "antigua-and-barbuda" + BS: "bahamas" + BB: "barbados" + CA: "canada" + CU: "cuba" + DM: "dominica" + DO: "dominican-republic" + GL: "greenland" + GD: "grenada" + HT: "haiti" + JM: "jamaica" + MX: "mexico" + US: "united-states-of-america" + PR: "puerto-rico" + KN: "saint-kitts-and-nevis" + LC: "saint-lucia" + VC: "saint-vincent-and-the-grenadines" + TT: "trinidad-and-tobago" + BZ: "belize" + CR: "costa-rica" + HN: "honduras" + GT: "guatemala" + NI: "nicaragua" + PA: "panama" + SV: "el-salvador" + SouthAmerica: + AR: "argentina" + BO: "bolivia" + BR: "brazil" + CL: "chile" + CO: "colombia" + EC: "ecuador" + FK: "falkland-islands" # Islands + GF: "french-guiana" + GY: "guyana" # No Data + PE: "peru" + PY: "paraguay" + SR: "suriname" + UY: "uruguay" + VE: "venezuela" + + + +# Based on: https://waml.org/waml-information-bulletin/46-3/index-to-lc-g-schedule/1-world/ +# Australasia region includes New Caledonia and Papua New Guinea +continent_regions: + SCR: ["DK", "NO", "SE", "FI", "IS"] # SCANDINAVIAN REGION + EER: ["BY", "PL", "CZ", "RU", "SK", "UA", "LT", "LV", "EE", "FI", "MD"] # EASTERN EUROPEAN REGION + CER: ["AT", "CH", "CZ", "DE", "HU", "PL", "SK", "LI"] # CENTRAL EUROPEAN REGION + BPR: ["AL", "BA", "BG", "GR", "HR", "ME", "RO", "SI", "RS", "ME", "MK"] # BALKAN PENISULAN REGION + WER: ["FR", "BE", "GB", "IE", "LU", "MC", "NL", "AD"] # WESTERN EUROPE + SER: ["ES", "AD", "IT", "PT", "SM", "MT"] # SOUTHERN EUROPEAN REGION + NAR: ["EG", "LY", "TN", "DZ", "MA", "EH", "SD", "SS"] # NORTHERN AFRICAN REGION + WAR: ["MR", "ML", "NE", "NG", "BJ", "BF", "TG", "GH", "CI", "LR", "SL", "GN", "SN", "GM"] # WESTERN AFRICAN REGION + CAR: ["TD", "CF", "CM", "GQ", "GA", "CD", "CG", "AO"] # CENTRAL AFRICAN REGION + EAR: ["ER", "ET", "UG", "KE", "RW", "BI", "TZ", "MZ", "DJ", "MG"] # EASTERN AFRICAN REGION + SAR: ["MW", "ZM", "ZW", "BW", "NA", "SZ", "LS", "ZA"] # SOUTHERN AFRICAN REGION + KVR: ["AZ", "GE", "AM"] # Asian regions + WAS: ["TR", "AM", "AZ", "BH", "CY", "GE", "IQ", "IL", "JO", "KW", "LB", "OM", "PS", "QA", "SA", "SY", "AE", "YE"] # WEST ASIAN REGION + FEAR: ["JP", "KP", "KR", "CN", "TW", "MN"] # FAR EASTERN ASIAN REGION + SEAR: ["LA", "TH", "KH", "VN", "PH", "MY", "SG", "BN", "ID"] # SOUTHEASTERN ASIAN REGION + CASR: ["KZ", "KG", "UZ", "TM", "TJ"] # CENTRAL ASIAN REGION + SASR: ["MM", "BD", "BT", "NP", "IN", "LK", "PK", "AF"] # SOUTHERN ASIAN REGION + ASEAN: ["VN", "TH", "ID", "PH", "MY", "MM", "KH", "LA", "SG", "BN"] # ASEAN + MEAR: ["TR", "SY", "LB", "CY", "IQ", "IR", "JO", "IL", "PS", "AE", "YE", "KW", "BH", "QA", "SA", "OM"] # MIDDLE EASTERN ASIAN REGION + NACR: ["CA", "GL", "MX", "US"] # American continent regions + LACR: ["AR", "BO", "BR", "CL", "CO", "EC", "GF", "PE", "PY", "SR", "UY", "VE"] # SOUTHERN LATIN AMERICAN REGION + CACR: ["BZ", "GT", "SV", "HN", "NI", "CR", "PA"] # CENTRAL AMERICAN REGION + AUO: ["AU", "NC", "NZ", "PG"] # Australasia + UnitedNations: ['AF', 'AL', 'DZ', 'AD', 'AO', 'AG', 'AR', 'AM', 'AU', 'AT', 'AZ', 'BS', 'BH', 'BD', 'BB', 'BY', 'BE', 'BZ', 'BJ', 'BT', 'BO', 'BA', 'BW', 'BR', 'BN', 'BG', 'BF', 'BI', 'CV', 'KH', 'CM', 'CA', 'CF', 'TD', 'CL', 'CN', 'CO', 'KM', 'CG', 'CR', 'CI', 'HR', 'CU', 'CY', 'CZ', 'DK', 'DJ', 'DM', 'DO', 'CD', 'EC', 'EG', 'SV', 'GQ', 'ER', 'EE', 'SZ', 'ET', 'FJ', 'FI', 'FR', 'GA', 'GM', 'GE', 'DE', 'GH', 'GR', 'GD', 'GT', 'GN', 'GW', 'GY', 'HT', 'HN', 'HU', 'IS', 'IN', 'ID', 'IR', 'IQ', 'IE', 'IL', 'IT', 'JM', 'JP', 'JO', 'KZ', 'KE', 'KI', 'KW', 'KG', 'LA', 'LV', 'LB', 'LS', 'LR', 'LY', 'LI', 'LT', 'LU', 'MK', 'MG', 'MW', 'MY', 'MV', 'ML', 'MT', 'MH', 'MR', 'MU', 'MX', 'FM', 'MD', 'MC', 'MN', 'ME', 'MA', 'MZ', 'MM', 'NA', 'NR', 'NP', 'NL', 'NZ', 'NI', 'NE', 'NG', 'KP', 'NO', 'OM', 'PK', 'PW', 'PA', 'PG', 'PY', 'PE', 'PH', 'PL', 'PT', 'QA', 'RO', 'RU', 'RW', 'WS', 'SM', 'ST', 'SA', 'SN', 'RS', 'SC', 'SL', 'SG', 'SK', 'SI', 'SB', 'SO', 'ZA', 'KR', 'SS', 'ES', 'LK', 'KN', 'LC', 'VC', 'SD', 'SR', 'SE', 'CH', 'SY', 'TJ', 'TZ', 'TH', 'TL', 'TG', 'TO', 'TT', 'TN', 'TR', 'TM', 'TV', 'UG', 'UA', 'AE', 'GB', 'US', 'UY', 'UZ', 'VU', 'VE', 'VN', 'YE', 'ZM', 'ZW'] # UnitedNations + +# Geofabrik and iso norm deviate for some countries and domains + +# dictionary of correspondence between iso country codes and geofabrik codes containing those information +# This dictionary instructs the script download_osm_data about how to successfully download data +# from countries that are aggregated into osm. +# For example, Senegal (SN) and Gambia (GM) cannot be downloaded from OSM separately, but only jointly as SN-GM +# That's the reason why in this dictionary they can be found the following entries: +# "SN": "SN-GM" +# "GM": "SN-GM" +# This instruct the workflow that when the country "SN" is requested, then it shall download the "SN-GM" file +iso_to_geofk_dict: + EH: "MA" # Western Sahara -> Morocco + SN: "SN-GM" # Senegal -> Senegal-Gambia + GM: "SN-GM" # Gambia -> Senegal-Gambia + KM: "comores" # Comores + IC: "canary-islands" # Canary islands + SG: "MY" # Singapore -> Malaysia-Singapore-Brunei + BN: "MY" # Brunei -> Malaysia-Singapore-Brunei + SA: "QA-AE-OM-BH-KW" # Saudi Arabia -> Gulf Cooperation Council + KW: "QA-AE-OM-BH-KW" # Kuwait -> Gulf Cooperation Council + BH: "QA-AE-OM-BH-KW" # Bahrain -> Gulf Cooperation Council + QA: "QA-AE-OM-BH-KW" # Qatar -> Gulf Cooperation Council + AE: "QA-AE-OM-BH-KW" # United Arab Emirates -> Gulf Cooperation Council + OM: "QA-AE-OM-BH-KW" # Oman -> Gulf Cooperation Council + PS: "PS-IL" # Israel and Palestine are merged in OSM + IL: "PS-IL" # Israel and Palestine are merged in OSM + SM: "IT" # San-Marino is merged to Italy + VA: "IT" # Vatican is merged to Italy + HT: "haiti-and-domrep" # Haiti and Dominican Republic are merged in OSM + DO: "haiti-and-domrep" # Haiti and Dominican Republic are merged in OSM + PA: "panama" # Panama + NF: "AU" # norfolk island is an AU territory + MP: "american-oceania" # northern mariana islands are US territory + GU: "american-oceania" # Guam is a US territory + AS: "american-oceania" # American Samoa is a US territory + CP: "ile-de-clipperton" # Ile de clipperton + PF: "polynesie-francaise" # Polynesie Francaise + VU: "vanuatu" # Vanuatu + TK: "tokelau" # Tokelau + MH: "marshall-islands" # Marshal islands + PN: "pitcairn-islands" # Pitcairn + WF: "wallis-et-futuna" # Wallis et Fortnuna + XK: "RS-KM" # Kosovo + BS: "bahamas" # Bahamas + BB: "central-america" # Barbados + CU: "cuba" # Cuba + RE: "reunion" # Reunion island (France) + YT: "mayotte" # "Mayotte island (France)" + GG: "guernsey-jersey" # Guernsey + JE: "guernsey-jersey" # Jersey + IM: "isle-of-man" # Isle of man + GP: "guadeloupe" # guadeloupe + JM: "jamaica" # jamaica + TT: "central-america" # Trinidad and Tobago + AG: "central-america" # Antigua e Barbuda + DM: "central-america" # Dominique + LC: "central-america" # Santa Lucia + VC: "central-america" # Saint Vincent e Grenadine + KN: "central-america" # Saint Kitts e Nevis + GD: "central-america" # Grenada diff --git a/doc/configtables/hydro.csv b/doc/configtables/hydro.csv index 7f10a4b5f..b4a563922 100644 --- a/doc/configtables/hydro.csv +++ b/doc/configtables/hydro.csv @@ -7,6 +7,7 @@ resource,,, carriers,--,"Any subset of {'ror', 'PHS', 'hydro'}","Specifies the types of hydro power plants to build per-unit availability time series for. 'ror' stands for run-of-river plants, 'PHS' represents pumped-hydro storage, and 'hydro' stands for hydroelectric dams." PHS_max_hours,h,float,"Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_." hydro_max_hours,h,"Any of {float, 'energy_capacity_totals_by_country', 'estimate_by_large_installations'}","Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom`` or heuristically determined. Cf. `PyPSA documentation `_." +hydro_max_hours_default,h,float,"(optional, default 6) Default value of max_hours for hydro plants with missing values" clip_min_inflow,MW,float,"To avoid too small values in the inflow time series, values below this threshold are set to zero." extendable, bool, "{True, False}", "True: In nodes where there is no hydro generation, adds a zero-capacity hydro generator so that hydro is considered for capacity expansion. It is done in the ``add_electricity`` rule." normalization,--,dict,"When specified, it describes how to normalize hydro time series to adhere to national statistics" diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 840bcf283..cb365a342 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -34,6 +34,11 @@ E.g. if a new rule becomes available describe how to use it `snakemake -j1 run_t * Add Asian cutout `PR #826 `__ +* Add osm_config yaml file `PR #822 `__ + +* Re-enable offshore wind and revise hydro `PR #830 `__ + + PyPSA-Earth 0.2.2 ================= diff --git a/envs/environment.yaml b/envs/environment.yaml index ea231f2ae..6730fa74c 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -43,6 +43,7 @@ dependencies: - matplotlib<=3.5.2 - reverse-geocode - country_converter +- py7zr # Keep in conda environment when calling ipython - ipython diff --git a/scripts/_helpers.py b/scripts/_helpers.py index aa449f9c4..6ba0bac1e 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -10,8 +10,8 @@ import country_converter as coco import geopandas as gpd -import numpy as np import pandas as pd +import yaml # list of recognised nan values (NA and na excluded as may be confused with Namibia 2-letter country code) NA_VALUES = ["NULL", "", "N/A", "NAN", "NaN", "nan", "Nan", "n/a", "null"] @@ -19,6 +19,51 @@ REGION_COLS = ["geometry", "name", "x", "y", "country"] +def read_osm_config(*args): + """ + Read values from the osm_config.yaml file based on provided key arguments. + + Parameters + ---------- + *args : str + One or more key arguments corresponding to the values to retrieve + from the config file. Typical arguments include "world_iso", + "continent_regions", "iso_to_geofk_dict", and "osm_clean_columns". + + Returns + ------- + tuple or str or dict + If a single key is provided, returns the corresponding value from the + osm_config.yaml file. If multiple keys are provided, returns a tuple + containing values corresponding to the provided keys. + + Examples + -------- + >>> values = read_osm_config("key1", "key2") + >>> print(values) + ('value1', 'value2') + + >>> world_iso = read_osm_config("world_iso") + >>> print(world_iso) + {"Africa": {"DZ": "algeria", ...}, ...} + """ + if "__file__" in globals(): + base_folder = os.path.dirname(__file__) + if not os.path.exists(os.path.join(base_folder, "configs")): + base_folder = os.path.dirname(base_folder) + else: + base_folder = os.getcwd() + osm_config_path = os.path.join(base_folder, "configs", "osm_config.yaml") + with open(osm_config_path, "r") as f: + osm_config = yaml.safe_load(f) + if len(args) == 0: + return osm_config + elif len(args) == 1: + return osm_config[args[0]] + else: + return tuple([osm_config[a] for a in args]) + + def sets_path_to_root(root_directory_name): """ Search and sets path to the given root directory (root/path/file). @@ -454,11 +499,11 @@ def getContinent(code): getContinent(code) >>> ["africa", "europe"] """ - from config_osm_data import world_iso continent_list = [] code_set = set(code) - for continent in world_iso: + world_iso = read_osm_config("world_iso") + for continent in world_iso.keys(): single_continent_set = set(world_iso[continent]) if code_set.intersection(single_continent_set): continent_list.append(continent) @@ -667,8 +712,6 @@ def create_country_list(input, iso_coding=True): """ import logging - from config_osm_data import continent_regions, world_iso - _logger = logging.getLogger(__name__) _logger.setLevel(logging.INFO) @@ -701,7 +744,7 @@ def filter_codes(c_list, iso_coding=True): for value1 in input: codes_list = [] - + world_iso, continent_regions = read_osm_config("world_iso", "continent_regions") # extract countries in world if value1 == "Earth": for continent in world_iso.keys(): diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 42a8485f7..6cc7b634e 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -35,6 +35,7 @@ hydro: carriers: hydro_max_hours: + hydro_max_hours_default: hydro_capital_cost: lines: @@ -303,10 +304,10 @@ def attach_wind_and_solar( n, costs, ppl, - input_profiles, + input_files, technologies, extendable_carriers, - line_length_factor=1, + line_length_factor, ): # TODO: rename tech -> carrier, technologies -> carriers _add_missing_carriers_from_costs(n, costs, technologies) @@ -323,29 +324,33 @@ def attach_wind_and_solar( df.carrier.mask(df.technology == "Onshore", "onwind", inplace=True) - with xr.open_dataset(getattr(snakemake.input, "profile_" + tech)) as ds: + with xr.open_dataset(getattr(input_files, "profile_" + tech)) as ds: if ds.indexes["bus"].empty: continue suptech = tech.split("-", 2)[0] if suptech == "offwind": - continue - # TODO: Uncomment out and debug. - # underwater_fraction = ds["underwater_fraction"].to_pandas() - # connection_cost = ( - # snakemake.params.length_factor * - # ds["average_distance"].to_pandas() * - # (underwater_fraction * - # costs.at[tech + "-connection-submarine", "capital_cost"] + - # (1.0 - underwater_fraction) * - # costs.at[tech + "-connection-underground", "capital_cost"] - # )) - # capital_cost = (costs.at["offwind", "capital_cost"] + - # costs.at[tech + "-station", "capital_cost"] + - # connection_cost) - # logger.info( - # "Added connection cost of {:0.0f}-{:0.0f} Eur/MW/a to {}". - # format(connection_cost.min(), connection_cost.max(), tech)) + underwater_fraction = ds["underwater_fraction"].to_pandas() + connection_cost = ( + line_length_factor + * ds["average_distance"].to_pandas() + * ( + underwater_fraction + * costs.at[tech + "-connection-submarine", "capital_cost"] + + (1.0 - underwater_fraction) + * costs.at[tech + "-connection-underground", "capital_cost"] + ) + ) + capital_cost = ( + costs.at["offwind", "capital_cost"] + + costs.at[tech + "-station", "capital_cost"] + + connection_cost + ) + logger.info( + "Added connection cost of {:0.0f}-{:0.0f} Eur/MW/a to {}".format( + connection_cost.min(), connection_cost.max(), tech + ) + ) else: capital_cost = costs.at[tech, "capital_cost"] @@ -390,11 +395,6 @@ def attach_conventional_generators( ) _add_missing_carriers_from_costs(n, costs, carriers) - # Replace carrier "natural gas" with the respective technology (OCGT or CCGT) to align with PyPSA names of "carriers" and avoid filtering "natural gas" powerplants in ppl.query("carrier in @carriers") - ppl.loc[ppl["carrier"] == "natural gas", "carrier"] = ppl.loc[ - ppl["carrier"] == "natural gas", "technology" - ] - ppl = ( ppl.query("carrier in @carriers") .join(costs, on="carrier", rsuffix="_r") @@ -457,9 +457,14 @@ def attach_hydro(n, costs, ppl): .rename(index=lambda s: str(s) + " hydro") ) - # TODO: remove this line to address nan when powerplantmatching is stable # Current fix, NaN technologies set to ROR - ppl.loc[ppl.technology.isna(), "technology"] = "Run-Of-River" + if ppl.technology.isna().any(): + n_nans = ppl.technology.isna().sum() + logger.warning( + f"Identified {n_nans} hydro powerplants with unknown technology.\n" + "Initialized to 'Run-Of-River'" + ) + ppl.loc[ppl.technology.isna(), "technology"] = "Run-Of-River" ror = ppl.query('technology == "Run-Of-River"') phs = ppl.query('technology == "Pumped Storage"') @@ -550,13 +555,20 @@ def attach_hydro(n, costs, ppl): if "hydro" in carriers and not hydro.empty: hydro_max_hours = c.get("hydro_max_hours") - hydro_stats = pd.read_csv( - snakemake.input.hydro_capacities, comment="#", na_values=["-"], index_col=0 + hydro_stats = ( + pd.read_csv( + snakemake.input.hydro_capacities, + comment="#", + na_values=["-"], + index_col=0, + ) + .groupby("Country") + .sum() ) e_target = hydro_stats["E_store[TWh]"].clip(lower=0.2) * 1e6 e_installed = hydro.eval("p_nom * max_hours").groupby(hydro.country).sum() e_missing = e_target - e_installed - missing_mh_i = hydro.query("max_hours == 0").index + missing_mh_i = hydro.query("max_hours.isnull()").index if hydro_max_hours == "energy_capacity_totals_by_country": max_hours_country = ( @@ -568,6 +580,8 @@ def attach_hydro(n, costs, ppl): hydro_stats["E_store[TWh]"] * 1e3 / hydro_stats["p_nom_discharge[GW]"] ) + max_hours_country.clip(lower=0, inplace=True) + missing_countries = pd.Index(hydro["country"].unique()).difference( max_hours_country.dropna().index ) @@ -577,9 +591,10 @@ def attach_hydro(n, costs, ppl): ", ".join(missing_countries) ) ) + hydro_max_hours_default = c.get("hydro_max_hours_default", 6.0) hydro_max_hours = hydro.max_hours.where( hydro.max_hours > 0, hydro.country.map(max_hours_country) - ).fillna(6) + ).fillna(hydro_max_hours_default) n.madd( "StorageUnit", diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 4d20cba20..dd70cbc1b 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -11,8 +11,13 @@ import geopandas as gpd import numpy as np import pandas as pd -from _helpers import configure_logging, read_geojson, sets_path_to_root, to_csv_nafix -from config_osm_data import osm_clean_columns +from _helpers import ( + configure_logging, + read_geojson, + read_osm_config, + sets_path_to_root, + to_csv_nafix, +) from shapely.geometry import LineString, Point from shapely.ops import linemerge, split from tqdm import tqdm @@ -813,7 +818,7 @@ def built_network( force_ac=False, ): logger.info("Stage 1/5: Read input data") - + osm_clean_columns = read_osm_config("osm_clean_columns") buses = read_geojson( inputs["substations"], osm_clean_columns["substation"].keys(), diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index f98ecff4d..b2f14d582 100644 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -253,16 +253,39 @@ def add_custom_powerplants(ppl, inputs, config): return add_ppls -def replace_natural_gas_technology(df): - mapping = {"Steam Turbine": "CCGT", "Combustion Engine": "OCGT"} - tech = df.Technology.replace(mapping).fillna("CCGT") - return df.Technology.mask(df.Fueltype == "Natural Gas", tech) - - -def replace_natural_gas_fueltype(df): - return df.Fueltype.mask( - (df.Technology == "OCGT") | (df.Technology == "CCGT"), "Natural Gas" +def replace_natural_gas_technology(df: pd.DataFrame): + """ + Maps and replaces gas technologies in the powerplants.csv onto model + compliant carriers. + """ + mapping = { + "Steam Turbine": "CCGT", + "Combustion Engine": "OCGT", + "NG": "CCGT", + "Ng": "CCGT", + "NG/FO": "OCGT", + "Ng/Fo": "OCGT", + "NG/D": "OCGT", + "LNG": "OCGT", + "CCGT/D": "CCGT", + "CCGT/FO": "CCGT", + "LCCGT": "CCGT", + "CCGT/Fo": "CCGT", + } + fueltype = df["Fueltype"] == "Natural Gas" + df.loc[fueltype, "Technology"] = ( + df.loc[fueltype, "Technology"].replace(mapping).fillna("CCGT") ) + unique_tech_with_ng = df.loc[fueltype, "Technology"].unique() + unknown_techs = np.setdiff1d(unique_tech_with_ng, ["CCGT", "OCGT"]) + if len(unknown_techs) > 0: + df.Technology.where( + fueltype, + df["Technology"].map({t: "CCGT" for t in unknown_techs}), + inplace=True, + ) + df["Fueltype"] = np.where(fueltype, df["Technology"], df["Fueltype"]) + return df if __name__ == "__main__": @@ -316,10 +339,7 @@ def replace_natural_gas_fueltype(df): .powerplant.fill_missing_decommissioning_years() .query('Fueltype not in ["Solar", "Wind"] and Country in @countries_names') .powerplant.convert_country_to_alpha2() - .assign( - Technology=replace_natural_gas_technology, - Fueltype=replace_natural_gas_fueltype, - ) + .pipe(replace_natural_gas_technology) ) ppl = add_custom_powerplants( diff --git a/scripts/config_osm_data.py b/scripts/config_osm_data.py deleted file mode 100644 index 13d1ddb9f..000000000 --- a/scripts/config_osm_data.py +++ /dev/null @@ -1,672 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors -# -# SPDX-License-Identifier: AGPL-3.0-or-later -# -*- coding: utf-8 -*- - -import country_converter as coco - -cc = coco.CountryConverter() - -# =============================== -# OSM FEATURE COLUMNS -# =============================== -# The feature category represents the final representation of the feature -# For node features: ways are converted to nodes -# For way features: only ways are used - -feature_category = { - "substation": "node", - "generator": "node", - "line": "way", - "tower": "node", - "cable": "way", -} - -# =============================== -# OSM FEATURE COLUMNS -# =============================== -# These configurations are used to specify which OSM tags are kept as columns in DataFrame. -# Follows the OSM Wiki: https://wiki.openstreetmap.org/wiki/Power - -# "Length" is added for way features -# "Area" is added for node features - -# ======================== -# BASIC INFO TAGS -# ======================== -# A list of tags that are relevant for most OSM keys - -columns_basic = [ - "id", - "lonlat", - "tags.power", - "Type", - "Country", - # "refs" -] - -# ======================== -# SUBSTATION TAGS -# ======================== - -# Default tags to keep as columns with substation -# Based on: https://wiki.openstreetmap.org/wiki/Key:substation -columns_substation = [ - "Area", - "tags.substation", - "tags.voltage", - # Other tags which are not kept by default - # ===================================== - # "TODO:ADD Tags not kept here", -] - -# ======================== -# GENERATOR TAGS -# ======================== - -# Default tags to keep as columns with generator -# Based on: https://wiki.openstreetmap.org/wiki/Key:generator - -columns_generator = [ - "Area", - "tags.name", - "tags.generator:type", - "tags.generator:method", - "tags.generator:source", - "tags.generator:output:electricity", - # Other tags which are not kept by default - # ===================================== - # "TODO:ADD Tags not kept here", -] - -# ======================== -# LINE TAGS -# ======================== - -# Default tags to keep as columns with line -# Based on: https://wiki.openstreetmap.org/wiki/Key:line - -columns_line = [ - "Length", - "tags.cables", - "tags.voltage", - "tags.circuits", - "tags.frequency", - # Other tags which are not kept by default - # ===================================== - # "TODO:ADD Tags not kept here", -] - -# ======================== -# CABLE TAGS -# ======================== - -# Default tags to keep as columns with substation -# Based on: https://wiki.openstreetmap.org/wiki/Key:cable - -columns_cable = [ - "Length", - "tags.cables", - "tags.voltage", - "tags.circuits", - "tags.frequency", - "tags.location", - # Other tags which are not kept by default - # ===================================== - # "TODO:ADD Tags not kept here", -] - -# ======================== -# TOWER TAGS -# ======================== - -# Default tags to keep as columns with tower -# Based on: https://wiki.openstreetmap.org/wiki/Key:tower - -columns_tower = [ - "Area", - "tags.tower", - "tags.material", - "tags.structure", - "tags.operator", - "tags.line_attachment", - "tags.line_management", - "tags.ref", - "tags.height", - # Other tags which are not kept by default - # ===================================== - # "TODO:ADD Tags not kept here", -] - -# FINAL DICTIONARY - -feature_columns = { - "substation": columns_basic + columns_substation, - "generator": columns_basic + columns_generator, - "line": columns_basic + columns_line, - "cable": columns_basic + columns_cable, - "tower": columns_basic + columns_tower, -} - -# Columns of CLEAN OSM DATA names and dtypes -osm_clean_columns = { - "substation": { - "bus_id": object, - "station_id": float, - "voltage": float, - "dc": bool, - "symbol": object, - "under_construction": bool, - "tag_substation": str, - "tag_area": str, - "lon": float, - "lat": float, - "country": str, - "geometry": object, - }, - "generator": { - "id": object, - "tags.power": object, - "Type": object, - "Country": str, - "Area": object, - "name": object, - "tags.generator:type": object, - "tags.generator:method": object, - "tags.generator:source": object, - "power_output_MW": object, - "geometry": object, - }, - "line": { - "line_id": object, - "bus0": object, - "bus1": object, - "voltage": float, - "circuits": float, - "length": float, - "underground": bool, - "under_construction": bool, - "tag_type": str, - "tag_frequency": float, - "dc": bool, - "country": object, - "geometry": object, - }, -} - -# Python dictionary of ISO 3166-1-alpha-2 codes, as per publicly -# available data on official ISO site in July 2015. -# -# Available under MIT license -# Dimitris Karagkasidis, https://github.com/pageflt - -continents = { - "LA": "NorthAmerica", - "SA": "SouthAmerica", - "AS": "Asia", - "OC": "Oceania", - "AF": "Africa", - "EU": "Europe", - # "AN": "antarctica" -} - -world_iso = { - "Africa": { - "DZ": "algeria", - "AO": "angola", - "BJ": "benin", - "BW": "botswana", - # "IO": "british-indian-ocean-territory", # Island - "BF": "burkina-faso", - "BI": "burundi", - "CM": "cameroon", - # "IC": "canary-islands", # Island - "CV": "cape-verde", - "CF": "central-african-republic", - "TD": "chad", - "KM": "comoros", - "CG": "congo-brazzaville", - "CD": "congo-democratic-republic", - "DJ": "djibouti", - "EG": "egypt", - "GQ": "equatorial-guinea", - "ER": "eritrea", - "ET": "ethiopia", - # "TF": "french-southern-territories", # Island - "GA": "gabon", - "GH": "ghana", - "GW": "guinea-bissau", # No Data - "GN": "guinea", - "CI": "ivory-coast", - "KE": "kenya", - "LS": "lesotho", - "LR": "liberia", - "LY": "libya", - "MG": "madagascar", - "MW": "malawi", - "ML": "mali", - "MR": "mauritania", - "MU": "mauritius", - # "YT": "mayotte", # Island - "MA": "morocco", - "MZ": "mozambique", - "NA": "namibia", - "NE": "niger", - "NG": "nigeria", - # "RE": "reunion", # Island - "RW": "rwanda", - # saint-helena-ascension-and-tristan-da-cunha # Islands - "ST": "sao-tome-and-principe", - "SN": "senegal", - "GM": "gambia", - "SC": "seychelles", - "SL": "sierra-leone", - "SO": "somalia", # No Data - # south-africa-and-lesotho - "ZA": "south-africa", - "SS": "south-sudan", - "SD": "sudan", - "SZ": "swaziland", - "TZ": "tanzania", - "TG": "togo", - "TN": "tunisia", - "UG": "uganda", - "ZM": "zambia", - "ZW": "zimbabwe", - "EH": "western-sahara", - }, - "Asia": { - "AF": "afghanistan", - "AM": "armenia", - "AZ": "azerbaijan", - "BH": "bahrain", - "BD": "bangladesh", - "BT": "bhutan", - # "IO": "british indian ocean territory", - "KH": "cambodia", - "CN": "china", - # "CX": "christmas island", # Island - # "CC": "cocos (keeling) islands", # Island - "CY": "cyprus", - # "EG": "egypt", # leads to bug -> missing ssp file when executing ["Africa"] - "GE": "georgia", - # "HK": "hong kong", # no more with gadm 4.1 - "IN": "india", - "ID": "indonesia", - "IR": "iran", - "IQ": "iraq", - "IL": "israel", - "JP": "japan", - "JO": "jordan", - "KZ": "kazakhstan", - "KP": "north-korea", - "KR": "south-korea", - "KW": "kuwait", - "KG": "kyrgyzstan", - "LA": "lao-people's-democratic-republic", - "LB": "lebanon", - "MY": "malaysia", - "BN": "brunei", - "MV": "maldives", - "MN": "mongolia", - "MM": "myanmar", - "NP": "nepal", - "OM": "oman", - "PK": "pakistan", - "PS": "palestine", - "PH": "philippines", - "QA": "qatar", - "RU": "russian-federation", - "SA": "saudi-arabia", - "SG": "singapore", # merged with MY - # "XS": "spratly-islands", #Island - "LK": "sri-lanka", - "SY": "syria", - "TW": "taiwan", - "TJ": "tajikistan", - "TH": "thailand", - "TL": "timor-leste", - "TR": "turkey", - "TM": "turkmenistan", - "AE": "united-arab-emirates", - # "XD": "united-nations-neutral-zone", - "UZ": "uzbekistan", - "VN": "vietnam", - "YE": "yemen", - }, - "Oceania": { - "AS": "american-samoa", # Island - "AU": "australia", - # "CP": "ile-de-clipperton", # Island In gadm as XCL - # "CK": "cook-islands", # Island - "FJ": "fiji", - # "PF": "french-polynesia", # Island - # "GU": "guam", # Island - "KI": "kiribati", # Island - "MH": "marshall-islands", - "FM": "micronesia", - "NR": "nauru", - "NC": "new-caledonia", # Island - "NZ": "new-zealand", - "NU": "niue", # Island - # "NF": "norfolk-island", # Island - # "MP": "northern-mariana-islands", - "PG": "papua-new-guinea", - # "PN": "pitcairn-islands", # Islands - "PW": "palau", - "WS": "samoa", - "SB": "solomon-islands", - # "TK": "tokelau", # Island - "TO": "tonga", - "TV": "tuvalu", - # "UM": "united-states-minor-outlying-islands", #Islands - "VU": "vanuatu", # Island - # "WF": "wallis-and-futuna", # Island - }, - "Europe": { - # "AX": "aland-islands", # Island - "AL": "albania", - "AD": "andorra", - "AM": "armenia", - "AT": "austria", - "AZ": "Azerbaijan", - "BY": "belarus", - "BE": "belgium", - "BA": "bosnia-herzegovina", - "BG": "bulgaria", - "HR": "croatia", - "CY": "cyprus", - "CZ": "czech-republic", - "DK": "denmark", - "EE": "estonia", - # "FO": "faroe islands", # Islands - "FI": "finland", - "FR": "france", - "GE": "georgia", - "DE": "germany", - # "GI": "gibraltar", # Island ? - "GR": "greece", - # "GG": "guernsey", # Island - "HU": "hungary", - "IS": "iceland", - "IE": "ireland-and-northern-ireland", - # "IM": "isle of man", # Island - "IT": "italy", - # "JE": "jersey", # Island - "KZ": "kazakhstan", - "XK": "kosovo", - "LV": "latvia", - "LI": "liechtenstein", - "LT": "lithuania", - "LU": "luxembourg", - "MK": "macedonia", - "MT": "malta", - "MD": "moldova", - "MC": "monaco", - "ME": "montenegro", - "NL": "netherlands", - "NO": "norway", - "PL": "poland", - "PT": "portugal", - "RO": "romania", - "RU": "russia", - "SM": "san-marino", - "RS": "serbia", - "SK": "slovakia", - "SI": "slovenia", - "ES": "spain", - # "SJ": "svalbard-and-jan-mayen", # Islands - "SE": "sweden", - "CH": "switzerland", - "UA": "ukraine", - "GB": "great-britain", - "TR": "turkey", - "VA": "vatican", - }, - "NorthAmerica": { - # "AI": "anguilla", #Island - "AG": "antigua-and-barbuda", - # "AW": "aruba", # Islands - "BS": "bahamas", - "BB": "barbados", - # "BM": "bermuda", # Islands - # "BQ": "bonaire", # Islands - # "VG": "british-virgin-islands", # Islands - "CA": "canada", - # "KY": "cayman-islands", # Islands - "CU": "cuba", - # "CW": "curacao", # Islands - "DM": "dominica", - "DO": "dominican-republic", - "GL": "greenland", - "GD": "grenada", - # "GP": "guadeloupe", # Islands - "HT": "haiti", - "JM": "jamaica", - # "MQ": "martinique", # Islands - "MX": "mexico", - # "MS": "montserrat", # Islands - "US": "united-states-of-america", - "PR": "puerto-rico", - # "BL": "saint-barthelemy", # Islands - "KN": "saint-kitts-and-nevis", - "LC": "saint-lucia", - # "MF": "saint-martin", # Islands - # "PM": "saint-pierre-and-miquelon", # Islands - "VC": "saint-vincent-and-the-grenadines", - # "SX": "saint-marteen", # Islands - "TT": "trinidad-and-tobago", - # "TC": "turks-and-caicos", # Islands - # "UM": "united-states-minor-outlying-islands", #Islands - # "VI": "united-states-virgin-islands", #Islands - "BZ": "belize", - "CR": "costa-rica", - "HN": "honduras", - "GT": "guatemala", - "NI": "nicaragua", - "PA": "panama", - "SV": "el-salvador", - }, - "SouthAmerica": { - "AR": "argentina", - "BO": "bolivia", - "BR": "brazil", - "CL": "chile", - "CO": "colombia", - "EC": "ecuador", - "FK": "falkland-islands", # Islands - "GF": "french-guiana", - "GY": "guyana", # No Data - "PE": "peru", - "PY": "paraguay", - "SR": "suriname", - "UY": "uruguay", - "VE": "venezuela", - }, - # "Antarctica": { - # "AQ": "antarctica", - # "BV": "bouvet-island", - # "HM": "heard-island-and-mcdonald-island", - # "GS": "south-georgia-and-the-south-sandwich-islands", - # }, -} - -# Based on: https://waml.org/waml-information-bulletin/46-3/index-to-lc-g-schedule/1-world/ -# Australasia region includes New Caledonia and Papua New Guinea -continent_regions = { - # European regions - "SCR": ["DK", "NO", "SE", "FI", "IS"], # SCANDINAVIAN REGION - # EASTERN EUROPEAN REGION - "EER": ["BY", "PL", "CZ", "RU", "SK", "UA", "LT", "LV", "EE", "FI", "MD"], - # CENTRAL EUROPEAN REGION - "CER": ["AT", "CH", "CZ", "DE", "HU", "PL", "SK", "LI"], - # BALKAN PENISULAN REGION - "BPR": ["AL", "BA", "BG", "GR", "HR", "ME", "RO", "SI", "RS", "ME", "MK"], - # WESTERN EUROPE - "WER": ["FR", "BE", "GB", "IE", "LU", "MC", "NL", "AD"], - # SOUTHERN EUROPEAN REGION - "SER": ["ES", "AD", "IT", "PT", "SM", "MT"], - # African regions - # NORTHERN AFRICAN REGION - "NAR": ["EG", "LY", "TN", "DZ", "MA", "EH", "SD", "SS"], - # WESTERN AFRICAN REGION - # Guinea-Bissau ["GW"] belongs to the region but power data are NA in OSM) - "WAR": [ - "MR", - "ML", - "NE", - "NG", - "BJ", - "BF", - "TG", - "GH", - "CI", - "LR", - "SL", - "GN", - "SN", - "GM", - ], - # CENTRAL AFRICAN REGION - "CAR": ["TD", "CF", "CM", "GQ", "GA", "CD", "CG", "AO"], - # EASTERN AFRICAN REGION - # Somalia ["SO"] belongs to the region but power data are NA in OSM) - "EAR": ["ER", "ET", "UG", "KE", "RW", "BI", "TZ", "MZ", "DJ", "MG"], - # SOUTHERN AFRICAN REGION - "SAR": ["MW", "ZM", "ZW", "BW", "NA", "SZ", "LS", "ZA"], - # Asian regions - "KVR": ["AZ", "GE", "AM"], - "WAS": [ - "TR", - "AM", - "AZ", - "BH", - "CY", - "GE", - "IQ", - "IL", - "JO", - "KW", - "LB", - "OM", - "PS", - "QA", - "SA", - "SY", - "AE", - "YE", - ], - # FAR EASTERN ASIAN REGION - "FEAR": ["JP", "KP", "KR", "CN", "TW", "MN"], # , "HK", "MO"], - # SOUTHEASTERN ASIAN REGION - "SEAR": ["LA", "TH", "KH", "VN", "PH", "MY", "SG", "BN", "ID"], - # CENTRAL ASIAN REGION - "CASR": ["KZ", "KG", "UZ", "TM", "TJ"], - # SOUTHERN ASIAN REGION - "SASR": ["MM", "BD", "BT", "NP", "IN", "LK", "PK", "AF"], - # ASEAN: Political denomination of Southeast Asian countries - "ASEAN": ["VN", "TH", "ID", "PH", "MY", "MM", "KH", "LA", "SG", "BN"], - # MIDDLE EASTERN ASIAN REGION - "MEAR": [ - "TR", - "SY", - "LB", - "CY", - "IQ", - "IR", - "JO", - "IL", - "PS", - "AE", - "YE", - "KW", - "BH", - "QA", - "SA", - "OM", - ], - # American continent regions - "NACR": ["CA", "GL", "MX", "US"], # NORTHERN AMERICAN CONTINENT REGION - # SOUTHERN LATIN AMERICAN REGION - "LACR": ["AR", "BO", "BR", "CL", "CO", "EC", "GF", "PE", "PY", "SR", "UY", "VE"], - # CENTRAL AMERICAN REGION - "CACR": ["BZ", "GT", "SV", "HN", "NI", "CR", "PA"], - # Australasia - "AUO": ["AU", "NC", "NZ", "PG"], - # UnitedNations - "UnitedNations": cc.convert(names=cc.UN.name_short, to="ISO2"), -} - -# Geofabrik and iso norm deviate for some countries and domains - -# dictionary of correspondence between iso country codes and geofabrik codes containing those information -# This dictionary instructs the script download_osm_data about how to successfully download data -# from countries that are aggregated into osm. -# For example, Senegal (SN) and Gambia (GM) cannot be downloaded from OSM separately, but only jointly as SN-GM -# That's the reason why in this dictionary they can be found the following entries: -# "SN": "SN-GM" -# "GM": "SN-GM" -# This instruct the workflow that when the country "SN" is requested, then it shall download the "SN-GM" file -iso_to_geofk_dict = { - "EH": "MA", # Western Sahara -> Morocco - "SN": "SN-GM", # Senegal -> Senegal-Gambia - "GM": "SN-GM", # Gambia -> Senegal-Gambia - "KM": "comores", # Comores - "IC": "canary-islands", # Canary islands - # "HK": "CN", # Hong Kong -> China # no more with gadm 4.1 - # "MO": "CN", # Macao -> China # no more with gadm 4.1 - "SG": "MY", # Singapore -> Malaysia-Singapore-Brunei - "BN": "MY", # Brunei -> Malaysia-Singapore-Brunei - "SA": "QA-AE-OM-BH-KW", # Saudi Arabia -> Gulf Cooperation Council - "KW": "QA-AE-OM-BH-KW", # Kuwait -> Gulf Cooperation Council - "BH": "QA-AE-OM-BH-KW", # Bahrain -> Gulf Cooperation Council - "QA": "QA-AE-OM-BH-KW", # Qatar -> Gulf Cooperation Council - "AE": "QA-AE-OM-BH-KW", # United Arab Emirates -> Gulf Cooperation Council - "OM": "QA-AE-OM-BH-KW", # Oman -> Gulf Cooperation Council - "PS": "PS-IL", # Israel and Palestine are merged in OSM - "IL": "PS-IL", # Israel and Palestine are merged in OSM - "SM": "IT", # San-Marino is merged to Italy - "VA": "IT", # Vatican is merged to Italy - "HT": "haiti-and-domrep", # Haiti and Dominican Republic are merged in OSM - "DO": "haiti-and-domrep", # Haiti and Dominican Republic are merged in OSM - "PA": "panama", # Panama - "NF": "AU", # norfolk island is an AU territory - "MP": "american-oceania", # northern mariana islands are US territory - "GU": "american-oceania", # Guam is a US territory - "AS": "american-oceania", # American Samoa is a US territory - "CP": "ile-de-clipperton", # Ile de clipperton - "PF": "polynesie-francaise", # Polynesie Francaise - "VU": "vanuatu", # Vanuatu - "TK": "tokelau", # Tokelau - "MH": "marshall-islands", # Marshal islands - "PN": "pitcairn-islands", # Pitcairn - "WF": "wallis-et-futuna", # Wallis et Fortnuna - "XK": "RS-KM", # Kosovo - "BS": "bahamas", # Bahamas - "BB": "central-america", # Barbados - "CU": "cuba", # Cuba - # "IC": "canary-islands", # Canary islands - "RE": "reunion", # Reunion island (France) - "YT": "mayotte", # "Mayotte island (France)" - "GG": "guernsey-jersey", # Guernsey - "JE": "guernsey-jersey", # Jersey - "IM": "isle-of-man", # Isle of man - "GP": "guadeloupe", # guadeloupe - "JM": "jamaica", # jamaica - "TT": "central-america", # Trinidad and Tobago - "AG": "central-america", # Antigua e Barbuda - "DM": "central-america", # Dominique - "LC": "central-america", # Santa Lucia - "VC": "central-america", # Saint Vincent e Grenadine - "KN": "central-america", # Saint Kitts e Nevis - "GD": "central-america", # Grenada - # "PM": "north-america", # Saint-Pierre e Miquelon - "FK": "south-america", # Falkland islands -} - -# data for some islands seem to be merged with some other areas data -# "FO": "faroe islands" -# "NF": "norfolk island", -# "PF": "french-polynesia" -# "GU": "guam" diff --git a/scripts/download_osm_data.py b/scripts/download_osm_data.py index 54e73ee2c..24afa4587 100644 --- a/scripts/download_osm_data.py +++ b/scripts/download_osm_data.py @@ -31,8 +31,7 @@ import shutil from pathlib import Path -from _helpers import configure_logging -from config_osm_data import iso_to_geofk_dict +from _helpers import configure_logging, read_osm_config from earth_osm import eo logger = logging.getLogger(__name__) @@ -62,7 +61,9 @@ def country_list_to_geofk(country_list): return full_codes_list -def convert_iso_to_geofk(iso_code, iso_coding=True, convert_dict=iso_to_geofk_dict): +def convert_iso_to_geofk( + iso_code, iso_coding=True, convert_dict=read_osm_config("iso_to_geofk_dict") +): """ Function to convert the iso code name of a country into the corresponding geofabrik In Geofabrik, some countries are aggregated, thus if a single diff --git a/scripts/make_statistics.py b/scripts/make_statistics.py index e07889c66..d87eb71c2 100644 --- a/scripts/make_statistics.py +++ b/scripts/make_statistics.py @@ -484,7 +484,7 @@ def collect_renewable_stats(rulename, technology): ), ) - add_computational_stats(df_RES_stats, snakemake, rulename) + add_computational_stats(df_RES_stats, snakemake, f"{rulename}_{technology}") return df_RES_stats else: