From b1a8a80c4d4780f93dcd1c469e5d172c427c58bd Mon Sep 17 00:00:00 2001 From: Thomas Kouroughli Date: Fri, 4 Aug 2023 21:53:03 +0100 Subject: [PATCH 01/32] remove config_osm_data --- scripts/config_osm_data.py | 672 ------------------------------------- 1 file changed, 672 deletions(-) delete mode 100644 scripts/config_osm_data.py diff --git a/scripts/config_osm_data.py b/scripts/config_osm_data.py deleted file mode 100644 index 73924057d..000000000 --- a/scripts/config_osm_data.py +++ /dev/null @@ -1,672 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors -# -# SPDX-License-Identifier: AGPL-3.0-or-later -# -*- coding: utf-8 -*- - -import country_converter as coco - -cc = coco.CountryConverter() - -# =============================== -# OSM FEATURE COLUMNS -# =============================== -# The feature category represents the final representation of the feature -# For node features: ways are converted to nodes -# For way features: only ways are used - -feature_category = { - "substation": "node", - "generator": "node", - "line": "way", - "tower": "node", - "cable": "way", -} - -# =============================== -# OSM FEATURE COLUMNS -# =============================== -# These configurations are used to specify which OSM tags are kept as columns in DataFrame. -# Follows the OSM Wiki: https://wiki.openstreetmap.org/wiki/Power - -# "Length" is added for way features -# "Area" is added for node features - -# ======================== -# BASIC INFO TAGS -# ======================== -# A list of tags that are relevant for most OSM keys - -columns_basic = [ - "id", - "lonlat", - "tags.power", - "Type", - "Country", - # "refs" -] - -# ======================== -# SUBSTATION TAGS -# ======================== - -# Default tags to keep as columns with substation -# Based on: https://wiki.openstreetmap.org/wiki/Key:substation -columns_substation = [ - "Area", - "tags.substation", - "tags.voltage", - # Other tags which are not kept by default - # ===================================== - # "TODO:ADD Tags not kept here", -] - -# ======================== -# GENERATOR TAGS -# ======================== - -# Default tags to keep as columns with generator -# Based on: https://wiki.openstreetmap.org/wiki/Key:generator - -columns_generator = [ - "Area", - "tags.name", - "tags.generator:type", - "tags.generator:method", - "tags.generator:source", - "tags.generator:output:electricity", - # Other tags which are not kept by default - # ===================================== - # "TODO:ADD Tags not kept here", -] - -# ======================== -# LINE TAGS -# ======================== - -# Default tags to keep as columns with line -# Based on: https://wiki.openstreetmap.org/wiki/Key:line - -columns_line = [ - "Length", - "tags.cables", - "tags.voltage", - "tags.circuits", - "tags.frequency", - # Other tags which are not kept by default - # ===================================== - # "TODO:ADD Tags not kept here", -] - -# ======================== -# CABLE TAGS -# ======================== - -# Default tags to keep as columns with substation -# Based on: https://wiki.openstreetmap.org/wiki/Key:cable - -columns_cable = [ - "Length", - "tags.cables", - "tags.voltage", - "tags.circuits", - "tags.frequency", - "tags.location", - # Other tags which are not kept by default - # ===================================== - # "TODO:ADD Tags not kept here", -] - -# ======================== -# TOWER TAGS -# ======================== - -# Default tags to keep as columns with tower -# Based on: https://wiki.openstreetmap.org/wiki/Key:tower - -columns_tower = [ - "Area", - "tags.tower", - "tags.material", - "tags.structure", - "tags.operator", - "tags.line_attachment", - "tags.line_management", - "tags.ref", - "tags.height", - # Other tags which are not kept by default - # ===================================== - # "TODO:ADD Tags not kept here", -] - -# FINAL DICTIONARY - -feature_columns = { - "substation": columns_basic + columns_substation, - "generator": columns_basic + columns_generator, - "line": columns_basic + columns_line, - "cable": columns_basic + columns_cable, - "tower": columns_basic + columns_tower, -} - -# Columns of CLEAN OSM DATA names and dtypes -osm_clean_columns = { - "substation": { - "bus_id": object, - "station_id": float, - "voltage": float, - "dc": bool, - "symbol": object, - "under_construction": bool, - "tag_substation": str, - "tag_area": str, - "lon": float, - "lat": float, - "country": str, - "geometry": object, - }, - "generator": { - "id": object, - "tags.power": object, - "Type": object, - "Country": str, - "Area": object, - "name": object, - "tags.generator:type": object, - "tags.generator:method": object, - "tags.generator:source": object, - "power_output_MW": object, - "geometry": object, - }, - "line": { - "line_id": object, - "bus0": object, - "bus1": object, - "voltage": float, - "circuits": float, - "length": float, - "underground": bool, - "under_construction": bool, - "tag_type": str, - "tag_frequency": float, - "dc": bool, - "country": object, - "geometry": object, - }, -} - -# Python dictionary of ISO 3166-1-alpha-2 codes, as per publicly -# available data on official ISO site in July 2015. -# -# Available under MIT license -# Dimitris Karagkasidis, https://github.com/pageflt - -continents = { - "LA": "NorthAmerica", - "SA": "SouthAmerica", - "AS": "Asia", - "OC": "Oceania", - "AF": "Africa", - "EU": "Europe", - # "AN": "antarctica" -} - -world_iso = { - "Africa": { - "DZ": "algeria", - "AO": "angola", - "BJ": "benin", - "BW": "botswana", - # "IO": "british-indian-ocean-territory", # Island - "BF": "burkina-faso", - "BI": "burundi", - "CM": "cameroon", - # "IC": "canary-islands", # Island - # "CV": "cape-verde", - "CF": "central-african-republic", - "TD": "chad", - # "KM": "comoros", - "CG": "congo-brazzaville", - "CD": "congo-democratic-republic", - "DJ": "djibouti", - "EG": "egypt", - "GQ": "equatorial-guinea", - "ER": "eritrea", - "ET": "ethiopia", - # "TF": "french-southern-territories", # Island - "GA": "gabon", - "GH": "ghana", - "GW": "guinea-bissau", # No Data - "GN": "guinea", - "CI": "ivory-coast", - "KE": "kenya", - "LS": "lesotho", - "LR": "liberia", - "LY": "libya", - "MG": "madagascar", - "MW": "malawi", - "ML": "mali", - "MR": "mauritania", - "MU": "mauritius", - # "YT": "mayotte", # Island - "MA": "morocco", - "MZ": "mozambique", - "NA": "namibia", - "NE": "niger", - "NG": "nigeria", - # "RE": "reunion", # Island - "RW": "rwanda", - # saint-helena-ascension-and-tristan-da-cunha # Islands - # "ST": "sao-tome-and-principe", - "SN": "senegal", - "GM": "gambia", - # "SC": "seychelles", - "SL": "sierra-leone", - "SO": "somalia", # No Data - # south-africa-and-lesotho - "ZA": "south-africa", - "SS": "south-sudan", - "SD": "sudan", - "SZ": "swaziland", - "TZ": "tanzania", - "TG": "togo", - "TN": "tunisia", - "UG": "uganda", - "ZM": "zambia", - "ZW": "zimbabwe", - "EH": "western-sahara", - }, - "Asia": { - "AF": "afghanistan", - "AM": "armenia", - "AZ": "azerbaijan", - "BH": "bahrain", - "BD": "bangladesh", - "BT": "bhutan", - # "IO": "british indian ocean territory", - "KH": "cambodia", - "CN": "china", - # "CX": "christmas island", # Island - # "CC": "cocos (keeling) islands", # Island - "CY": "cyprus", - # "EG": "egypt", # leads to bug -> missing ssp file when executing ["Africa"] - "GE": "georgia", - # "HK": "hong kong", # no more with gadm 4.1 - "IN": "india", - "ID": "indonesia", - "IR": "iran", - "IQ": "iraq", - "IL": "israel", - "JP": "japan", - "JO": "jordan", - "KZ": "kazakhstan", - "KP": "north-korea", - "KR": "south-korea", - "KW": "kuwait", - "KG": "kyrgyzstan", - "LA": "lao-people's-democratic-republic", - "LB": "lebanon", - "MY": "malaysia", - "BN": "brunei", - "MV": "maldives", - "MN": "mongolia", - "MM": "myanmar", - "NP": "nepal", - "OM": "oman", - "PK": "pakistan", - "PS": "palestine", - "PH": "philippines", - "QA": "qatar", - "RU": "russian-federation", - "SA": "saudi-arabia", - "SG": "singapore", # merged with MY - # "XS": "spratly-islands", #Island - "LK": "sri-lanka", - "SY": "syria", - "TW": "taiwan", - "TJ": "tajikistan", - "TH": "thailand", - "TL": "timor-leste", - "TR": "turkey", - "TM": "turkmenistan", - "AE": "united-arab-emirates", - # "XD": "united-nations-neutral-zone", - "UZ": "uzbekistan", - "VN": "vietnam", - "YE": "yemen", - }, - "Oceania": { - "AS": "american-samoa", # Island - "AU": "australia", - # "CP": "ile-de-clipperton", # Island In gadm as XCL - # "CK": "cook-islands", # Island - "FJ": "fiji", - # "PF": "french-polynesia", # Island - # "GU": "guam", # Island - "KI": "kiribati", # Island - "MH": "marshall-islands", - "FM": "micronesia", - "NR": "nauru", - "NC": "new-caledonia", # Island - "NZ": "new-zealand", - "NU": "niue", # Island - # "NF": "norfolk-island", # Island - # "MP": "northern-mariana-islands", - "PG": "papua-new-guinea", - # "PN": "pitcairn-islands", # Islands - "PW": "palau", - "WS": "samoa", - "SB": "solomon-islands", - # "TK": "tokelau", # Island - "TO": "tonga", - "TV": "tuvalu", - # "UM": "united-states-minor-outlying-islands", #Islands - "VU": "vanuatu", # Island - # "WF": "wallis-and-futuna", # Island - }, - "Europe": { - # "AX": "aland-islands", # Island - "AL": "albania", - "AD": "andorra", - "AM": "armenia", - "AT": "austria", - "AZ": "Azerbaijan", - "BY": "belarus", - "BE": "belgium", - "BA": "bosnia-herzegovina", - "BG": "bulgaria", - "HR": "croatia", - "CY": "cyprus", - "CZ": "czech-republic", - "DK": "denmark", - "EE": "estonia", - # "FO": "faroe islands", # Islands - "FI": "finland", - "FR": "france", - "GE": "georgia", - "DE": "germany", - # "GI": "gibraltar", # Island ? - "GR": "greece", - # "GG": "guernsey", # Island - "HU": "hungary", - "IS": "iceland", - # "IE": "ireland-and-northern-ireland", - # "IM": "isle of man", # Island - "IT": "italy", - # "JE": "jersey", # Island - "KZ": "kazakhstan", - "XK": "kosovo", - "LV": "latvia", - "LI": "liechtenstein", - "LT": "lithuania", - "LU": "luxembourg", - "MK": "macedonia", - "MT": "malta", - "MD": "moldova", - "MC": "monaco", - "ME": "montenegro", - "NL": "netherlands", - "NO": "norway", - "PL": "poland", - "PT": "portugal", - "RO": "romania", - "RU": "russia", - "SM": "san-marino", - "RS": "serbia", - "SK": "slovakia", - "SI": "slovenia", - "ES": "spain", - # "SJ": "svalbard-and-jan-mayen", # Islands - "SE": "sweden", - "CH": "switzerland", - "UA": "ukraine", - "GB": "great-britain", - "TR": "turkey", - "VA": "vatican", - }, - "NorthAmerica": { - # "AI": "anguilla", #Island - "AG": "antigua-and-barbuda", - # "AW": "aruba", # Islands - "BS": "bahamas", - "BB": "barbados", - # "BM": "bermuda", # Islands - # "BQ": "bonaire", # Islands - # "VG": "british-virgin-islands", # Islands - "CA": "canada", - # "KY": "cayman-islands", # Islands - "CU": "cuba", - # "CW": "curacao", # Islands - "DM": "dominica", - "DO": "dominican-republic", - "GL": "greenland", - "GD": "grenada", - # "GP": "guadeloupe", # Islands - "HT": "haiti", - "JM": "jamaica", - # "MQ": "martinique", # Islands - "MX": "mexico", - # "MS": "montserrat", # Islands - "US": "united-states-of-america", - "PR": "puerto-rico", - # "BL": "saint-barthelemy", # Islands - "KN": "saint-kitts-and-nevis", - "LC": "saint-lucia", - # "MF": "saint-martin", # Islands - # "PM": "saint-pierre-and-miquelon", # Islands - "VC": "saint-vincent-and-the-grenadines", - # "SX": "saint-marteen", # Islands - "TT": "trinidad-and-tobago", - # "TC": "turks-and-caicos", # Islands - # "UM": "united-states-minor-outlying-islands", #Islands - # "VI": "united-states-virgin-islands", #Islands - "BZ": "belize", - "CR": "costa-rica", - "HN": "honduras", - "GT": "guatemala", - "NI": "nicaragua", - "PA": "panama", - "SV": "el-salvador", - }, - "SouthAmerica": { - "AR": "argentina", - "BO": "bolivia", - "BR": "brazil", - "CL": "chile", - "CO": "colombia", - "EC": "ecuador", - "FK": "falkland-islands", # Islands - "GF": "french-guiana", - "GY": "guyana", # No Data - "PE": "peru", - "PY": "paraguay", - "SR": "suriname", - "UY": "uruguay", - "VE": "venezuela", - }, - # "Antarctica": { - # "AQ": "antarctica", - # "BV": "bouvet-island", - # "HM": "heard-island-and-mcdonald-island", - # "GS": "south-georgia-and-the-south-sandwich-islands", - # }, -} - -# Based on: https://waml.org/waml-information-bulletin/46-3/index-to-lc-g-schedule/1-world/ -# Australasia region includes New Caledonia and Papua New Guinea -continent_regions = { - # European regions - "SCR": ["DK", "NO", "SE", "FI", "IS"], # SCANDINAVIAN REGION - # EASTERN EUROPEAN REGION - "EER": ["BY", "PL", "CZ", "RU", "SK", "UA", "LT", "LV", "EE", "FI", "MD"], - # CENTRAL EUROPEAN REGION - "CER": ["AT", "CH", "CZ", "DE", "HU", "PL", "SK", "LI"], - # BALKAN PENISULAN REGION - "BPR": ["AL", "BA", "BG", "GR", "HR", "ME", "RO", "SI", "RS", "ME", "MK"], - # WESTERN EUROPE - "WER": ["FR", "BE", "GB", "IE", "LU", "MC", "NL", "AD"], - # SOUTHERN EUROPEAN REGION - "SER": ["ES", "AD", "IT", "PT", "SM", "MT"], - # African regions - # NORTHERN AFRICAN REGION - "NAR": ["EG", "LY", "TN", "DZ", "MA", "EH", "SD", "SS"], - # WESTERN AFRICAN REGION - # Guinea-Bissau ["GW"] belongs to the region but power data are NA in OSM) - "WAR": [ - "MR", - "ML", - "NE", - "NG", - "BJ", - "BF", - "TG", - "GH", - "CI", - "LR", - "SL", - "GN", - "SN", - "GM", - ], - # CENTRAL AFRICAN REGION - "CAR": ["TD", "CF", "CM", "GQ", "GA", "CD", "CG", "AO"], - # EASTERN AFRICAN REGION - # Somalia ["SO"] belongs to the region but power data are NA in OSM) - "EAR": ["ER", "ET", "UG", "KE", "RW", "BI", "TZ", "MZ", "DJ", "MG"], - # SOUTHERN AFRICAN REGION - "SAR": ["MW", "ZM", "ZW", "BW", "NA", "SZ", "LS", "ZA"], - # Asian regions - "KVR": ["AZ", "GE", "AM"], - "WAS": [ - "TR", - "AM", - "AZ", - "BH", - "CY", - "GE", - "IQ", - "IL", - "JO", - "KW", - "LB", - "OM", - "PS", - "QA", - "SA", - "SY", - "AE", - "YE", - ], - # FAR EASTERN ASIAN REGION - "FEAR": ["JP", "KP", "KR", "CN", "TW", "MN"], # , "HK", "MO"], - # SOUTHEASTERN ASIAN REGION - "SEAR": ["LA", "TH", "KH", "VN", "PH", "MY", "SG", "BN", "ID"], - # CENTRAL ASIAN REGION - "CASR": ["KZ", "KG", "UZ", "TM", "TJ"], - # SOUTHERN ASIAN REGION - "SASR": ["MM", "BD", "BT", "NP", "IN", "LK", "PK", "AF"], - # ASEAN: Political denomination of Southeast Asian countries - "ASEAN": ["VN", "TH", "ID", "PH", "MY", "MM", "KH", "LA", "SG", "BN"], - # MIDDLE EASTERN ASIAN REGION - "MEAR": [ - "TR", - "SY", - "LB", - "CY", - "IQ", - "IR", - "JO", - "IL", - "PS", - "AE", - "YE", - "KW", - "BH", - "QA", - "SA", - "OM", - ], - # American continent regions - "NACR": ["CA", "GL", "MX", "US"], # NORTHERN AMERICAN CONTINENT REGION - # SOUTHERN LATIN AMERICAN REGION - "LACR": ["AR", "BO", "BR", "CL", "CO", "EC", "GF", "PE", "PY", "SR", "UY", "VE"], - # CENTRAL AMERICAN REGION - "CACR": ["BZ", "GT", "SV", "HN", "NI", "CR", "PA"], - # Australasia - "AUO": ["AU", "NC", "NZ", "PG"], - # UnitedNations - "UnitedNations": cc.convert(names=cc.UN.name_short, to="ISO2"), -} - -# Geofabrik and iso norm deviate for some countries and domains - -# dictionary of correspondence between iso country codes and geofabrik codes containing those information -# This dictionary instructs the script download_osm_data about how to successfully download data -# from countries that are aggregated into osm. -# For example, Senegal (SN) and Gambia (GM) cannot be downloaded from OSM separately, but only jointly as SN-GM -# That's the reason why in this dictionary they can be found the following entries: -# "SN": "SN-GM" -# "GM": "SN-GM" -# This instruct the workflow that when the country "SN" is requested, then it shall download the "SN-GM" file -iso_to_geofk_dict = { - "EH": "MA", # Western Sahara -> Morocco - "SN": "SN-GM", # Senegal -> Senegal-Gambia - "GM": "SN-GM", # Gambia -> Senegal-Gambia - "KM": "comores", # Comores - "IC": "canary-islands", # Canary islands - # "HK": "CN", # Hong Kong -> China # no more with gadm 4.1 - # "MO": "CN", # Macao -> China # no more with gadm 4.1 - "SG": "MY", # Singapore -> Malaysia-Singapore-Brunei - "BN": "MY", # Brunei -> Malaysia-Singapore-Brunei - "SA": "QA-AE-OM-BH-KW", # Saudi Arabia -> Gulf Cooperation Council - "KW": "QA-AE-OM-BH-KW", # Kuwait -> Gulf Cooperation Council - "BH": "QA-AE-OM-BH-KW", # Bahrain -> Gulf Cooperation Council - "QA": "QA-AE-OM-BH-KW", # Qatar -> Gulf Cooperation Council - "AE": "QA-AE-OM-BH-KW", # United Arab Emirates -> Gulf Cooperation Council - "OM": "QA-AE-OM-BH-KW", # Oman -> Gulf Cooperation Council - "PS": "PS-IL", # Israel and Palestine are merged in OSM - "IL": "PS-IL", # Israel and Palestine are merged in OSM - "SM": "IT", # San-Marino is merged to Italy - "VA": "IT", # Vatican is merged to Italy - "HT": "haiti-and-domrep", # Haiti and Dominican Republic are merged in OSM - "DO": "haiti-and-domrep", # Haiti and Dominican Republic are merged in OSM - "PA": "panama", # Panama - "NF": "AU", # norfolk island is an AU territory - "MP": "american-oceania", # northern mariana islands are US territory - "GU": "american-oceania", # Guam is a US territory - "AS": "american-oceania", # American Samoa is a US territory - "CP": "ile-de-clipperton", # Ile de clipperton - "PF": "polynesie-francaise", # Polynesie Francaise - "VU": "vanuatu", # Vanuatu - "TK": "tokelau", # Tokelau - "MH": "marshall-islands", # Marshal islands - "PN": "pitcairn-islands", # Pitcairn - "WF": "wallis-et-futuna", # Wallis et Fortnuna - "XK": "RS-KM", # Kosovo - "BS": "bahamas", # Bahamas - "BB": "central-america", # Barbados - "CU": "cuba", # Cuba - # "IC": "canary-islands", # Canary islands - "RE": "reunion", # Reunion island (France) - "YT": "mayotte", # "Mayotte island (France)" - "GG": "guernsey-jersey", # Guernsey - "JE": "guernsey-jersey", # Jersey - "IM": "isle-of-man", # Isle of man - "GP": "guadeloupe", # guadeloupe - "JM": "jamaica", # jamaica - "TT": "central-america", # Trinidad and Tobago - "AG": "central-america", # Antigua e Barbuda - "DM": "central-america", # Dominique - "LC": "central-america", # Santa Lucia - "VC": "central-america", # Saint Vincent e Grenadine - "KN": "central-america", # Saint Kitts e Nevis - "GD": "central-america", # Grenada - # "PM": "north-america", # Saint-Pierre e Miquelon - "FK": "south-america", # Falkland islands -} - -# data for some islands seem to be merged with some other areas data -# "FO": "faroe islands" -# "NF": "norfolk island", -# "PF": "french-polynesia" -# "GU": "guam" From cd514cffd7b2cd0874310bced92cf4fe127a97d3 Mon Sep 17 00:00:00 2001 From: Thomas Kouroughli Date: Fri, 4 Aug 2023 21:53:15 +0100 Subject: [PATCH 02/32] add osm_config.yaml --- configs/osm_config.yaml | 299 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 299 insertions(+) create mode 100644 configs/osm_config.yaml diff --git a/configs/osm_config.yaml b/configs/osm_config.yaml new file mode 100644 index 000000000..8d0777c7a --- /dev/null +++ b/configs/osm_config.yaml @@ -0,0 +1,299 @@ +osm_clean_columns: + substation: + bus_id: object + station_id: float + voltage: float + dc: bool + symbol: object + under_construction: bool + tag_substation: str + tag_area: str + lon: float + lat: float + country: str + geometry: object + generator: + id: object + tags.power: object + Type: object + Country: str + Area: object + name: object + tags.generator:type: object + tags.generator:method: object + tags.generator:source: object + power_output_MW: object + geometry: object + line: + line_id: object + bus0: object + bus1: object + voltage: float + circuits: float + length: float + underground: bool + under_construction: bool + tag_type: str + tag_frequency: float + dc: bool + country: object + geometry: object + +continents: + LA: NorthAmerica + SA: SouthAmerica + AS: Asia + OC: Oceania + AF: Africa + EU: Europe + +world_iso: + Africa: + DZ: algeria + AO: angola + BJ: benin + BW: botswana + BF: burkina-faso + BI: burundi + CM: cameroon + CF: central-african-republic + TD: chad + CG: congo-brazzaville + CD: congo-democratic-republic + DJ: djibouti + EG: egypt + GQ: equatorial-guinea + ER: eritrea + ET: ethiopia + GA: gabon + GH: ghana + GW: guinea-bissau + GN: guinea + CI: ivory-coast + KE: kenya + LS: lesotho + LR: liberia + LY: libya + MG: madagascar + MW: malawi + ML: mali + MR: mauritania + MU: mauritius + MA: morocco + MZ: mozambique + NA: namibia + NE: niger + NG: nigeria + RW: rwanda + SN: senegal + GM: gambia + SL: sierra-leone + SO: somalia + ZA: south-africa + SS: south-sudan + SD: sudan + SZ: swaziland + TZ: tanzania + TG: togo + TN: tunisia + UG: uganda + ZM: zambia + ZW: zimbabwe + EH: western-sahara + Asia: + AF: afghanistan + AM: armenia + AZ: azerbaijan + BH: bahrain + BD: bangladesh + BT: bhutan + KH: cambodia + CN: china + CY: cyprus + GE: georgia + IN: india + ID: indonesia + IR: iran + IQ: iraq + IL: israel + JP: japan + JO: jordan + KZ: kazakhstan + KP: north-korea + KR: south-korea + KW: kuwait + KG: kyrgyzstan + LA: lao-people's-democratic-republic + LB: lebanon + MY: malaysia + BN: brunei + MV: maldives + MN: mongolia + MM: myanmar + NP: nepal + OM: oman + PK: pakistan + PS: palestine + PH: philippines + QA: qatar + RU: russian-federation + SA: saudi-arabia + SG: singapore + LK: sri-lanka + SY: syria + TW: taiwan + TJ: tajikistan + TH: thailand + TL: timor-leste + TR: turkey + TM: turkmenistan + AE: united-arab-emirates + UZ: uzbekistan + VN: vietnam + YE: yemen + Oceania: + AS: american-samoa + AU: australia + FJ: fiji + KI: kiribati + # "MP": "northern-mariana-islands", +# "PN": "pitcairn-islands", +# "AX": "aland-islands" +# "MS": "montserrat" +# "VI": "united-states-virgin-islands", +# "GI": "gibraltar", +# "IO": "british-indian-ocean-territory", +# "IM": "isle of man", # Island +# "VG": "british-virgin-islands", # Islands +# "WS": "samoa" +# "KP": "north-korea" +# "SX": "sint-maarten" +# "TL": "east-timor" +# "VC": "saint-vincent-and-the-grenadines" +# "ST": "sao-tome-and-principe", +# "SC": "seychelles", +# "TC": "turks-and-caicos", +# "UM": "united-states-minor-outlying-islands", #Islands +# "WF": "wallis-and-futuna", # Island +# "PN": "pitcairn-islands", # Islands +# "CI": "cote-d-ivoire", +# "IR": "iran", +# "TF": "french-southern-territories", +# "SZ": "swaziland", +# "BV": "bouvet-island", +# "CG": "congo-brazzaville", +# "TK": "tokelau", +# "CX": "christmas-island", +# "PG": "papua-new-guinea", +# "VU": "vanuatu", +# "NU": "niue", +# "KI": "kiribati", +# "NF": "norfolk-island", +# "MP": "northern-mariana-islands", +# "FO": "faroe-islands", +# "CC": "cocos-islands", +# "TL": "east-timor", +# "XK": "kosovo", +# "TC": "turks-and-caicos-islands", +# "MF": "saint-martin", +# "SX": "sint-maarten", +# "BL": "saint-barthelemy", +# "CZ": "czech-republic", +# "SM": "san-marino", +# "VA": "vatican-city", +# "LA": "laos", +# "BQ": "bonaire-sint-eustatius-saba", +# "CW": "curacao", +# "PM": "saint-pierre-and-miquelon", +# "XS": "spratly-islands", +# "MF": "saint-martin", +# "TF": "french-southern-territories", +# "RE": "reunion", +# "CI": "cote-d-ivoire", +# "IR": "iran", +# "TF": "french-southern-territories", +# "SZ": "swaziland", +# "BV": "bouvet-island", +# "CG": "congo-brazzaville", +# "TK": "tokelau", +# "CX": "christmas-island", +# "PG": "papua-new-guinea", +# "VU": "vanuatu", +# "NU": "niue", +# "KI": "kiribati", +# "NF": "norfolk-island", +# "MP": "northern-mariana-islands", +# "FO": "faroe-islands", +# "CC": "cocos-islands", +# "TL": "east-timor", +# "XK": "kosovo", +# "TC": "turks-and-caicos-islands", +# "MF": "saint-martin", +# "SX": "sint-maarten", +# "BL": "saint-barthelemy", +# "CZ": "czech-republic", +# "SM": "san-marino", +# "VA": "vatican-city", +# "LA": "laos", +# "BQ": "bonaire-sint-eustatius-saba", +# "CW": "curacao", +# "PM": "saint-pierre-and-miquelon", +# "XS": "spratly-islands", +# "MF": "saint-martin", +# "TF": "french-southern-territories", +# "RE": "reunion", +# "CI": "cote-d-ivoire", +# "IR": "iran", +# "TF": "french-southern-territories", +# "SZ": "swaziland", +# "BV": "bouvet-island", +# "CG": "congo-brazzaville", +# "TK": "tokelau", +# "CX": "christmas-island", +# "PG": "papua-new-guinea", +# "VU": "vanuatu", +# "NU": "niue", +# "KI": "kiribati", +# "NF": "norfolk-island", +# "MP": "northern-mariana-islands", +# "FO": "faroe-islands", +# "CC": "cocos-islands", +# "TL": "east-timor", +# "XK": "kosovo", +# "TC": "turks-and-caicos-islands", +# "MF": "saint-martin", +# "SX": "sint-maarten", +# "BL": "saint-barthelemy", +# "CZ": "czech-republic", +# "SM": "san-marino", +# "VA": "vatican-city", +# "LA": "laos", +# "BQ": "bonaire-sint-eustatius-saba", +# "CW": "curacao", +# "PM": "saint-pierre-and-miquelon", +# "XS": "spratly-islands", +# "MF": "saint-martin", +# "TF": "french-southern-territories", +# "RE": "reunion", +# "CI": "cote-d-ivoire", +# "IR": "iran", +# "TF": "french-southern-territories", +# "SZ": "swaziland", +# "BV": "bouvet-island", +# "CG": "congo-brazzaville", +# "TK": "tokelau", +# "CX": "christmas-island", +# "PG": "papua-new-guinea", +# "VU": "vanuatu", +# "NU": "niue", +# "KI": "kiribati", +# "NF": "norfolk-island", +# "MP": "northern-mariana-islands", +# "FO": "faroe-islands", +# "CC": "cocos-islands", +# "TL": "east-timor", +# "XK": "kosovo", +# "TC": "turks-and-caicos-islands", +# "MF": "saint-martin", +# "SX": "sint-maarten", +# "BL": "saint-barthelemy From 9a3ed3819b8b950008ffb0573216c5559df927c7 Mon Sep 17 00:00:00 2001 From: Thomas Kouroughli Date: Sat, 5 Aug 2023 00:23:44 +0100 Subject: [PATCH 03/32] working version of clean_osm --- configs/osm_config.yaml | 595 ++++++++++++++++++++--------------- scripts/_helpers.py | 11 +- scripts/build_osm_network.py | 9 +- scripts/download_osm_data.py | 4 +- 4 files changed, 365 insertions(+), 254 deletions(-) diff --git a/configs/osm_config.yaml b/configs/osm_config.yaml index 8d0777c7a..a017b556e 100644 --- a/configs/osm_config.yaml +++ b/configs/osm_config.yaml @@ -48,252 +48,355 @@ continents: EU: Europe world_iso: + # world_iso maps ISO country codes to country names Africa: - DZ: algeria - AO: angola - BJ: benin - BW: botswana - BF: burkina-faso - BI: burundi - CM: cameroon - CF: central-african-republic - TD: chad - CG: congo-brazzaville - CD: congo-democratic-republic - DJ: djibouti - EG: egypt - GQ: equatorial-guinea - ER: eritrea - ET: ethiopia - GA: gabon - GH: ghana - GW: guinea-bissau - GN: guinea - CI: ivory-coast - KE: kenya - LS: lesotho - LR: liberia - LY: libya - MG: madagascar - MW: malawi - ML: mali - MR: mauritania - MU: mauritius - MA: morocco - MZ: mozambique - NA: namibia - NE: niger - NG: nigeria - RW: rwanda - SN: senegal - GM: gambia - SL: sierra-leone - SO: somalia - ZA: south-africa - SS: south-sudan - SD: sudan - SZ: swaziland - TZ: tanzania - TG: togo - TN: tunisia - UG: uganda - ZM: zambia - ZW: zimbabwe - EH: western-sahara + DZ: "algeria" # Algeria + AO: "angola" + BJ: "benin" + BW: "botswana" + # IO: "british-indian-ocean-territory" # Island + BF: "burkina-faso" + BI: "burundi" + CM: "cameroon" + # IC: "canary-islands" # Island + # CV: "cape-verde" + CF: "central-african-republic" + TD: "chad" + # KM: "comoros" + CG: "congo-brazzaville" + CD: "congo-democratic-republic" + DJ: "djibouti" + EG: "egypt" + GQ: "equatorial-guinea" + ER: "eritrea" + ET: "ethiopia" + # TF: "french-southern-territories" # Island + GA: "gabon" + GH: "ghana" + GW: "guinea-bissau" # No Data + GN: "guinea" + CI: "ivory-coast" + KE: "kenya" + LS: "lesotho" + LR: "liberia" + LY: "libya" + MG: "madagascar" + MW: "malawi" + ML: "mali" + MR: "mauritania" + MU: "mauritius" + # YT: "mayotte" # Island + MA: "morocco" + MZ: "mozambique" + NA: "namibia" + NE: "niger" + NG: "nigeria" + # RE: "reunion" # Island + RW: "rwanda" + # saint-helena-ascension-and-tristan-da-cunha # Islands + # ST: "sao-tome-and-principe" + SN: "senegal" + GM: "gambia" + # SC: "seychelles" + SO: "somalia" # No Data + # south-africa-and-lesotho + ZA: "south-africa" + SS: "south-sudan" + SD: "sudan" + SZ: "swaziland" + TZ: "tanzania" + TG: "togo" + TN: "tunisia" + UG: "uganda" + ZM: "zambia" + ZW: "zimbabwe" + EH: "western-sahara" + Asia: - AF: afghanistan - AM: armenia - AZ: azerbaijan - BH: bahrain - BD: bangladesh - BT: bhutan - KH: cambodia - CN: china - CY: cyprus - GE: georgia - IN: india - ID: indonesia - IR: iran - IQ: iraq - IL: israel - JP: japan - JO: jordan - KZ: kazakhstan - KP: north-korea - KR: south-korea - KW: kuwait - KG: kyrgyzstan - LA: lao-people's-democratic-republic - LB: lebanon - MY: malaysia - BN: brunei - MV: maldives - MN: mongolia - MM: myanmar - NP: nepal - OM: oman - PK: pakistan - PS: palestine - PH: philippines - QA: qatar - RU: russian-federation - SA: saudi-arabia - SG: singapore - LK: sri-lanka - SY: syria - TW: taiwan - TJ: tajikistan - TH: thailand - TL: timor-leste - TR: turkey - TM: turkmenistan - AE: united-arab-emirates - UZ: uzbekistan - VN: vietnam - YE: yemen + AF: "afghanistan" + AM: "armenia" + AZ: "azerbaijan" + BH: "bahrain" + BD: "bangladesh" + BT: "bhutan" + # IO: "british-indian-ocean-territory" + KH: "cambodia" + CN: "china" + # CX: "christmas-island" # Island + # CC: "cocos-keeling-islands" # Island + CY: "cyprus" + # EG: "egypt" # leads to bug -> missing ssp file when executing ["Africa"] + GE: "georgia" + # HK: "hong-kong" # no more with gadm 4.1 + IN: "india" + ID: "indonesia" + IR: "iran" + IQ: "iraq" + IL: "israel" + JP: "japan" + JO: "jordan" + KZ: "kazakhstan" + KP: "north-korea" + KR: "south-korea" + KW: "kuwait" + KG: "kyrgyzstan" + LA: "lao-peoples-democratic-republic" + LB: "lebanon" + MY: "malaysia" + BN: "brunei" + MV: "maldives" + MN: "mongolia" + MM: "myanmar" + NP: "nepal" + OM: "oman" + PK: "pakistan" + PS: "palestine" + PH: "philippines" + QA: "qatar" + RU: "russian-federation" + SA: "saudi-arabia" + SG: "singapore" # merged with MY + # XS: "spratly-islands" # Island + LK: "sri-lanka" + SY: "syria" + TW: "taiwan" + TJ: "tajikistan" + TH: "thailand" + TL: "timor-leste" + TR: "turkey" + TM: "turkmenistan" + AE: "united-arab-emirates" + # XD: "united-nations-neutral-zone" + UZ: "uzbekistan" + VN: "vietnam" + YE: "yemen" + Oceania: - AS: american-samoa - AU: australia - FJ: fiji - KI: kiribati - # "MP": "northern-mariana-islands", -# "PN": "pitcairn-islands", -# "AX": "aland-islands" -# "MS": "montserrat" -# "VI": "united-states-virgin-islands", -# "GI": "gibraltar", -# "IO": "british-indian-ocean-territory", -# "IM": "isle of man", # Island -# "VG": "british-virgin-islands", # Islands -# "WS": "samoa" -# "KP": "north-korea" -# "SX": "sint-maarten" -# "TL": "east-timor" -# "VC": "saint-vincent-and-the-grenadines" -# "ST": "sao-tome-and-principe", -# "SC": "seychelles", -# "TC": "turks-and-caicos", -# "UM": "united-states-minor-outlying-islands", #Islands -# "WF": "wallis-and-futuna", # Island -# "PN": "pitcairn-islands", # Islands -# "CI": "cote-d-ivoire", -# "IR": "iran", -# "TF": "french-southern-territories", -# "SZ": "swaziland", -# "BV": "bouvet-island", -# "CG": "congo-brazzaville", -# "TK": "tokelau", -# "CX": "christmas-island", -# "PG": "papua-new-guinea", -# "VU": "vanuatu", -# "NU": "niue", -# "KI": "kiribati", -# "NF": "norfolk-island", -# "MP": "northern-mariana-islands", -# "FO": "faroe-islands", -# "CC": "cocos-islands", -# "TL": "east-timor", -# "XK": "kosovo", -# "TC": "turks-and-caicos-islands", -# "MF": "saint-martin", -# "SX": "sint-maarten", -# "BL": "saint-barthelemy", -# "CZ": "czech-republic", -# "SM": "san-marino", -# "VA": "vatican-city", -# "LA": "laos", -# "BQ": "bonaire-sint-eustatius-saba", -# "CW": "curacao", -# "PM": "saint-pierre-and-miquelon", -# "XS": "spratly-islands", -# "MF": "saint-martin", -# "TF": "french-southern-territories", -# "RE": "reunion", -# "CI": "cote-d-ivoire", -# "IR": "iran", -# "TF": "french-southern-territories", -# "SZ": "swaziland", -# "BV": "bouvet-island", -# "CG": "congo-brazzaville", -# "TK": "tokelau", -# "CX": "christmas-island", -# "PG": "papua-new-guinea", -# "VU": "vanuatu", -# "NU": "niue", -# "KI": "kiribati", -# "NF": "norfolk-island", -# "MP": "northern-mariana-islands", -# "FO": "faroe-islands", -# "CC": "cocos-islands", -# "TL": "east-timor", -# "XK": "kosovo", -# "TC": "turks-and-caicos-islands", -# "MF": "saint-martin", -# "SX": "sint-maarten", -# "BL": "saint-barthelemy", -# "CZ": "czech-republic", -# "SM": "san-marino", -# "VA": "vatican-city", -# "LA": "laos", -# "BQ": "bonaire-sint-eustatius-saba", -# "CW": "curacao", -# "PM": "saint-pierre-and-miquelon", -# "XS": "spratly-islands", -# "MF": "saint-martin", -# "TF": "french-southern-territories", -# "RE": "reunion", -# "CI": "cote-d-ivoire", -# "IR": "iran", -# "TF": "french-southern-territories", -# "SZ": "swaziland", -# "BV": "bouvet-island", -# "CG": "congo-brazzaville", -# "TK": "tokelau", -# "CX": "christmas-island", -# "PG": "papua-new-guinea", -# "VU": "vanuatu", -# "NU": "niue", -# "KI": "kiribati", -# "NF": "norfolk-island", -# "MP": "northern-mariana-islands", -# "FO": "faroe-islands", -# "CC": "cocos-islands", -# "TL": "east-timor", -# "XK": "kosovo", -# "TC": "turks-and-caicos-islands", -# "MF": "saint-martin", -# "SX": "sint-maarten", -# "BL": "saint-barthelemy", -# "CZ": "czech-republic", -# "SM": "san-marino", -# "VA": "vatican-city", -# "LA": "laos", -# "BQ": "bonaire-sint-eustatius-saba", -# "CW": "curacao", -# "PM": "saint-pierre-and-miquelon", -# "XS": "spratly-islands", -# "MF": "saint-martin", -# "TF": "french-southern-territories", -# "RE": "reunion", -# "CI": "cote-d-ivoire", -# "IR": "iran", -# "TF": "french-southern-territories", -# "SZ": "swaziland", -# "BV": "bouvet-island", -# "CG": "congo-brazzaville", -# "TK": "tokelau", -# "CX": "christmas-island", -# "PG": "papua-new-guinea", -# "VU": "vanuatu", -# "NU": "niue", -# "KI": "kiribati", -# "NF": "norfolk-island", -# "MP": "northern-mariana-islands", -# "FO": "faroe-islands", -# "CC": "cocos-islands", -# "TL": "east-timor", -# "XK": "kosovo", -# "TC": "turks-and-caicos-islands", -# "MF": "saint-martin", -# "SX": "sint-maarten", -# "BL": "saint-barthelemy + AS: "american-samoa" # Island + AU: "australia" + # CP: "ile-de-clipperton" # Island In gadm as XCL + # CK: "cook-islands" # Island + FJ: "fiji" + # PF: "french-polynesia" # Island + # GU: "guam" # Island + KI: "kiribati" # Island + MH: "marshall-islands" + FM: "micronesia" + NR: "nauru" + NC: "new-caledonia" # Island + NZ: "new-zealand" + NU: "niue" # Island + # NF: "norfolk-island" # Island + # MP: "northern-mariana-islands" + PG: "papua-new-guinea" + # PN: "pitcairn-islands" # Islands + PW: "palau" + WS: "samoa" + SB: "solomon-islands" + # TK: "tokelau" # Island + TO: "tonga" + TV: "tuvalu" + # UM: "united-states-minor-outlying-islands" # Islands + VU: "vanuatu" # Island + # WF: "wallis-and-futuna" # Island + + Europe: + # AX: "aland-islands" # Island + AL: "albania" + AD: "andorra" + AM: "armenia" + AT: "austria" + AZ: "Azerbaijan" + BY: "belarus" + BE: "belgium" + BA: "bosnia-herzegovina" + BG: "bulgaria" + HR: "croatia" + CY: "cyprus" + CZ: "czech-republic" + DK: "denmark" + EE: "estonia" + # FO: "faroe-islands" # Islands + FI: "finland" + FR: "france" + GE: "georgia" + DE: "germany" + # GI: "gibraltar" # Island ? + GR: "greece" + # GG: "guernsey" # Island + HU: "hungary" + IS: "iceland" + # IE: "ireland-and-northern-ireland" + # IM: "isle-of-man" # Island + IT: "italy" + # JE: "jersey" # Island + KZ: "kazakhstan" + XK: "kosovo" + LV: "latvia" + LI: "liechtenstein" + LT: "lithuania" + LU: "luxembourg" + MK: "macedonia" + MT: "malta" + MD: "moldova" + MC: "monaco" + ME: "montenegro" + NL: "netherlands" + NO: "norway" + PL: "poland" + PT: "portugal" + RO: "romania" + RU: "russia" + SM: "san-marino" + RS: "serbia" + SK: "slovakia" + SI: "slovenia" + ES: "spain" + # SJ: "svalbard-and-jan-mayen" # Islands + SE: "sweden" + CH: "switzerland" + UA: "ukraine" + GB: "great-britain" + TR: "turkey" + VA: "vatican" + + NorthAmerica: + # AI: "anguilla" # Island + AG: "antigua-and-barbuda" + # AW: "aruba" # Islands + BS: "bahamas" + BB: "barbados" + # BM: "bermuda" # Islands + # BQ: "bonaire" # Islands + # VG: "british-virgin-islands" # Islands + CA: "canada" + # KY: "cayman-islands" # Islands + CU: "cuba" + # CW: "curacao" # Islands + DM: "dominica" + DO: "dominican-republic" + GL: "greenland" + GD: "grenada" + # GP: "guadeloupe" # Islands + HT: "haiti" + JM: "jamaica" + # MQ: "martinique" # Islands + MX: "mexico" + # MS: "montserrat" # Islands + US: "united-states-of-america" + PR: "puerto-rico" + # BL: "saint-barthelemy" # Islands + KN: "saint-kitts-and-nevis" + LC: "saint-lucia" + # MF: "saint-martin" # Islands + # PM: "saint-pierre-and-miquelon" # Islands + VC: "saint-vincent-and-the-grenadines" + # SX: "saint-marteen" # Islands + TT: "trinidad-and-tobago" + # TC: "turks-and-caicos" # Islands + # UM: "united-states-minor-outlying-islands" # Islands + # VI: "united-states-virgin-islands" # Islands + BZ: "belize" + CR: "costa-rica" + HN: "honduras" + GT: "guatemala" + NI: "nicaragua" + PA: "panama" + SV: "el-salvador" + + SouthAmerica: + AR: "argentina" + BO: "bolivia" + BR: "brazil" + CL: "chile" + CO: "colombia" + EC: "ecuador" + FK: "falkland-islands" # Islands + GF: "french-guiana" + GY: "guyana" # No Data + PE: "peru" + PY: "paraguay" + SR: "suriname" + UY: "uruguay" + VE: "venezuela" + + + +# Based on: https://waml.org/waml-information-bulletin/46-3/index-to-lc-g-schedule/1-world/ +continent_regions: + SCR: ["DK", "NO", "SE", "FI", "IS"] # SCANDINAVIAN REGION + EER: ["BY", "PL", "CZ", "RU", "SK", "UA", "LT", "LV", "EE", "FI", "MD"] # EASTERN EUROPEAN REGION + CER: ["AT", "CH", "CZ", "DE", "HU", "PL", "SK", "LI"] # CENTRAL EUROPEAN REGION + BPR: ["AL", "BA", "BG", "GR", "HR", "ME", "RO", "SI", "RS", "ME", "MK"] # BALKAN PENISULAN REGION + WER: ["FR", "BE", "GB", "IE", "LU", "MC", "NL", "AD"] # WESTERN EUROPE + SER: ["ES", "AD", "IT", "PT", "SM", "MT"] # SOUTHERN EUROPEAN REGION + NAR: ["EG", "LY", "TN", "DZ", "MA", "EH", "SD", "SS"] # NORTHERN AFRICAN REGION + WAR: ["MR", "ML", "NE", "NG", "BJ", "BF", "TG", "GH", "CI", "LR", "SL", "GN", "SN", "GM"] # WESTERN AFRICAN REGION + CAR: ["TD", "CF", "CM", "GQ", "GA", "CD", "CG", "AO"] # CENTRAL AFRICAN REGION + EAR: ["ER", "ET", "UG", "KE", "RW", "BI", "TZ", "MZ", "DJ", "MG"] # EASTERN AFRICAN REGION + SAR: ["MW", "ZM", "ZW", "BW", "NA", "SZ", "LS", "ZA"] # SOUTHERN AFRICAN REGION + KVR: ["AZ", "GE", "AM"] # Asian regions + WAS: ["TR", "AM", "AZ", "BH", "CY", "GE", "IQ", "IL", "JO", "KW", "LB", "OM", "PS", "QA", "SA", "SY", "AE", "YE"] # WEST ASIAN REGION + FEAR: ["JP", "KP", "KR", "CN", "TW", "MN"] # FAR EASTERN ASIAN REGION + SEAR: ["LA", "TH", "KH", "VN", "PH", "MY", "SG", "BN", "ID"] # SOUTHEASTERN ASIAN REGION + CASR: ["KZ", "KG", "UZ", "TM", "TJ"] # CENTRAL ASIAN REGION + SASR: ["MM", "BD", "BT", "NP", "IN", "LK", "PK", "AF"] # SOUTHERN ASIAN REGION + ASEAN: ["VN", "TH", "ID", "PH", "MY", "MM", "KH", "LA", "SG", "BN"] # ASEAN + MEAR: ["TR", "SY", "LB", "CY", "IQ", "IR", "JO", "IL", "PS", "AE", "YE", "KW", "BH", "QA", "SA", "OM"] # MIDDLE EASTERN ASIAN REGION + NACR: ["CA", "GL", "MX", "US"] # American continent regions + LACR: ["AR", "BO", "BR", "CL", "CO", "EC", "GF", "PE", "PY", "SR", "UY", "VE"] # SOUTHERN LATIN AMERICAN REGION + CACR: ["BZ", "GT", "SV", "HN", "NI", "CR", "PA"] # CENTRAL AMERICAN REGION + AUO: ["AU", "NC", "NZ", "PG"] # Australasia + UnitedNations: "cc.convert(names=cc.UN.name_short, to='ISO2')" # UnitedNations + +iso_to_geofk_dict: + EH: "MA" # Western Sahara -> Morocco + SN: "SN-GM" # Senegal -> Senegal-Gambia + GM: "SN-GM" # Gambia -> Senegal-Gambia + KM: "comores" # Comores + IC: "canary-islands" # Canary islands + SG: "MY" # Singapore -> Malaysia-Singapore-Brunei + BN: "MY" # Brunei -> Malaysia-Singapore-Brunei + SA: "QA-AE-OM-BH-KW" # Saudi Arabia -> Gulf Cooperation Council + KW: "QA-AE-OM-BH-KW" # Kuwait -> Gulf Cooperation Council + BH: "QA-AE-OM-BH-KW" # Bahrain -> Gulf Cooperation Council + QA: "QA-AE-OM-BH-KW" # Qatar -> Gulf Cooperation Council + AE: "QA-AE-OM-BH-KW" # United Arab Emirates -> Gulf Cooperation Council + OM: "QA-AE-OM-BH-KW" # Oman -> Gulf Cooperation Council + PS: "PS-IL" # Israel and Palestine are merged in OSM + IL: "PS-IL" # Israel and Palestine are merged in OSM + SM: "IT" # San-Marino is merged to Italy + VA: "IT" # Vatican is merged to Italy + HT: "haiti-and-domrep" # Haiti and Dominican Republic are merged in OSM + DO: "haiti-and-domrep" # Haiti and Dominican Republic are merged in OSM + PA: "panama" # Panama + NF: "AU" # norfolk island is an AU territory + MP: "american-oceania" # northern mariana islands are US territory + GU: "american-oceania" # Guam is a US territory + AS: "american-oceania" # American Samoa is a US territory + CP: "ile-de-clipperton" # Ile de clipperton + PF: "polynesie-francaise" # Polynesie Francaise + VU: "vanuatu" # Vanuatu + TK: "tokelau" # Tokelau + MH: "marshall-islands" # Marshal islands + PN: "pitcairn-islands" # Pitcairn + WF: "wallis-et-futuna" # Wallis et Fortnuna + XK: "RS-KM" # Kosovo + BS: "bahamas" # Bahamas + BB: "central-america" # Barbados + CU: "cuba" # Cuba + RE: "reunion" # Reunion island (France) + YT: "mayotte" # "Mayotte island (France)" + GG: "guernsey-jersey" # Guernsey + JE: "guernsey-jersey" # Jersey + IM: "isle-of-man" # Isle of man + GP: "guadeloupe" # guadeloupe + JM: "jamaica" # jamaica + TT: "central-america" # Trinidad and Tobago + AG: "central-america" # Antigua e Barbuda + DM: "central-america" # Dominique + LC: "central-america" # Santa Lucia + VC: "central-america" # Saint Vincent e Grenadine + KN: "central-america" # Saint Kitts e Nevis + GD: "central-america" # Grenada + diff --git a/scripts/_helpers.py b/scripts/_helpers.py index a132997ce..6823f9940 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -10,10 +10,16 @@ import country_converter as coco import geopandas as gpd -import numpy as np import pandas as pd +import yaml REGION_COLS = ["geometry", "name", "x", "y", "country"] +with open("configs/osm_config.yaml", "r") as f: + osm_config = yaml.safe_load(f) +world_iso = osm_config["world_iso"] +continent_regions = osm_config["continent_regions"] +iso_to_geofk_dict = osm_config["iso_to_geofk_dict"] +osm_clean_columns = osm_config["osm_clean_columns"] def sets_path_to_root(root_directory_name): @@ -449,7 +455,6 @@ def getContinent(code): getContinent(code) >>> ["africa", "europe"] """ - from config_osm_data import world_iso continent_list = [] code_set = set(code) @@ -665,8 +670,6 @@ def create_country_list(input, iso_coding=True): """ import logging - from config_osm_data import continent_regions, world_iso - _logger = logging.getLogger(__name__) _logger.setLevel(logging.INFO) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index b89180c94..cdb93092a 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -11,8 +11,13 @@ import geopandas as gpd import numpy as np import pandas as pd -from _helpers import configure_logging, read_geojson, sets_path_to_root, to_csv_nafix -from config_osm_data import osm_clean_columns +from _helpers import ( + configure_logging, + osm_clean_columns, + read_geojson, + sets_path_to_root, + to_csv_nafix, +) from shapely.geometry import LineString, Point from shapely.ops import linemerge, split from tqdm import tqdm diff --git a/scripts/download_osm_data.py b/scripts/download_osm_data.py index f53bb7c93..458b1a10a 100644 --- a/scripts/download_osm_data.py +++ b/scripts/download_osm_data.py @@ -31,8 +31,7 @@ import shutil from pathlib import Path -from _helpers import configure_logging -from config_osm_data import iso_to_geofk_dict +from _helpers import configure_logging, iso_to_geofk_dict from earth_osm import eo logger = logging.getLogger(__name__) @@ -57,6 +56,7 @@ def country_list_to_geofk(country_list): full_codes_list : list Example ["NG","ZA"] """ + full_codes_list = [convert_iso_to_geofk(c_code) for c_code in set(country_list)] return full_codes_list From d8ef8c9e8a1a785f1afc7f38d40ed5bac3a0ed44 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 4 Aug 2023 23:24:02 +0000 Subject: [PATCH 04/32] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- configs/osm_config.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/configs/osm_config.yaml b/configs/osm_config.yaml index a017b556e..13a6bf0e8 100644 --- a/configs/osm_config.yaml +++ b/configs/osm_config.yaml @@ -111,7 +111,7 @@ world_iso: ZM: "zambia" ZW: "zimbabwe" EH: "western-sahara" - + Asia: AF: "afghanistan" AM: "armenia" @@ -170,7 +170,7 @@ world_iso: UZ: "uzbekistan" VN: "vietnam" YE: "yemen" - + Oceania: AS: "american-samoa" # Island AU: "australia" @@ -199,7 +199,7 @@ world_iso: # UM: "united-states-minor-outlying-islands" # Islands VU: "vanuatu" # Island # WF: "wallis-and-futuna" # Island - + Europe: # AX: "aland-islands" # Island AL: "albania" @@ -259,7 +259,7 @@ world_iso: GB: "great-britain" TR: "turkey" VA: "vatican" - + NorthAmerica: # AI: "anguilla" # Island AG: "antigua-and-barbuda" @@ -303,7 +303,7 @@ world_iso: NI: "nicaragua" PA: "panama" SV: "el-salvador" - + SouthAmerica: AR: "argentina" BO: "bolivia" @@ -399,4 +399,3 @@ iso_to_geofk_dict: VC: "central-america" # Saint Vincent e Grenadine KN: "central-america" # Saint Kitts e Nevis GD: "central-america" # Grenada - From 325189f672d4785c56b521833c6db44cb5c3b72e Mon Sep 17 00:00:00 2001 From: Thomas Kouroughli Date: Tue, 8 Aug 2023 14:36:52 +0100 Subject: [PATCH 05/32] change str to os.path object --- scripts/_helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 6823f9940..cd2b04888 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -14,7 +14,8 @@ import yaml REGION_COLS = ["geometry", "name", "x", "y", "country"] -with open("configs/osm_config.yaml", "r") as f: +osm_config_path = os.path.join("configs", "osm_config.yaml") +with open(osm_config_path, "r") as f: osm_config = yaml.safe_load(f) world_iso = osm_config["world_iso"] continent_regions = osm_config["continent_regions"] From ff965d07552b4a4ca02d2bc8218b30a98d4300c0 Mon Sep 17 00:00:00 2001 From: Thomas Kouroughli Date: Tue, 8 Aug 2023 14:39:10 +0100 Subject: [PATCH 06/32] uncomment necessary countries --- configs/osm_config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/configs/osm_config.yaml b/configs/osm_config.yaml index 13a6bf0e8..137648e02 100644 --- a/configs/osm_config.yaml +++ b/configs/osm_config.yaml @@ -59,10 +59,10 @@ world_iso: BI: "burundi" CM: "cameroon" # IC: "canary-islands" # Island - # CV: "cape-verde" + CV: "cape-verde" CF: "central-african-republic" TD: "chad" - # KM: "comoros" + KM: "comoros" CG: "congo-brazzaville" CD: "congo-democratic-republic" DJ: "djibouti" @@ -94,10 +94,10 @@ world_iso: # RE: "reunion" # Island RW: "rwanda" # saint-helena-ascension-and-tristan-da-cunha # Islands - # ST: "sao-tome-and-principe" + ST: "sao-tome-and-principe" SN: "senegal" GM: "gambia" - # SC: "seychelles" + SC: "seychelles" SO: "somalia" # No Data # south-africa-and-lesotho ZA: "south-africa" @@ -226,7 +226,7 @@ world_iso: # GG: "guernsey" # Island HU: "hungary" IS: "iceland" - # IE: "ireland-and-northern-ireland" + IE: "ireland-and-northern-ireland" # IM: "isle-of-man" # Island IT: "italy" # JE: "jersey" # Island From e868681c048faee1e6ba690d5d42211ed5e62316 Mon Sep 17 00:00:00 2001 From: Thomas Kouroughli Date: Tue, 8 Aug 2023 14:41:42 +0100 Subject: [PATCH 07/32] add back descriptive comments --- configs/osm_config.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/configs/osm_config.yaml b/configs/osm_config.yaml index 137648e02..fe7876296 100644 --- a/configs/osm_config.yaml +++ b/configs/osm_config.yaml @@ -1,3 +1,4 @@ +# Columns of CLEAN OSM DATA names and dtypes osm_clean_columns: substation: bus_id: object @@ -38,7 +39,11 @@ osm_clean_columns: dc: bool country: object geometry: object - +# Python dictionary of ISO 3166-1-alpha-2 codes, as per publicly +# available data on official ISO site in July 2015. +# +# Available under MIT license +# Dimitris Karagkasidis, https://github.com/pageflt continents: LA: NorthAmerica SA: SouthAmerica @@ -323,6 +328,7 @@ world_iso: # Based on: https://waml.org/waml-information-bulletin/46-3/index-to-lc-g-schedule/1-world/ +# Australasia region includes New Caledonia and Papua New Guinea continent_regions: SCR: ["DK", "NO", "SE", "FI", "IS"] # SCANDINAVIAN REGION EER: ["BY", "PL", "CZ", "RU", "SK", "UA", "LT", "LV", "EE", "FI", "MD"] # EASTERN EUROPEAN REGION From d37d94e6b76592b0fc2a09a7c4517f942a5db536 Mon Sep 17 00:00:00 2001 From: Thomas Kouroughli Date: Wed, 9 Aug 2023 11:04:14 +0100 Subject: [PATCH 08/32] add read_osm_config_function --- configs/osm_config.yaml | 15 ++++++++++++ scripts/_helpers.py | 47 +++++++++++++++++++++++++++++------- scripts/build_osm_network.py | 4 +-- scripts/download_osm_data.py | 6 +++-- 4 files changed, 59 insertions(+), 13 deletions(-) diff --git a/configs/osm_config.yaml b/configs/osm_config.yaml index fe7876296..36ed6d07c 100644 --- a/configs/osm_config.yaml +++ b/configs/osm_config.yaml @@ -1,3 +1,8 @@ +# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC0-1.0 + + # Columns of CLEAN OSM DATA names and dtypes osm_clean_columns: substation: @@ -355,6 +360,16 @@ continent_regions: AUO: ["AU", "NC", "NZ", "PG"] # Australasia UnitedNations: "cc.convert(names=cc.UN.name_short, to='ISO2')" # UnitedNations +# Geofabrik and iso norm deviate for some countries and domains + +# dictionary of correspondence between iso country codes and geofabrik codes containing those information +# This dictionary instructs the script download_osm_data about how to successfully download data +# from countries that are aggregated into osm. +# For example, Senegal (SN) and Gambia (GM) cannot be downloaded from OSM separately, but only jointly as SN-GM +# That's the reason why in this dictionary they can be found the following entries: +# "SN": "SN-GM" +# "GM": "SN-GM" +# This instruct the workflow that when the country "SN" is requested, then it shall download the "SN-GM" file iso_to_geofk_dict: EH: "MA" # Western Sahara -> Morocco SN: "SN-GM" # Senegal -> Senegal-Gambia diff --git a/scripts/_helpers.py b/scripts/_helpers.py index cd2b04888..7b03c3b6c 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -14,13 +14,41 @@ import yaml REGION_COLS = ["geometry", "name", "x", "y", "country"] -osm_config_path = os.path.join("configs", "osm_config.yaml") -with open(osm_config_path, "r") as f: - osm_config = yaml.safe_load(f) -world_iso = osm_config["world_iso"] -continent_regions = osm_config["continent_regions"] -iso_to_geofk_dict = osm_config["iso_to_geofk_dict"] -osm_clean_columns = osm_config["osm_clean_columns"] + + +# world_iso = osm_config["world_iso"] +# continent_regions = osm_config["continent_regions"] +# iso_to_geofk_dict = osm_config["iso_to_geofk_dict"] +# osm_clean_columns = osm_config["osm_clean_columns"] + + +def read_osm_config(*args): + """ + Reads the osm_config.yaml file and returns values based on the provided key + arguments. + + Parameters + ---------- + *args : str + One or more key arguments corresponding to the values + to retrieve from the config file. + + Returns + ------- + tuple + A tuple containing values from the osm_config.yaml + file corresponding to the provided keys. + + Examples + -------- + >>> values = read_osm_config("key1", "key2") + >>> print(values) + ('value1', 'value2') + """ + osm_config_path = os.path.join("configs", "osm_config.yaml") + with open(osm_config_path, "r") as f: + osm_config = yaml.safe_load(f) + return tuple([osm_config[a] for a in args]) def sets_path_to_root(root_directory_name): @@ -459,7 +487,7 @@ def getContinent(code): continent_list = [] code_set = set(code) - for continent in world_iso: + for continent in read_osm_config("world_iso"): single_continent_set = set(world_iso[continent]) if code_set.intersection(single_continent_set): continent_list.append(continent) @@ -682,6 +710,7 @@ def filter_codes(c_list, iso_coding=True): geofabrik-specific ones. When geofabrik codes are selected(iso_coding=False), ignore iso-specific names. """ + if ( iso_coding ): # if country lists are in iso coding, then check if they are 2-string @@ -703,7 +732,7 @@ def filter_codes(c_list, iso_coding=True): for value1 in input: codes_list = [] - + world_iso, continent_regions = read_osm_config("world_iso", "continent_regions") # extract countries in world if value1 == "Earth": for continent in world_iso.keys(): diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index cdb93092a..14aac6601 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -13,8 +13,8 @@ import pandas as pd from _helpers import ( configure_logging, - osm_clean_columns, read_geojson, + read_osm_config, sets_path_to_root, to_csv_nafix, ) @@ -810,7 +810,7 @@ def add_buses_to_empty_countries(country_list, fp_country_shapes, buses): def built_network(inputs, outputs, config, geo_crs, distance_crs, force_ac=False): logger.info("Stage 1/5: Read input data") - + osm_clean_columns = read_osm_config("osm_clean_columns") buses = read_geojson( inputs["substations"], osm_clean_columns["substation"].keys(), diff --git a/scripts/download_osm_data.py b/scripts/download_osm_data.py index 458b1a10a..50e045704 100644 --- a/scripts/download_osm_data.py +++ b/scripts/download_osm_data.py @@ -31,7 +31,7 @@ import shutil from pathlib import Path -from _helpers import configure_logging, iso_to_geofk_dict +from _helpers import configure_logging, iso_to_geofk_dict, read_osm_config from earth_osm import eo logger = logging.getLogger(__name__) @@ -62,7 +62,9 @@ def country_list_to_geofk(country_list): return full_codes_list -def convert_iso_to_geofk(iso_code, iso_coding=True, convert_dict=iso_to_geofk_dict): +def convert_iso_to_geofk( + iso_code, iso_coding=True, convert_dict=read_osm_config("iso_to_geofk_dict") +): """ Function to convert the iso code name of a country into the corresponding geofabrik In Geofabrik, some countries are aggregated, thus if a single From d5eee0d85263b9ef057813e7ca693e183c245b5a Mon Sep 17 00:00:00 2001 From: Thomas Kouroughli Date: Wed, 9 Aug 2023 13:44:58 +0100 Subject: [PATCH 09/32] add UnitedNations list --- configs/osm_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/osm_config.yaml b/configs/osm_config.yaml index 36ed6d07c..5bad450b7 100644 --- a/configs/osm_config.yaml +++ b/configs/osm_config.yaml @@ -358,7 +358,7 @@ continent_regions: LACR: ["AR", "BO", "BR", "CL", "CO", "EC", "GF", "PE", "PY", "SR", "UY", "VE"] # SOUTHERN LATIN AMERICAN REGION CACR: ["BZ", "GT", "SV", "HN", "NI", "CR", "PA"] # CENTRAL AMERICAN REGION AUO: ["AU", "NC", "NZ", "PG"] # Australasia - UnitedNations: "cc.convert(names=cc.UN.name_short, to='ISO2')" # UnitedNations + UnitedNations: ['AF', 'AL', 'DZ', 'AD', 'AO', 'AG', 'AR', 'AM', 'AU', 'AT', 'AZ', 'BS', 'BH', 'BD', 'BB', 'BY', 'BE', 'BZ', 'BJ', 'BT', 'BO', 'BA', 'BW', 'BR', 'BN', 'BG', 'BF', 'BI', 'CV', 'KH', 'CM', 'CA', 'CF', 'TD', 'CL', 'CN', 'CO', 'KM', 'CG', 'CR', 'CI', 'HR', 'CU', 'CY', 'CZ', 'DK', 'DJ', 'DM', 'DO', 'CD', 'EC', 'EG', 'SV', 'GQ', 'ER', 'EE', 'SZ', 'ET', 'FJ', 'FI', 'FR', 'GA', 'GM', 'GE', 'DE', 'GH', 'GR', 'GD', 'GT', 'GN', 'GW', 'GY', 'HT', 'HN', 'HU', 'IS', 'IN', 'ID', 'IR', 'IQ', 'IE', 'IL', 'IT', 'JM', 'JP', 'JO', 'KZ', 'KE', 'KI', 'KW', 'KG', 'LA', 'LV', 'LB', 'LS', 'LR', 'LY', 'LI', 'LT', 'LU', 'MK', 'MG', 'MW', 'MY', 'MV', 'ML', 'MT', 'MH', 'MR', 'MU', 'MX', 'FM', 'MD', 'MC', 'MN', 'ME', 'MA', 'MZ', 'MM', 'NA', 'NR', 'NP', 'NL', 'NZ', 'NI', 'NE', 'NG', 'KP', 'NO', 'OM', 'PK', 'PW', 'PA', 'PG', 'PY', 'PE', 'PH', 'PL', 'PT', 'QA', 'RO', 'RU', 'RW', 'WS', 'SM', 'ST', 'SA', 'SN', 'RS', 'SC', 'SL', 'SG', 'SK', 'SI', 'SB', 'SO', 'ZA', 'KR', 'SS', 'ES', 'LK', 'KN', 'LC', 'VC', 'SD', 'SR', 'SE', 'CH', 'SY', 'TJ', 'TZ', 'TH', 'TL', 'TG', 'TO', 'TT', 'TN', 'TR', 'TM', 'TV', 'UG', 'UA', 'AE', 'GB', 'US', 'UY', 'UZ', 'VU', 'VE', 'VN', 'YE', 'ZM', 'ZW'] # UnitedNations # Geofabrik and iso norm deviate for some countries and domains From 8526aad14b0d8ab3af4efc16e6869d8e5e647dcb Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Sat, 12 Aug 2023 00:46:07 +0200 Subject: [PATCH 10/32] Enable offshore wind --- scripts/add_electricity.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 6129a0de8..4a05b5ef8 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -329,23 +329,21 @@ def attach_wind_and_solar( suptech = tech.split("-", 2)[0] if suptech == "offwind": - continue - # TODO: Uncomment out and debug. - # underwater_fraction = ds["underwater_fraction"].to_pandas() - # connection_cost = ( - # snakemake.config["lines"]["length_factor"] * - # ds["average_distance"].to_pandas() * - # (underwater_fraction * - # costs.at[tech + "-connection-submarine", "capital_cost"] + - # (1.0 - underwater_fraction) * - # costs.at[tech + "-connection-underground", "capital_cost"] - # )) - # capital_cost = (costs.at["offwind", "capital_cost"] + - # costs.at[tech + "-station", "capital_cost"] + - # connection_cost) - # logger.info( - # "Added connection cost of {:0.0f}-{:0.0f} Eur/MW/a to {}". - # format(connection_cost.min(), connection_cost.max(), tech)) + underwater_fraction = ds["underwater_fraction"].to_pandas() + connection_cost = ( + snakemake.config["lines"]["length_factor"] * + ds["average_distance"].to_pandas() * + (underwater_fraction * + costs.at[tech + "-connection-submarine", "capital_cost"] + + (1.0 - underwater_fraction) * + costs.at[tech + "-connection-underground", "capital_cost"] + )) + capital_cost = (costs.at["offwind", "capital_cost"] + + costs.at[tech + "-station", "capital_cost"] + + connection_cost) + logger.info( + "Added connection cost of {:0.0f}-{:0.0f} Eur/MW/a to {}". + format(connection_cost.min(), connection_cost.max(), tech)) else: capital_cost = costs.at[tech, "capital_cost"] From 3daf756e98b3a2f599ad9b13f6d99b4c5d3a6a1e Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Sat, 12 Aug 2023 14:53:20 +0200 Subject: [PATCH 11/32] Remove and fix hydro in add_electricity --- scripts/add_electricity.py | 45 ++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 4a05b5ef8..ed0ec719a 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -331,19 +331,25 @@ def attach_wind_and_solar( if suptech == "offwind": underwater_fraction = ds["underwater_fraction"].to_pandas() connection_cost = ( - snakemake.config["lines"]["length_factor"] * - ds["average_distance"].to_pandas() * - (underwater_fraction * - costs.at[tech + "-connection-submarine", "capital_cost"] + - (1.0 - underwater_fraction) * - costs.at[tech + "-connection-underground", "capital_cost"] - )) - capital_cost = (costs.at["offwind", "capital_cost"] + - costs.at[tech + "-station", "capital_cost"] + - connection_cost) + snakemake.config["lines"]["length_factor"] + * ds["average_distance"].to_pandas() + * ( + underwater_fraction + * costs.at[tech + "-connection-submarine", "capital_cost"] + + (1.0 - underwater_fraction) + * costs.at[tech + "-connection-underground", "capital_cost"] + ) + ) + capital_cost = ( + costs.at["offwind", "capital_cost"] + + costs.at[tech + "-station", "capital_cost"] + + connection_cost + ) logger.info( - "Added connection cost of {:0.0f}-{:0.0f} Eur/MW/a to {}". - format(connection_cost.min(), connection_cost.max(), tech)) + "Added connection cost of {:0.0f}-{:0.0f} Eur/MW/a to {}".format( + connection_cost.min(), connection_cost.max(), tech + ) + ) else: capital_cost = costs.at[tech, "capital_cost"] @@ -455,9 +461,14 @@ def attach_hydro(n, costs, ppl): .rename(index=lambda s: str(s) + " hydro") ) - # TODO: remove this line to address nan when powerplantmatching is stable # Current fix, NaN technologies set to ROR - ppl.loc[ppl.technology.isna(), "technology"] = "Run-Of-River" + if ppl.technology.isna().any(): + n_nans = ppl.technology.isna().sum() + logger.warning( + f"Identified {n_nans} hydro powerplants with unknown technology.\n" + "Initialized to 'Run-Of-River'" + ) + ppl.loc[ppl.technology.isna(), "technology"] = "Run-Of-River" ror = ppl.query('technology == "Run-Of-River"') phs = ppl.query('technology == "Pumped Storage"') @@ -550,11 +561,11 @@ def attach_hydro(n, costs, ppl): hydro_max_hours = c.get("hydro_max_hours") hydro_stats = pd.read_csv( snakemake.input.hydro_capacities, comment="#", na_values=["-"], index_col=0 - ) + ).set_index("Country") e_target = hydro_stats["E_store[TWh]"].clip(lower=0.2) * 1e6 e_installed = hydro.eval("p_nom * max_hours").groupby(hydro.country).sum() e_missing = e_target - e_installed - missing_mh_i = hydro.query("max_hours == 0").index + missing_mh_i = hydro.query("max_hours.isnull()").index if hydro_max_hours == "energy_capacity_totals_by_country": max_hours_country = ( @@ -566,6 +577,8 @@ def attach_hydro(n, costs, ppl): hydro_stats["E_store[TWh]"] * 1e3 / hydro_stats["p_nom_discharge[GW]"] ) + max_hours_country.clip(0, inplace=True) + missing_countries = pd.Index(hydro["country"].unique()).difference( max_hours_country.dropna().index ) From 25fdd67b597f7c43e73d752b6cdf7ff21994db92 Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Sat, 12 Aug 2023 16:09:21 +0200 Subject: [PATCH 12/32] bugfix: sumup duplicated entries --- scripts/add_electricity.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index ed0ec719a..2bdf40a51 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -559,9 +559,16 @@ def attach_hydro(n, costs, ppl): if "hydro" in carriers and not hydro.empty: hydro_max_hours = c.get("hydro_max_hours") - hydro_stats = pd.read_csv( - snakemake.input.hydro_capacities, comment="#", na_values=["-"], index_col=0 - ).set_index("Country") + hydro_stats = ( + pd.read_csv( + snakemake.input.hydro_capacities, + comment="#", + na_values=["-"], + index_col=0, + ) + .groupby("Country") + .sum() + ) e_target = hydro_stats["E_store[TWh]"].clip(lower=0.2) * 1e6 e_installed = hydro.eval("p_nom * max_hours").groupby(hydro.country).sum() e_missing = e_target - e_installed From 98a45db1faf648f5351f7af8b217328a834057c2 Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Sat, 12 Aug 2023 16:09:58 +0200 Subject: [PATCH 13/32] Add release note --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index af2c1f6ce..7d53d49be 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -32,6 +32,8 @@ E.g. if a new rule becomes available describe how to use it `snakemake -j1 run_t * Add Asian cutout `PR #826 `__ +* Re-enable offshore wind and revise hydro `PR #830 `__ + PyPSA-Earth 0.2.2 ================= From 6cebf3b6336d410279bcd5d5e79c64c8b81005f7 Mon Sep 17 00:00:00 2001 From: "Katherine M. Antonio" Date: Mon, 14 Aug 2023 08:51:42 -0400 Subject: [PATCH 14/32] Add cutout South America (#835) * Update bundle_config.yaml * Update bundle_config.yaml I made changes to be consistent with the NorthAmerica, Europe bundle. I see the tabs aligning well with other bundles. Let me know if I'm missing something. * Update bundle_config.yaml Removed "databundles:" on line 341 --- configs/bundle_config.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/configs/bundle_config.yaml b/configs/bundle_config.yaml index 978824b80..c6064ebc5 100644 --- a/configs/bundle_config.yaml +++ b/configs/bundle_config.yaml @@ -336,3 +336,15 @@ databundles: # output: [cutouts/cutout-2013-era5.nc] # disable_by_opt: # build_cutout: [all] + + # Cutout for South America, approx 18 GB + bundle_cutouts_southamerica: + countries: [SouthAmerica] + category: cutouts + destination: "cutouts" + urls: + # zenodo: + gdrive: https://drive.google.com/file/d/1Jeu2Vzoq4mNDUKSvIviN8HqSFx5gl61b/view?usp=sharing + output: [cutouts/cutout-2013-era5.nc] + disable_by_opt: + build_cutout: [all] From a1583bc57c2bf958562faa6a0c35461569a68383 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 14 Aug 2023 14:54:08 +0200 Subject: [PATCH 15/32] docs(contributor): contrib-readme-action has updated readme (#836) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index c4d9284b8..3eab8b0c8 100644 --- a/README.md +++ b/README.md @@ -376,6 +376,13 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe + + + kma33 +
+ Katherine M. Antonio + + pitmonticone From 988dc1aae8e684715e4197d7d386d97b33d60988 Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Thu, 17 Aug 2023 13:33:27 +0200 Subject: [PATCH 16/32] Add max_hours default and minor revisions --- config.default.yaml | 1 + config.tutorial.yaml | 1 + doc/configtables/hydro.csv | 1 + scripts/add_electricity.py | 6 ++++-- 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index 40a00662d..c2d62bafc 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -293,6 +293,7 @@ renewable: carriers: [ror, PHS, hydro] PHS_max_hours: 6 hydro_max_hours: "energy_capacity_totals_by_country" # not active + hydro_max_hours_default: 6.0 # (optional, default 6) Default value of max_hours for hydro when NaN values are found clip_min_inflow: 1.0 extendable: true normalization: diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 2115c00df..b31691be9 100644 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -291,6 +291,7 @@ renewable: carriers: [ror, PHS, hydro] PHS_max_hours: 6 hydro_max_hours: "energy_capacity_totals_by_country" # one of energy_capacity_totals_by_country, estimate_by_large_installations or a float + hydro_max_hours_default: 6.0 # (optional, default 6) Default value of max_hours for hydro when NaN values are found clip_min_inflow: 1.0 normalization: method: hydro_capacities # 'hydro_capacities' to rescale country hydro production by using hydro_capacities, 'eia' to rescale by eia data, false for no rescaling diff --git a/doc/configtables/hydro.csv b/doc/configtables/hydro.csv index 7f10a4b5f..b4a563922 100644 --- a/doc/configtables/hydro.csv +++ b/doc/configtables/hydro.csv @@ -7,6 +7,7 @@ resource,,, carriers,--,"Any subset of {'ror', 'PHS', 'hydro'}","Specifies the types of hydro power plants to build per-unit availability time series for. 'ror' stands for run-of-river plants, 'PHS' represents pumped-hydro storage, and 'hydro' stands for hydroelectric dams." PHS_max_hours,h,float,"Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_." hydro_max_hours,h,"Any of {float, 'energy_capacity_totals_by_country', 'estimate_by_large_installations'}","Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom`` or heuristically determined. Cf. `PyPSA documentation `_." +hydro_max_hours_default,h,float,"(optional, default 6) Default value of max_hours for hydro plants with missing values" clip_min_inflow,MW,float,"To avoid too small values in the inflow time series, values below this threshold are set to zero." extendable, bool, "{True, False}", "True: In nodes where there is no hydro generation, adds a zero-capacity hydro generator so that hydro is considered for capacity expansion. It is done in the ``add_electricity`` rule." normalization,--,dict,"When specified, it describes how to normalize hydro time series to adhere to national statistics" diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 2bdf40a51..fcb204928 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -35,6 +35,7 @@ hydro: carriers: hydro_max_hours: + hydro_max_hours_default: hydro_capital_cost: lines: @@ -584,7 +585,7 @@ def attach_hydro(n, costs, ppl): hydro_stats["E_store[TWh]"] * 1e3 / hydro_stats["p_nom_discharge[GW]"] ) - max_hours_country.clip(0, inplace=True) + max_hours_country.clip(lower=0, inplace=True) missing_countries = pd.Index(hydro["country"].unique()).difference( max_hours_country.dropna().index @@ -595,9 +596,10 @@ def attach_hydro(n, costs, ppl): ", ".join(missing_countries) ) ) + hydro_max_hours_default = c.get("hydro_max_hours_default", 6.0) hydro_max_hours = hydro.max_hours.where( hydro.max_hours > 0, hydro.country.map(max_hours_country) - ).fillna(6) + ).fillna(hydro_max_hours_default) n.madd( "StorageUnit", From ce4d686aa908093bce8d10c46e9f229424c5c261 Mon Sep 17 00:00:00 2001 From: Hazem <87850910+hazemakhalek@users.noreply.github.com> Date: Thu, 17 Aug 2023 22:04:54 +0200 Subject: [PATCH 17/32] Update environment.yaml (#838) --- envs/environment.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/envs/environment.yaml b/envs/environment.yaml index ea231f2ae..6730fa74c 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -43,6 +43,7 @@ dependencies: - matplotlib<=3.5.2 - reverse-geocode - country_converter +- py7zr # Keep in conda environment when calling ipython - ipython From 26f69ffea1d9a05d8f00b2745528065873e59169 Mon Sep 17 00:00:00 2001 From: Thomas Kouroughli Date: Mon, 21 Aug 2023 12:16:50 +0100 Subject: [PATCH 18/32] add world_iso to getContinent --- scripts/_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index df551bd3b..dbe8065ce 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -466,7 +466,7 @@ def make_accessable(*ios): return snakemake -def getContinent(code): +def getContinent(code, world_iso: read_osm_config("world_iso")): """ Returns continent names that contains list of iso-code countries. From 2279f924fb89622e9da036076040437b53b5e65a Mon Sep 17 00:00:00 2001 From: Thomas Kouroughli Date: Mon, 21 Aug 2023 13:09:39 +0100 Subject: [PATCH 19/32] fix osm_config implementation --- configs/osm_config.yaml | 57 +----------------------------------- scripts/_helpers.py | 11 +++++-- scripts/build_osm_network.py | 2 +- scripts/download_osm_data.py | 2 +- 4 files changed, 11 insertions(+), 61 deletions(-) diff --git a/configs/osm_config.yaml b/configs/osm_config.yaml index 5bad450b7..d4d70f33b 100644 --- a/configs/osm_config.yaml +++ b/configs/osm_config.yaml @@ -58,17 +58,14 @@ continents: EU: Europe world_iso: - # world_iso maps ISO country codes to country names Africa: DZ: "algeria" # Algeria AO: "angola" BJ: "benin" BW: "botswana" - # IO: "british-indian-ocean-territory" # Island BF: "burkina-faso" BI: "burundi" CM: "cameroon" - # IC: "canary-islands" # Island CV: "cape-verde" CF: "central-african-republic" TD: "chad" @@ -80,7 +77,6 @@ world_iso: GQ: "equatorial-guinea" ER: "eritrea" ET: "ethiopia" - # TF: "french-southern-territories" # Island GA: "gabon" GH: "ghana" GW: "guinea-bissau" # No Data @@ -95,21 +91,16 @@ world_iso: ML: "mali" MR: "mauritania" MU: "mauritius" - # YT: "mayotte" # Island - MA: "morocco" MZ: "mozambique" NA: "namibia" NE: "niger" NG: "nigeria" - # RE: "reunion" # Island RW: "rwanda" - # saint-helena-ascension-and-tristan-da-cunha # Islands ST: "sao-tome-and-principe" SN: "senegal" GM: "gambia" SC: "seychelles" - SO: "somalia" # No Data - # south-africa-and-lesotho + SO: "somalia" ZA: "south-africa" SS: "south-sudan" SD: "sudan" @@ -121,7 +112,6 @@ world_iso: ZM: "zambia" ZW: "zimbabwe" EH: "western-sahara" - Asia: AF: "afghanistan" AM: "armenia" @@ -129,15 +119,10 @@ world_iso: BH: "bahrain" BD: "bangladesh" BT: "bhutan" - # IO: "british-indian-ocean-territory" KH: "cambodia" CN: "china" - # CX: "christmas-island" # Island - # CC: "cocos-keeling-islands" # Island CY: "cyprus" - # EG: "egypt" # leads to bug -> missing ssp file when executing ["Africa"] GE: "georgia" - # HK: "hong-kong" # no more with gadm 4.1 IN: "india" ID: "indonesia" IR: "iran" @@ -166,7 +151,6 @@ world_iso: RU: "russian-federation" SA: "saudi-arabia" SG: "singapore" # merged with MY - # XS: "spratly-islands" # Island LK: "sri-lanka" SY: "syria" TW: "taiwan" @@ -176,19 +160,13 @@ world_iso: TR: "turkey" TM: "turkmenistan" AE: "united-arab-emirates" - # XD: "united-nations-neutral-zone" UZ: "uzbekistan" VN: "vietnam" YE: "yemen" - Oceania: AS: "american-samoa" # Island AU: "australia" - # CP: "ile-de-clipperton" # Island In gadm as XCL - # CK: "cook-islands" # Island FJ: "fiji" - # PF: "french-polynesia" # Island - # GU: "guam" # Island KI: "kiribati" # Island MH: "marshall-islands" FM: "micronesia" @@ -196,22 +174,14 @@ world_iso: NC: "new-caledonia" # Island NZ: "new-zealand" NU: "niue" # Island - # NF: "norfolk-island" # Island - # MP: "northern-mariana-islands" PG: "papua-new-guinea" - # PN: "pitcairn-islands" # Islands PW: "palau" WS: "samoa" SB: "solomon-islands" - # TK: "tokelau" # Island TO: "tonga" TV: "tuvalu" - # UM: "united-states-minor-outlying-islands" # Islands VU: "vanuatu" # Island - # WF: "wallis-and-futuna" # Island - Europe: - # AX: "aland-islands" # Island AL: "albania" AD: "andorra" AM: "armenia" @@ -226,20 +196,15 @@ world_iso: CZ: "czech-republic" DK: "denmark" EE: "estonia" - # FO: "faroe-islands" # Islands FI: "finland" FR: "france" GE: "georgia" DE: "germany" - # GI: "gibraltar" # Island ? GR: "greece" - # GG: "guernsey" # Island HU: "hungary" IS: "iceland" IE: "ireland-and-northern-ireland" - # IM: "isle-of-man" # Island IT: "italy" - # JE: "jersey" # Island KZ: "kazakhstan" XK: "kosovo" LV: "latvia" @@ -262,50 +227,31 @@ world_iso: SK: "slovakia" SI: "slovenia" ES: "spain" - # SJ: "svalbard-and-jan-mayen" # Islands SE: "sweden" CH: "switzerland" UA: "ukraine" GB: "great-britain" TR: "turkey" VA: "vatican" - NorthAmerica: - # AI: "anguilla" # Island AG: "antigua-and-barbuda" - # AW: "aruba" # Islands BS: "bahamas" BB: "barbados" - # BM: "bermuda" # Islands - # BQ: "bonaire" # Islands - # VG: "british-virgin-islands" # Islands CA: "canada" - # KY: "cayman-islands" # Islands CU: "cuba" - # CW: "curacao" # Islands DM: "dominica" DO: "dominican-republic" GL: "greenland" GD: "grenada" - # GP: "guadeloupe" # Islands HT: "haiti" JM: "jamaica" - # MQ: "martinique" # Islands MX: "mexico" - # MS: "montserrat" # Islands US: "united-states-of-america" PR: "puerto-rico" - # BL: "saint-barthelemy" # Islands KN: "saint-kitts-and-nevis" LC: "saint-lucia" - # MF: "saint-martin" # Islands - # PM: "saint-pierre-and-miquelon" # Islands VC: "saint-vincent-and-the-grenadines" - # SX: "saint-marteen" # Islands TT: "trinidad-and-tobago" - # TC: "turks-and-caicos" # Islands - # UM: "united-states-minor-outlying-islands" # Islands - # VI: "united-states-virgin-islands" # Islands BZ: "belize" CR: "costa-rica" HN: "honduras" @@ -313,7 +259,6 @@ world_iso: NI: "nicaragua" PA: "panama" SV: "el-salvador" - SouthAmerica: AR: "argentina" BO: "bolivia" diff --git a/scripts/_helpers.py b/scripts/_helpers.py index dbe8065ce..c0624bd3f 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -51,7 +51,7 @@ def read_osm_config(*args): osm_config_path = os.path.join("configs", "osm_config.yaml") with open(osm_config_path, "r") as f: osm_config = yaml.safe_load(f) - return tuple([osm_config[a] for a in args]) + return [osm_config[a] for a in args] def sets_path_to_root(root_directory_name): @@ -466,7 +466,7 @@ def make_accessable(*ios): return snakemake -def getContinent(code, world_iso: read_osm_config("world_iso")): +def getContinent(code): """ Returns continent names that contains list of iso-code countries. @@ -490,7 +490,8 @@ def getContinent(code, world_iso: read_osm_config("world_iso")): continent_list = [] code_set = set(code) - for continent in read_osm_config("world_iso"): + world_iso = read_osm_config("world_iso")[0] + for continent in world_iso.keys(): single_continent_set = set(world_iso[continent]) if code_set.intersection(single_continent_set): continent_list.append(continent) @@ -757,3 +758,7 @@ def filter_codes(c_list, iso_coding=True): full_codes_list = filter_codes(list(set(full_codes_list)), iso_coding=iso_coding) return full_codes_list + + +if __name__ == "__main__": + getContinent(["PH"]) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 14aac6601..c7c615f23 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -810,7 +810,7 @@ def add_buses_to_empty_countries(country_list, fp_country_shapes, buses): def built_network(inputs, outputs, config, geo_crs, distance_crs, force_ac=False): logger.info("Stage 1/5: Read input data") - osm_clean_columns = read_osm_config("osm_clean_columns") + osm_clean_columns = read_osm_config("osm_clean_columns")[0] buses = read_geojson( inputs["substations"], osm_clean_columns["substation"].keys(), diff --git a/scripts/download_osm_data.py b/scripts/download_osm_data.py index 50e045704..508f06359 100644 --- a/scripts/download_osm_data.py +++ b/scripts/download_osm_data.py @@ -31,7 +31,7 @@ import shutil from pathlib import Path -from _helpers import configure_logging, iso_to_geofk_dict, read_osm_config +from _helpers import configure_logging, read_osm_config from earth_osm import eo logger = logging.getLogger(__name__) From 254c63f0c9a81a150ead58e7d66ebbbe90383160 Mon Sep 17 00:00:00 2001 From: Thomas Kouroughli Date: Wed, 23 Aug 2023 11:43:38 +0100 Subject: [PATCH 20/32] Fix missing gas generators (#824) * add additional gas tech mapping * add docstrings to replace_natural_gas_technology * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add new ignore words to pre-commit * add logger and catchall * final implementation * add natural gas condition to technology replace. * add fueltype condition to unknown_techs * remove add_electricity duplication and add fueltype --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- scripts/add_electricity.py | 5 ---- scripts/build_powerplants.py | 46 ++++++++++++++++++++++++++---------- 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4b9ad36b5..8bb9d2738 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: rev: v2.2.5 hooks: - id: codespell - args: ['--ignore-regex="(\b[A-Z]+\b)"', '--ignore-words-list=fom,appartment,bage,ore,setis,tabacco,berfore'] # Ignore capital case words, e.g. country codes + args: ['--ignore-regex="(\b[A-Z]+\b)"', '--ignore-words-list=fom,appartment,bage,ore,setis,tabacco,berfore,fo,FO'] types_or: [python, rst, markdown] files: ^(scripts|doc)/ diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index fcb204928..2429e9375 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -395,11 +395,6 @@ def attach_conventional_generators( ) _add_missing_carriers_from_costs(n, costs, carriers) - # Replace carrier "natural gas" with the respective technology (OCGT or CCGT) to align with PyPSA names of "carriers" and avoid filtering "natural gas" powerplants in ppl.query("carrier in @carriers") - ppl.loc[ppl["carrier"] == "natural gas", "carrier"] = ppl.loc[ - ppl["carrier"] == "natural gas", "technology" - ] - ppl = ( ppl.query("carrier in @carriers") .join(costs, on="carrier", rsuffix="_r") diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index c37fd9e28..622061443 100644 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -253,16 +253,39 @@ def add_custom_powerplants(ppl, inputs, config): return add_ppls -def replace_natural_gas_technology(df): - mapping = {"Steam Turbine": "CCGT", "Combustion Engine": "OCGT"} - tech = df.Technology.replace(mapping).fillna("CCGT") - return df.Technology.mask(df.Fueltype == "Natural Gas", tech) - - -def replace_natural_gas_fueltype(df): - return df.Fueltype.mask( - (df.Technology == "OCGT") | (df.Technology == "CCGT"), "Natural Gas" +def replace_natural_gas_technology(df: pd.DataFrame): + """ + Maps and replaces gas technologies in the powerplants.csv onto model + compliant carriers. + """ + mapping = { + "Steam Turbine": "CCGT", + "Combustion Engine": "OCGT", + "NG": "CCGT", + "Ng": "CCGT", + "NG/FO": "OCGT", + "Ng/Fo": "OCGT", + "NG/D": "OCGT", + "LNG": "OCGT", + "CCGT/D": "CCGT", + "CCGT/FO": "CCGT", + "LCCGT": "CCGT", + "CCGT/Fo": "CCGT", + } + fueltype = df["Fueltype"] == "Natural Gas" + df.loc[fueltype, "Technology"] = ( + df.loc[fueltype, "Technology"].replace(mapping).fillna("CCGT") ) + unique_tech_with_ng = df.loc[fueltype, "Technology"].unique() + unknown_techs = np.setdiff1d(unique_tech_with_ng, ["CCGT", "OCGT"]) + if len(unknown_techs) > 0: + df.Technology.where( + fueltype, + df["Technology"].map({t: "CCGT" for t in unknown_techs}), + inplace=True, + ) + df["Fueltype"] = np.where(fueltype, df["Technology"], df["Fueltype"]) + return df if __name__ == "__main__": @@ -316,10 +339,7 @@ def replace_natural_gas_fueltype(df): .powerplant.fill_missing_decommissioning_years() .query('Fueltype not in ["Solar", "Wind"] and Country in @countries_names') .powerplant.convert_country_to_alpha2() - .assign( - Technology=replace_natural_gas_technology, - Fueltype=replace_natural_gas_fueltype, - ) + .pipe(replace_natural_gas_technology) ) ppl = add_custom_powerplants( From f86db51e07fbec1cc1a71a677c50105686163ea6 Mon Sep 17 00:00:00 2001 From: Ekaterina Date: Thu, 24 Aug 2023 01:57:41 +0300 Subject: [PATCH 21/32] Keep cutout in case it still exists (#839) * Keep cutout in case it still exists * Fix definitions * Improve an error message * Implement Davide's suggestion * Add a docstring and improve naming --- Snakefile | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Snakefile b/Snakefile index 4c384113e..ed7d489fb 100644 --- a/Snakefile +++ b/Snakefile @@ -289,7 +289,27 @@ rule build_bus_regions: "scripts/build_bus_regions.py" +def terminate_if_cutout_exists(config=config): + """ + Check if any of the requested cutout files exist. + If that's the case, terminate execution to avoid data loss. + """ + config_cutouts = [ + d_value["cutout"] for tc, d_value in config["renewable"].items() + ] + list(config["atlite"]["cutouts"].keys()) + + for ct in set(config_cutouts): + cutout_fl = "cutouts/" + CDIR + ct + ".nc" + if os.path.exists(cutout_fl): + raise Exception( + "An option `build_cutout` is enabled, while a cutout file '" + + cutout_fl + + "' still exists and risks to be overwritten. If this is an intended behavior, please move or delete this file and re-run the rule. Otherwise, just disable the `build_cutout` rule in the config file." + ) + + if config["enable"].get("build_cutout", False): + terminate_if_cutout_exists(config) rule build_cutout: input: From c9ee798d8b8bebc57acc89f32ac31cb83c484be6 Mon Sep 17 00:00:00 2001 From: Davide Fioriti <67809479+davide-f@users.noreply.github.com> Date: Fri, 25 Aug 2023 19:53:48 +0200 Subject: [PATCH 22/32] Fix databundle earth (#844) * Fix databundle earth * Add hydrobasins to outputs --- configs/bundle_config.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/configs/bundle_config.yaml b/configs/bundle_config.yaml index c6064ebc5..2a5ebb30d 100644 --- a/configs/bundle_config.yaml +++ b/configs/bundle_config.yaml @@ -147,6 +147,14 @@ databundles: - data/gebco/GEBCO_2021_TID.nc - data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif - data/ssp2-2.6/2030/era5_2013/Africa.nc + - data/ssp2-2.6/2030/era5_2013/Asia.nc + - data/ssp2-2.6/2030/era5_2013/Europe.nc + - data/ssp2-2.6/2030/era5_2013/NorthAmerica.nc + - data/ssp2-2.6/2030/era5_2013/SouthAmerica.nc + - data/ssp2-2.6/2030/era5_2013/Oceania.nc + - data/hydrobasins/hybas_world_lev04_v1c.shp + - data/hydrobasins/hybas_world_lev05_v1c.shp + - data/hydrobasins/hybas_world_lev06_v1c.shp # resources bundle containing the resources folder for Africa only bundle_natura_earth: From eaabd3b740caeaf67e7900b039e6f5d1834ff7af Mon Sep 17 00:00:00 2001 From: Davide Fioriti <67809479+davide-f@users.noreply.github.com> Date: Sun, 27 Aug 2023 01:42:02 +0200 Subject: [PATCH 23/32] Fix build_ren name column (#847) --- scripts/make_statistics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/make_statistics.py b/scripts/make_statistics.py index 009986006..ce6614067 100644 --- a/scripts/make_statistics.py +++ b/scripts/make_statistics.py @@ -486,7 +486,7 @@ def collect_renewable_stats(rulename, technology): ), ) - add_computational_stats(df_RES_stats, snakemake, rulename) + add_computational_stats(df_RES_stats, snakemake, f"{rulename}_{technology}") return df_RES_stats else: From 504ffba002dd94ff9339a53eadcefd8092206666 Mon Sep 17 00:00:00 2001 From: Thomas Kouroughli Date: Sun, 27 Aug 2023 11:18:53 +0100 Subject: [PATCH 24/32] remove redundant comments --- scripts/_helpers.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index c0624bd3f..93d888470 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -19,12 +19,6 @@ REGION_COLS = ["geometry", "name", "x", "y", "country"] -# world_iso = osm_config["world_iso"] -# continent_regions = osm_config["continent_regions"] -# iso_to_geofk_dict = osm_config["iso_to_geofk_dict"] -# osm_clean_columns = osm_config["osm_clean_columns"] - - def read_osm_config(*args): """ Reads the osm_config.yaml file and returns values based on the provided key From 1d2ab7c39c7a49d6b0f1bcedfc8dabfd792047ba Mon Sep 17 00:00:00 2001 From: Thomas Kouroughli Date: Sun, 27 Aug 2023 11:33:58 +0100 Subject: [PATCH 25/32] add comments to config and conditional returns --- scripts/_helpers.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 93d888470..26d0946ba 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -19,7 +19,7 @@ REGION_COLS = ["geometry", "name", "x", "y", "country"] -def read_osm_config(*args): +def read_osm_config(*args) -> tuple: """ Reads the osm_config.yaml file and returns values based on the provided key arguments. @@ -28,7 +28,9 @@ def read_osm_config(*args): ---------- *args : str One or more key arguments corresponding to the values - to retrieve from the config file. + to retrieve from the config file. The typical arguments used are + "world_iso", "continent_regions", "iso_to_geofk_dict" and + "osm_clean_columns" Returns ------- @@ -41,11 +43,19 @@ def read_osm_config(*args): >>> values = read_osm_config("key1", "key2") >>> print(values) ('value1', 'value2') + >>> world_iso = read_osm_config("world_iso") + >>> print(world_iso) + {"Africa":{"DZ":"algeria",...},...} """ osm_config_path = os.path.join("configs", "osm_config.yaml") with open(osm_config_path, "r") as f: osm_config = yaml.safe_load(f) - return [osm_config[a] for a in args] + if len(args) == 0: + return osm_config + elif len(args) == 1: + return osm_config[args[0]] + else: + tuple([osm_config[a] for a in args]) def sets_path_to_root(root_directory_name): @@ -484,7 +494,7 @@ def getContinent(code): continent_list = [] code_set = set(code) - world_iso = read_osm_config("world_iso")[0] + world_iso = read_osm_config("world_iso") for continent in world_iso.keys(): single_continent_set = set(world_iso[continent]) if code_set.intersection(single_continent_set): From e106ba3687b6e1cbf5d1bb39c816645d1adb2b10 Mon Sep 17 00:00:00 2001 From: Thomas Kouroughli Date: Sun, 27 Aug 2023 17:04:44 +0100 Subject: [PATCH 26/32] fix read_osm_config --- scripts/_helpers.py | 2 +- scripts/build_osm_network.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 26d0946ba..f17171bda 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -55,7 +55,7 @@ def read_osm_config(*args) -> tuple: elif len(args) == 1: return osm_config[args[0]] else: - tuple([osm_config[a] for a in args]) + return tuple([osm_config[a] for a in args]) def sets_path_to_root(root_directory_name): diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index c7c615f23..14aac6601 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -810,7 +810,7 @@ def add_buses_to_empty_countries(country_list, fp_country_shapes, buses): def built_network(inputs, outputs, config, geo_crs, distance_crs, force_ac=False): logger.info("Stage 1/5: Read input data") - osm_clean_columns = read_osm_config("osm_clean_columns")[0] + osm_clean_columns = read_osm_config("osm_clean_columns") buses = read_geojson( inputs["substations"], osm_clean_columns["substation"].keys(), From 260d49a55e1283948df2c3b56a3198c4d78890f1 Mon Sep 17 00:00:00 2001 From: Thomas Kouroughli Date: Sun, 27 Aug 2023 17:07:07 +0100 Subject: [PATCH 27/32] remove spaces --- scripts/_helpers.py | 1 - scripts/download_osm_data.py | 1 - 2 files changed, 2 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index f17171bda..0530837ea 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -715,7 +715,6 @@ def filter_codes(c_list, iso_coding=True): geofabrik-specific ones. When geofabrik codes are selected(iso_coding=False), ignore iso-specific names. """ - if ( iso_coding ): # if country lists are in iso coding, then check if they are 2-string diff --git a/scripts/download_osm_data.py b/scripts/download_osm_data.py index 508f06359..83d8aa3d0 100644 --- a/scripts/download_osm_data.py +++ b/scripts/download_osm_data.py @@ -56,7 +56,6 @@ def country_list_to_geofk(country_list): full_codes_list : list Example ["NG","ZA"] """ - full_codes_list = [convert_iso_to_geofk(c_code) for c_code in set(country_list)] return full_codes_list From d678a3ce2f4b7550e0813042af1ca6256636571a Mon Sep 17 00:00:00 2001 From: Thomas Kouroughli Date: Sun, 27 Aug 2023 17:22:04 +0100 Subject: [PATCH 28/32] add relative path --- scripts/_helpers.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 0530837ea..2dd07b55f 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -47,7 +47,13 @@ def read_osm_config(*args) -> tuple: >>> print(world_iso) {"Africa":{"DZ":"algeria",...},...} """ - osm_config_path = os.path.join("configs", "osm_config.yaml") + if "__file__" in globals(): + base_folder = os.path.dirname(__file__) + if not os.path.exists(os.path.join(base_folder, "configs")): + base_folder = os.path.dirname(base_folder) + else: + base_folder = os.getcwd() + osm_config_path = os.path.join(base_folder, "configs", "osm_config.yaml") with open(osm_config_path, "r") as f: osm_config = yaml.safe_load(f) if len(args) == 0: From 799f21bbe55da3f1271f29ea07111c8224120172 Mon Sep 17 00:00:00 2001 From: Thomas Kouroughli Date: Sun, 27 Aug 2023 18:52:24 +0100 Subject: [PATCH 29/32] remove typing --- scripts/_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 2dd07b55f..2d8232281 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -19,7 +19,7 @@ REGION_COLS = ["geometry", "name", "x", "y", "country"] -def read_osm_config(*args) -> tuple: +def read_osm_config(*args): """ Reads the osm_config.yaml file and returns values based on the provided key arguments. From 18a373668a320f3c75964c21d4eec7eea3f8ab7e Mon Sep 17 00:00:00 2001 From: Thomas Kouroughli Date: Sun, 27 Aug 2023 18:52:56 +0100 Subject: [PATCH 30/32] remove mock snakemake test --- scripts/_helpers.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 2d8232281..df67e2a53 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -767,7 +767,3 @@ def filter_codes(c_list, iso_coding=True): full_codes_list = filter_codes(list(set(full_codes_list)), iso_coding=iso_coding) return full_codes_list - - -if __name__ == "__main__": - getContinent(["PH"]) From e2cfb0f300babf09375ff43a21308b9d10588b61 Mon Sep 17 00:00:00 2001 From: Thomas Kouroughli Date: Mon, 28 Aug 2023 03:26:17 +0100 Subject: [PATCH 31/32] update release notes. --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index af2c1f6ce..57e918942 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -32,6 +32,8 @@ E.g. if a new rule becomes available describe how to use it `snakemake -j1 run_t * Add Asian cutout `PR #826 `__ +* Add osm_config yaml file `PR #822 `__ + PyPSA-Earth 0.2.2 ================= From a0cda00f8f1af168a04167ed1690d49c4a03a1da Mon Sep 17 00:00:00 2001 From: Thomas Kouroughli Date: Mon, 28 Aug 2023 06:15:46 +0100 Subject: [PATCH 32/32] adjusted docstrings of read_osm_config --- scripts/_helpers.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index df67e2a53..063552cbf 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -21,31 +21,31 @@ def read_osm_config(*args): """ - Reads the osm_config.yaml file and returns values based on the provided key - arguments. + Read values from the osm_config.yaml file based on provided key arguments. Parameters ---------- *args : str - One or more key arguments corresponding to the values - to retrieve from the config file. The typical arguments used are - "world_iso", "continent_regions", "iso_to_geofk_dict" and - "osm_clean_columns" + One or more key arguments corresponding to the values to retrieve + from the config file. Typical arguments include "world_iso", + "continent_regions", "iso_to_geofk_dict", and "osm_clean_columns". Returns ------- - tuple - A tuple containing values from the osm_config.yaml - file corresponding to the provided keys. + tuple or str or dict + If a single key is provided, returns the corresponding value from the + osm_config.yaml file. If multiple keys are provided, returns a tuple + containing values corresponding to the provided keys. Examples -------- >>> values = read_osm_config("key1", "key2") >>> print(values) ('value1', 'value2') + >>> world_iso = read_osm_config("world_iso") >>> print(world_iso) - {"Africa":{"DZ":"algeria",...},...} + {"Africa": {"DZ": "algeria", ...}, ...} """ if "__file__" in globals(): base_folder = os.path.dirname(__file__)