Fix islanded buses and add clustering by networks - clean branch

config.default.yaml

@@ -62,6 +62,8 @@ cluster_options:
     exclude_carriers: []
     remove_stubs: true
     remove_stubs_across_borders: true
+    p_threshold_drop_isolated: 10 # [MW] isolated buses are being discarded if bus mean power is below the specified threshold 
+    p_threshold_merge_isolated: 300 # [MW] isolated buses are being merged into a single isolated bus if bus mean power is above the specified threshold 
   cluster_network:
     algorithm: kmeans
     feature: solar+onwind-time
@@ -91,6 +93,7 @@ build_shape_options:
   worldpop_method: "standard"  # "standard" pulls from web 1kmx1km raster, "api" pulls from API 100mx100m raster, false (not "false") no pop addition to shape which is useful when generating only cutout
   gdp_method: "standard"  # "standard" pulls from web 1x1km raster, false (not "false") no gdp addition to shape which useful when generating only cutout
   contended_flag: "set_by_country" # "set_by_country" assigns the contended areas to the countries according to the GADM database, "drop" drops these contended areas from the model
+
 clean_osm_data_options:  # osm = OpenStreetMap
   names_by_shapes: true  # Set the country name based on the extended country shapes
   threshold_voltage: 35000  # [V] assets below that voltage threshold will not be used (cable, line, generator, etc.)
@@ -103,6 +106,7 @@ build_osm_network:  # Options of the build_osm_network script; osm = OpenStreetM
   group_tolerance_buses: 500  # [m] (default 500) Tolerance in meters of the close buses to merge
   split_overpassing_lines: true  # When True, lines overpassing buses are splitted and connected to the bueses
   overpassing_lines_tolerance: 1  # [m] (default 1) Tolerance to identify lines overpassing buses
+  force_ac: false  # When true, it forces all components (lines and substation) to be AC-only. To be used if DC assets create problem.
 
 base_network:
   min_voltage_substation_offshore: 35000  # [V] minimum voltage of the offshore substations
@@ -168,7 +172,6 @@ lines:
 links:
   p_max_pu: 1.0
   p_nom_max: .inf
-  include_tyndp: true
   under_construction: "zero" # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity
 
 transformers:

config.tutorial.yaml

@@ -74,6 +74,8 @@ cluster_options:
     exclude_carriers: []
     remove_stubs: true
     remove_stubs_across_borders: true
+    p_threshold_drop_isolated: 10 # [MW] isolated buses are being discarded if bus mean power is below the specified threshold 
+    p_threshold_merge_isolated: 300 # [MW] isolated buses are being merged into a single isolated bus if bus mean power is above the specified threshold 
   cluster_network:
     algorithm: kmeans
     feature: solar+onwind-time

scripts/build_demand_profiles.py

@@ -182,7 +182,7 @@ def upsample(cntry, group):
     demand_profiles = demand_profiles.loc[start_date:end_date]
     demand_profiles.to_csv(out_path, header=True)
 
-    logger.info(f"Demand_profiles csv file created for the corrisponding snapshots.")
+    logger.info(f"Demand_profiles csv file created for the corresponding snapshots.")
 
 
 if __name__ == "__main__":

scripts/cluster_network.py

@@ -197,7 +197,7 @@ def get_feature_for_hac(n, buses_i=None, feature=None):
     if "offwind" in carriers:
         carriers.remove("offwind")
         carriers = np.append(
-            carriers, network.generators.carrier.filter(like="offwind").unique()
+            carriers, n.generators.carrier.filter(like="offwind").unique()
         )
 
     if feature.split("-")[1] == "cap":
@@ -249,13 +249,14 @@ def distribute_clusters(
             n.loads_t.p_set.mean()
             .groupby(n.loads.bus)
             .sum()
-            .groupby([n.buses.country])
+            .groupby([n.buses.country, n.buses.sub_network])
             .sum()
             .pipe(normed)
         )
-        assert len(L.index) == len(n.buses.country.unique()), (
+        countries_in_L = pd.unique(L.index.get_level_values(0))
+        assert len(countries_in_L) == len(n.buses.country.unique()), (
             "The following countries have no load: "
-            f"{list(set(L.index).symmetric_difference(set(n.buses.country.unique())))}"
+            f"{list(set(countries_in_L).symmetric_difference(set(n.buses.country.unique())))}"
         )
         distribution_factor = L
 
@@ -264,11 +265,17 @@ def distribute_clusters(
             columns={"name": "country"}
         )
         add_population_data(
-            df_pop_c, country_list, year, update, out_logging, nprocesses=nprocesses
+            df_pop_c, country_list, "standard", year, update, out_logging
         )
         P = df_pop_c.loc[:, ("country", "pop")]
-        P = P.groupby(P["country"]).sum().pipe(normed).squeeze()
-        distribution_factor = P
+        n_df = n.buses.copy()[["country", "sub_network"]]
+
+        pop_dict = P.set_index("country")["pop"].to_dict()
+        n_df["pop"] = n_df["country"].map(pop_dict)
+
+        distribution_factor = (
+            n_df.groupby(["country", "sub_network"]).sum().pipe(normed).squeeze()
+        )
 
     if distribution_cluster == ["gdp"]:
         df_gdp_c = gpd.read_file(inputs.country_shapes).rename(
@@ -281,12 +288,19 @@ def distribute_clusters(
             out_logging,
             name_file_nc="GDP_PPP_1990_2015_5arcmin_v2.nc",
         )
+
         G = df_gdp_c.loc[:, ("country", "gdp")]
-        G = G.groupby(df_gdp_c["country"]).sum().pipe(normed).squeeze()
-        distribution_factor = G
+        n_df = n.buses.copy()[["country", "sub_network"]]
+
+        gdp_dict = G.set_index("country")["gdp"].to_dict()
+        n_df["gdp"] = n_df["country"].map(gdp_dict)
+
+        distribution_factor = (
+            n_df.groupby(["country", "sub_network"]).sum().pipe(normed).squeeze()
+        )
 
     # TODO: 1. Check if sub_networks can be added here i.e. ["country", "sub_network"]
-    N = n.buses.groupby(["country"]).size()
+    N = n.buses.groupby(["country", "sub_network"]).size()
 
     assert (
         n_clusters >= len(N) and n_clusters <= N.sum()
@@ -318,7 +332,7 @@ def distribute_clusters(
 
     m = po.ConcreteModel()
 
-    def n_bounds(model, n_id):
+    def n_bounds(model, *n_id):
         """
         Create a function that makes a bound pair for pyomo
 
@@ -379,7 +393,11 @@ def locate_bus(coords, co):
     buses["gadm_{}".format(gadm_level)] = buses[["x", "y", "country"]].apply(
         lambda bus: locate_bus(bus[["x", "y"]], bus["country"]), axis=1
     )
-    busmap = buses["gadm_{}".format(gadm_level)]
+
+    buses["gadm_subnetwork"] = (
+        buses["gadm_{}".format(gadm_level)] + "_" + buses["carrier"].astype(str)
+    )
+    busmap = buses["gadm_subnetwork"]
 
     return busmap
 
@@ -476,11 +494,10 @@ def reduce_network(n, buses):
     def busmap_for_country(x):
         # A number of the countries in the clustering can be > 1
         if isinstance(n_clusters, pd.Series):
+            n_cluster_c = n_clusters[x.name]
             if isinstance(x.name, tuple):
-                n_cluster_c = n_clusters[x.name[0]]
                 prefix = x.name[0] + x.name[1] + " "
             else:
-                n_cluster_c = n_clusters[x.name]
                 prefix = x.name + " "
         else:
             n_cluster_c = n_clusters
@@ -519,8 +536,8 @@ def busmap_for_country(x):
 
     return (
         n.buses.groupby(
-            ["country"],
-            # ["country", "sub_network"] # TODO: 2. Add sub_networks (see previous TODO)
+            # ["country"],
+            ["country", "sub_network"],  # TODO: 2. Add sub_networks (see previous TODO)
             group_keys=False,
         )
         .apply(busmap_for_country)