Merge pull request #93 from NREL/gb/rrp_shapes

Gb/rrp shapes

reVX/cli.py

@@ -13,7 +13,7 @@
 from reVX.utilities.exclusions_converter import ExclusionsConverter
 from reVX.utilities.forecasts import Forecasts
 from reVX.utilities.output_extractor import output_extractor
-from reVX.utilities.region import RegionClassifier
+from reVX.utilities.region_classifier import RegionClassifier
 from reVX.wind_setbacks.setbacks_converter import SetbacksConverter
 
 from reVX import __version__
@@ -70,7 +70,7 @@ def region_classifier(meta_path, regions_path, regions_label, fout,
     - Used to classify meta points with a label from a shapefile
     """
     RegionClassifier.run(meta_path=meta_path,
-                         regions_path=regions_path,
+                         regions=regions_path,
                          regions_label=regions_label,
                          force=force, fout=fout)
 

reVX/plexos/rev_reeds_plexos_cli.py

@@ -8,7 +8,7 @@
 import logging
 
 from rex.utilities.hpc import SLURM
-from rex.utilities.cli_dtypes import STR, INT, INTLIST
+from rex.utilities.cli_dtypes import STR, INT, INTLIST, STRLIST
 from rex.utilities.loggers import init_mult
 
 from reVX.plexos.rev_reeds_plexos import Manager
@@ -30,11 +30,23 @@
               help='REEDS build years to aggregate profiles for.')
 @click.option('--scenario', '-s', default=None, type=STR, show_default=True,
               help='Optional filter to run just one scenario from job input.')
+@click.option('--plexos_columns', '-pc', default=None, type=STRLIST,
+              show_default=True,
+              help='Optional list of additional columns to pass through from '
+              'the plexos input table')
+@click.option('-ffb', '--force_full_build', is_flag=True,
+              help='Flag to ensure the full requested buildout is built at '
+              'each SC point. If True, the remainder of the requested build '
+              'will always be built at the last resource gid in the sc point.')
+@click.option('-fsm', '--force_shape_map', is_flag=True,
+              help='Flag to force the mapping of supply curve points to the '
+              'plexos node shape file input (if a shape file is input) via '
+              'nearest neighbor to shape centroid.')
 @click.option('-v', '--verbose', is_flag=True,
               help='Flag to turn on debug logging. Default is not verbose.')
 @click.pass_context
 def main(ctx, name, job_input, out_dir, cf_years, build_years,
-         scenario, verbose):
+         scenario, plexos_columns, force_full_build, force_shape_map, verbose):
     """reV-ReEDS-PLEXOS Command Line Interface"""
 
     ctx.ensure_object(dict)
@@ -44,6 +56,9 @@ def main(ctx, name, job_input, out_dir, cf_years, build_years,
     ctx.obj['CF_YEARS'] = cf_years
     ctx.obj['BUILD_YEARS'] = build_years
     ctx.obj['SCENARIO'] = scenario
+    ctx.obj['PLEXOS_COLUMNS'] = plexos_columns
+    ctx.obj['FORCE_FULL_BUILD'] = force_full_build
+    ctx.obj['FORCE_SHAPE_MAP'] = force_shape_map
     ctx.obj['VERBOSE'] = verbose
 
     if ctx.invoked_subcommand is None:
@@ -56,11 +71,15 @@ def main(ctx, name, job_input, out_dir, cf_years, build_years,
         logger.info('Aggregating plexos scenario "{}".'.format(scenario))
         for cf_year in cf_years:
             Manager.run(job_input, out_dir, scenario=scenario,
-                        cf_year=cf_year, build_years=build_years)
+                        cf_year=cf_year, build_years=build_years,
+                        plexos_columns=plexos_columns,
+                        force_full_build=force_full_build,
+                        force_shape_map=force_shape_map)
 
 
 def get_node_cmd(name, job_input, out_dir, cf_year, build_year,
-                 scenario, verbose):
+                 scenario, plexos_columns, force_full_build,
+                 force_shape_map, verbose):
     """Get a CLI call command for the plexos CLI."""
 
     args = ['-n {}'.format(SLURM.s(name)),
@@ -69,8 +88,15 @@ def get_node_cmd(name, job_input, out_dir, cf_year, build_year,
             '-y [{}]'.format(SLURM.s(cf_year)),
             '-by [{}]'.format(SLURM.s(build_year)),
             '-s {}'.format(SLURM.s(scenario)),
+            '-pc {}'.format(SLURM.s(plexos_columns)),
             ]
 
+    if force_full_build:
+        args.append('-ffb')
+
+    if force_shape_map:
+        args.append('-fsm')
+
     if verbose:
         args.append('-v')
 
@@ -104,6 +130,9 @@ def eagle(ctx, alloc, memory, walltime, feature, stdout_path):
     cf_years = ctx.obj['CF_YEARS']
     build_years = ctx.obj['BUILD_YEARS']
     scenario = ctx.obj['SCENARIO']
+    plexos_columns = ctx.obj['PLEXOS_COLUMNS']
+    force_full_build = ctx.obj['FORCE_FULL_BUILD']
+    force_shape_map = ctx.obj['FORCE_SHAPE_MAP']
     verbose = ctx.obj['VERBOSE']
 
     if stdout_path is None:
@@ -127,7 +156,8 @@ def eagle(ctx, alloc, memory, walltime, feature, stdout_path):
                              .format(name, scenario.replace(' ', '_'),
                                      build_year, cf_year))
                 cmd = get_node_cmd(node_name, job_input, out_dir, cf_year,
-                                   build_year, scenario, verbose)
+                                   build_year, scenario, plexos_columns,
+                                   force_full_build, force_shape_map, verbose)
 
                 logger.info('Running reVX plexos aggregation on Eagle with '
                             'node name "{}"'.format(node_name))

reVX/plexos/utilities.py

@@ -104,7 +104,8 @@ class DataCleaner:
                     'resource_ids_cnts': 'gid_counts'}
 
     PLEXOS_META_COLS = ('sc_gid', 'plexos_id', 'latitude', 'longitude',
-                        'voltage', 'interconnect', 'built_capacity')
+                        'voltage', 'interconnect', 'built_capacity',
+                        'geometry')
 
     def __init__(self, plexos_meta, profiles, name_map=None):
         """
@@ -132,15 +133,16 @@ def rename_cols(df, name_map=None):
         df : pd.DataFrame
             Input df with bad or inconsistent column names.
         name_map : dictionary, optional
-            Column rename mapping, by default None -> {'gid': 'sc_gid'}
+            Column rename mapping, by default None -> {'lat': 'latitude',
+            'lon': 'longitude'}
 
         Parameters
         ----------
         df : pd.DataFrame
             Same as inputs but with better col names.
         """
         if name_map is None:
-            name_map = {'gid': 'sc_gid'}
+            name_map = {'lat': 'latitude', 'lon': 'longitude'}
 
         df = df.rename(columns=name_map)
         return df
@@ -191,10 +193,15 @@ def pre_filter_plexos_meta(cls, plexos_meta, name_map=None):
 
         # Several plexos nodes share the same location. As of 8/2019
         # Josh Novacheck suggests that the duplicate locations can be dropped.
-        plexos_meta = plexos_meta.sort_values(by='voltage', ascending=False)
+        if 'voltage' in plexos_meta:
+            plexos_meta = plexos_meta.sort_values(by='voltage',
+                                                  ascending=False)
+
         plexos_meta = plexos_meta.drop_duplicates(
             subset=['latitude', 'longitude'], keep='first')
-        plexos_meta = plexos_meta.sort_values(by='sc_gid')
+
+        if 'gid' in plexos_meta:
+            plexos_meta = plexos_meta.sort_values(by='gid')
 
         return plexos_meta
 
@@ -318,21 +325,18 @@ def merge_extent(self, new_meta, new_profiles, name_map=None):
                             len(self._plexos_meta) + len(new_meta)))
 
         for i, ind in enumerate(new_meta.index.values):
-            lookup = (self._plexos_meta['sc_gid'].values
-                      == new_meta.loc[ind, 'sc_gid'])
+            lookup = (self._plexos_meta['plexos_id'].values
+                      == new_meta.loc[ind, 'plexos_id'])
             if any(lookup):
                 i_self = np.where(lookup)[0]
                 if len(i_self) > 1:
                     warn('Duplicate PLEXOS node GIDs in base plexos meta!')
                 else:
                     i_self = i_self[0]
 
-                logger.debug('Merging plexos node IDs {} and {} '
-                             '(gids {} and {})'.format(
-                                 self._plexos_meta.iloc[i_self]['plexos_id'],
-                                 new_meta.iloc[i]['plexos_id'],
-                                 self._plexos_meta.iloc[i_self]['sc_gid'],
-                                 new_meta.iloc[i]['sc_gid']))
+                logger.debug('Merging plexos node IDs {} and {} '.format(
+                             self._plexos_meta.iloc[i_self]['plexos_id'],
+                             new_meta.iloc[i]['plexos_id']))
 
                 self._merge_plexos_meta(self._plexos_meta, new_meta, i_self, i)
                 self._profiles[:, i_self] += new_profiles[:, i]

reVX/utilities/__init__.py

@@ -5,4 +5,4 @@
 from .exclusions_converter import ExclusionsConverter
 from .forecasts import Forecasts
 from .output_extractor import output_extractor
-from .region import RegionClassifier
+from .region_classifier import RegionClassifier

reVX/utilities/region.py → reVX/utilities/region_classifier.py

@@ -38,22 +38,35 @@ class RegionClassifier():
     CRS = "EPSG:4326"
     DEFAULT_REGIONS_LABEL = 'regions_index'
 
-    def __init__(self, meta_path, regions_path, regions_label=None):
+    def __init__(self, meta_path, regions, regions_label=None):
         """
         Parameters
         ----------
         meta_path : str | pandas.DataFrame
             Path to meta CSV file, resource .h5 file, or pre-loaded meta
             DataFrame containing lat/lon points
-        regions_path : str
-            Path to regions shapefile containing labeled geometries
+        regions : str | GeoDataFrame
+            Path to regions shapefile containing labeled geometries or
+            a pre-loaded GeoDataFrame
         regions_label : str
             Attribute to use as label in the regions shapefile
         """
         self._regions_label = regions_label
+        if self._regions_label is None:
+            self._regions_label = self.DEFAULT_REGIONS_LABEL
 
         self._meta = self._get_meta(meta_path)
-        self._regions = self._get_regions(regions_path, regions_label)
+        self._regions = self._get_regions(regions, self._regions_label)
+
+    @property
+    def regions(self):
+        """Get the regions GeoDataFrame
+
+        Returns
+        -------
+        GeoDataFrame
+        """
+        return self._regions
 
     @staticmethod
     def output_to_csv(gdf, path):
@@ -73,25 +86,29 @@ def output_to_csv(gdf, path):
         output_gdf.to_csv(path, index=False)
 
     @classmethod
-    def _get_regions(cls, regions_path, regions_label):
+    def _get_regions(cls, regions, regions_label):
         """ Load the regions shapefile into geopandas dataframe
 
         Parameters
         ----------
-        regions_path : str
-            Path to regions shapefile containing labeled geometries
+        regions : str | GeoDataFrame
+            Path to regions shapefile containing labeled geometries or
+            a pre-loaded GeoDataFrame
         regions_label : str
             Attribute to use as label in the regions shapefile
         """
-        regions = gpd.read_file(regions_path).to_crs(cls.CRS)
-        if regions_label not in regions.columns:
-            regions_label = cls.DEFAULT_REGIONS_LABEL
-            regions[regions_label] = regions.index
-            logger.warning('Setting regions label: {}'.format(regions_label))
+
+        if not isinstance(regions, gpd.GeoDataFrame):
+            regions = gpd.read_file(regions).to_crs(cls.CRS)
+            if regions_label not in regions.columns:
+                regions_label = cls.DEFAULT_REGIONS_LABEL
+                regions[regions_label] = regions.index
+                logger.warning('Setting regions label: {}'
+                               .format(regions_label))
 
         centroids = regions.geometry.centroid
-        regions['lon'] = centroids.x
-        regions['lat'] = centroids.y
+        regions['longitude'] = centroids.x
+        regions['latitude'] = centroids.y
 
         return regions
 
@@ -216,7 +233,7 @@ def classify(self, force=False):
             logger.warning('The following points are outliers:')
             logger.warning(outlier_inds)
             cols = self._get_lat_lon_labels(self._meta)
-            lookup = self._regions[['lat', 'lon']]
+            lookup = self._regions[['latitude', 'longitude']]
             target = self._meta.loc[outlier_inds][cols]
             region_inds += list(self._nearest(target, lookup))
 
@@ -261,7 +278,7 @@ def _check_geometry(self):
         return list(self._regions[isvalid == 0].index)
 
     @classmethod
-    def run(cls, meta_path, regions_path, regions_label=None,
+    def run(cls, meta_path, regions, regions_label=None,
             force=False, fout=None):
         """ Run full classification
 
@@ -270,8 +287,9 @@ def run(cls, meta_path, regions_path, regions_label=None,
         meta_path : str | pandas.DataFrame
             Path to meta CSV file, resource .h5 file, or pre-loaded meta
             DataFrame containing lat/lon points
-        regions_path : str
-            Path to regions shapefile containing labeled geometries
+        regions : str | GeoDataFrame
+            Path to regions shapefile containing labeled geometries or
+            a pre-loaded GeoDataFrame
         regions_label : str
             Attribute to use a label in the regions shapefile
         force : str
@@ -292,7 +310,7 @@ def run(cls, meta_path, regions_path, regions_label=None,
                                  regions_label=regions_label
                                  force=force, fout=fout)
         """
-        classifier = cls(meta_path=meta_path, regions_path=regions_path,
+        classifier = cls(meta_path=meta_path, regions=regions,
                          regions_label=regions_label)
         classification = classifier.classify(force=force)
         if fout:

reVX/version.py

@@ -3,4 +3,4 @@
 reVX version number
 """
 
-__version__ = "0.3.24"
+__version__ = "0.3.25"

tests/data/reeds_pca_regions_test/NA_PCA_Map.prj

@@ -0,0 +1 @@
+PROJCS["WGS_1984_Web_Mercator_Auxiliary_Sphere",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Mercator_Auxiliary_Sphere"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],PARAMETER["Auxiliary_Sphere_Type",0.0],UNIT["Meter",1.0]]

tests/data/reeds_pca_regions_test/README.txt

@@ -0,0 +1 @@
+These have an error with MISO-N and SPP: North and South Dakota should be in SPP instead of MISO-N