CLI eagle bugs

.github/workflows/pull_request_tests.yml

@@ -6,10 +6,12 @@ jobs:
   build:
     runs-on: ${{ matrix.os }}
     strategy:
-      max-parallel: 3
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: [3.7, 3.8]
+        python-version: [3.8]
+        include:
+          - os: ubuntu-latest
+            python-version: 3.7
 
     steps:
     - uses: actions/checkout@v2

reVX/cli.py

@@ -137,16 +137,9 @@ def exclusions(ctx, excl_h5):
 
 @exclusions.command()
 @click.option('--layers', '-l', required=True, type=click.Path(exists=True),
-              help=(".json file containing list of geotiffs to load or "
-                    "mapping of layer names to geotiffs"))
-@click.option('--descriptions', '-d', default=None,
-              type=click.Path(exists=True), show_default=True,
-              help=(".json file containing layer descriptions as a list or "
-                    "mapping to layers"))
-@click.option('--scale_factors', '-scale', default=None,
-              type=click.Path(exists=True), show_default=True,
-              help=(".json file containing layer scale_factors and final "
-                    "dtypes as a mapping to layers"))
+              help=(".json file containing mapping of layer names to geotiffs."
+                    " Json can also contain layer descriptions and/or "
+                    "scale factors"))
 @click.option('-check_tiff', '-ct', is_flag=True,
               help=("Flag to check tiff profile and coordinates against "
                     "exclusion .h5 profile and coordinates"))
@@ -164,27 +157,19 @@ def exclusions(ctx, excl_h5):
 @click.option('--purge', '-r', is_flag=True,
               help="Remove existing .h5 file before loading layers")
 @click.pass_context
-def layers_to_h5(ctx, layers, descriptions, scale_factors, check_tiff,
-                 setbacks, transform_atol, coord_atol, purge):
+def layers_to_h5(ctx, layers, check_tiff, setbacks, transform_atol, coord_atol,
+                 purge):
     """
     Add layers to exclusions .h5 file
     """
     excl_h5 = ctx.obj['EXCL_H5']
     if purge and os.path.isfile(excl_h5):
         os.remove(excl_h5)
 
-    layers = safe_json_load(layers)
-    if 'layers' in layers:
-        layers = layers['layers']
-
-    if descriptions is not None:
-        descriptions = safe_json_load(descriptions)
-        if 'descriptions' in descriptions:
-            descriptions = {os.path.basename(layer).split('.')[0]: d
-                            for layer, d in zip(layers, descriptions)}
-
-    if scale_factors is not None:
-        scale_factors = safe_json_load(scale_factors)
+    inputs = safe_json_load(layers)
+    layers = inputs['layers']
+    descriptions = inputs.get('descriptions')
+    scale_factors = inputs.get('scale_factors')
 
     if setbacks:
         SetbacksConverter.layers_to_h5(excl_h5, layers,

reVX/plexos/plexos_plants_cli.py

@@ -78,7 +78,8 @@ def main(ctx, name, plexos_table, sc_table, cf_fpath, out_dir, dist_percentile,
     ctx.obj['VERBOSE'] = verbose
 
     if ctx.invoked_subcommand is None:
-        init_mult(name, out_dir, modules=['reVX', 'reV', 'rex'],
+        log_modules = [__name__, 'reVX', 'reV', 'rex']
+        init_mult(name, out_dir, modules=log_modules,
                   verbose=verbose)
         logger.info('Aggregating Plant for buses in PLEXOS table: {}'
                     .format(plexos_table))

reVX/plexos/rev_reeds_plexos_cli.py

@@ -47,7 +47,8 @@ def main(ctx, name, job_input, out_dir, cf_years, build_years,
     ctx.obj['VERBOSE'] = verbose
 
     if ctx.invoked_subcommand is None:
-        init_mult(name, out_dir, modules=['reVX', 'reV', 'rex'],
+        log_modules = [__name__, 'reVX', 'reV', 'rex']
+        init_mult(name, out_dir, modules=log_modules,
                   verbose=verbose)
         logger.info('Running reV to PLEXOS pipeline using job input: {}'
                     .format(job_input))

reVX/reeds/reeds_cli.py

@@ -157,7 +157,7 @@ def local(ctx, out_dir, log_dir, verbose):
     if 'VERBOSE' in ctx.obj:
         verbose = any((ctx.obj['VERBOSE'], verbose))
 
-    log_modules = ['reVX', 'reV', 'rex']
+    log_modules = [__name__, 'reVX', 'reV', 'rex']
     init_mult(name, log_dir, modules=log_modules, verbose=verbose)
 
     logger.info('Running reV to ReEDS pipeline\n'
@@ -460,8 +460,8 @@ def eagle(config):
                 'node name "{}"'.format(name))
     slurm_manager = SLURM()
     out = slurm_manager.sbatch(cmd,
-                               alloc=config.execution_control.alloc,
-                               memory=config.execution_control.node_mem,
+                               alloc=config.execution_control.allocation,
+                               memory=config.execution_control.memory,
                                walltime=config.execution_control.walltime,
                                feature=config.execution_control.feature,
                                name=name, stdout_path=stdout_path,

reVX/rpm/rpm_cli.py

@@ -149,7 +149,7 @@ def local(ctx, out_dir, cf_profiles, log_dir, max_workers, verbose):
     if 'VERBOSE' in ctx.obj:
         verbose = any((ctx.obj['VERBOSE'], verbose))
 
-    log_modules = ['reVX', 'reV', 'rex']
+    log_modules = [__name__, 'reVX', 'reV', 'rex']
     init_mult(name, log_dir, modules=log_modules, verbose=verbose)
 
     logger.info('Running reV to RPM pipeline\n'
@@ -346,8 +346,8 @@ def eagle(config):
     out = slurm_manager.sbatch(cmd,
                                name=name,
                                stdout_path=stdout_path,
-                               alloc=config.execution_control.alloc,
-                               memory=config.execution_control.node_mem,
+                               alloc=config.execution_control.allocation,
+                               memory=config.execution_control.memory,
                                walltime=config.execution_control.walltime,
                                feature=config.execution_control.feature,
                                module=config.execution_control.module,

reVX/utilities/exclusions_converter.py

@@ -375,6 +375,11 @@ def _write_geotiff(geotiff, profile, values):
         values : ndarray
             Geotiff data
         """
+        out_dir = os.path.dirname(geotiff)
+        if not os.path.exists(out_dir):
+            logger.debug("Creating {}".format(out_dir))
+            os.makedirs(out_dir)
+
         if values.shape[0] != 1:
             values = np.expand_dims(values, 0)
 

reVX/version.py

@@ -3,4 +3,4 @@
 reVX version number
 """
 
-__version__ = "0.3.19"
+__version__ = "0.3.20"

reVX/wind_dirs/mean_wind_dirs_cli.py

@@ -182,7 +182,7 @@ def local(ctx, res_h5_fpath, excl_fpath, wdir_dsets, out_dir, tm_dset,
     if 'VERBOSE' in ctx.obj:
         verbose = any((ctx.obj['VERBOSE'], verbose))
 
-    log_modules = ['reVX', 'reV', 'rex']
+    log_modules = [__name__, 'reVX', 'reV', 'rex']
     init_mult(name, log_dir, modules=log_modules, verbose=verbose)
 
     logger.info('Averaging Wind Directions \n'
@@ -262,8 +262,8 @@ def eagle(config):
     logger.info('Averaging wind directions on Eagle with '
                 'node name "{}"'.format(name))
     out = slurm_manager.sbatch(cmd,
-                               alloc=config.execution_control.alloc,
-                               memory=config.execution_control.node_mem,
+                               alloc=config.execution_control.allocation,
+                               memory=config.execution_control.memory,
                                walltime=config.execution_control.walltime,
                                feature=config.execution_control.feature,
                                name=name, stdout_path=stdout_path,

reVX/wind_dirs/prominent_wind_dirs_cli.py

@@ -163,7 +163,7 @@ def local(ctx, powerrose_h5_fpath, excl_fpath, out_dir, agg_dset, tm_dset,
     if 'VERBOSE' in ctx.obj:
         verbose = any((ctx.obj['VERBOSE'], verbose))
 
-    log_modules = ['reVX', 'reV', 'rex']
+    log_modules = [__name__, 'reVX', 'reV', 'rex']
     init_mult(name, log_dir, modules=log_modules, verbose=verbose)
 
     logger.info('Aggregating Prominent Wind Directions \n'
@@ -236,8 +236,8 @@ def eagle(config):
     logger.info('Running prominent wind directions computation on Eagle with '
                 'node name "{}"'.format(name))
     out = slurm_manager.sbatch(cmd,
-                               alloc=config.execution_control.alloc,
-                               memory=config.execution_control.node_mem,
+                               alloc=config.execution_control.allocation,
+                               memory=config.execution_control.memory,
                                walltime=config.execution_control.walltime,
                                feature=config.execution_control.feature,
                                name=name, stdout_path=stdout_path,

reVX/wind_setbacks/setbacks_converter.py

@@ -129,9 +129,8 @@ def layers_to_h5(cls, excl_h5, layers, chunks=(128, 128),
         ----------
         excl_h5 : str
             Path to .h5 file containing or to contain exclusion layers
-        layers : list | dict
-            List of setbacks to load
-            or dictionary mapping goetiffs to the layers to load
+        layers : dict
+            Dictionary mapping goetiffs to the layers to load
         chunks : tuple, optional
             Chunk size of exclusions in Geotiff, by default (128, 128)
         replace : bool, optional