Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow run functions to accept pathlib.Path objects #386

Closed
wants to merge 4 commits into from
Closed

Allow run functions to accept pathlib.Path objects #386

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8ea3e03
Allow run_* scripts to take Path object; refactored to make code more…
aloukina Feb 26, 2020
fdb79e2
Updated doc strings
aloukina Feb 26, 2020
8d9b2e9
Fixed indentation
aloukina Feb 26, 2020
d8bf8f0
PEP8 fixes and more comments
aloukina Feb 26, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
92 changes: 87 additions & 5 deletions rsmtool/configuration_parser.py
View file
Open in desktop
Expand Up @@ -26,12 +26,14 @@
join,
splitext)

from pathlib import Path
from ruamel import yaml

from rsmtool import HAS_RSMEXTRA
from rsmtool.utils import parse_json_with_comments
from rsmtool.utils import (DEFAULTS,
CHECK_FIELDS,
CONTEXT_TO_FUNCTION,
LIST_FIELDS,
BOOLEAN_FIELDS,
MODEL_NAME_MAPPING,
Expand Down Expand Up @@ -764,15 +766,15 @@ def _check_config_is_loaded(self):


@classmethod
def get_configparser(cls, filepath, *args, **kwargs):
def get_configparser(cls, config_file_or_obj_or_dict, *args, **kwargs):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, might it make more sense to just move this logic to the ConfigurationParser() initializer instead of having this classmethod since all we are doing is getting the args and then calling the initializer anyway?

"""
Get the correct `ConfigurationParser` object,
based on the file extension.
based on the input.

Parameters
----------
filepath : str
The path to the configuration file.
config_file_or_obj_or_dict: str or pathlib.Path or dict or Configuration
Input object used to construct the configuration parser

Returns
-------
Expand All @@ -784,14 +786,28 @@ def get_configparser(cls, filepath, *args, **kwargs):
ValueError
If config file is not .json or .cfg.
"""
_, extension = splitext(filepath)
# If we received a path to a file,
# let's convert it to Path
if isinstance(config_file_or_obj_or_dict, Path):
filepath = config_file_or_obj_or_dict
elif isinstance(config_file_or_obj_or_dict, str):
filepath = Path(config_file_or_obj_or_dict)
# if we got anything else, we initialize the base ConfigurationParser
else:
return ConfigurationParser(*args, **kwargs)

# For files we initialize one of the subclasses.
# Get the file extension to determine which.
extension = filepath.suffix
if extension.lower() not in CONFIG_TYPE:
raise ValueError('Configuration file must be '
'in either `.json` or `.cfg`'
'format. You specified: {}.'.format(extension))

return CONFIG_TYPE[extension.lower()](*args, **kwargs)



@staticmethod
def check_id_fields(id_field_values):
"""
Expand Down Expand Up @@ -1363,6 +1379,72 @@ def read_normalize_validate_and_process_config(self,
return self.normalize_validate_and_process_config(context=context)


def get_configuration_from_file_obj_or_dict(self,
config_file_or_obj_or_dict,
context='rsmtool'):
'''
Construct Configuration object from file, object or dictionary
Parameters
----------
config_file_or_obj_or_dict : str or Pathlib.Path or
Configuration or Dictionary
Path to the experiment configuration file.
Users can also pass a `Configuration` object that is in memory
or a Python dictionary with keys corresponding to fields in the
configuration file.
For configuration object `.configdir` needs to be set to indicate the reference path. If
the user passes a dictionary, the reference path will be set to
the current directory and all relative paths will be resolved relative
to this path.
to the current directory.
context : str
context used to validate configuration object
Defaults to `rsmtool`

Returns
-------
configuration : Configuration
Configuration object constructed from the input
'''

# check what sort of input we got
# if we got a string we consider this to be path to config file
if isinstance(config_file_or_obj_or_dict, str) or \
isinstance(config_file_or_obj_or_dict, Path):

# read configuration from file
configuration = self.read_normalize_validate_and_process_config(config_file_or_obj_or_dict,
context=context)

elif isinstance(config_file_or_obj_or_dict, dict):

# read configuration from dictionary
configuration = self.load_normalize_and_validate_config_from_dict(config_file_or_obj_or_dict,
context=context)

elif isinstance(config_file_or_obj_or_dict, Configuration):

configuration = config_file_or_obj_or_dict
# raise an error if we are passed a Configuration object
# without a configdir attribute. This can only
# happen if the object was constructed using an earlier version
# of RSMTool and stored
if configuration.configdir is None:
raise AttributeError("Configuration object must have configdir attribute.")

else:
raise ValueError("The input to {} must be "
"a path to the file (str or pathlib.Path), a dictionary, "
"or a configuration object. You passed "
"{}.".format(CONTEXT_TO_FUNCTION[context],
type(config_file_or_obj_or_dict)))

return configuration





class JSONConfigurationParser(ConfigurationParser):
"""
A subclass of `ConfigurationParser` for parsing
Expand Down
39 changes: 6 additions & 33 deletions rsmtool/rsmcompare.py
View file
Open in desktop
Expand Up @@ -69,7 +69,7 @@ def run_comparison(config_file_or_obj_or_dict, output_dir):

Parameters
----------
config_file_or_obj_or_dict : str or Configuration or Dictionary
config_file_or_obj_or_dict : str or pathlib.Path or Configuration or Dictionary
Path to the experiment configuration file.
Users can also pass a `Configuration` object that is in memory
or a Python dictionary with keys corresponding to fields in the
Expand All @@ -91,38 +91,11 @@ def run_comparison(config_file_or_obj_or_dict, output_dir):

logger = logging.getLogger(__name__)

# check what sort of input we got
# if we got a string we consider this to be path to config file
if isinstance(config_file_or_obj_or_dict, str):

# Instantiate configuration parser object
parser = ConfigurationParser.get_configparser(config_file_or_obj_or_dict)
configuration = parser.read_normalize_validate_and_process_config(config_file_or_obj_or_dict,
context='rsmcompare')

elif isinstance(config_file_or_obj_or_dict, dict):

# initialize the parser from dict
parser = ConfigurationParser()
configuration = parser.load_normalize_and_validate_config_from_dict(config_file_or_obj_or_dict,
context='rsmcompare')

elif isinstance(config_file_or_obj_or_dict, Configuration):

configuration = config_file_or_obj_or_dict
# raise an error if we are passed a Configuration object
# without a configdir attribute. This can only
# happen if the object was constructed using an earlier version
# of RSMTool and stored
if configuration.configdir is None:
raise AttributeError("Configuration object must have configdir attribute.")

else:
raise ValueError("The input to run_comparison must be "
"a path to the file (str), a dictionary, "
"or a configuration object. You passed "
"{}.".format(type(config_file_or_obj_or_dict)))

# initialize a correct configparser
parser = ConfigurationParser.get_configparser(config_file_or_obj_or_dict)
# create a configuration object from input
configuration = parser.get_configuration_from_file_obj_or_dict(config_file_or_obj_or_dict,
context='rsmcompare')
logger.info('Saving configuration file.')
configuration.save(output_dir)

Expand Down
38 changes: 6 additions & 32 deletions rsmtool/rsmeval.py
View file
Open in desktop
Expand Up @@ -36,7 +36,7 @@ def run_evaluation(config_file_or_obj_or_dict, output_dir):

Parameters
----------
config_file_or_obj_or_dict : str or Configuration or Dictionary
config_file_or_obj_or_dict : str or pathlib.Path or Configuration or Dictionary
Path to the experiment configuration file.
Users can also pass a `Configuration` object that is in memory
or a Python dictionary with keys corresponding to fields in the
Expand Down Expand Up @@ -67,37 +67,11 @@ def run_evaluation(config_file_or_obj_or_dict, output_dir):
os.makedirs(figdir, exist_ok=True)
os.makedirs(reportdir, exist_ok=True)

# check what sort of input we got
# if we got a string we consider this to be path to config file
if isinstance(config_file_or_obj_or_dict, str):

# Instantiate configuration parser object
parser = ConfigurationParser.get_configparser(config_file_or_obj_or_dict)
configuration = parser.read_normalize_validate_and_process_config(config_file_or_obj_or_dict,
context='rsmeval')

elif isinstance(config_file_or_obj_or_dict, dict):

# initialize the parser from dict
parser = ConfigurationParser()
configuration = parser.load_normalize_and_validate_config_from_dict(config_file_or_obj_or_dict,
context='rsmeval')

elif isinstance(config_file_or_obj_or_dict, Configuration):

configuration = config_file_or_obj_or_dict
# raise an error if we are passed a Configuration object
# without a configdir attribute. This can only
# happen if the object was constructed using an earlier version
# of RSMTool and stored
if configuration.configdir is None:
raise AttributeError("Configuration object must have configdir attribute.")

else:
raise ValueError("The input to run_evaluation must be "
"a path to the file (str), a dictionary, "
"or a configuration object. You passed "
"{}.".format(type(config_file_or_obj_or_dict)))
# initialize a correct configparser
parser = ConfigurationParser.get_configparser(config_file_or_obj_or_dict)
# create a configuration object from input
configuration = parser.get_configuration_from_file_obj_or_dict(config_file_or_obj_or_dict,
context='rsmeval')

logger.info('Saving configuration file.')
configuration.save(output_dir)
Expand Down
39 changes: 6 additions & 33 deletions rsmtool/rsmpredict.py
View file
Open in desktop
Expand Up @@ -43,7 +43,7 @@ def compute_and_save_predictions(config_file_or_obj_or_dict,

Parameters
----------
config_file_or_obj : str or configuration_parser.Configuration
config_file_or_obj : str or pathlib.Path or Configuration or Dictionary
Path to the experiment configuration file.
Users can also pass a `Configuration` object that is in memory.
Relative paths in the configuration file will be interpreted relative
Expand All @@ -65,38 +65,11 @@ def compute_and_save_predictions(config_file_or_obj_or_dict,

logger = logging.getLogger(__name__)

# check what sort of input we got
# if we got a string we consider this to be path to config file
if isinstance(config_file_or_obj_or_dict, str):

# Instantiate configuration parser object
parser = ConfigurationParser.get_configparser(config_file_or_obj_or_dict)
configuration = parser.read_normalize_validate_and_process_config(config_file_or_obj_or_dict,
context='rsmpredict')

elif isinstance(config_file_or_obj_or_dict, dict):

# initialize the parser from dict
parser = ConfigurationParser()
configuration = parser.load_normalize_and_validate_config_from_dict(config_file_or_obj_or_dict,
context='rsmpredict')

elif isinstance(config_file_or_obj_or_dict, Configuration):

configuration = config_file_or_obj_or_dict
# raise an error if we are passed a Configuration object
# without a configdir attribute. This can only
# happen if the object was constructed using an earlier version
# of RSMTool and stored
if configuration.configdir is None:
raise AttributeError("Configuration object must have configdir attribute.")

else:
raise ValueError("The input to compute_and_save_predictions must be "
"a path to the file (str), a dictionary, "
"or a configuration object. You passed "
"{}.".format(type(config_file_or_obj_or_dict)))

# initialize a correct configparser
parser = ConfigurationParser.get_configparser(config_file_or_obj_or_dict)
# create a configuration object from input
configuration = parser.get_configuration_from_file_obj_or_dict(config_file_or_obj_or_dict,
context='rsmpredict')
# get the experiment ID
experiment_id = configuration['experiment_id']

Expand Down
40 changes: 6 additions & 34 deletions rsmtool/rsmsummarize.py
View file
Open in desktop
Expand Up @@ -99,7 +99,7 @@ def run_summary(config_file_or_obj_or_dict,

Parameters
----------
config_file_or_obj_or_dict : str or Configuration or dict
config_file_or_obj_or_dict : str or pathlib.Path or Configuration or Dictionary
Path to the experiment configuration file.
Users can also pass a `Configuration` object that is in memory
or a Python dictionary with keys corresponding to fields in the
Expand Down Expand Up @@ -131,39 +131,11 @@ def run_summary(config_file_or_obj_or_dict,
os.makedirs(figdir, exist_ok=True)
os.makedirs(reportdir, exist_ok=True)

# check what sort of input we got
# if we got a string we consider this to be path to config file
if isinstance(config_file_or_obj_or_dict, str):

# Instantiate configuration parser object
parser = ConfigurationParser.get_configparser(config_file_or_obj_or_dict)
configuration = parser.read_normalize_validate_and_process_config(config_file_or_obj_or_dict,
context='rsmsummarize')

elif isinstance(config_file_or_obj_or_dict, dict):

# initialize the parser from dict
parser = ConfigurationParser()
configuration = parser.load_normalize_and_validate_config_from_dict(config_file_or_obj_or_dict,
context='rsmsummarize')

elif isinstance(config_file_or_obj_or_dict, Configuration):

configuration = config_file_or_obj_or_dict
# raise an error if we are passed a Configuration object
# without a configdir attribute. This can only
# happen if the object was constructed using an earlier version
# of RSMTool and stored
if configuration.configdir is None:
raise AttributeError("Configuration object must have configdir attribute.")

else:
raise ValueError("The input to run_summary must be "
"a path to the file (str), a dictionary, "
"or a configuration object. You passed "
"{}.".format(type(config_file_or_obj_or_dict)))
logger.info('Saving configuration file.')
configuration.save(output_dir)
# initialize a correct configparser
parser = ConfigurationParser.get_configparser(config_file_or_obj_or_dict)
# create a configuration object from input
configuration = parser.get_configuration_from_file_obj_or_dict(config_file_or_obj_or_dict,
context='rsmsummarize')

# get the list of the experiment dirs
experiment_dirs = configuration['experiment_dirs']
Expand Down
35 changes: 5 additions & 30 deletions rsmtool/rsmtool.py
View file
Open in desktop
Expand Up @@ -36,7 +36,7 @@ def run_experiment(config_file_or_obj_or_dict,

Parameters
----------
config_file_or_obj_or_dict : str or Configuration or Dictionary
config_file_or_obj_or_dict : str or pathlib.Path or Configuration or Dictionary
Path to the experiment configuration file.
Users can also pass a `Configuration` object that is in memory
or a Python dictionary with keys corresponding to fields in the
Expand Down Expand Up @@ -74,35 +74,10 @@ def run_experiment(config_file_or_obj_or_dict,
makedirs(figdir, exist_ok=True)
makedirs(reportdir, exist_ok=True)

# check what sort of input we got
# if we got a string we consider this to be path to config file
if isinstance(config_file_or_obj_or_dict, str):

# Instantiate configuration parser object
parser = ConfigurationParser.get_configparser(config_file_or_obj_or_dict)
configuration = parser.read_normalize_validate_and_process_config(config_file_or_obj_or_dict)

elif isinstance(config_file_or_obj_or_dict, dict):

# initialize the parser from dict
parser = ConfigurationParser()
configuration = parser.load_normalize_and_validate_config_from_dict(config_file_or_obj_or_dict)

elif isinstance(config_file_or_obj_or_dict, Configuration):

configuration = config_file_or_obj_or_dict
# raise an error if we are passed a Configuration object
# without a configdir attribute. This can only
# happen if the object was constructed using an earlier version
# of RSMTool and stored
if configuration.configdir is None:
raise AttributeError("Configuration object must have configdir attribute.")

else:
raise ValueError("The input to run_experiment must be "
"a path to the file (str), a dictionary, "
"or a configuration object. You passed "
"{}.".format(type(config_file_or_obj_or_dict)))
# initialize a correct configparser
parser = ConfigurationParser.get_configparser(config_file_or_obj_or_dict)
# create a configuration object from input
configuration = parser.get_configuration_from_file_obj_or_dict(config_file_or_obj_or_dict)

logger.info('Saving configuration file.')
configuration.save(output_dir)
Expand Down