Skip to content

Commit

Permalink
Merge pull request #28 from lahwaacz/master
Browse files Browse the repository at this point in the history 
Refactor config file parsing
  • Loading branch information
@bw2
bw2 committed Nov 9, 2015
2 parents 55dc9e8 + 5931ee1 commit de246ae
Show file tree
Hide file tree
Showing 2 changed files with 97 additions and 67 deletions.
145 changes: 86 additions & 59 deletions configargparse.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ def __init__(self,

auto_env_var_prefix=None,

config_file_parser=None,
default_config_files=[],
ignore_unknown_config_file_keys=False,
allow_unknown_config_file_keys=False, # deprecated
Expand Down Expand Up @@ -124,6 +125,8 @@ def __init__(self,
variables whose names are this prefix followed by the config
file key, all in upper case. (eg. setting this to "foo_" will
allow an arg like "--arg1" to also be set via env. var FOO_ARG1)
config_file_parser: An instance of a parser to be used for parsing
config files. Default: ConfigFileParser()
default_config_files: When specified, this list of config files will
be parsed in order, with the values from each config file
taking precedence over pervious ones. This allows an application
Expand Down Expand Up @@ -177,6 +180,10 @@ def __init__(self,
argparse.ArgumentParser.__init__(self, **kwargs_for_super)

# parse the additionial args
if config_file_parser is None:
self._config_file_parser = ConfigFileParser()
else:
self._config_file_parser = config_file_parser
self._default_config_files = default_config_files
self._ignore_unknown_config_file_keys = ignore_unknown_config_file_keys \
or allow_unknown_config_file_keys
Expand Down Expand Up @@ -283,7 +290,9 @@ def parse_known_args(self, args = None, namespace = None,
# parse each config file
for stream in config_streams[::-1]:
try:
config_settings = self.parse_config_file(stream)
config_settings = self._config_file_parser.parse(stream)
except ConfigFileParserException as e:
self.error(e)
finally:
if hasattr(stream, "close"):
stream.close()
Expand Down Expand Up @@ -357,8 +366,9 @@ def parse_known_args(self, args = None, namespace = None,

if output_file_paths:
# generate the config file contents
contents = self.convert_parsed_args_to_config_file_contents(
config_items = self.get_items_for_config_file_output(
self._source_to_settings, namespace)
contents = self._config_file_parser.serialize(config_items)
for output_file_path in output_file_paths:
with open(output_file_path, "w") as output_file:
output_file.write(contents)
Expand All @@ -367,37 +377,6 @@ def parse_known_args(self, args = None, namespace = None,
self.exit(0, "Wrote config file to " + str(output_file_paths))
return namespace, unknown_args

def parse_config_file(self, stream):
"""Parses a config file and return a dictionary of settings"""

settings = OrderedDict()
for i, line in enumerate(stream):
line = line.strip()
if not line or line[0] in ["#", ";", "["] or line.startswith("---"):
continue
white_space = "\\s*"
key = "(?P<key>[^:=;#\s]+?)"
value1 = white_space+"[:=]"+white_space+"(?P<value>[^;#]+?)"
value2 = white_space+"[\s]"+white_space+"(?P<value>[^;#\s]+?)"
comment = white_space+"(?P<comment>\\s[;#].*)?"

key_only_match = re.match("^" + key + comment + "$", line)
if key_only_match:
key = key_only_match.group("key")
settings[key] = "true"
continue

key_value_match = re.match("^"+key+value1+comment+"$", line) or \
re.match("^"+key+value2+comment+"$", line)
if key_value_match:
key = key_value_match.group("key")
value = key_value_match.group("value")
settings[key] = value
continue

self.error("Unexpected line %s in %s: %s" % (i, stream.name, line))
return settings

def get_command_line_key_for_unknown_config_file_setting(self, key):
"""Compute a commandline arg key to be used for a config file setting
that doesn't correspond to any defined configargparse arg (and so
Expand All @@ -411,19 +390,19 @@ def get_command_line_key_for_unknown_config_file_setting(self, key):

return command_line_key

def convert_parsed_args_to_config_file_contents(self, source_to_settings,
parsed_namespace):
def get_items_for_config_file_output(self, source_to_settings,
parsed_namespace):
"""Does the inverse of config parsing by taking parsed values and
converting them back to a string representing config file contents.
Args:
source_to_settings: the dictionary created within parse_known_args()
parsed_namespace: namespace object created within parse_known_args()
Returns:
contents of config file as a string
an OrderedDict with the items to be written to the config file
"""
r = StringIO()
for source, settings in self._source_to_settings.items():
config_file_items = OrderedDict()
for source, settings in source_to_settings.items():
if source == _COMMAND_LINE_SOURCE_KEY:
_, existing_command_line_args = settings['']
for action in self._actions:
Expand All @@ -437,28 +416,26 @@ def convert_parsed_args_to_config_file_contents(self, source_to_settings,
value = str(value).lower()
elif type(value) is list:
value = "["+", ".join(map(str, value))+"]"
r.write("%s = %s\n" % (config_file_keys[0], value))
config_file_items[config_file_keys[0]] = value

elif source == _ENV_VAR_SOURCE_KEY:
for key, (action, value) in settings.items():
config_file_keys = self.get_possible_config_keys(action)
if config_file_keys:
value = getattr(parsed_namespace, action.dest, None)
if value is not None:
r.write("%s = %s\n" % (config_file_keys[0], value))
config_file_items[config_file_keys[0]] = value
elif source.startswith(_CONFIG_FILE_SOURCE_KEY):
for key, (action, value) in settings.items():
r.write("%s = %s\n" % (key, value))
config_file_items[key] = value
elif source == _DEFAULTS_SOURCE_KEY:
for key, (action, value) in settings.items():
config_file_keys = self.get_possible_config_keys(action)
if config_file_keys:
value = getattr(parsed_namespace, action.dest, None)
if value is not None:
r.write("%s = %s\n" % (config_file_keys[0], value))

return r.getvalue()

config_file_items[config_file_keys[0]] = value
return config_file_items

def convert_setting_to_command_line_arg(self, action, key, value):
"""Converts a config file or env var key/value to a list of
Expand Down Expand Up @@ -625,25 +602,14 @@ def format_help(self):

msg += ("Args that start with '%s' (eg. %s) can also be set in "
"a config file") % (cc, config_settable_args[0][0])
config_arg_string = " or one ".join(a.option_strings[0]
config_arg_string = " or ".join(a.option_strings[0]
for a in config_path_actions if a.option_strings)
if config_arg_string:
config_arg_string = "specified via " + config_arg_string
if default_config_files or config_arg_string:
msg += " (%s)" % " or ".join(default_config_files +
msg += " (%s)." % " or ".join(default_config_files +
[config_arg_string])
msg += " by using .ini or .yaml-style syntax "
examples = []
key_value_args = [arg for arg, a in config_settable_args
if type(a) not in ACTION_TYPES_THAT_DONT_NEED_A_VALUE]
if key_value_args:
examples += ["%s=value" % key_value_args[0].strip(cc)]
flag_args = [arg for arg, a in config_settable_args
if type(a) in ACTION_TYPES_THAT_DONT_NEED_A_VALUE]
if flag_args:
examples += ["%s=TRUE" % flag_args[0].strip(cc)]
if examples:
msg += "(eg. %s)." % " or ".join(examples)
msg += " " + self._config_file_parser.get_syntax_description()

if self._add_env_var_help:
env_var_actions = [(a.env_var, a) for a in self._actions
Expand Down Expand Up @@ -672,6 +638,67 @@ def format_help(self):
return argparse.ArgumentParser.format_help(self)


class ConfigFileParser(object):

def parse(self, stream):
"""Parses a config file and return a dictionary of settings"""

settings = OrderedDict()
for i, line in enumerate(stream):
line = line.strip()
if not line or line[0] in ["#", ";", "["] or line.startswith("---"):
continue
white_space = "\\s*"
key = "(?P<key>[^:=;#\s]+?)"
value1 = white_space+"[:=]"+white_space+"(?P<value>[^;#]+?)"
value2 = white_space+"[\s]"+white_space+"(?P<value>[^;#\s]+?)"
comment = white_space+"(?P<comment>\\s[;#].*)?"

key_only_match = re.match("^" + key + comment + "$", line)
if key_only_match:
key = key_only_match.group("key")
settings[key] = "true"
continue

key_value_match = re.match("^"+key+value1+comment+"$", line) or \
re.match("^"+key+value2+comment+"$", line)
if key_value_match:
key = key_value_match.group("key")
value = key_value_match.group("value")
settings[key] = value
continue

raise ConfigFileParserException("Unexpected line %s in %s: %s" % \
(i, stream.name, line))
return settings

def serialize(self, items):
"""Does the inverse of config parsing by taking parsed values and
converting them back to a string representing config file contents.
Args:
items: an OrderedDict with items to be written to the config file
Returns:
contents of config file as a string
"""
r = StringIO()
for key, value in items.items():
r.write("%s = %s\n" % (key, value))
return r.getvalue()

def get_syntax_description(self):
msg = ("The recognized syntax for setting (key, value) pairs is based "
"on the INI and YAML formats (e.g. key=value or foo=TRUE). "
"For full documentation of the differences from the standards "
"please refer to the ConfigArgParse documentation.")
return msg

class ConfigFileParserException(Exception):
"""Raised when config file parsing failed.
"""
pass



def add_argument(self, *args, **kwargs):
"""
Expand Down
19 changes: 11 additions & 8 deletions tests/test_configargparse.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ def testBasicCase2(self, use_groups=False):
self.assertRegex(self.format_help(),
'usage: .* \[-h\] --genome GENOME \[-v\] -g MY_CFG_FILE'
' \[-d DBSNP\]\s+\[-f FRMT\]\s+vcf \[vcf ...\]\n\n' +
7*'(.+\s+)'+ # repeated 7 times because .+ matches atmost 1 line
8*'(.+\s+)'+ # repeated 8 times because .+ matches atmost 1 line
'positional arguments:\n'
' vcf \s+ Variant file\(s\)\n\n'
'optional arguments:\n'
Expand All @@ -240,7 +240,7 @@ def testBasicCase2(self, use_groups=False):
self.assertRegex(self.format_help(),
'usage: .* \[-h\] --genome GENOME \[-v\] -g MY_CFG_FILE'
' \[-d DBSNP\]\s+\[-f FRMT\]\s+vcf \[vcf ...\]\n\n'+
7*'.+\s+'+ # repeated 7 times because .+ matches atmost 1 line
8*'.+\s+'+ # repeated 8 times because .+ matches atmost 1 line
'positional arguments:\n'
' vcf \s+ Variant file\(s\)\n\n'
'optional arguments:\n'
Expand Down Expand Up @@ -305,7 +305,7 @@ def testMutuallyExclusiveArgs(self):
self.assertRegex(self.format_help(),
'usage: .* \[-h\] --genome GENOME \[-v\]\s+ \(-f1 TYPE1_CFG_FILE \|'
' \s*-f2 TYPE2_CFG_FILE\)\s+\(-f FRMT \| -b\)\n\n' +
5*'.+\s+'+ # repeated 5 times because .+ matches atmost 1 line
7*'.+\s+'+ # repeated 7 times because .+ matches atmost 1 line
'optional arguments:\n'
' -h, --help show this help message and exit\n'
' -f1 TYPE1_CFG_FILE, --type1-cfg-file TYPE1_CFG_FILE\n'
Expand Down Expand Up @@ -584,7 +584,7 @@ def testConstructor_ConfigFileArgs(self):

self.assertRegex(self.format_help(),
'usage: .* \[-h\] -c CONFIG_FILE --genome GENOME\n\n'+
5*'.+\s+'+ # repeated 5 times because .+ matches atmost 1 line
7*'.+\s+'+ # repeated 7 times because .+ matches atmost 1 line
'optional arguments:\n'
' -h, --help\s+ show this help message and exit\n'
' -c CONFIG_FILE, --config CONFIG_FILE\s+ my config file\n'
Expand Down Expand Up @@ -622,10 +622,13 @@ def test_FormatHelp(self):
'usage: .* \[-h\] -c CONFIG_FILE\s+'
'\[-w CONFIG_OUTPUT_PATH\]\s* --arg1 ARG1\s* \[--flag\]\s*'
'Args that start with \'--\' \(eg. --arg1\) can also be set in a '
'config file\s*\(~/.myconfig or specified via -c\)\s* by using '
'.ini or .yaml-style syntax \(eg.\s*arg1=value or flag=TRUE\).\s* '
'If an arg is specified in more than one place, then\s*'
'commandline values override config file values which override '
'config file\s*\(~/.myconfig or specified via -c\).\s*'
'The recognized syntax for setting \(key,\s*value\) pairs is based on '
'the INI and YAML formats \(e.g. key=value or\s*foo=TRUE\). For full '
'documentation of the differences from the standards please\s*'
'refer to the ConfigArgParse documentation.\s*'
'If an arg is specified in more than\s*one place, then '
'commandline values override config file values which override\s*'
'defaults.\s*'
'optional arguments:\s*'
'-h, --help \s* show this help message and exit\n\s*'
Expand Down

0 comments on commit de246ae

Please sign in to comment.