From 9fdfd934d61aff6ae3683ed9755803a2bfd795ee Mon Sep 17 00:00:00 2001 From: Lockefox Date: Mon, 9 Jul 2018 14:23:34 -0700 Subject: [PATCH 1/2] adding functionality to ignore jinja templated values as None --- prosper/common/prosper_config.py | 37 +++++++++++++++++++++++++++++--- tests/test_config.cfg.j2 | 1 + tests/test_prosper_config.py | 17 +++++++++++++++ 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/prosper/common/prosper_config.py b/prosper/common/prosper_config.py index 75e3d8f..1ec66d7 100644 --- a/prosper/common/prosper_config.py +++ b/prosper/common/prosper_config.py @@ -8,11 +8,12 @@ import configparser from configparser import ExtendedInterpolation import logging +import re import anyconfig import anytemplate - +JINJA_PATTERN = re.compile(r'.*{{\S*}}.*') def render_secrets( config_path, secret_path, @@ -44,6 +45,36 @@ def render_secrets( return p_config +def check_value( + config, + section, + option, + jinja_pattern=JINJA_PATTERN, +): + """try to figure out if value is valid or jinja2 template value + + Args: + config (:obj:`configparser.ConfigParser`): config object to read key from + section (str): name of section in configparser + option (str): name of option in configparser + jinja_pattern (:obj:`_sre.SRE_Pattern`): a `re.compile()` pattern to match on + + Returns: + str: value if value, else None + + Raises: + KeyError: + configparser.NoOptionError: + configparser.NoSectionError: + + """ + value = config[section][option] + print(value) + print(re.match(jinja_pattern, str(value))) + if re.match(jinja_pattern, value): + return None + + return value class ProsperConfig(object): """configuration handler for all prosper projects @@ -160,7 +191,7 @@ def get_option( option = None try: - option = self.local_config[section_name][key_name] + option = check_value(self.local_config, section_name, key_name) self.logger.debug('-- using local config') if option: return option @@ -168,7 +199,7 @@ def get_option( self.logger.debug('`%s` not found in local config', section_info) try: - option = self.global_config[section_name][key_name] + option = check_value(self.global_config, section_name, key_name) self.logger.debug('-- using global config') if option: return option diff --git a/tests/test_config.cfg.j2 b/tests/test_config.cfg.j2 index f060396..0e8b847 100644 --- a/tests/test_config.cfg.j2 +++ b/tests/test_config.cfg.j2 @@ -1,5 +1,6 @@ [TEST] secret = {{test.secret}} + partial_secret = secret_{{test.partial_secret}} key1 = vals key2 = 100 key3 = diff --git a/tests/test_prosper_config.py b/tests/test_prosper_config.py index 3aab8f7..c424f00 100644 --- a/tests/test_prosper_config.py +++ b/tests/test_prosper_config.py @@ -3,6 +3,7 @@ Pytest functions for exercising prosper.common.prosper_config """ +import configparser import os from os import path import json @@ -29,6 +30,22 @@ def test_setup_environment(): expected_val = prosper_config.get_value_from_environment('TEST', 'dummy_val') assert expected_val == ENV_TEST_1 +def test_check_value(): + """validate check_value happypath""" + config = prosper_config.read_config(path.join(HERE, 'test_config.cfg.j2')) + + assert prosper_config.check_value(config, 'TEST', 'secret') == None + assert prosper_config.check_value(config, 'TEST', 'partial_secret') == None + assert prosper_config.check_value(config, 'TEST', 'key1') == 'vals' +def test_skip_jinja(): + """validate system treats jinja formatted blocks as blanks""" + config = prosper_config.ProsperConfig(path.join(HERE, 'test_config.cfg.j2')) + + print(config.get_option('TEST', 'secret')) + assert config.get_option('TEST', 'secret') == None + assert config.get_option('TEST', 'partial_secret') == None + assert config.get_option('TEST', 'key1') == 'vals' + def test_render_secrets(): """happypath test for p_config.render_secrets""" config_path = path.join(HERE, 'test_config.cfg.j2') From 067f14510a922085674fb0513699a742f9d00f28 Mon Sep 17 00:00:00 2001 From: Lockefox Date: Mon, 9 Jul 2018 15:00:49 -0700 Subject: [PATCH 2/2] trimming up code for code_climate tests --- prosper/common/prosper_config.py | 39 ++++++++------------------------ 1 file changed, 10 insertions(+), 29 deletions(-) diff --git a/prosper/common/prosper_config.py b/prosper/common/prosper_config.py index 1ec66d7..3f5e7f5 100644 --- a/prosper/common/prosper_config.py +++ b/prosper/common/prosper_config.py @@ -69,8 +69,6 @@ def check_value( """ value = config[section][option] - print(value) - print(re.match(jinja_pattern, str(value))) if re.match(jinja_pattern, value): return None @@ -88,8 +86,8 @@ class ProsperConfig(object): 5. args_default -- function default w/o global config Args: - config_filename (str): path to config file - local_filepath_override (str): path modifier for private config file + config_filename (str): path to config + local_filepath_override (str, optional): path to alternate private config file Attributes: global_config (:obj:`configparser.ConfigParser`) @@ -104,15 +102,6 @@ def __init__( config_filename, local_filepath_override='', ): - """get the config filename for initializing data structures - - Args: - config_filename (str): path to config - local_filepath_override (str, optional): path to alternate private config file - logger (:obj:`logging.Logger`, optional): capture messages to logger - debug_mode (bool, optional): enable debug modes for config helper - - """ self.config_filename = config_filename self.local_config_filename = get_local_config_filepath(config_filename) if local_filepath_override: @@ -135,7 +124,10 @@ def get( key_name (str): key name in config.section_name Returns: - (str): do not check defaults, only return local value + str: do not check defaults, only return local value + + Raises: + KeyError: unable to find option in either local or global config """ value = None @@ -179,7 +171,7 @@ def get_option( args_default (any): arg default given by a function Returns: - (str) appropriate response as per priority order + str: appropriate response as per priority order """ if args_option != args_default and\ @@ -214,9 +206,6 @@ def get_option( self.logger.debug('-- using default argument') return args_default #If all esle fails return the given default - def attach_logger(self, logger): - """because load orders might be weird, add logger later""" - self.logger = logger ENVNAME_PAD = 'PROSPER' def get_value_from_environment( @@ -283,9 +272,6 @@ def read_config( config_filepath (str): path to config file. abspath > relpath logger (:obj:`logging.Logger`): logger to catch error msgs - Raises: - FileNotFound: file access issues - """ config_parser = configparser.ConfigParser( interpolation=ExtendedInterpolation(), @@ -294,14 +280,9 @@ def read_config( inline_comment_prefixes=('#') ) logger.debug('config_filepath=%s', config_filepath) - try: - with open(config_filepath, 'r') as filehandle: - config_parser.read_file(filehandle) - except Exception as error_msg: - logger.error( - 'Unable to parse config file: %s', config_filepath, exc_info=True - ) - raise error_msg + + with open(config_filepath, 'r') as filehandle: + config_parser.read_file(filehandle) return config_parser