From c127325ecc5fb63a6aae1658e52ca5e807fe1baf Mon Sep 17 00:00:00 2001 From: Daniel Brotsky Date: Tue, 2 May 2017 21:28:10 -0700 Subject: [PATCH 1/2] Fix #159. Allow config files to be output from external commands. * Convention: a filename of $(...) is actually a process specification. * The process is executed in the current directory of the containing config file. * The standard output of the process is used as the config file content and parsed. * The process specification is handed to the platform shell for execution. --- user_sync/config.py | 59 ++++++++++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/user_sync/config.py b/user_sync/config.py index c3b693672..a15fb8ced 100644 --- a/user_sync/config.py +++ b/user_sync/config.py @@ -21,6 +21,7 @@ import logging import os import re +import subprocess import types import keyring @@ -641,6 +642,7 @@ def load_sub_config(cls, filename): def load_other_config(cls, filename): ''' same as load_root_config, but does no post-processing. + :type filename: str ''' return cls.load_from_yaml(filename, {}) @@ -667,25 +669,41 @@ def load_from_yaml(cls, filename, path_keys): does the key have a default value so that must be added to the dictionary if there is not already a value found. ''' - cls.filepath = os.path.abspath(filename) - cls.filename = os.path.split(cls.filepath)[1] - cls.dirpath = os.path.dirname(cls.filepath) - if not os.path.isfile(cls.filepath): - raise AssertionException('No such configuration file: %s' % (cls.filepath,)) - - # read the dict from the YAML file - try: - with open(filename, 'r', 1) as input_file: - yml = yaml.load(input_file) - except IOError as e: - # if a file operation error occurred while loading the - # configuration file, swallow up the exception and re-raise this - # as an configuration loader exception. - raise AssertionException('Error reading configuration file: %s' % e) - except yaml.error.MarkedYAMLError as e: - # same as above, but indicate this problem has to do with - # parsing the configuration file. - raise AssertionException('Error parsing configuration file: %s' % e) + if filename.startswith('$(') and filename.endswith(')'): + # it's a command line to execute and read standard output + dir_end = filename.index(']') + if filename.startswith('$([') and dir_end > 0: + dir = filename[3:dir_end] + cmd = filename[dir_end+1:-1] + else: + dir = os.path.abspath(".") + cmd = filename[3:-1] + try: + bytes = subprocess.check_output(cmd, cwd=dir, shell=True) + yml = yaml.load(bytes) + except subprocess.CalledProcessError as e: + raise AssertionException("Error executing process '%s' in dir '%s': %s" % (cmd, dir, e)) + except yaml.error.MarkedYAMLError as e: + raise AssertionException('Error parsing process YAML data: %s' % e) + else: + # it's a pathname to a configuration file to read + cls.filepath = os.path.abspath(filename) + if not os.path.isfile(cls.filepath): + raise AssertionException('No such configuration file: %s' % (cls.filepath,)) + cls.filename = os.path.split(cls.filepath)[1] + cls.dirpath = os.path.dirname(cls.filepath) + try: + with open(filename, 'r', 1) as input_file: + yml = yaml.load(input_file) + except IOError as e: + # if a file operation error occurred while loading the + # configuration file, swallow up the exception and re-raise this + # as an configuration loader exception. + raise AssertionException('Error reading configuration file: %s' % e) + except yaml.error.MarkedYAMLError as e: + # same as above, but indicate this problem has to do with + # parsing the configuration file. + raise AssertionException('Error parsing configuration file: %s' % e) # process the content of the dict for path_key, options in path_keys.iteritems(): @@ -773,6 +791,9 @@ def relative_path(cls, val, must_exist): if not isinstance(val, types.StringTypes): raise AssertionException("Expected pathname for setting %s in config file %s" % (cls.key_path, cls.filename)) + if val.startswith('$(') and val.endswith(')'): + # this presumes + return "$([" + cls.dirpath + "]" + val[2:-1] + ")" if cls.dirpath and not os.path.isabs(val): val = os.path.abspath(os.path.join(cls.dirpath, val)) if must_exist and not os.path.isfile(val): From df6d2d80754561f1a5fbb955ccdd51d4093543ee Mon Sep 17 00:00:00 2001 From: Daniel Brotsky Date: Tue, 2 May 2017 22:02:11 -0700 Subject: [PATCH 2/2] Fix #175: users doesn't default to all, instead no users are read * This was introduced when we first "defaulted" the `--users` argument (a long time ago). * The fix is to act as if *no* `--users` argument is `--users` by itself, because what the defaulting code really did was to default the argument to `all`. --- user_sync/app.py | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/user_sync/app.py b/user_sync/app.py index ff11ead8d..fd9e73b82 100644 --- a/user_sync/app.py +++ b/user_sync/app.py @@ -233,25 +233,24 @@ def create_config_loader_options(args): # --users users_args = args.users - if users_args is not None: - users_action = None if len(users_args) == 0 else user_sync.helper.normalize_string(users_args.pop(0)) - if (users_action == None or users_action == 'all'): - config_options['directory_connector_module_name'] = 'user_sync.connector.directory_ldap' - elif (users_action == 'file'): - if len(users_args) == 0: - raise AssertionException('Missing file path for --users %s [file_path]' % users_action) - config_options['directory_connector_module_name'] = 'user_sync.connector.directory_csv' - config_options['directory_connector_overridden_options'] = {'file_path': users_args.pop(0)} - elif (users_action == 'mapped'): - config_options['directory_connector_module_name'] = 'user_sync.connector.directory_ldap' - config_options['directory_group_mapped'] = True - elif (users_action == 'group'): - if len(users_args) == 0: - raise AssertionException('Missing groups for --users %s [groups]' % users_action) - config_options['directory_connector_module_name'] = 'user_sync.connector.directory_ldap' - config_options['directory_group_filter'] = users_args.pop(0).split(',') - else: - raise AssertionException('Unknown argument --users %s' % users_action) + users_action = None if not users_args else user_sync.helper.normalize_string(users_args.pop(0)) + if (users_action == None or users_action == 'all'): + config_options['directory_connector_module_name'] = 'user_sync.connector.directory_ldap' + elif (users_action == 'file'): + if len(users_args) == 0: + raise AssertionException('Missing file path for --users %s [file_path]' % users_action) + config_options['directory_connector_module_name'] = 'user_sync.connector.directory_csv' + config_options['directory_connector_overridden_options'] = {'file_path': users_args.pop(0)} + elif (users_action == 'mapped'): + config_options['directory_connector_module_name'] = 'user_sync.connector.directory_ldap' + config_options['directory_group_mapped'] = True + elif (users_action == 'group'): + if len(users_args) == 0: + raise AssertionException('Missing groups for --users %s [groups]' % users_action) + config_options['directory_connector_module_name'] = 'user_sync.connector.directory_ldap' + config_options['directory_group_filter'] = users_args.pop(0).split(',') + else: + raise AssertionException('Unknown argument --users %s' % users_action) username_filter_pattern = args.username_filter_pattern if (username_filter_pattern):