Merge pull request #146 from adobe-apiplatform/v2

Bring branch up to date

Author: phil-levy

tests/config_file_test.py

@@ -21,6 +21,7 @@
 import types
 import unittest
 import os
+import os.path
 import yaml
 
 import mock
@@ -43,14 +44,17 @@ def assert_eq(self, result, expected, error_message):
 
     @mock.patch('__builtin__.open')
     @mock.patch('yaml.load')
-    def test_load_root_config(self, mock_yaml, mock_open):
+    @mock.patch('os.path.isfile')
+    def test_load_root_config(self, mock_isfile, mock_yaml, mock_open):
         '''
         tests ConfigFileLoader.load_root_config by inputing a root configuration
         file path, and asserts that the resulting processed content has the
         file references properly updated
+        :type mock_isfile: Mock
         :type mock_yaml: Mock
         :type mock_open: Mock
         '''
+        mock_isfile.return_value = True
         mocked_open = mock_open('test')
         mocked_open_name = '%s.open' % __name__
         with patch(mocked_open_name, mocked_open, create=True):
@@ -72,9 +76,7 @@ def test_load_root_config(self, mock_yaml, mock_open):
                                 ]
                         }
                 }
-            with mock.patch(os.path.isfile) as m_isfile:
-                m_isfile.return_value = True
-                yml = ConfigFileLoader.load_root_config('config-test/user-sync-config-test.yml')
+            yml = ConfigFileLoader.load_root_config('config-test/user-sync-config-test.yml')
 
             # test path updating
             self.assert_eq(yml['logging']['file_log_directory'], os.path.abspath('config-test/log-test-1'),
@@ -92,24 +94,25 @@ def test_load_root_config(self, mock_yaml, mock_open):
 
     @mock.patch('__builtin__.open')
     @mock.patch('yaml.load')
-    def test_load_root_default_config(self, mock_yaml, mock_open):
+    @mock.patch('os.path.isfile')
+    def test_load_root_default_config(self, mock_isfile, mock_yaml, mock_open):
         '''
         tests ConfigFileLoader.load_root_config by inputing a root configuration
         file path, and asserts that the resulting processed content has the
         file references properly updated
+        :type mock_isfile: Mock
         :type mock_yaml: Mock
         :type mock_open: Mock
         '''
+        mock_isfile.return_value = True
         mocked_open = mock_open('test')
         mocked_open_name = '%s.open' % __name__
         with patch(mocked_open_name, mocked_open, create=True):
             mock_yaml.return_value = {
                     'adobe_users': {'connectors': {'umapi': 'test-123'}},
                     'logging': {'log_to_file': True},
                 }
-            with mock.patch(os.path.isfile) as m_isfile:
-                m_isfile.return_value = True
-                yml = ConfigFileLoader.load_root_config('config-test-2/user-sync-config-test.yml')
+            yml = ConfigFileLoader.load_root_config('config-test-2/user-sync-config-test.yml')
 
             # assert default values are preserved
             self.assert_eq(yml['logging']['file_log_directory'], os.path.abspath('config-test-2/logs'),
@@ -121,14 +124,17 @@ def test_load_root_default_config(self, mock_yaml, mock_open):
 
     @mock.patch('__builtin__.open')
     @mock.patch('yaml.load')
-    def test_load_sub_config(self, mock_yaml, mock_open):
+    @mock.patch('os.path.isfile')
+    def test_load_sub_config(self, mock_isfile, mock_yaml, mock_open):
         '''
         same purpose as test_load_root_config, but tests against sub
         configuration path updates (which is currently only the private key
         path in the umapi configuration file)
+        :type mock_isfile: Mock
         :type mock_yaml: Mock
         :type mock_open: Mock
         '''
+        mock_isfile.return_value = True
         mocked_open = mock_open('test')
         mocked_open_name = '%s.open' % __name__
         with patch(mocked_open_name, mocked_open, create=True):
@@ -141,9 +147,7 @@ def test_load_sub_config(self, mock_yaml, mock_open):
                             'test-2': 123
                         }
                 }
-            with mock.patch(os.path.isfile) as m_isfile:
-                m_isfile.return_value = True
-                yml = ConfigFileLoader.load_sub_config('sub-config-test/user-sync-config-test.yml')
+            yml = ConfigFileLoader.load_sub_config('sub-config-test/user-sync-config-test.yml')
 
             # test path updating
             self.assert_eq(yml['enterprise']['priv_key_path'], os.path.abspath('keys/test-key.key'),

tests/config_test.py

@@ -194,6 +194,7 @@ def mock_send_commands(commands, callback = None):
 
         action_manager = mock.mock.create_autospec(user_sync.connector.umapi.ActionManager)
         action_manager.has_work = lambda: False
+        action_manager.get_statistics = lambda: (len(commands_list_output), 0)
         mock_connector = mock.mock.create_autospec(user_sync.connector.umapi)
         mock_connector.iter_users = lambda: list(users_to_return)
         mock_connector.send_commands = mock_send_commands

tests/connector/dashboard_test.py renamed to tests/connector/umapi_test.py

@@ -206,11 +206,29 @@ def create_config_loader(args):
 
 
 def create_config_loader_options(args):
+    '''
+    This is where all the command-line arguments get set as options in the main config
+    so that it appears as if they were loaded as part of the main configuration file.
+    If you add an option that is supposed to be set from the command line here, you
+    had better make sure you add it to the ones read in get_rule_options.
+    :param args: the command-line args as parsed
+    :return: the configured options for the config loader.
+    '''
     config_options = {
-        'test_mode': args.test_mode,
+        'delete_strays': False,
+        'directory_connector_module_name': None,
+        'directory_connector_overridden_options': None,
+        'directory_group_filter': None,
+        'directory_group_mapped': False,
+        'disentitle_strays': False,
+        'exclude_strays': False,
         'manage_groups': args.manage_groups,
+        'remove_strays': False,
+        'stray_list_input_path': None,
+        'stray_list_output_path': None,
+        'test_mode': args.test_mode,
         'update_user_info': args.update_user_info,
-        'directory_group_mapped': False,
+        'username_filter_regex': None,
     }
 
     # --users
@@ -256,7 +274,6 @@ def create_config_loader_options(args):
             if not adobe_action_args:
                 raise AssertionException('Missing file path for --adobe-only-user-action %s [file_path]' % adobe_action)
             config_options['stray_list_output_path'] = adobe_action_args.pop(0)
-            logger.info('Writing unmatched users to: %s', config_options['stray_list_output_path'])
         elif (adobe_action == 'delete'):
             config_options['delete_strays'] = True
         elif (adobe_action == 'remove'):
@@ -289,9 +306,9 @@ def log_parameters(args):
     '''
     logger.info('------- Invocation parameters -------')
     logger.info(' '.join(sys.argv))
-    logger.info('-------- Internal parameters --------')
+    logger.debug('-------- Internal parameters --------')
     for parameter_name, parameter_value in args.__dict__.iteritems():
-        logger.info('  %s: %s', parameter_name, parameter_value)
+        logger.debug('  %s: %s', parameter_name, parameter_value)
     logger.info('-------------------------------------')
 
 
@@ -323,7 +340,7 @@ def main():
             finally:
                 lock.unlock()
         else:
-            logger.info("Process is already locked")
+            logger.critical("A different User Sync process is currently running.")
 
     except AssertionException as e:
         if (not e.is_reported()):

tests/rules_test.py

@@ -127,6 +127,10 @@ def get_directory_connector_configs(self):
         directory_config = self.main_config.get_dict_config('directory_users', True)
         if directory_config != None:
             connectors_config = directory_config.get_dict_config('connectors', True)
+        # make sure neither ldap nor csv connectors get reported as unused
+        if connectors_config:
+            connectors_config.get_list('ldap', True)
+            connectors_config.get_list('csv', True)
         return connectors_config
 
     def get_directory_connector_options(self, connector_name):
@@ -171,8 +175,8 @@ def get_directory_groups(self):
             if groups == None:
                 adobe_groups_by_directory_group[directory_group] = groups = []
 
-            adobe_groups_config = item.get_list_config('adobe_groups')
-            for adobe_group in adobe_groups_config.iter_values(types.StringTypes):
+            adobe_groups = item.get_list('adobe_groups', True)
+            for adobe_group in adobe_groups or []:
                 group = user_sync.rules.AdobeGroup.create(adobe_group)
                 if group is None:
                     validation_message = 'Bad adobe group: "%s" in directory group: "%s"' % (adobe_group, directory_group)
@@ -268,17 +272,17 @@ def get_rule_options(self):
             default_country_code = directory_config.get_string('default_country_code', True)
         if not new_account_type:
             new_account_type = user_sync.identity_type.ENTERPRISE_IDENTITY_TYPE
-            self.logger.info("Using default for new_account_type: %s", new_account_type)
+            self.logger.debug("Using default for new_account_type: %s", new_account_type)
 
         # process exclusion configuration options
         exclude_identity_types = exclude_identity_type_names = []
         exclude_users = exclude_users_regexps = []
         exclude_groups = exclude_group_names = []
-        umapi_config = self.main_config.get_dict_config('adobe_users', True)
-        if umapi_config:
-            exclude_identity_type_names = umapi_config.get_list('exclude_identity_types', True) or []
-            exclude_users_regexps = umapi_config.get_list('exclude_users', True) or []
-            exclude_group_names = umapi_config.get_list('exclude_groups', True) or []
+        adobe_config = self.main_config.get_dict_config('adobe_users', True)
+        if adobe_config:
+            exclude_identity_type_names = adobe_config.get_list('exclude_identity_types', True) or []
+            exclude_users_regexps = adobe_config.get_list('exclude_users', True) or []
+            exclude_group_names = adobe_config.get_list('exclude_adobe_groups', True) or []
         for name in exclude_identity_type_names:
             message_format = 'Illegal value in exclude_identity_types: %s'
             identity_type = user_sync.identity_type.parse_identity_type(name, message_format)
@@ -335,6 +339,7 @@ def get_rule_options(self):
             'disentitle_strays': options['disentitle_strays'],
             'exclude_groups': exclude_groups,
             'exclude_identity_types': exclude_identity_types,
+            'exclude_strays': options['exclude_strays'],
             'exclude_users': exclude_users,
             'extended_attributes': extended_attributes,
             'manage_groups': options['manage_groups'],

user_sync/app.py

@@ -65,7 +65,7 @@ def __init__(self, caller_options):
         builder.set_string_value('domain_column_name', 'domain')
         builder.set_string_value('identity_type_column_name', 'type')
         builder.set_string_value('user_identity_type', None)
-        builder.set_string_value('logger_name', 'connector.' + CSVDirectoryConnector.name)
+        builder.set_string_value('logger_name', CSVDirectoryConnector.name)
         builder.require_string_value('file_path')
         options = builder.get_options()
 
@@ -85,9 +85,9 @@ def load_users_and_groups(self, groups, extended_attributes):
         '''        
         options = self.options
         file_path = options['file_path']
-        self.logger.info('Reading from: %s', file_path)
+        self.logger.debug('Reading from: %s', file_path)
         self.users = users = self.read_users(file_path, extended_attributes)
-        self.logger.info('Number of users loaded: %d', len(users))
+        self.logger.debug('Number of users loaded: %d', len(users))
         return users.itervalues()
 
     def read_users(self, file_path, extended_attributes):

user_sync/config.py

@@ -66,7 +66,7 @@ def __init__(self, caller_options):
         builder.set_string_value('user_domain_format', None)
         builder.set_string_value('user_identity_type', None)
         builder.set_int_value('search_page_size', 200)
-        builder.set_string_value('logger_name', 'connector.' + LDAPDirectoryConnector.name)
+        builder.set_string_value('logger_name', LDAPDirectoryConnector.name)
         host = builder.require_string_value('host')
         username = builder.require_string_value('username')
         builder.require_string_value('base_dn')
@@ -86,18 +86,19 @@ def __init__(self, caller_options):
         require_tls_cert = options['require_tls_cert']
         logger.debug('Initialized with options: %s', options)            
 
-        logger.info('Connecting to: %s using username: %s', host, username)            
+        logger.debug('Connecting to: %s using username: %s', host, username)
         if not require_tls_cert:
             ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER)
-        connection = ldap.initialize(host)
-        connection.protocol_version = ldap.VERSION3
-        connection.set_option(ldap.OPT_REFERRALS, 0)
+        
         try:
+            connection = ldap.initialize(host)
+            connection.protocol_version = ldap.VERSION3
+            connection.set_option(ldap.OPT_REFERRALS, 0)
             connection.simple_bind_s(username, password)
         except Exception as e:
             raise user_sync.error.AssertionException(repr(e))
         self.connection = connection
-        logger.info('Connected')            
+        logger.debug('Connected')
 
     def load_users_and_groups(self, groups, extended_attributes):
         '''
@@ -116,7 +117,7 @@ def load_users_and_groups(self, groups, extended_attributes):
                 user_by_uid[uid] = user
             user_by_dn[user_dn] = user
 
-        self.logger.info('Total users loaded: %d', len(user_by_dn))
+        self.logger.debug('Total users loaded: %d', len(user_by_dn))
 
         for group in groups:
             total_group_members = 0