Added better error checks in investigator.

doc/enums.rst

@@ -25,6 +25,7 @@ Host Statuses
 * **active**: The host is part of the cluster and is registered with the Container Manager.
 * **inactive**: The host exists but is currently not actively working as a node for the Container Manager.
 * **disassociated**: The host exists but is not associated with the Container Manager.
+* **failed**: Unable to access the system.
 
 
 .. _upgrade-statuses:

src/commissaire/handlers/hosts.py

@@ -128,7 +128,6 @@ def on_put(self, req, resp, address):
             data = req.stream.read().decode()
             host_creation = json.loads(data)
             ssh_priv_key = host_creation['ssh_priv_key']
-            INVESTIGATE_QUEUE.put((host_creation, ssh_priv_key))
             host_creation['address'] = address
             host_creation['os'] = ''
             host_creation['status'] = 'investigating'
@@ -141,6 +140,7 @@ def on_put(self, req, resp, address):
             new_host = self.store.set(
                 '/commissaire/hosts/{0}'.format(
                     address), host.to_json(secure=True))
+            INVESTIGATE_QUEUE.put((host_creation, ssh_priv_key))
             resp.status = falcon.HTTP_201
             req.context['model'] = Host(**json.loads(new_host.value))
 

src/commissaire/jobs/investigator.py

@@ -55,26 +55,31 @@ def investigator(queue, store, run_once=False):
         logger.debug('Wrote key for {0}'.format(address))
         f.close()
 
-        result, facts = transport.get_info(address, key_file)
-        try:
-            f.unlink(key_file)
-            logger.debug('Removed temporary key file {0}'.format(key_file))
-        except:
-            logger.warn(
-                'Unable to remove the temporary key file: {0}'.format(
-                    key_file))
         key = '/commissaire/hosts/{0}'.format(address)
         data = json.loads(store.get(key).value)
-        data.update(facts)
-        data['last_check'] = datetime.datetime.utcnow().isoformat()
-        data['status'] = 'bootstrapping'
-        logger.info('Facts for {0} retrieved'.format(address))
 
+        try:
+            result, facts = transport.get_info(address, key_file)
+            data.update(facts)
+            data['last_check'] = datetime.datetime.utcnow().isoformat()
+            data['status'] = 'bootstrapping'
+            logger.info('Facts for {0} retrieved'.format(address))
+        except:
+            logger.warn('Getting info failed for {0}'.format(address))
+            data['status'] = 'failed'
+        finally:
+            try:
+                f.unlink(key_file)
+                logger.debug('Removed temporary key file {0}'.format(key_file))
+            except:
+                logger.warn(
+                    'Unable to remove the temporary key file: {0}'.format(
+                        key_file))
         store.set(key, json.dumps(data))
-        logging.debug('Investigation update for {0}: {1}'.format(
+        logging.debug('Finished investigation update for {0}: {1}'.format(
             address, data))
         logger.info(
-            'Finished and stored investigation for {0}'.format(address))
+            'Finished and stored investigation data for {0}'.format(address))
 
         if run_once:
             logger.info('Exiting due to run_once request.')

src/commissaire/transport/ansibleapi.py

@@ -262,4 +262,5 @@ def get_info(self, ip, key_file):
             facts['space'] = space
 
             return (result, facts)
+        # TODO: Make specific exceptions
         raise Exception('Can not get info for {0}'.format(ip))

test/test_jobs_clusterexec.py

@@ -39,42 +39,44 @@ def test_clusterexec(self):
         """
         Verify the clusterexec.
         """
-        with mock.patch('commissaire.transport.ansibleapi.Transport') as _tp:
-            _tp().restart.return_value = (0, {})
+        for cmd in ('restart', 'upgrade'):
+            with mock.patch('commissaire.transport.ansibleapi.Transport') as _tp:
+                getattr(_tp(), cmd).return_value = (0, {})
 
-            child = {'value': self.etcd_host}
-            return_value = MagicMock(_children=[child])
-            return_value.leaves = return_value._children
+                child = {'value': self.etcd_host}
+                return_value = MagicMock(_children=[child])
+                return_value.leaves = return_value._children
 
-            store = etcd.Client()
-            store.get = MagicMock('get')
-            store.get.return_value = return_value
-            store.set = MagicMock('set')
+                store = etcd.Client()
+                store.get = MagicMock('get')
+                store.get.return_value = return_value
+                store.set = MagicMock('set')
 
-            clusterexec('default', 'restart', store)
+                clusterexec('default', cmd, store)
 
-            self.assertEquals(1, store.get.call_count)
-            # We should have 4 sets for 1 host
-            self.assertEquals(4, store.set.call_count)
+                self.assertEquals(1, store.get.call_count)
+                # We should have 4 sets for 1 host
+                self.assertEquals(4, store.set.call_count)
 
     def test_clusterexec_stops_on_failure(self):
         """
         Verify the clusterexec will stop on first failure.
         """
-        with mock.patch('commissaire.transport.ansibleapi.Transport') as _tp:
-            _tp().restart.return_value = (1, {})
+        for cmd in ('restart', 'upgrade'):
+            with mock.patch('commissaire.transport.ansibleapi.Transport') as _tp:
+                getattr(_tp(), cmd).return_value = (1, {})
 
-            child = {'value': self.etcd_host}
-            return_value = MagicMock(_children=[child])
-            return_value.leaves = return_value._children
+                child = {'value': self.etcd_host}
+                return_value = MagicMock(_children=[child])
+                return_value.leaves = return_value._children
 
-            store = etcd.Client()
-            store.get = MagicMock('get')
-            store.get.return_value = return_value
-            store.set = MagicMock('set')
+                store = etcd.Client()
+                store.get = MagicMock('get')
+                store.get.return_value = return_value
+                store.set = MagicMock('set')
 
-            clusterexec('default', 'restart', store)
+                clusterexec('default', cmd, store)
 
-            self.assertEquals(1, store.get.call_count)
-            # We should have 4 sets for 1 host
-            self.assertEquals(3, store.set.call_count)
+                self.assertEquals(1, store.get.call_count)
+                # We should have 4 sets for 1 host
+                self.assertEquals(3, store.set.call_count)