Catch InstanceLimitExceeded errors from Nimbus, and log nicely

ooici · Mar 8, 2013 · ed64a88 · ed64a88
1 parent d836b91
commit ed64a88
Show file tree

Hide file tree

Showing 3 changed files with 35 additions and 3 deletions.
diff --git a/epu/exceptions.py b/epu/exceptions.py
@@ -33,6 +33,12 @@ class GeneralIaaSException(Exception):
     """
 
 
+class IaaSIsFullException(GeneralIaaSException):
+    """
+    IaaS site can not start any more VMs with the parameters specified
+    """
+
+
 # Exceptions used by DTRS
 class DeployableTypeLookupError(Exception):
     pass

diff --git a/epu/provisioner/core.py b/epu/provisioner/core.py
@@ -23,7 +23,7 @@
 from epu.provisioner.store import group_records, sanitize_record, VERSION_KEY
 from epu.exceptions import DeployableTypeLookupError, DeployableTypeValidationError
 from epu.states import InstanceState
-from epu.exceptions import WriteConflictError, UserNotPermittedError, GeneralIaaSException
+from epu.exceptions import WriteConflictError, UserNotPermittedError, GeneralIaaSException, IaaSIsFullException
 from epu import cei_events
 from epu.util import check_user
 from epu.domain_log import EpuLoggerThreadSpecific
@@ -346,6 +346,12 @@ def _really_execute_provision_request(self, launch, nodes, caller):
                          spec, nodes)
                 self._launch_one_group(spec, nodes, caller=caller)
 
+            except IaaSIsFullException, iif:
+                log.warning('Problem launching group %s: %s',
+                        spec.name, str(iif))
+                newstate = states.FAILED
+                has_failed = True
+                failure_message = "IAAS_FULL: " + str(iif)
             except GeneralIaaSException, gie:
                 log.exception('Problem launching group %s: %s',
                         spec.name, str(gie))
@@ -444,8 +450,11 @@ def _launch_one_group(self, spec, nodes, caller=None):
                 raise timeout('IAAS_TIMEOUT')
         except Exception, e:
             # XXX TODO introspect the exception to get more specific error information
-            log.exception('Error launching nodes: ' + str(e))
-            raise GeneralIaaSException(str(e))
+            exp_as_str = str(e)
+            if "InstanceLimitExceeded" in exp_as_str:
+                raise IaaSIsFullException(exp_as_str)
+            else:
+                raise GeneralIaaSException(exp_as_str)
 
         # underlying node driver may return a list or an object
         if not hasattr(iaas_nodes, '__iter__'):

diff --git a/epu/provisioner/test/test_core.py b/epu/provisioner/test/test_core.py
@@ -588,6 +588,23 @@ def x():
         with patch.object(FakeNodeDriver, 'list_nodes', side_effect=x):
             self.core.query_one_site('site1', nodes, caller=caller)
 
+    def test_launch_one_iaas_full(self):
+        def x(**kwargs):
+            raise Exception("InstanceLimitExceeded: too many vms :(")
+
+        with patch.object(FakeNodeDriver, 'create_node', side_effect=x):
+            self.core._IAAS_DEFAULT_TIMEOUT = 0.5
+
+            node_id = _new_id()
+            launch_id = _new_id()
+
+            self._prepare_execute(launch_id=launch_id, instance_ids=[node_id])
+
+            self.assertTrue(self.notifier.assure_state(states.FAILED))
+            self.assertIn('IAAS_FULL', self.notifier.nodes[node_id]['state_desc'])
+            launch = self.store.get_launch(launch_id)
+            self.assertEqual(launch['state'], states.FAILED)
+
     def test_launch_one_iaas_timeout(self):
         def x(**kwargs):
             raise timeout("Launch took too long")