Add force and max_wait parameters to destroy_model()

unit_tests/test_zaza_controller.py

@@ -37,7 +37,9 @@ async def _list_models():
         async def _add_model(model_name, config=None):
             return self.model1
 
-        async def _destroy_model(model_name):
+        async def _destroy_model(model_name, force=False, max_wait=None):
+            if model_name in self.models:
+                self.models.remove(model_name)
             return
 
         async def _get_cloud():
@@ -88,7 +90,7 @@ def test_add_model_config(self):
     def test_destroy_model(self):
         controller.destroy_model(self.model1.info.name)
         self.Controller_mock.destroy_model.assert_called_once_with(
-            self.model1.info.name)
+            self.model1.info.name, force=True, max_wait=600)
 
     def test_get_cloud(self):
         self.assertEqual(

zaza/controller.py

@@ -17,7 +17,9 @@
 import logging
 from juju.controller import Controller
 import subprocess
+import tenacity
 from zaza import sync_wrapper
+import zaza.utilities.exceptions
 
 
 async def async_add_model(model_name, config=None, region=None):
@@ -52,7 +54,23 @@ async def async_destroy_model(model_name):
     controller = Controller()
     await controller.connect()
     logging.debug("Destroying model {}".format(model_name))
-    await controller.destroy_model(model_name)
+    await controller.destroy_model(model_name, force=True, max_wait=600)
+    # The model ought to be destroyed by now.  Let's make sure, and if not,
+    # raise an error.  Even if the model has been destroyed, it's still hangs
+    # around in the .list_models() for a little while; retry until it goes
+    # away, or that fails.
+    for attempt in tenacity.Retrying(
+            stop=tenacity.stop_after_attempt(20),
+            wait=tenacity.wait_exponential(
+                multiplier=1, min=2, max=20),
+            retry=tenacity.retry_if_exception_type(
+                zaza.utilities.exceptions.DestroyModelFailed)):
+        with attempt:
+            remaining_models = await controller.list_models()
+            if model_name in remaining_models:
+                raise zaza.utilities.exceptions.DestroyModelFailed(
+                    "Destroying model {} failed.".format(model_name))
+    logging.debug("Model {} destroyed.".format(model_name))
     await controller.disconnect()
 
 destroy_model = sync_wrapper(async_destroy_model)

zaza/utilities/exceptions.py

@@ -184,3 +184,9 @@ class NovaGuestRestartFailed(Exception):
     """Nova guest restart failed."""
 
     pass
+
+
+class DestroyModelFailed(Exception):
+    """The controller.destroy_model() failed in some interesting way."""
+
+    pass