Object retrier

unit_tests/utilities/test_utilities.py

@@ -0,0 +1,188 @@
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+
+import unit_tests.utils as ut_utils
+
+import zaza.openstack.utilities as utilities
+
+
+class SomeException(Exception):
+    pass
+
+
+class SomeException2(Exception):
+    pass
+
+
+class SomeException3(Exception):
+    pass
+
+
+class TestObjectRetrierWraps(ut_utils.BaseTestCase):
+
+    def test_object_wrap(self):
+
+        class A:
+
+            def func(self, a, b=1):
+                return a + b
+
+        a = A()
+        wrapped_a = utilities.ObjectRetrierWraps(a)
+        self.assertEqual(wrapped_a.func(3), 4)
+
+    def test_object_multilevel_wrap(self):
+
+        class A:
+
+            def f1(self, a, b):
+                return a * b
+
+        class B:
+
+            @property
+            def f2(self):
+
+                return A()
+
+        b = B()
+        wrapped_b = utilities.ObjectRetrierWraps(b)
+        self.assertEqual(wrapped_b.f2.f1(5, 6), 30)
+
+    def test_object_wrap_number(self):
+
+        class A:
+
+            class_a = 5
+
+            def __init__(self):
+                self.instance_a = 10
+
+            def f1(self, a, b):
+                return a * b
+
+        a = A()
+        wrapped_a = utilities.ObjectRetrierWraps(a)
+        self.assertEqual(wrapped_a.class_a, 5)
+        self.assertEqual(wrapped_a.instance_a, 10)
+
+    @mock.patch("time.sleep")
+    def test_object_wrap_exception(self, mock_sleep):
+
+        class A:
+
+            def func(self):
+                raise SomeException()
+
+        a = A()
+        # retry on a specific exception
+        wrapped_a = utilities.ObjectRetrierWraps(
+            a, num_retries=1, retry_exceptions=[SomeException])
+        with self.assertRaises(SomeException):
+            wrapped_a.func()
+
+        mock_sleep.assert_called_once_with(5)
+
+        # also retry on any exception if none specified
+        wrapped_a = utilities.ObjectRetrierWraps(a, num_retries=1)
+        mock_sleep.reset_mock()
+        with self.assertRaises(SomeException):
+            wrapped_a.func()
+
+        mock_sleep.assert_called_once_with(5)
+
+        # no retry if exception isn't listed.
+        wrapped_a = utilities.ObjectRetrierWraps(
+            a, num_retries=1, retry_exceptions=[SomeException2])
+        mock_sleep.reset_mock()
+        with self.assertRaises(SomeException):
+            wrapped_a.func()
+
+        mock_sleep.assert_not_called()
+
+    @mock.patch("time.sleep")
+    def test_log_called(self, mock_sleep):
+
+        class A:
+
+            def func(self):
+                raise SomeException()
+
+        a = A()
+        mock_log = mock.Mock()
+        wrapped_a = utilities.ObjectRetrierWraps(
+            a, num_retries=1, log=mock_log)
+        with self.assertRaises(SomeException):
+            wrapped_a.func()
+
+        # there should be two calls; one for the single retry and one for the
+        # failure.
+        self.assertEqual(mock_log.call_count, 2)
+
+    @mock.patch("time.sleep")
+    def test_back_off_maximum(self, mock_sleep):
+
+        class A:
+
+            def func(self):
+                raise SomeException()
+
+        a = A()
+        wrapped_a = utilities.ObjectRetrierWraps(a, num_retries=3, backoff=2)
+        with self.assertRaises(SomeException):
+            wrapped_a.func()
+        # Note third call hits maximum wait time of 15.
+        mock_sleep.assert_has_calls([mock.call(5),
+                                     mock.call(10),
+                                     mock.call(15)])
+
+    @mock.patch("time.sleep")
+    def test_total_wait(self, mock_sleep):
+
+        class A:
+
+            def func(self):
+                raise SomeException()
+
+        a = A()
+        wrapped_a = utilities.ObjectRetrierWraps(
+            a, num_retries=3, total_wait=9)
+        with self.assertRaises(SomeException):
+            wrapped_a.func()
+        # Note only two calls, as total wait is 9, so a 3rd retry would exceed
+        # that.
+        mock_sleep.assert_has_calls([mock.call(5),
+                                     mock.call(5)])
+
+    @mock.patch("time.sleep")
+    def test_retry_on_connect_failure(self, mock_sleep):
+
+        class A:
+
+            def func1(self):
+                raise SomeException()
+
+            def func2(self):
+                raise utilities.ConnectFailure()
+
+        a = A()
+        wrapped_a = utilities.retry_on_connect_failure(a, num_retries=2)
+        with self.assertRaises(SomeException):
+            wrapped_a.func1()
+        mock_sleep.assert_not_called()
+        with self.assertRaises(utilities.ConnectFailure):
+            wrapped_a.func2()
+        mock_sleep.assert_has_calls([mock.call(5)])

zaza/openstack/charm_tests/octavia/tests.py

@@ -25,6 +25,8 @@
 import zaza.openstack.charm_tests.test_utils as test_utils
 import zaza.openstack.utilities.openstack as openstack_utils
 
+from zaza.openstack.utilities import ObjectRetrierWraps
+
 
 class CharmOperationTest(test_utils.OpenStackBaseTest):
     """Charm operation tests."""
@@ -63,12 +65,12 @@ class LBAASv2Test(test_utils.OpenStackBaseTest):
     def setUpClass(cls):
         """Run class setup for running LBaaSv2 service tests."""
         super(LBAASv2Test, cls).setUpClass()
-        cls.keystone_client = openstack_utils.get_keystone_session_client(
-            cls.keystone_session)
-        cls.neutron_client = openstack_utils.get_neutron_session_client(
-            cls.keystone_session)
-        cls.octavia_client = openstack_utils.get_octavia_session_client(
-            cls.keystone_session)
+        cls.keystone_client = ObjectRetrierWraps(
+            openstack_utils.get_keystone_session_client(cls.keystone_session))
+        cls.neutron_client = ObjectRetrierWraps(
+            openstack_utils.get_neutron_session_client(cls.keystone_session))
+        cls.octavia_client = ObjectRetrierWraps(
+            openstack_utils.get_octavia_session_client(cls.keystone_session))
         cls.RESOURCE_PREFIX = 'zaza-octavia'
 
         # NOTE(fnordahl): in the event of a test failure we do not want to run

zaza/openstack/utilities/__init__.py

@@ -13,3 +13,155 @@
 # limitations under the License.
 
 """Collection of utilities to support zaza tests etc."""
+
+
+import time
+
+from keystoneauth1.exceptions.connection import ConnectFailure
+
+
+class ObjectRetrierWraps(object):
+    """An automatic retrier for an object.
+
+    This is designed to be used with an instance of an object.  Basically, it
+    wraps the object and any attributes that are fetched.  Essentially, it is
+    used to provide retries on method calls on openstack client objects in
+    tests to increase robustness of tests.
+
+    Although, technically this is bad, retries can be logged with the optional
+    log method.
+
+    Usage:
+
+        # get a client that does 3 retries, waits 5 seconds between retries and
+        # retries on any error.
+        some_client = ObjectRetrierWraps(get_some_client)
+        # this gets retried up to 3 times.
+        things = some_client.list_things()
+
+    Note, it is quite simple.  It wraps the object and on a getattr(obj, name)
+    it finds the name and then returns a wrapped version of that name.  On a
+    call, it returns the value of that call.  It only wraps objects in the
+    chain that are either callable or have a __getattr__() method.  i.e. one
+    that can then be retried or further fetched.  This means that if a.b.c() is
+    a chain of objects, and we just wrap 'a', then 'b' and 'c' will both be
+    wrapped that the 'c' object __call__() method will be the one that is
+    actually retried.
+
+    Note: this means that properties that do method calls won't be retried.
+    This is a limitation that may be addressed in the future, if it is needed.
+    """
+
+    def __init__(self, obj, num_retries=3, initial_interval=5.0, backoff=1.0,
+                 max_interval=15.0, total_wait=30.0, retry_exceptions=None,
+                 log=None):
+        """Initialise the retrier object.
+
+        :param obj: The object to wrap. Ought to be an instance of something
+            that you want to get methods on to call or be called itself.
+        :type obj: Any
+        :param num_retries: The (maximum) number of retries.  May not be hit if
+            the total_wait time is exceeded.
+        :type num_retries: int
+        :param initial_interval: The initial or starting interval between
+            retries.
+        :type initial_interval: float
+        :param backoff: The exponential backoff multiple.  1 is linear.
+        :type backoff: float
+        :param max_interval: The maximum interval between retries.
+            If backoff is >1 then the initial_interval will never grow larger
+            than max_interval.
+        :type max_interval: float
+        :param retry_exceptions: The list of exceptions to retry on, or None.
+            If a list, then it will only retry if the exception is one of the
+            ones in the list.
+        :type retry_exceptions: List[Exception]
+        """
+        # Note we use semi-private variable names that shouldn't clash with any
+        # on the actual object.
+        self.__obj = obj
+        self.__kwargs = {
+            'num_retries': num_retries,
+            'initial_interval': initial_interval,
+            'backoff': backoff,
+            'max_interval': max_interval,
+            'total_wait': total_wait,
+            'retry_exceptions': retry_exceptions,
+            'log': log or (lambda x: None),
+        }
+
+    def __getattr__(self, name):
+        """Get attribute; delegates to wrapped object."""
+        # Note the above may generate an attribute error; we expect this and
+        # will fail with an attribute error.
+        attr = getattr(self.__obj, name)
+        if callable(attr) or hasattr(attr, "__getattr__"):
+            return ObjectRetrierWraps(attr, **self.__kwargs)
+        else:
+            return attr
+        # TODO(ajkavanagh): Note detecting a property is a bit trickier.  we
+        # can do isinstance(attr, property), but then the act of accessing it
+        # is what calls it.  i.e. it would fail at the getattr(self.__obj,
+        # name) stage.  The solution is to check first, and if it's a property,
+        # then treat it like the retrier.  However, I think this is too
+        # complex for the first go, and to use manual retries in that instance.
+
+    def __call__(self, *args, **kwargs):
+        """Call the object; delegates to the wrapped object."""
+        obj = self.__obj
+        retry = 0
+        wait = self.__kwargs['initial_interval']
+        max_interval = self.__kwargs['max_interval']
+        log = self.__kwargs['log']
+        backoff = self.__kwargs['backoff']
+        total_wait = self.__kwargs['total_wait']
+        num_retries = self.__kwargs['num_retries']
+        retry_exceptions = self.__kwargs['retry_exceptions']
+        wait_so_far = 0
+        while True:
+            try:
+                return obj(*args, **kwargs)
+            except Exception as e:
+                # if retry_exceptions is not None, or the type of the exception
+                # is not in the list of retries, then raise an exception
+                # immediately.  This means that if retry_exceptions is None,
+                # then the method is always retried.
+                if (retry_exceptions is not None and
+                        type(e) not in retry_exceptions):
+                    raise
+                retry += 1
+                if retry > num_retries:
+                    log("{}: exceeded number of retries, so erroring out"
+                        .format(str(obj)))
+                    raise e
+                log("{}: call failed: retrying in {} seconds"
+                    .format(str(obj), wait))
+                time.sleep(wait)
+                wait_so_far += wait
+                if wait_so_far >= total_wait:
+                    raise e
+                wait = wait * backoff
+                if wait > max_interval:
+                    wait = max_interval
+
+
+def retry_on_connect_failure(client, **kwargs):
+    """Retry an object that eventually gets resolved to a call.
+
+    Specifically, this uses ObjectRetrierWraps but only against the
+    keystoneauth1.exceptions.connection.ConnectFailure exeception.
+
+    :params client: the object that may throw and exception when called.
+    :type client: Any
+    :params **kwargs: the arguments supplied to the ObjectRetrierWraps init
+                      method
+    :type **kwargs: Dict[Any]
+    :returns: client wrapped in an ObjectRetrierWraps instance
+    :rtype: ObjectRetrierWraps[client]
+    """
+    kwcopy = kwargs.copy()
+    if 'retry_exceptions' not in kwcopy:
+        kwcopy['retry_exceptions'] = []
+    if ConnectFailure not in kwcopy['retry_exceptions']:
+        kwcopy['retry_exceptions'].append(ConnectFailure)
+    return ObjectRetrierWraps(client, **kwcopy)