From 6010fb3fff11fc961c5c8038543122b909e15666 Mon Sep 17 00:00:00 2001
From: Dmitry Mescheryakov <dmescheryakov@mirantis.com>
Date: Fri, 2 Oct 2015 13:04:43 +0300
Subject: [PATCH] Make forever_retry_uncaught_exceptions handle its own
 failures

When an exception occurs inside 'except' clause, it is not handled.
As a result, forever_retry_uncaught_exceptions fails with exception,
while by definition it should not.

For instance, oslo.messaging's RPC server relies on that
function to process any exception. When forever_retry_... fails
to do so, the server thread dies. An example could be
found in referenced bug.

Change-Id: I415a0f49b25b80a264f0bc951f4b926d57a9c9a8
Closes-Bug: #1502092
---
 oslo_utils/excutils.py | 49 ++++++++++++++++++++++++++----------------
 1 file changed, 30 insertions(+), 19 deletions(-)

diff --git a/oslo_utils/excutils.py b/oslo_utils/excutils.py
index dbaf0f6b..33a9431e 100644
--- a/oslo_utils/excutils.py
+++ b/oslo_utils/excutils.py
@@ -204,23 +204,34 @@ def inner_func(*args, **kwargs):
             try:
                 return infunc(*args, **kwargs)
             except Exception as exc:
-                this_exc_message = six.u(str(exc))
-                if this_exc_message == last_exc_message:
-                    exc_count += 1
-                else:
-                    exc_count = 1
-                # Do not log any more frequently than once a minute unless
-                # the exception message changes
-                cur_time = int(time.time())
-                if (cur_time - last_log_time > 60 or
-                        this_exc_message != last_exc_message):
-                    logging.exception(
-                        _LE('Unexpected exception occurred %d time(s)... '
-                            'retrying.') % exc_count)
-                    last_log_time = cur_time
-                    last_exc_message = this_exc_message
-                    exc_count = 0
-                # This should be a very rare event. In case it isn't, do
-                # a sleep.
-                time.sleep(1)
+                try:
+                    this_exc_message = six.u(str(exc))
+                    if this_exc_message == last_exc_message:
+                        exc_count += 1
+                    else:
+                        exc_count = 1
+                    # Do not log any more frequently than once a minute unless
+                    # the exception message changes
+                    cur_time = int(time.time())
+                    if (cur_time - last_log_time > 60 or
+                            this_exc_message != last_exc_message):
+                        logging.exception(
+                            _LE('Unexpected exception occurred %d time(s)... '
+                                'retrying.') % exc_count)
+                        last_log_time = cur_time
+                        last_exc_message = this_exc_message
+                        exc_count = 0
+                    # This should be a very rare event. In case it isn't, do
+                    # a sleep.
+                    time.sleep(1)
+                except Exception:
+                    try:
+                        logging.exception(
+                            _LE('Unexpected error occurred while '
+                                'processing Exception'))
+                    except Exception:
+                        # In case either serialization of the last exception
+                        # or logging failed, ignore the error
+                        pass
+
     return inner_func