[ADDON-43460] retry and fail if mib unavailable

splunk_connect_for_snmp_poller/manager/mib_server_client.py

@@ -21,6 +21,9 @@
 import backoff as backoff
 import requests as requests
 from aiohttp import ClientSession
+from requests.adapters import HTTPAdapter
+from urllib3.exceptions import MaxRetryError
+from urllib3.util.retry import Retry
 
 logger = logging.getLogger(__name__)
 
@@ -105,13 +108,45 @@ def format_value_for_mib_server(value, value_type):
         return str(value)
 
 
+def requests_retry_session(
+    retries=6,
+    backoff_factor=10,
+    session=None,
+):
+    session = session or requests.Session()
+    retry = Retry(
+        total=retries,
+        read=retries,
+        connect=retries,
+        backoff_factor=backoff_factor,
+    )
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount("http://", adapter)
+    session.mount("https://", adapter)
+    return session
+
+
+def retry_getting_mib_profile(profiles_url: str):
+    try:
+        return requests_retry_session().get(profiles_url, timeout=3).text
+    except ConnectionRefusedError:
+        logger.exception(f"Connection to MIB refused for {profiles_url}")
+        raise
+    except MaxRetryError:
+        logger.exception(
+            f"Failed to connect to MIB - {profiles_url} after exponential retries"
+        )
+        raise
+    except Exception:
+        logger.exception(
+            f"Encountered unexpected exception while connecting to MIB - {profiles_url}"
+        )
+        raise
+
+
 def get_mib_profiles():
     mib_server_url = os.environ["MIBS_SERVER_URL"]
     endpoint = "profiles"
     profiles_url = os.path.join(mib_server_url.strip("/"), endpoint)
 
-    try:
-        return requests.get(profiles_url, timeout=3).text
-    except Exception:
-        logger.exception("Error getting profiles from MIB server")
-        return {}
+    return retry_getting_mib_profile(profiles_url)