Merge branch 'master' into no-libmhash2

fence/config-default.yaml

@@ -480,6 +480,10 @@ PRIVACY_POLICY_URL: null
 # In the absence of this OVERRIDE prefixed config, the legacy NGINX_RATE_LIMIT from the k8s deployment yaml is applied
 OVERRIDE_NGINX_RATE_LIMIT: 18
 
+# This is the maximum numbers of retries when exponentially backing off against an
+# error from an external API.
+DEFAULT_BACKOFF_SETTINGS_MAX_TRIES: 3
+
 # //////////////////////////////////////////////////////////////////////////////////////
 # SUPPORT INFO
 # //////////////////////////////////////////////////////////////////////////////////////

fence/resources/google/access_utils.py

@@ -2,13 +2,13 @@
 Utilities for determine access and validity for service account
 registration.
 """
+import backoff
 import time
 import flask
 from urllib.parse import unquote
 import traceback
 
 from cirrus.google_cloud.iam import GooglePolicyMember
-
 from cirrus.google_cloud.errors import GoogleAPIError
 from cirrus.google_cloud.iam import GooglePolicy
 from cirrus import GoogleCloudManager
@@ -33,7 +33,7 @@
     get_monitoring_service_account_email,
     is_google_managed_service_account,
 )
-from fence.utils import get_valid_expiration_from_request
+from fence.utils import get_valid_expiration_from_request, DEFAULT_BACKOFF_SETTINGS
 
 logger = get_logger(__name__)
 
@@ -58,7 +58,7 @@ def bulk_update_google_groups(google_bulk_mapping):
             members_from_google = []
 
             try:
-                members_from_google = gcm.get_group_members(group)
+                members_from_google = _get_members_from_google_group(group)
             except Exception as exc:
                 logging.error(
                     f"ERROR: FAILED TO GET MEMBERS FROM GOOGLE GROUP {group}! "
@@ -71,6 +71,7 @@ def bulk_update_google_groups(google_bulk_mapping):
                 google_update_failures = True
 
             google_members = set(member.get("email") for member in members_from_google)
+
             logger.debug(f"Google membership for {group}: {google_members}")
             logger.debug(f"Expected membership for {group}: {expected_members}")
 
@@ -84,8 +85,9 @@ def bulk_update_google_groups(google_bulk_mapping):
             # do add
             for member_email in to_add:
                 logger.info(f"Adding to group {group}: {member_email}")
+
                 try:
-                    gcm.add_member_to_group(member_email, group)
+                    _add_member_to_google_group(member_email, group)
                 except Exception as exc:
                     logging.error(
                         f"ERROR: FAILED TO ADD MEMBER {member_email} TO GOOGLE "
@@ -100,8 +102,9 @@ def bulk_update_google_groups(google_bulk_mapping):
             # do remove
             for member_email in to_delete:
                 logger.info(f"Removing from group {group}: {member_email}")
+
                 try:
-                    gcm.remove_member_from_group(member_email, group)
+                    _remove_member_to_google_group(member_email, group)
                 except Exception as exc:
                     logging.error(
                         f"ERROR: FAILED TO REMOVE MEMBER {member_email} FROM "
@@ -118,6 +121,19 @@ def bulk_update_google_groups(google_bulk_mapping):
                     f"FAILED TO UPDATE GOOGLE GROUPS (see previous errors)."
                 )
 
+@backoff.on_exception(backoff.expo, Exception, **DEFAULT_BACKOFF_SETTINGS)
+def _get_members_from_google_group(gcm, group):
+    return gcm.get_group_members(group)
+
+
+@backoff.on_exception(backoff.expo, Exception, **DEFAULT_BACKOFF_SETTINGS)
+def _add_member_to_google_group(gcm, add_member_to_group, group):
+    gcm.add_member_to_group(member_email, group)
+
+
+@backoff.on_exception(backoff.expo, Exception, **DEFAULT_BACKOFF_SETTINGS)
+def _remove_member_to_google_group(gcm, add_member_to_group, group):
+    gcm.remove_member_from_group(member_email, group)
 
 def get_google_project_number(google_project_id, google_cloud_manager):
     """

fence/utils.py

@@ -421,6 +421,6 @@ def get_from_cache(item_id, memory_cache, db_cache_table, db_cache_table_id_fiel
 DEFAULT_BACKOFF_SETTINGS = {
     "on_backoff": log_backoff_retry,
     "on_giveup": log_backoff_giveup,
-    "max_tries": 3,
+    "max_tries": config["DEFAULT_BACKOFF_SETTINGS_MAX_TRIES"],
     "giveup": exception_do_not_retry,
 }