updating BuildConfig retry

Signed-off-by: Robert Cerven <rcerven@redhat.com>

osbs/api.py

@@ -34,6 +34,7 @@
                              OsbsOrchestratorNotEnabled)
 # import utils in this way, so that we can mock standalone functions with flexmock
 from osbs import utils
+from osbs.utils import retry_on_conflict
 
 from six.moves import http_client
 
@@ -358,6 +359,40 @@ def _get_image_stream_info_for_build_request(self, build_request):
 
         return image_stream, image_stream_tag_name
 
+    @retry_on_conflict
+    def _update_build_config_when_exist(self, build_json):
+        existing_bc = self._get_existing_build_config(build_json)
+        self._verify_labels_match(build_json, existing_bc)
+        # Existing build config may have a different name if matched by
+        # git-repo-name and git-branch labels. Continue using existing
+        # build config name.
+        build_config_name = existing_bc['metadata']['name']
+        logger.debug('existing build config name to be used "%s"',
+                     build_config_name)
+        self._verify_no_running_builds(build_config_name)
+
+        # Remove nodeSelector, will be set from build_json for worker build
+        old_nodeselector = existing_bc['spec'].pop('nodeSelector', None)
+        logger.debug("removing build config's nodeSelector %s", old_nodeselector)
+
+        utils.buildconfig_update(existing_bc, build_json)
+        # Reset name change that may have occurred during
+        # update above, since renaming is not supported.
+        existing_bc['metadata']['name'] = build_config_name
+        logger.debug('build config for %s already exists, updating...',
+                     build_config_name)
+
+        self.os.update_build_config(build_config_name, json.dumps(existing_bc))
+        return existing_bc
+
+    @retry_on_conflict
+    def _update_build_config_with_triggers(self, build_json, triggers):
+        existing_bc = self._get_existing_build_config(build_json)
+        existing_bc['spec']['triggers'] = triggers
+        build_config_name = existing_bc['metadata']['name']
+        self.os.update_build_config(build_config_name, json.dumps(existing_bc))
+        return existing_bc
+
     def _create_build_config_and_build(self, build_request):
         build_json = build_request.render()
         api_version = build_json['apiVersion']
@@ -379,30 +414,8 @@ def _create_build_config_and_build(self, build_request):
         triggers = build_json['spec'].pop('triggers', None)
 
         if existing_bc:
-            self._verify_labels_match(build_json, existing_bc)
-            # Existing build config may have a different name if matched by
-            # git-repo-name and git-branch labels. Continue using existing
-            # build config name.
             build_config_name = existing_bc['metadata']['name']
-            logger.debug('existing build config name to be used "%s"',
-                         build_config_name)
-            self._verify_no_running_builds(build_config_name)
-
-            # Remove nodeSelector, will be set from build_json for worker build
-            old_nodeselector = existing_bc['spec'].pop('nodeSelector', None)
-            logger.debug("removing build config's nodeSelector %s", old_nodeselector)
-
-            utils.buildconfig_update(existing_bc, build_json)
-            # Reset name change that may have occurred during
-            # update above, since renaming is not supported.
-            existing_bc['metadata']['name'] = build_config_name
-            logger.debug('build config for %s already exists, updating...',
-                         build_config_name)
-
-            self.os.update_build_config(build_config_name, json.dumps(existing_bc))
-            if triggers:
-                # Retrieve updated version to pick up lastVersion
-                existing_bc = self._get_existing_build_config(existing_bc)
+            existing_bc = self._update_build_config_when_exist(build_json)
 
         else:
             logger.debug("build config for %s doesn't exist, creating...",
@@ -416,8 +429,7 @@ def _create_build_config_and_build(self, build_request):
             logger.debug('Changed parent ImageStreamTag? %s', changed_ist)
 
         if triggers:
-            existing_bc['spec']['triggers'] = triggers
-            self.os.update_build_config(build_config_name, json.dumps(existing_bc))
+            existing_bc = self._update_build_config_with_triggers(build_json, triggers)
 
         if image_stream and triggers:
             prev_version = existing_bc['status']['lastVersion']

osbs/constants.py

@@ -91,6 +91,12 @@
 # requests timeout in seconds
 HTTP_REQUEST_TIMEOUT = 600
 
+# number of retries on openshift conflict
+OS_CONFLICT_MAX_RETRIES = 8
+
+# number of seconds to wait, before retrying on openshift conflict
+OS_CONFLICT_WAIT = 5
+
 BUILD_TYPE_ORCHESTRATOR = object()
 BUILD_TYPE_WORKER = object()
 

osbs/core.py

@@ -6,7 +6,6 @@
 of the BSD license. See the LICENSE file for details.
 """
 from __future__ import print_function, unicode_literals, absolute_import
-from functools import wraps
 import json
 import os
 import numbers
@@ -22,7 +21,7 @@
                             SERVICEACCOUNT_CACRT)
 from osbs.exceptions import (OsbsResponseException, OsbsException,
                              OsbsWatchBuildNotFound, OsbsAuthException)
-from osbs.utils import graceful_chain_get
+from osbs.utils import graceful_chain_get, retry_on_conflict
 from requests.exceptions import ConnectionError
 from requests.utils import guess_json_utf
 
@@ -46,28 +45,6 @@ def check_response(response, log_level=logging.ERROR):
         raise OsbsResponseException(message=content, status_code=response.status_code)
 
 
-def retry_on_conflict(func, sleep_seconds=0.5, max_attempts=10):
-    @wraps(func)
-    def retry(*args, **kwargs):
-        last_exception = None
-        for attempt in range(max_attempts):
-            if attempt != 0:
-                time.sleep(sleep_seconds)
-
-            logger.debug("attempt %d to call %s", attempt + 1, func.__name__)
-            try:
-                return func(*args, **kwargs)
-            except OsbsResponseException as ex:
-                if ex.status_code != http_client.CONFLICT:
-                    raise
-
-                last_exception = ex
-
-        raise last_exception or RuntimeError("operation not attempted")
-
-    return retry
-
-
 # TODO: error handling: create function which handles errors in response object
 class Openshift(object):
     def __init__(self, openshift_api_url, openshift_api_version, openshift_oauth_url,

osbs/utils.py

@@ -6,7 +6,7 @@
 of the BSD license. See the LICENSE file for details.
 """
 from __future__ import print_function, absolute_import, unicode_literals
-
+from functools import wraps
 import contextlib
 import copy
 import logging
@@ -19,12 +19,15 @@
 import sys
 import tempfile
 import tarfile
+import time
 import requests
 from collections import namedtuple
 from datetime import datetime
 from io import BytesIO
 from hashlib import sha256
 from osbs.repo_utils import RepoConfiguration, RepoInfo, AdditionalTagsConfig
+from osbs.constants import OS_CONFLICT_MAX_RETRIES, OS_CONFLICT_WAIT
+from six.moves import http_client
 
 try:
     # py3
@@ -554,3 +557,22 @@ def get_name_and_value(self, label_type):
             return self._label_values[label_type]
         else:
             return (label_type, self._df_labels[label_type])
+
+
+def retry_on_conflict(func,
+                      retry_times=OS_CONFLICT_MAX_RETRIES,
+                      retry_delay=OS_CONFLICT_WAIT):
+    @wraps(func)
+    def retry(*args, **kwargs):
+        for counter in range(retry_times + 1):
+            try:
+                return func(*args, **kwargs)
+            except OsbsResponseException as ex:
+                if ex.status_code == http_client.CONFLICT and counter != retry_times:
+                    logger.info("retrying on conflict %s", ex.message)
+                    logger.debug("attempt %d to call %s", counter + 1, func.__name__)
+                    time.sleep(retry_delay * (2 ** counter))
+                else:
+                    raise
+
+    return retry