sktm should make multiple attempts to query Jenkins

README.md

@@ -265,6 +265,7 @@ Create the `~/.sktmrc` file telling sktm how to access Jenkins:
     jurl = http://localhost:8080
     jlogin = sktm
     jpass = sesame
+    jretry = 30
     EOF
 
 Usage

sktm/executable.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2017 Red Hat, Inc. All rights reserved. This copyrighted
+# Copyright (c) 2017-2018 Red Hat, Inc. All rights reserved. This copyrighted
 # material is made available to anyone wishing to use, modify, copy, or
 # redistribute it subject to the terms and conditions of the GNU General
 # Public License v.2 or later.
@@ -30,6 +30,8 @@
     '../templates/report.footer'
 )
 
+DEFAULT_JENKINS_RETRY_COUNT = 30
+
 
 def setup_parser():
     """
@@ -47,6 +49,9 @@ def setup_parser():
     parser.add_argument("--jurl", help="Jenkins URL")
     parser.add_argument("--jlogin", help="Jenkins login")
     parser.add_argument("--jpass", help="Jenkins password")
+    parser.add_argument("--jretry", type=int,
+                        help="Counter to retry Jenkins, default to %d" %
+                        DEFAULT_JENKINS_RETRY_COUNT)
     parser.add_argument("--jjname", help="Jenkins job name")
     parser.add_argument("--makeopts", help="Specify options for make")
     parser.add_argument("--cfgurl", type=str, help="Kernel config URL")
@@ -205,6 +210,11 @@ def load_config(args):
     else:
         cfg['report_footer'] = os.path.abspath(cfg['report_footer'])
 
+    if not cfg.get('jretry'):
+        cfg['jretry'] = DEFAULT_JENKINS_RETRY_COUNT
+    else:
+        cfg['jretry'] = int(cfg.get('jretry'))
+
     return cfg
 
 
@@ -223,7 +233,8 @@ def main():
         jenkins_project = sktm.jenkins.JenkinsProject(cfg.get("jjname"),
                                                       cfg.get("jurl"),
                                                       cfg.get("jlogin"),
-                                                      cfg.get("jpass"))
+                                                      cfg.get("jpass"),
+                                                      cfg.get("jretry"))
 
         sw = sktm.watcher(jenkins_project, cfg.get("db"),
                           cfg.get("filter"), cfg.get("makeopts"))

sktm/jenkins.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2017 Red Hat, Inc. All rights reserved. This copyrighted
+# Copyright (c) 2017-2018 Red Hat, Inc. All rights reserved. This copyrighted
 # material is made available to anyone wishing to use, modify, copy, or
 # redistribute it subject to the terms and conditions of the GNU General
 # Public License v.2 or later.
@@ -23,7 +23,8 @@
 
 class JenkinsProject(object):
     """Jenkins project interface"""
-    def __init__(self, name, url, username=None, password=None):
+    def __init__(self, name, url, username=None, password=None,
+                 retry_cnt=None):
         """
         Initialize a Jenkins project interface.
 
@@ -32,19 +33,69 @@ def __init__(self, name, url, username=None, password=None):
             url:         Jenkins instance URL.
             username:    Jenkins user name.
             password:    Jenkins user password.
+            retry_cnt:   Counter to retry Jenkins in case of temporary network
+                         failures.
         """
         self.name = name
+
         # Initialize Jenkins server interface
         # TODO Add support for CSRF protection
         self.server = jenkinsapi.jenkins.Jenkins(url, username, password)
 
+        self.retry_cnt = retry_cnt
+
+    def __get_job(self):
+        """
+        Get Jenkens job by job name. Retry Jenkins self.retry_cnt times
+        in case of temporary network failures.
+
+        Return:
+            job if succeed, else raise the last exception.
+        """
+        for i in range(self.retry_cnt):
+            try:
+                job = self.server.get_job(self.name)
+                return job
+            except Exception as e:
+                logging.warning("catch %s: %s" % (type(e), e))
+                logging.info("now sleep 60s and try again")
+                time.sleep(60)
+
+        logging.error("fail to get job after retry %d times" % self.retry_cnt)
+        raise e
+
+    def __get_build(self, job, buildid):
+        """
+        Get Jenkins build by build ID. Retry Jenkins self.retry_cnt times
+        in case of temporary network failures.
+
+        Args:
+            job:        Jenkins job.
+            buildid:    Jenkins build ID.
+
+        Return:
+            build if succeed, else raise the last exception.
+        """
+        for i in range(self.retry_cnt):
+            try:
+                build = job.get_build(buildid)
+                return build
+            except Exception as e:
+                logging.warning("catch %s: %s" % (type(e), e))
+                logging.info("now sleep 60s and try again")
+                time.sleep(60)
+
+        logging.error("fail to get build after retry %d times" %
+                      self.retry_cnt)
+        raise e
+
     def _wait_and_get_build(self, buildid):
-        job = self.server.get_job(self.name)
-        build = job.get_build(buildid)
+        job = self.__get_job()
+        build = self.__get_build(job, buildid)
         build.block_until_complete(delay=60)
 
-        # call get_build again to ensure we have the results
-        build = job.get_build(buildid)
+        # call self.__get_build() again to ensure we have the results
+        build = self.__get_build(job, buildid)
 
         return build
 
@@ -313,8 +364,8 @@ def build(self, baserepo=None, ref=None, baseconfig=None,
             params["makeopts"] = makeopts
 
         logging.debug(params)
-        self.server.get_job(self.name)
-        expected_id = self.server.get_job(self.name).get_next_build_number()
+        job = self.__get_job()
+        expected_id = job.get_next_build_number()
         self.server.build_job(self.name, params)
         build = self.find_build(params, expected_id)
         logging.info("submitted build: %s", build)
@@ -330,8 +381,8 @@ def is_build_complete(self, buildid):
         Return:
             True if the build is complete, False if not.
         """
-        job = self.server.get_job(self.name)
-        build = job.get_build(buildid)
+        job = self.__get_job()
+        build = self.__get_build(job, buildid)
 
         return not build.is_running()
 
@@ -349,7 +400,7 @@ def _params_eq(self, build, params):
         return True
 
     def find_build(self, params, eid=None):
-        job = self.server.get_job(self.name)
+        job = self.__get_job()
         lbuild = None
 
         while not lbuild:
@@ -367,7 +418,7 @@ def find_build(self, params, eid=None):
 
         # slowpath
         for bid in job.get_build_ids():
-            build = job.get_build(bid)
+            build = self.__get_build(job, bid)
             if self._params_eq(build, params):
                 return build
         return None