handle connection being closed while streaming logs

When the connection is closed, if the time since the last activity is
less than a minute, finish.

Otherwise, ask again for logs starting from just after we last saw
activity.

osbs/core.py

@@ -382,6 +382,51 @@ def wait_for_new_build_config_instance(self, build_config_id, prev_version):
 
         raise OsbsResponseException("New BuildConfig instance not found")
 
+    def stream_logs(self, build_id):
+        """
+        stream logs from build
+
+        :param bulid_id: str
+        :return: iterator
+        """
+        kwargs = {'follow': 1}
+
+        # If connection is closed within this many seconds, give up:
+        min_idle_timeout = 60
+
+        while True:
+            buildlogs_url = self._build_url("builds/%s/log/" % build_id,
+                                            **kwargs)
+            response = self._get(buildlogs_url, stream=1,
+                                 headers={'Connection': 'close'})
+            check_response(response)
+
+            # Stream logs, but be careful of the connection closing
+            # due to idle timeout. In that case, try again until the
+            # call returns more quickly than a reasonable timeout
+            # would be set to.
+            last_activity = time.time()
+
+            for line in response.iter_lines():
+                last_activity = time.time()
+                yield line
+
+            idle = time.time() - last_activity
+            logger.debug("connection closed after %ds", idle)
+            if idle < min_idle_timeout:
+                # Finish output
+                raise StopIteration
+
+            since = int(idle - 1)
+            logger.debug("fetching logs starting from %ds ago", since)
+            kwargs['sinceSeconds'] = since
+
+            buildlogs_url = self._build_url("builds/%s/log/" % build_id,
+                                            **kwargs)
+            response = self._get(buildlogs_url, stream=True,
+                                 headers={'Connection': 'close'})
+            check_response(response)
+
     def logs(self, build_id, follow=False, build_json=None, wait_if_missing=False):
         """
         provide logs from build
@@ -411,13 +456,12 @@ def logs(self, build_id, follow=False, build_json=None, wait_if_missing=False):
         if br.is_pending():
             return
 
-        buildlogs_url = self._build_url("builds/%s/log/" % build_id,
-                                        follow=(1 if follow else 0))
-        response = self._get(buildlogs_url, stream=follow, headers={'Connection': 'close'})
-        check_response(response)
-
         if follow:
-            return response.iter_lines()
+            return self.stream_logs(build_id)
+
+        buildlogs_url = self._build_url("builds/%s/log/" % build_id)
+        response = self._get(buildlogs_url, headers={'Connection': 'close'})
+        check_response(response)
         return response.content
 
     def list_builds(self, build_config_id=None, field_selector=None):