cli,api: rewrite retrieving logs

new arg:

 * wait_if_missing -- wait for build to get created

new api method:

 * get_docker_build_logs -- return logs from `docker build` instead from
   build container

`BuildResponse.get_logs` is capable of decoding logs now.

Fixes #131
Fixes #132

osbs/api.py

@@ -234,27 +234,45 @@ def create_build(self, namespace=DEFAULT_NAMESPACE, **kwargs):
             raise OsbsException("Unknown build type: '%s'" % build_type)
 
     @osbsapi
-    def get_build_logs(self, build_id, follow=False, build_json=None, namespace=DEFAULT_NAMESPACE):
-        if follow:
-            return self.os.logs(build_id, follow=follow, build_json=build_json,
-                                namespace=namespace)
-        try:
+    def get_build_logs(self, build_id, follow=False, build_json=None, wait_if_missing=False,
+                       namespace=DEFAULT_NAMESPACE):
+        """
+        provide logs from build
+
+        :param build_id: str
+        :param follow: bool, fetch logs as they come?
+        :param build_json: dict, to save one get-build query
+        :param wait_if_missing: bool, if build doesn't exist, wait
+        :param namespace: str
+        :return: None, str or iterator
+        """
+        return self.os.logs(build_id, follow=follow, build_json=build_json,
+                            wait_if_missing=wait_if_missing, namespace=namespace)
+
+    @osbsapi
+    def get_docker_build_logs(self, build_id, decode_logs=True, build_json=None,
+                              namespace=DEFAULT_NAMESPACE):
+        """
+        get logs provided by "docker build"
+
+        :param build_id: str
+        :param decode_logs: bool, docker by default output logs in simple json structure:
+            { "stream": "line" }
+            if this arg is set to True, it decodes logs to human readable form
+        :param build_json: dict, to save one get-build query
+        :param namespace: str
+        :return: str
+        """
+        if not build_json:
             build = self.os.get_build(build_id, namespace=namespace)
-        except OsbsResponseException as ex:
-            if ex.status_code != 404:
-                raise
-        else:
             build_response = BuildResponse(build)
-            logs = None
-            if build_response.is_finished():
-                metadata = build_response.json.get("metadata", {})
-                md = metadata.get("annotations", metadata.get("labels", {}))
-                logs = md.get("logs", None)
-
-            if logs:
-                return logs
+        else:
+            build_response = BuildResponse(None, build_json)
 
-            return self.os.logs(build_id, follow=False, build_json=build_json, namespace=namespace)
+        if build_response.is_finished():
+            logs = build_response.get_logs(decode_logs=decode_logs)
+            return logs
+        logger.warning("build haven't finished yet")
 
     @osbsapi
     def wait_for_build_to_finish(self, build_id, namespace=DEFAULT_NAMESPACE):

osbs/build/build_response.py

@@ -82,8 +82,35 @@ def get_rpm_packages(self):
     def get_dockerfile(self):
         return graceful_chain_get(self.get_annotations_or_labels(), "dockerfile")
 
-    def get_logs(self):
-        return graceful_chain_get(self.get_annotations_or_labels(), "logs")
+    def get_logs(self, decode_logs=True):
+        """
+        :param decode_logs: bool, docker by default output logs in simple json structure:
+            { "stream": "line" }
+            if this arg is set to True, it decodes logs to human readable form
+        :return: str
+        """
+        logs = graceful_chain_get(self.get_annotations_or_labels(), "logs")
+        if not logs:
+            logger.error("no logs")
+            return ""
+        if decode_logs:
+            output = []
+            for line in logs.split("\n"):
+                try:
+                    decoded_line = json.loads(line)
+                except ValueError:
+                    continue
+                output += [decoded_line.get("stream", "").strip()]
+                error = decoded_line.get("error", "").strip()
+                if error:
+                    output += [error]
+                error_detail = decoded_line.get("errorDetail", "").strip()
+                if error_detail:
+                    output += [error_detail]
+            output += "\n"
+            return "\n".join(output)
+        else:
+            return logs
 
     def get_commit_id(self):
         return graceful_chain_get(self.get_annotations_or_labels(), "commit_id")

osbs/cli/main.py

@@ -154,12 +154,22 @@ def cmd_build_logs(args, osbs):
     build_id = args.BUILD_ID[0]
     follow = args.follow
 
-    if follow:
-        for line in osbs.get_build_logs(build_id, follow=True, namespace=args.namespace):
-            print(line)
+    if follow and args.from_docker_build:
+        print("Can't use --follow and --from-docker-build. "
+              "Logs from docker build are part of metadata of a already built image.")
+        return
+
+    if args.from_docker_build:
+        logs = osbs.get_docker_build_logs(build_id)
     else:
-        logs = osbs.get_build_logs(build_id, follow=False, namespace=args.namespace)
-        print(logs, end="")
+        logs = osbs.get_build_logs(build_id, follow=follow,
+                                   ignore_if_missing=args.ignore_if_missing,
+                                   namespace=args.namespace)
+        if follow:
+            for line in logs:
+                print(line)
+            return
+    print(logs, end="")
 
 
 def cmd_watch_build(args, osbs):
@@ -234,6 +244,10 @@ def cli():
     build_logs_parser.add_argument("BUILD_ID", help="build ID", nargs=1)
     build_logs_parser.add_argument("-f", "--follow", help="follow logs as they come", action="store_true",
                                    default=False)
+    build_logs_parser.add_argument("--wait-if-missing", help="if build is not created yet, wait", action="store_true",
+                                   default=False)
+    build_logs_parser.add_argument("--from-docker-build", help="return logs from `docker build` instead",
+                                   action="store_true", default=False)
     build_logs_parser.set_defaults(func=cmd_build_logs)
 
     build_parser = subparsers.add_parser('build', help='build an image in OSBS')

osbs/core.py

@@ -9,7 +9,8 @@
 import json
 
 import logging
-from osbs.constants import DEFAULT_NAMESPACE, BUILD_FINISHED_STATES, BUILD_RUNNING_STATES, BUILD_PENDING_STATES
+from osbs.build.build_response import BuildResponse
+from osbs.constants import DEFAULT_NAMESPACE, BUILD_FINISHED_STATES, BUILD_RUNNING_STATES
 from osbs.exceptions import OsbsResponseException, OsbsException, OsbsWatchBuildNotFound
 
 try:
@@ -161,19 +162,36 @@ def start_build(self, build_config_id, namespace=DEFAULT_NAMESPACE):
         build_config["Kind"] = "Build"
         return self.create_build(build_config, namespace=namespace)
 
-    def logs(self, build_id, follow=False, build_json=None, namespace=DEFAULT_NAMESPACE):
+    def logs(self, build_id, follow=False, build_json=None, wait_if_missing=False,
+             namespace=DEFAULT_NAMESPACE):
         """
+        provide logs from build
 
-        :param follow:
-        :return:
+        :param build_id: str
+        :param follow: bool, fetch logs as they come?
+        :param build_json: dict, to save one get-build query
+        :param wait_if_missing: bool, if build doesn't exist, wait
+        :param namespace: str
+        :return: None, str or iterator
         """
-        if follow:
-            self.wait_for_build_to_get_scheduled(build_id, namespace=namespace)
-        else:
-            # When build is in new or pending state, openshift responds with 500
+        # does build exist?
+        try:
             build_json = build_json or self.get_build(build_id, namespace=namespace).json()
-            if build_json['status'].lower() in BUILD_PENDING_STATES:
-                return
+        except OsbsResponseException as ex:
+            if ex.status_code == 404:
+                if not wait_if_missing:
+                    raise OsbsException("Build '%s' doesn't exist." % build_id)
+            else:
+                raise
+
+        if follow or wait_if_missing:
+            build_json = self.wait_for_build_to_get_scheduled(build_id, namespace=namespace)
+
+        br = BuildResponse(None, build_json=build_json)
+
+        # When build is in new or pending state, openshift responds with 500
+        if br.is_pending():
+            return
 
         # 0.5+
         buildlogs_url = self._build_url("buildLogs/%s/" % build_id,

tests/fake_api.py

@@ -7,7 +7,6 @@
 """
 from __future__ import absolute_import, unicode_literals, print_function
 
-import json
 import os
 import re
 import pytest
@@ -66,6 +65,11 @@ def process_authorize(content):
             "file": "build_test-build-123.json",
         }
     },
+    "/osapi/v1beta1/buildLogs/%s/" % TEST_BUILD: {
+        "get": {
+            "file": "build_test-build-123_logs.txt",
+        },
+    },
 
     "/oauth/authorize": {
         "get": {

tests/mock_jsons/0.4.1/build_test-build-123.json

@@ -6,7 +6,7 @@
         "labels": {
             "artefacts": "",
             "dockerfile": "FROM 834629358fe214f210b0ed606fba2c17827d7a46dd74bd3309afc2a103ad0e89\nRUN uname -a && env\n",
-            "logs": "{\"stream\":\"Step 0 : FROM 834629358fe214f210b0ed606fba2c17827d7a46dd74bd3309afc2a103ad0e89\\n\"}\r\n\n{\"stream\":\" ---\\u003e 834629358fe2\\n\"}\r\n\n{\"stream\":\"Step 1 : RUN uname -a \\u0026\\u0026 env\\n\"}\r\n\n{\"stream\":\" ---\\u003e Running in ebae6e171c85\\n\"}\r\n\n{\"stream\":\"Linux c4e263145f81 3.10.0-121.el7.x86_64 #1 SMP Tue Apr 8 10:48:19 EDT 2014 x86_64 x86_64 x86_64 GNU/Linux\\n\"}\r\n\n{\"stream\":\"HOSTNAME=c4e263145f81\\nPATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\\nPWD=/\\nSHLVL=1\\nHOME=/root\\n_=/usr/bin/env\\n\"}\r\n\n{\"stream\":\" ---\\u003e 52c4b6965b00\\n\"}\r\n\n{\"stream\":\"Removing intermediate container ebae6e171c85\\n\"}\r\n\n{\"stream\":\"Successfully built 52c4b6965b00\\n\"}\r\n",
+            "logs": "{\"stream\":\"Step 0 : FROM 834629358fe214f210b0ed606fba2c17827d7a46dd74bd3309afc2a103ad0e89\\n\"}\r\n\n{\"stream\":\" ---\\u003e 834629358fe2\\n\"}\r\n\n{\"stream\":\"Step 1 : RUN uname -a \\u0026\\u0026 env\\n\"}\r\n\n{\"stream\":\" ---\\u003e Running in ebae6e171c85\\n\"}\r\n\n{\"stream\":\"Linux c4e263145f81 3.10.0-121.el7.x86_64 #1 SMP Tue Apr 8 10:48:19 EDT 2014 x86_64 x86_64 x86_64 GNU/Linux\\n\"}\r\n\n{\"stream\":\"HOSTNAME=c4e263145f81\\nPATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\\nPWD=/\\nSHLVL=1\\nHOME=/root\\n_=/usr/bin/env\\n\"}\r\n\n{\"stream\":\" ---\\u003e 52c4b6965b00\\n\"}\r\n\n{\"stream\":\"Removing intermediate container ebae6e171c85\\n\"}\r\n\n{\"stream\":\"Successfully built 52c4b6965b00\\n\"}",
             "rpm-packages": "setup-2.9.0-2.fc21.noarch\nbasesystem-10.0-10.fc21.noarch"
         },
         "name": "test-build-123",

tests/mock_jsons/0.4.1/build_test-build-123_logs.txt

@@ -0,0 +1 @@
+line 1

tests/unit_test.py

@@ -11,6 +11,7 @@
 import json
 import os
 import sys
+from types import GeneratorType
 
 import pytest
 import logging
@@ -565,11 +566,24 @@ def test_get_user_api(osbs):
 
 
 def test_build_logs_api(osbs):
-    response = osbs.get_build_logs(TEST_BUILD)
-    # We should get a response.
-    assert response is not None
-    # The first line of the logs should be 'Step 0 : FROM ...'
-    assert response.split('\n')[0].find("Step ") != -1
+    logs = osbs.get_build_logs(TEST_BUILD)
+    assert isinstance(logs, tuple(list(six.string_types) + [bytes]))
+    assert logs == "line 1"
+
+
+def test_build_logs_api_follow(osbs):
+    logs = osbs.get_build_logs(TEST_BUILD, follow=True)
+    assert isinstance(logs, GeneratorType)
+    assert next(logs) == "line 1"
+    with pytest.raises(StopIteration):
+        assert next(logs)
+
+
+@pytest.mark.parametrize('decode_docker_logs', [True, False])
+def test_build_logs_api_from_docker(osbs, decode_docker_logs):
+    logs = osbs.get_docker_build_logs(TEST_BUILD, decode_logs=decode_docker_logs)
+    assert isinstance(logs, tuple(list(six.string_types) + [bytes]))
+    assert logs.split('\n')[0].find("Step ") != -1
 
 
 @pytest.mark.skipif(sys.version_info[0] >= 3,