Merge pull request #12 from scrapinghub/workaround-for-docker-py-issue

Docker client returns invalid json

shub_image/utils.py

@@ -1,9 +1,9 @@
 import os
 import re
-import json
 import click
 import importlib
 
+from six import string_types
 import ruamel.yaml as yaml
 
 from shub import config as shub_config
@@ -88,6 +88,20 @@ def get_docker_client():
         import docker
     except ImportError:
         raise ImportError('You need docker-py installed for the cmd')
+
+    class CustomDockerClient(docker.Client):
+
+        # XXX: workaround for https://github.com/docker/docker-py/issues/1059
+        def _stream_helper(self, response, decode=False):
+            it = super(CustomDockerClient, self)._stream_helper(response, decode=decode)
+            for data in it:
+                if not isinstance(data, string_types):
+                    yield data
+                for line in data.split('\r\n'):
+                    line = line.strip()
+                    if line:
+                        yield line
+
     docker_host = os.environ.get('DOCKER_HOST')
     tls_config = None
     if os.environ.get('DOCKER_TLS_VERIFY', False):
@@ -102,9 +116,10 @@ def get_docker_client():
             assert_hostname=False)
         docker_host = docker_host.replace('tcp://', 'https://')
     version = os.environ.get('DOCKER_VERSION', DEFAULT_DOCKER_VERSION)
-    return docker.Client(base_url=docker_host,
-                         version=version,
-                         tls=tls_config)
+    return CustomDockerClient(
+        base_url=docker_host,
+        version=version,
+        tls=tls_config)
 
 
 def format_image_name(image_name, image_tag):

tests/test_utils.py

@@ -1,12 +1,12 @@
+import json
 import os
 import sys
+
+import docker
 import mock
-import errno
-import shutil
 import StringIO
 import tempfile
-from unittest import TestCase, main
-from shub import config as shub_config
+from unittest import TestCase
 from shub import exceptions as shub_exceptions
 
 from shub_image.utils import missing_modules
@@ -23,6 +23,7 @@
 
 from .utils import FakeProjectDirectory, add_scrapy_fake_config
 
+
 class ReleaseUtilsTest(TestCase):
 
     def test_missing_modules(self):
@@ -42,22 +43,31 @@ def test_get_project_dir(self):
     def test_get_docker_client(self):
         mocked_docker = mock.Mock()
         sys.modules['docker'] = mocked_docker
+        client_mock = mock.Mock()
+
+        class DockerClientMock(object):
+
+            def __init__(self, *args, **kwargs):
+                client_mock(*args, **kwargs)
+
+        mocked_docker.Client = DockerClientMock
+
         assert get_docker_client()
-        mocked_docker.Client.assert_called_with(
+        client_mock.assert_called_with(
             base_url=None, tls=None, version='1.17')
         # set basic test environment
         os.environ['DOCKER_HOST'] = 'http://127.0.0.1'
         os.environ['DOCKER_VERSION'] = '1.18'
         assert get_docker_client()
-        mocked_docker.Client.assert_called_with(
+        client_mock.assert_called_with(
             base_url='http://127.0.0.1', tls=None, version='1.18')
         # test for tls
         os.environ['DOCKER_TLS_VERIFY'] = '1'
         os.environ['DOCKER_CERT_PATH'] = '/tmp/cert/path'
         mocked_tls = mock.Mock()
         mocked_docker.tls.TLSConfig.return_value = mocked_tls
         assert get_docker_client()
-        mocked_docker.Client.assert_called_with(
+        client_mock.assert_called_with(
             base_url='http://127.0.0.1',
             tls=mocked_tls,
             version='1.18')
@@ -81,6 +91,22 @@ def test_format_image_name(self):
             mocked.return_value = config
             assert format_image_name('test', None) == 'test:test-version'
 
+    def test_custom_docker_client_workaround(self):
+        """Test workaround for https://github.com/docker/docker-py/issues/1059."""
+        line = (
+            '{"status":"Pulling from library/python","id":"2.7"}\r\n'
+            '{"status":"Pulling fs layer","progressDetail":{},"id":"5c90d4a2d1a8"}\r\n'
+        )
+
+        # mocked_docker.Client._stream_helper.return_value = (line,)
+        client = get_docker_client()
+        with mock.patch.object(docker.Client, '_stream_helper', return_value=(line,)):
+            result = list(client._stream_helper(mock.Mock(), decode=False))
+        assert len(result) == 2
+        assert json.loads(result[0])['id'] == '2.7'
+        assert json.loads(result[1])['id'] == '5c90d4a2d1a8'
+
+
 class ReleaseConfigTest(TestCase):
 
     def test_init(self):

tests/utils.py

@@ -29,7 +29,7 @@
 
 @contextmanager
 def FakeProjectDirectory():
-    tmpdir = tempfile.mkdtemp()
+    tmpdir = os.path.realpath(tempfile.mkdtemp())
     current = os.getcwd()
     os.chdir(tmpdir)
     try: