libs: Add envoy.docker.utils (#30)

Signed-off-by: Ryan Northey <ryan@synca.io>

.github/workflows/ci.yml

@@ -16,6 +16,7 @@ jobs:
         - envoy.base.utils
         - envoy.base.runner
         - envoy.base.checker
+        - envoy.docker.utils
         - envoy.github.abstract
         - envoy.github.release
         - envoy.gpg.identity
@@ -93,6 +94,7 @@ jobs:
         - envoy.base.utils
         - envoy.base.runner
         - envoy.base.checker
+        - envoy.docker.utils
         - envoy.github.abstract
         - envoy.github.release
         - envoy.gpg.identity

envoy.docker.utils/README.rst

@@ -0,0 +1,5 @@
+
+envoy.docker.utils
+==================
+
+Docker utils used in Envoy proxy's CI

envoy.docker.utils/VERSION

@@ -0,0 +1 @@
+0.0.1

envoy.docker.utils/envoy/docker/utils/__init__.py

@@ -0,0 +1,11 @@
+
+from .utils import (
+    build_image,
+    BuildError,
+    docker_client)
+
+
+__all__ = (
+    "build_image",
+    "BuildError",
+    "docker_client")

envoy.docker.utils/envoy/docker/utils/utils.py

@@ -0,0 +1,111 @@
+import tarfile
+import tempfile
+from contextlib import asynccontextmanager
+from typing import AsyncIterator, Callable, Optional
+
+import aiodocker
+
+
+class BuildError(Exception):
+    pass
+
+
+async def _build_image(
+        tar,  #: IO[bytes] (`docker.images.build` expects `BinaryIO`)
+        docker: aiodocker.Docker,
+        context: str,
+        tag: str,
+        buildargs: Optional[dict] = None,
+        stream: Optional[Callable] = None,
+        **kwargs) -> None:
+    """Docker image builder
+
+    if a `stream` callable arg is supplied, logs are output there.
+
+    raises `tools.docker.utils.BuildError` with any error output.
+    """
+    # create a tarfile from the supplied directory
+    with tarfile.open(tar.name, fileobj=tar, mode="w") as tarball:
+        tarball.add(context, arcname=".")
+    tar.seek(0)
+
+    # build the docker image
+    build = docker.images.build(
+        fileobj=tar,
+        encoding="gzip",
+        tag=tag,
+        stream=True,
+        buildargs=buildargs or {},
+        **kwargs)
+
+    async for line in build:
+        if line.get("errorDetail"):
+            raise BuildError(
+                f"Docker image failed to build {tag} {buildargs}\n"
+                f"{line['errorDetail']['message']}")
+        if stream and "stream" in line:
+            stream(line["stream"].strip())
+
+
+async def build_image(*args, **kwargs) -> None:
+    """Creates a Docker context by tarballing a directory, and then building
+    an image with it
+
+    aiodocker doesn't provide an in-built way to build docker images from a
+    directory, only a file, so you can't include artefacts.
+
+    this adds the ability to include artefacts.
+
+    as an example, assuming you have a directory containing a `Dockerfile` and
+    some artefacts at `/tmp/mydockercontext` - and wanted to build the image
+    `envoy:foo` you could:
+
+    ```python
+
+    import asyncio
+
+    from tools.docker import utils
+
+
+    async def myimage():
+        async with utils.docker_client() as docker:
+            await utils.build_image(
+                docker,
+                "/tmp/mydockerbuildcontext",
+                "envoy:foo",
+                buildargs={})
+
+    asyncio.run(myimage())
+    ```
+    """
+    with tempfile.NamedTemporaryFile() as tar:
+        await _build_image(tar, *args, **kwargs)
+
+
+@asynccontextmanager
+async def docker_client(
+        url: Optional[str] = "") -> AsyncIterator[aiodocker.Docker]:
+    """Aiodocker client
+
+    For example to dump the docker image data:
+
+    ```python
+
+    import asyncio
+
+    from tools.docker import utils
+
+
+    async def docker_images():
+        async with utils.docker_client() as docker:
+            print(await docker.images.list())
+
+    asyncio.run(docker_images())
+    ```
+    """
+
+    docker = aiodocker.Docker(url)
+    try:
+        yield docker
+    finally:
+        await docker.close()

envoy.docker.utils/setup.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+
+import os
+import codecs
+from setuptools import find_namespace_packages, setup  # type:ignore
+
+
+def read(fname):
+    file_path = os.path.join(os.path.dirname(__file__), fname)
+    return codecs.open(file_path, encoding='utf-8').read()
+
+
+setup(
+    name='envoy.docker.utils',
+    version=read("VERSION"),
+    author='Ryan Northey',
+    author_email='ryan@synca.io',
+    maintainer='Ryan Northey',
+    maintainer_email='ryan@synca.io',
+    license='Apache Software License 2.0',
+    url='https://github.com/envoyproxy/pytooling/envoy.docker.utils',
+    description="GPG identity util used in Envoy proxy's CI",
+    long_description=read('README.rst'),
+    py_modules=['envoy.docker.utils'],
+    packages=find_namespace_packages(),
+    package_data={'envoy.docker.utils': ['py.typed']},
+    python_requires='>=3.5',
+    extras_require={
+        "test": [
+            "pytest",
+            "pytest-asyncio",
+            "pytest-coverage",
+            "pytest-patches"],
+        "lint": ['flake8'],
+        "types": [
+            'mypy'],
+        "publish": ['wheel'],
+    },
+    install_requires=[
+        "aiodocker",
+    ],
+    classifiers=[
+        'Development Status :: 4 - Beta',
+        'Framework :: Pytest',
+        'Intended Audience :: Developers',
+        'Topic :: Software Development :: Testing',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3 :: Only',
+        'Programming Language :: Python :: Implementation :: CPython',
+        'Operating System :: OS Independent',
+        'License :: OSI Approved :: Apache Software License',
+    ],
+)

envoy.docker.utils/tests/test_utils.py

@@ -0,0 +1,146 @@
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from envoy.docker import utils
+
+
+class MockAsyncIterator:
+    def __init__(self, seq):
+        self.iter = iter(seq)
+        self.count = 0
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+        self.count += 1
+        try:
+            return next(self.iter)
+        except StopIteration:
+            raise StopAsyncIteration
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("args", [(), ("ARG1", ), ("ARG1", "ARG2")])
+@pytest.mark.parametrize("kwargs", [{}, dict(kkey1="VVAR1", kkey2="VVAR2")])
+async def test_util_build_image(patches, args, kwargs):
+    patched = patches(
+        "_build_image",
+        "tempfile",
+        prefix="envoy.docker.utils.utils")
+
+    with patched as (m_build, m_temp):
+        assert not await utils.build_image(*args, **kwargs)
+
+    temp_file = m_temp.NamedTemporaryFile
+    assert (
+        list(temp_file.call_args)
+        == [(), {}])
+
+    assert (
+        list(m_build.call_args)
+        == [(temp_file.return_value.__enter__.return_value, ) + args,
+            kwargs])
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("stream", [True, False])
+@pytest.mark.parametrize("buildargs", [None, dict(key1="VAR1", key2="VAR2")])
+@pytest.mark.parametrize("error", [None, "SOMETHING WENT WRONG"])
+async def test_util__build_image(patches, stream, buildargs, error):
+    lines = (
+        dict(notstream=f"NOTLINE{i}",
+             stream=f"LINE{i}")
+        for i in range(1, 4))
+
+    if error:
+        lines = list(lines)
+        lines[1]["errorDetail"] = dict(message=error)
+        lines = iter(lines)
+
+    docker = AsyncMock()
+    docker.images.build = MagicMock(return_value=MockAsyncIterator(lines))
+
+    _stream = MagicMock()
+    tar = MagicMock()
+    patched = patches(
+        "tarfile",
+        prefix="envoy.docker.utils.utils")
+
+    with patched as (m_tar, ):
+        args = (tar, docker, "CONTEXT", "TAG")
+        kwargs = {}
+        if stream:
+            kwargs["stream"] = _stream
+        if buildargs:
+            kwargs["buildargs"] = buildargs
+
+        if error:
+            with pytest.raises(utils.BuildError):
+                await utils.utils._build_image(*args, **kwargs)
+        else:
+            assert not await utils.utils._build_image(*args, **kwargs)
+
+    assert (
+        list(m_tar.open.call_args)
+        == [(tar.name,), {'fileobj': tar, 'mode': 'w'}])
+    assert (
+        list(m_tar.open.return_value.__enter__.return_value.add.call_args)
+        == [('CONTEXT',), {'arcname': '.'}])
+    assert (
+        list(tar.seek.call_args)
+        == [(0,), {}])
+    assert (
+        list(docker.images.build.call_args)
+        == [(),
+            {'fileobj': tar,
+             'encoding': 'gzip',
+             'tag': 'TAG',
+             'stream': True,
+             'buildargs': buildargs or {}}])
+    if stream and error:
+        assert (
+            list(list(c) for c in _stream.call_args_list)
+            == [[('LINE1',), {}]])
+        return
+    elif stream:
+        assert (
+            list(list(c) for c in _stream.call_args_list)
+            == [[(f'LINE{i}',), {}] for i in range(1, 4)])
+        return
+    # the iterator should be called n + 1 for the n of items
+    # if there was an error it should stop at the error
+    assert docker.images.build.return_value.count == 2 if error else 4
+    assert not _stream.called
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("raises", [True, False])
+@pytest.mark.parametrize("url", [None, "URL"])
+async def test_util_docker_client(patches, raises, url):
+
+    class DummyError(Exception):
+        pass
+
+    patched = patches(
+        "aiodocker",
+        prefix="envoy.docker.utils.utils")
+
+    with patched as (m_docker, ):
+        m_docker.Docker.return_value.close = AsyncMock()
+        if raises:
+            with pytest.raises(DummyError):
+                async with utils.docker_client(url) as docker:
+                    raise DummyError()
+        else:
+            async with utils.docker_client(url) as docker:
+                pass
+
+    assert (
+        list(m_docker.Docker.call_args)
+        == [(url,), {}])
+    assert docker == m_docker.Docker.return_value
+    assert (
+        list(m_docker.Docker.return_value.close.call_args)
+        == [(), {}])

envoy.gpg.identity/VERSION

@@ -1 +1 @@
-0.0.1
+0.0.2-dev