Release 3.2.0

.travis.yml

@@ -0,0 +1,9 @@
+os: osx
+language: generic
+
+before_script:
+  - pip install tox
+
+script:
+  - make test-py27
+  - make test-py36

cekit/builder.py

@@ -29,10 +29,15 @@ def execute(self):
     def prepare(self):
         pass
 
+    def before_run(self):
+        pass
+
     def run(self):
         raise CekitError(
             "Command.run() method is not implemented for '{}' command and '{}' type. Please report it!".format(self._command, self._type))
 
+    def after_run(self):
+        pass
 
 class Builder(Command):
     """
@@ -57,8 +62,16 @@ def execute(self):
             LOGGER.info("The --dry-run parameter was specified, build will not be executed")
             return
 
+        self.before_run()
+
         self.run()
 
+        self.after_run()
+
+    def before_run(self):
+        LOGGER.debug("Checking CEKit build dependencies...")
+        self.dependency_handler.handle(self)
+
     def prepare(self):
         if self.build_engine == 'docker' or self.build_engine == 'buildah' or self.build_engine == "podman":
             from cekit.generator.docker import DockerGenerator as generator_impl
@@ -87,6 +100,3 @@ def prepare(self):
 
         self.generator.init()
         self.generator.generate(self.build_engine)
-
-        LOGGER.debug("Checking CEKit build dependencies...")
-        self.dependency_handler.handle(self)

cekit/builders/docker_builder.py

@@ -4,8 +4,6 @@
 import sys
 import traceback
 
-import yaml
-
 from cekit.builder import Builder
 from cekit.errors import CekitError
 
@@ -20,7 +18,8 @@
     pass
 
 try:
-    # Docker Python library, the old one
+    # Docker Python library, the new one
+    import docker
     from docker.api.client import APIClient as APIClientClass
 except ImportError:
     pass
@@ -34,7 +33,8 @@
     pass
 
 try:
-    # Docker Python library, the new one
+    # Docker Python library, the old one
+    import docker  # pylint: disable=ungrouped-imports
     from docker.client import Client as APIClientClass  # pylint: disable=ungrouped-imports
 except ImportError:
     pass
@@ -48,19 +48,6 @@ class DockerBuilder(Builder):
     def __init__(self, common_params, params):
         super(DockerBuilder, self).__init__('docker', common_params, params)
 
-        # Default Docker daemon connection timeout 10 minutes
-        # It needs to be high enough to allow Docker daemon to export the
-        # image for squashing.
-        try:
-            self.timeout = int(os.getenv('DOCKER_TIMEOUT', '600'))
-        except ValueError:
-            raise CekitError("Provided timeout value: %s cannot be parsed as integer, exiting." %
-                             os.getenv('DOCKER_TIMEOUT'))
-
-        if self.timeout <= 0:
-            raise CekitError(
-                "Provided timeout value needs to be greater than zero, currently: %s, exiting." % self.timeout)
-
     @staticmethod
     def dependencies():
         deps = {}
@@ -109,9 +96,13 @@ def _build_with_docker(self, docker_client):
                     # of container images.
                     continue
 
-                # This prevents poluting CEKit log with dowloading/extracting msgs
+                # This prevents polluting CEKit log with dowloading/extracting messages
                 messages = ANSI_ESCAPE.sub('', messages).strip()
 
+                # Python 2 compatibility
+                if sys.version_info[0] == 2:
+                    messages = messages.encode("utf-8", errors="ignore")
+
                 for message in messages.split('\n'):
                     LOGGER.info('Docker: {}'.format(message))
 
@@ -153,7 +144,8 @@ def _build_with_docker(self, docker_client):
                 msg = "Image build failed with a yum error and you don't " \
                     "have any yum repository configured, please check " \
                     "your image/module descriptor for proper repository " \
-                    " definitions."
+                    "definitions."
+
             raise CekitError(msg, ex)
 
         return docker_layer_ids[-1]
@@ -176,6 +168,64 @@ def _tag(self, docker_client, image_id, tags):
             else:
                 docker_client.tag(image_id, tag)
 
+    def _docker_client(self):
+        LOGGER.debug("Preparing Docker client...")
+
+        # Default Docker daemon connection timeout 10 minutes
+        # It needs to be high enough to allow Docker daemon to export the
+        # image for squashing.
+        try:
+            timeout = int(os.getenv('DOCKER_TIMEOUT', '600'))
+        except ValueError:
+            raise CekitError("Provided timeout value: '{}' cannot be parsed as integer, exiting.".format(
+                os.getenv('DOCKER_TIMEOUT')))
+
+        if timeout <= 0:
+            raise CekitError(
+                "Provided timeout value needs to be greater than zero, currently: '{}', exiting.".format(timeout))
+
+        params = {"version": "1.22"}
+        params.update(docker.utils.kwargs_from_env())
+        params["timeout"] = timeout
+
+        try:
+            client = APIClientClass(**params)
+        except docker.errors.DockerException as e:
+            LOGGER.error("Could not create Docker client, please make sure that you "
+                         "specified valid parameters in the 'DOCKER_HOST' environment variable, "
+                         "examples: 'unix:///var/run/docker.sock', 'tcp://192.168.22.33:1234'")
+            raise CekitError("Error while creating the Docker client", e)
+
+        if client and self._valid_docker_connection(client):
+            LOGGER.debug("Docker client ready and working")
+            LOGGER.debug(client.version())
+            return client
+
+        LOGGER.error(
+            "Could not connect to the Docker daemon at '{}', please make sure the Docker "
+            "daemon is running.".format(client.base_url))
+
+        if client.base_url.startswith('unix'):
+            LOGGER.error(
+                "Please make sure the Docker socket has correct permissions.")
+
+        if os.environ.get('DOCKER_HOST'):
+            LOGGER.error("If Docker daemon is running, please make sure that you specified valid "
+                         "parameters in the 'DOCKER_HOST' environment variable, examples: "
+                         "'unix:///var/run/docker.sock', 'tcp://192.168.22.33:1234'. You may "
+                         "also need to specify 'DOCKER_TLS_VERIFY', and 'DOCKER_CERT_PATH' "
+                         "environment variables.")
+
+        raise CekitError("Cannot connect to Docker daemon")
+
+    def _valid_docker_connection(self, client):
+        try:
+            return client.ping()
+        except requests.exceptions.ConnectionError:
+            pass
+
+        return False
+
     def run(self):
         tags = self.params.tags
 
@@ -185,7 +235,7 @@ def run(self):
         LOGGER.info("Building container image using Docker...")
         LOGGER.debug("Building image with tags: '%s'" % "', '".join(tags))
 
-        docker_client = APIClientClass(version="1.22", timeout=self.timeout)
+        docker_client = self._docker_client()
 
         # Build image
         image_id = self._build_with_docker(docker_client)

cekit/builders/osbs.py

@@ -80,10 +80,10 @@ def dependencies():
 
         return deps
 
-    def prepare(self):
+    def before_run(self):
         """Prepares dist-git repository for OSBS build."""
 
-        super(OSBSBuilder, self).prepare()
+        super(OSBSBuilder, self).before_run()
 
         self.prepare_dist_git()
         self.copy_to_dist_git()
@@ -356,7 +356,7 @@ def clean(self):
                 shutil.rmtree(d, ignore_errors=True)
 
                 if d in git_files:
-                    subprocess.check_output(["git", "rm", "-rf", d])
+                    subprocess.check_call(["git", "rm", "-rf", d])
 
     def add(self, artifacts):
         LOGGER.debug("Adding files to git stage...")
@@ -366,6 +366,10 @@ def add(self, artifacts):
                 LOGGER.debug("Skipping staging '{}' in git because it is an artifact".format(obj))
                 continue
 
+            if obj == ".git":
+                LOGGER.debug("Skipping '.git' directory")
+                continue
+
             LOGGER.debug("Staging '{}'...".format(obj))
 
             subprocess.check_call(["git", "add", "--all", obj])

cekit/descriptor/base.py

@@ -1,4 +1,10 @@
 import collections
+
+try:
+    collectionsAbc = collections.abc
+except AttributeError:
+    collectionsAbc = collections
+
 import logging
 import os
 
@@ -10,7 +16,7 @@
 logger = logging.getLogger('cekit')
 
 
-class Descriptor(collections.MutableMapping):
+class Descriptor(collectionsAbc.MutableMapping):
     """Class serving as parent for any descriptor in cekit.
 
     Class implement collections.MutableMapping so it can be used as a dictionary.

cekit/descriptor/image.py

@@ -59,8 +59,7 @@ def _prepare(self):
                                          for a in self._descriptor.get('artifacts', [])]
         self._descriptor['modules'] = Modules(self._descriptor.get('modules', {}), self.path)
         self._descriptor['packages'] = Packages(self._descriptor.get('packages', {}), self.path)
-        if 'osbs' in self._descriptor:
-            self._descriptor['osbs'] = Osbs(self._descriptor['osbs'], self.path)
+        self._descriptor['osbs'] = Osbs(self._descriptor.get('osbs', {}), self.path)
         self._descriptor['volumes'] = [Volume(x) for x in self._descriptor.get('volumes', [])]
 
         # make sure image declarations override any module definitions
@@ -163,7 +162,7 @@ def packages(self):
 
     @property
     def osbs(self):
-        return self.get('osbs', Osbs({}, self.path))
+        return self.get('osbs')
 
     @osbs.setter
     def osbs(self, value):
@@ -241,11 +240,26 @@ def apply_image_overrides(self, overrides):
             if override.packages.manager:
                 self.packages._descriptor['manager'] = override.packages.manager
 
-            if override.packages.content_sets:
-                self.packages.content_sets = _merge_descriptors(
-                    override.packages.content_sets, self.packages.content_sets)
-            elif override.packages.content_sets_file:
-                self.packages.content_sets_file = override.packages.content_sets_file
+            # If the key is set (value could be None!)
+            if 'content_sets' in override.packages:
+                # If value is different than None, merge it
+                if override.packages.content_sets:
+                    self.packages.content_sets = _merge_descriptors(
+                        override.packages.content_sets, self.packages.content_sets)
+                else:
+                    # If key exists and value is None - remove it from the main packages section
+                    self.packages._descriptor.pop('content_sets', None)
+
+            # If the key is set (value could be None!)
+            if 'content_sets_file' in override.packages:
+                # If value is different than None, set it
+                if override.packages.content_sets_file:
+                    self.packages.content_sets_file = override.packages.content_sets_file
+                else:
+                    # If key exists and value is None - remove it from the main packages section
+                    # Remove 'content_sets' too, because this is where it should already be translated into
+                    self.packages._descriptor.pop('content_sets_file', None)
+                    self.packages._descriptor.pop('content_sets', None)
 
             if override.osbs != None:
                 self.osbs = override.osbs.merge(self.osbs)

cekit/descriptor/packages.py

@@ -42,7 +42,8 @@
 
 
 class Packages(Descriptor):
-    """Object representing Pakcages
+    """
+    Object representing packages
 
     Args:
       descriptor - yaml containing Packages section
@@ -52,10 +53,16 @@ def __init__(self, descriptor, descriptor_path):
         self.schemas = packages_schema
         self.descriptor_path = descriptor_path
         super(Packages, self).__init__(descriptor)
-        if ('content_sets_file' in descriptor and 'content_sets' in descriptor):
-            raise CekitError("You cannot specify content_sets and content_sets_file together!")
 
-        if 'content_sets_file' in descriptor:
+        # If 'content_sets' and 'content_sets_file' are defined at the same time
+        if set(['content_sets', 'content_sets_file']).issubset(set(descriptor.keys())):
+            raise CekitError(
+                "You cannot specify 'content_sets' and 'content_sets_file' together in the packages section!")
+
+        # If the 'content_sets_file' key is set and is not None we need to read the specified
+        # file and make it available in the 'content_sets' key. The 'content_sets_file' key is removed
+        # afterwards.
+        if descriptor.get('content_sets_file', None):
             content_sets_file = os.path.join(self.descriptor_path, descriptor['content_sets_file'])
 
             if not os.path.exists(content_sets_file):

cekit/generator/base.py

@@ -86,6 +86,7 @@ def init(self):
 
         LOGGER.debug("Removing old target directory")
         shutil.rmtree(self.target, ignore_errors=True)
+        os.makedirs(os.path.join(self.target, 'image'))
 
         # Read the main image descriptor and create an Image object from it
         descriptor = tools.load_descriptor(self._descriptor_path)

cekit/version.py

@@ -1,2 +1,2 @@
-version = "3.1.0"
+version = "3.2.0"
 schema_version = 2

docs/getting-started/modules.rst

@@ -139,8 +139,8 @@ Tomcat
    # Defined artifacts that are used to build the image
    artifacts:
      - name: tomcat.tar.gz
-        url: https://archive.apache.org/dist/tomcat/tomcat-8/v8.5.24/bin/apache-tomcat-8.5.24.tar.gz
-        md5: 080075877a66adf52b7f6d0013fa9730
+       url: https://archive.apache.org/dist/tomcat/tomcat-8/v8.5.24/bin/apache-tomcat-8.5.24.tar.gz
+       md5: 080075877a66adf52b7f6d0013fa9730
    execute:
      - script: install.sh
 

docs/handbook/building/builder-engines.rst

@@ -30,6 +30,40 @@ Example
 
         $ cekit build docker
 
+Docker environment variables
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+It is possible to use environment variables to let CEKit know where is the Docker daemon
+located it should connect to.
+
+.. note::
+    Read more about `Docker daemon settings related to exposing it to clients <https://docs.docker.com/engine/reference/commandline/dockerd/#daemon-socket-option>`__.
+
+By default, if you do not specify anything, **CEKit will try to use a locally running Docker daemon**.
+
+If you need to customize this behavior (for example when you want to use Docker daemon
+running in a VM) you can set following environment variables:
+
+``DOCKER_HOST``
+    The ``DOCKER_HOST`` environment variable is where you specify where the Daemon is running. It supports
+    multiple protocols, but the most widely used ones are: ``unix://`` (where you specify path to a local
+    socket) and ``tcp://`` (where you can define host location and port).
+
+    Examples of ``DOCKER_HOST``: ``unix:///var/run/docker.sock``, ``tcp://192.168.22.33:1234``.
+
+    Depending how your daemon is configured you may need to configure settings related to encryption.
+
+    .. code-block:: bash
+
+        # Connect to a remote Docker daemon
+        $ DOCKER_HOST="tcp://192.168.22.33:1234" cekit build docker
+``DOCKER_TLS_VERIFY``
+    You can set ``DOCKER_TLS_VERIFY`` to a non-empty value to indicate that the TLS verification should
+    take place. By default certificate verification is **disabled**.
+``DOCKER_CERT_PATH``
+    You can point ``DOCKER_CERT_PATH`` environment variable to a directory containing certificates to use when
+    connecting to the Docker daemon.
+
 
 OSBS builder
 ---------------------------