Allow for image additions archives (as well as plugins)

It is quite useful to be able to add additional items (such
as jenkins build information) into the image but the only
way to do this seems to be via plugins archives. Using these
for such information ends badly when building neutron (because
those *plugin* archives need to contain setup.py files) so this
proposes a new mechanism to put files that are not python plugins but
are auxilary useful data to put in the image (such as the jenkins
data mentioned above).

Change-Id: Ica8b5750bd239e023f14225190e68dab75519d36

doc/image-building.rst

@@ -267,7 +267,7 @@ image, add the following to the ``template-override`` file::
         && pip --no-cache-dir install networking-cisco
     {% endblock %}
 
-Acute readers may notice there is one problem with this however. Assuming
+Astute readers may notice there is one problem with this however. Assuming
 nothing else in the Dockerfile changes for a period of time, the above ``RUN``
 statement will be cached by Docker, meaning new commits added to the Git
 repository may be missed on subsequent builds. To solve this the Kolla build
@@ -309,6 +309,68 @@ The template now becomes::
     pip --no-cache-dir install /plugins/*
     {% endblock %}
 
+Additions Functionality
+-----------------------
+
+The Dockerfile customisation mechanism is also useful for adding/installing
+additions into images. An example of this is adding your jenkins job build
+metadata (say formatted into a jenkins.json file) into the image.
+
+The bottom of each Dockerfile contains two blocks, ``image_name_footer``, and
+``footer``. The ``image_name_footer`` is intended for image specific
+modifications, while the ``footer`` can be used to apply a common set of
+modifications to every Dockerfile.
+
+For example, to add the ``jenkins.json`` additions to the ``neutron_server``
+image, add the following to the ``template-override`` file::
+
+    {% extends parent_template %}
+
+    {% block neutron_server_footer %}
+    RUN cp /additions/jenkins/jenkins.json /jenkins.json
+    {% endblock %}
+
+Astute readers may notice there is one problem with this however. Assuming
+nothing else in the Dockerfile changes for a period of time, the above ``RUN``
+statement will be cached by Docker, meaning new commits added to the Git
+repository may be missed on subsequent builds. To solve this the Kolla build
+tool also supports cloning additional repositories at build time, which will be
+automatically made available to the build, within an archive named
+``additions-archive``.
+
+.. note::
+
+    The following is available for source build types only.
+
+To use this, add a section to ``/etc/kolla/kolla-build.conf`` in the following
+format::
+
+    [<image>-additions-<additions-name>]
+
+Where ``<image>`` is the image that the plugin should be installed into, and
+``<additions-name>`` is the chosen additions identifier.
+
+Continuing with the above example, add the following to
+``/etc/kolla/kolla-build.conf``::
+
+    [neutron-server-jenkins]
+    type = local
+    location = /path/to/your/jenkins/data
+
+The build will copy the directory, resulting in the following archive
+structure::
+
+    additions-archive.tar
+    |__ additions
+        |__jenkins
+
+The template now becomes::
+
+    {% block neutron_server_footer %}
+    ADD additions-archive /
+    RUN cp /additions/jenkins/jenkins.json /jenkins.json
+    {% endblock %}
+
 Custom Repos
 ------------
 

kolla/image/build.py

@@ -94,6 +94,10 @@ def make_a_logger(conf=None, image_name=None):
                  STATUS_ERROR, STATUS_PARENT_ERROR)
 
 
+class ArchivingError(Exception):
+    pass
+
+
 @contextlib.contextmanager
 def join_many(threads):
     try:
@@ -148,6 +152,7 @@ def __init__(self, name, canonical_name, path, parent_name='',
         self.logger = logger
         self.children = []
         self.plugins = []
+        self.additions = []
 
     def copy(self):
         c = Image(self.name, self.canonical_name, self.path,
@@ -159,6 +164,8 @@ def copy(self):
             c.children = list(self.children)
         if self.plugins:
             c.plugins = list(self.plugins)
+        if self.additions:
+            c.additions = list(self.additions)
         return c
 
     def __repr__(self):
@@ -354,6 +361,39 @@ def update_buildargs(self):
         return buildargs
 
     def builder(self, image):
+
+        def make_an_archive(items, arcname, item_child_path=None):
+            if not item_child_path:
+                item_child_path = arcname
+            archives = list()
+            items_path = os.path.join(image.path, item_child_path)
+            for item in items:
+                archive_path = self.process_source(image, item)
+                if image.status in STATUS_ERRORS:
+                    raise ArchivingError
+                archives.append(archive_path)
+            if archives:
+                for archive in archives:
+                    with tarfile.open(archive, 'r') as archive_tar:
+                        archive_tar.extractall(path=items_path)
+            else:
+                try:
+                    os.mkdir(items_path)
+                except OSError as e:
+                    if e.errno == errno.EEXIST:
+                        self.logger.info(
+                            'Directory %s already exist. Skipping.',
+                            items_path)
+                    else:
+                        self.logger.error('Failed to create directory %s: %s',
+                                          items_path, e)
+                        image.status = STATUS_CONNECTION_ERROR
+                        raise ArchivingError
+            arc_path = os.path.join(image.path, '%s-archive' % arcname)
+            with tarfile.open(arc_path, 'w') as tar:
+                tar.add(items_path, arcname=arcname)
+            return len(os.listdir(items_path))
+
         self.logger.debug('Processing')
         if image.status == STATUS_UNMATCHED:
             return
@@ -373,32 +413,26 @@ def builder(self, image):
             if image.status in STATUS_ERRORS:
                 return
 
-        plugin_archives = list()
-        plugins_path = os.path.join(image.path, 'plugins')
-        for plugin in image.plugins:
-            archive_path = self.process_source(image, plugin)
-            if image.status in STATUS_ERRORS:
-                return
-            plugin_archives.append(archive_path)
-        if plugin_archives:
-            for plugin_archive in plugin_archives:
-                with tarfile.open(plugin_archive, 'r') as plugin_archive_tar:
-                    plugin_archive_tar.extractall(path=plugins_path)
+        try:
+            plugins_am = make_an_archive(image.plugins, 'plugins')
+        except ArchivingError:
+            self.logger.error(
+                "Failed turning any plugins into a plugins archive")
+            return
         else:
-            try:
-                os.mkdir(plugins_path)
-            except OSError as e:
-                if e.errno == errno.EEXIST:
-                    self.logger.info('Directory %s already exist. Skipping.',
-                                     plugins_path)
-                else:
-                    self.logger.error('Failed to create directory %s: %s',
-                                      plugins_path, e)
-                    image.status = STATUS_CONNECTION_ERROR
-                    return
-        with tarfile.open(os.path.join(image.path, 'plugins-archive'),
-                          'w') as tar:
-            tar.add(plugins_path, arcname='plugins')
+            self.logger.debug(
+                "Turned %s plugins into plugins archive",
+                plugins_am)
+        try:
+            additions_am = make_an_archive(image.additions, 'additions')
+        except ArchivingError:
+            self.logger.error(
+                "Failed turning any additions into a additions archive")
+            return
+        else:
+            self.logger.debug(
+                "Turned %s additions into additions archive",
+                additions_am)
 
         # Pull the latest image for the base distro only
         pull = self.conf.pull if image.parent is None else False
@@ -848,6 +882,20 @@ def process_source_installation(image, section):
                                   plugin)
                     image.plugins.append(
                         process_source_installation(image, plugin))
+                for addition in [
+                    match.group(0) for match in
+                    (re.search('^{}-additions-.+'.format(image.name),
+                     section) for section in all_sections) if match]:
+                    try:
+                        self.conf.register_opts(
+                            common_config.get_source_opts(),
+                            addition
+                        )
+                    except cfg.DuplicateOptError:
+                        LOG.debug('Addition %s already registered in config',
+                                  addition)
+                    image.additions.append(
+                        process_source_installation(image, addition))
 
             self.images.append(image)