Merge pull request #9 from shakenfist/mounts

Add overlay mounts as an output renderer.

.github/workflows/functional-tests.yml

@@ -56,7 +56,7 @@ jobs:
           cd /srv/github/_work/occystrap/occystrap/occystrap
           rm -f dist/*
           python3 setup.py sdist bdist_wheel
-          pip3 install dist/occystrap*.whl
+          sudo pip3 install dist/occystrap*.whl
 
       - name: Run a local docker registry to talk to, and populate it with test data
         run: |
@@ -75,4 +75,7 @@ jobs:
           cd /srv/github/_work/occystrap/occystrap/occystrap/deploy
           sudo pip3 install -r requirements.txt
           sudo pip3 install -r test-requirements.txt
-          stestr run --concurrency=5
+
+          # This needs to run as root because some of the tests require
+          # escalated permissions.
+          sudo stestr run --concurrency=5

deploy/occystrap_ci/tests/test_whiteout.py

@@ -1,20 +1,22 @@
 
 import logging
 import os
+from oslo_concurrency import processutils
 import tempfile
 import testtools
 
 
 from occystrap import docker_registry
 from occystrap import output_ocibundle
+from occystrap import output_mounts
 
 
 logging.basicConfig(level=logging.INFO, format='%(message)s')
 LOG = logging.getLogger()
 
 
 class WhiteoutsTestCase(testtools.TestCase):
-    def test_whiteouts(self):
+    def test_whiteouts_ocibundle(self):
         image = 'occystrap_deletion_layers'
         tag = 'latest'
 
@@ -43,3 +45,36 @@ def test_whiteouts(self):
             self.assertFalse(
                 os.path.exists(os.path.join(tempdir,
                                             'rootfs/anotherdirectory/.wh..wh..opq')))
+
+    def test_whiteouts_mounts(self):
+        image = 'occystrap_deletion_layers'
+        tag = 'latest'
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            oci = output_mounts.MountWriter(image, tag, tempdir)
+            img = docker_registry.Image(
+                'localhost:5000', image, tag, 'linux', 'amd64', '',
+                secure=False)
+            for image_element in img.fetch(fetch_callback=oci.fetch_callback):
+                oci.process_image_element(*image_element)
+            oci.finalize()
+            oci.write_bundle()
+
+            self.assertTrue(
+                os.path.exists(os.path.join(tempdir, 'rootfs')))
+            self.assertFalse(
+                os.path.exists(os.path.join(tempdir, 'rootfs/file')))
+            self.assertFalse(
+                os.path.exists(os.path.join(tempdir, 'rootfs/.wh.file')))
+            self.assertFalse(
+                os.path.exists(os.path.join(tempdir, 'rootfs/directory')))
+            self.assertTrue(
+                os.path.exists(os.path.join(tempdir, 'rootfs/anotherfile')))
+            self.assertTrue(
+                os.path.exists(os.path.join(tempdir, 'rootfs/anotherdirectory')))
+            self.assertFalse(
+                os.path.exists(os.path.join(tempdir,
+                                            'rootfs/anotherdirectory/.wh..wh..opq')))
+
+            processutils.execute('umount %s' % os.path.join(tempdir, 'rootfs'),
+                                 shell=True)

occystrap/common.py

@@ -0,0 +1,40 @@
+import json
+
+from occystrap.constants import RUNC_SPEC_TEMPLATE
+
+
+def write_container_config(container_config_filename, runtime_config_filename,
+                           container_template=RUNC_SPEC_TEMPLATE,
+                           container_values=None):
+    if not container_values:
+        container_values = {}
+
+    # Read the container config
+    with open(container_config_filename) as f:
+        image_conf = json.loads(f.read())
+
+    # Write a runc specification for the container
+    container_conf = json.loads(container_template)
+
+    container_conf['process']['terminal'] = True
+    cwd = image_conf['config']['WorkingDir']
+    if cwd == '':
+        cwd = '/'
+    container_conf['process']['cwd'] = cwd
+
+    entrypoint = image_conf['config'].get('Entrypoint', [])
+    if not entrypoint:
+        entrypoint = []
+    cmd = image_conf['config'].get('Cmd', [])
+    if cmd:
+        entrypoint.extend(cmd)
+    container_conf['process']['args'] = entrypoint
+
+    # terminal = false means "pass through existing file descriptors"
+    container_conf['process']['terminal'] = False
+
+    container_conf['hostname'] = container_values.get(
+        'hostname', 'occystrap')
+
+    with open(runtime_config_filename, 'w') as f:
+        f.write(json.dumps(container_conf, indent=4, sort_keys=True))

occystrap/main.py

@@ -1,15 +1,17 @@
 import click
 import logging
+import os
+from shakenfist_utilities import logs
+import sys
 
 from occystrap import docker_registry
 from occystrap import output_directory
+from occystrap import output_mounts
 from occystrap import output_ocibundle
 from occystrap import output_tarfile
 
-logging.basicConfig(level=logging.INFO)
 
-LOG = logging.getLogger(__name__)
-LOG.setLevel(logging.INFO)
+LOG = logs.setup_console(__name__)
 
 
 @click.group()
@@ -94,6 +96,30 @@ def fetch_to_tarfile(ctx, registry, image, tag, tarfile, insecure):
 cli.add_command(fetch_to_tarfile)
 
 
+@click.command()
+@click.argument('registry')
+@click.argument('image')
+@click.argument('tag')
+@click.argument('path')
+@click.option('--insecure', is_flag=True, default=False)
+@click.pass_context
+def fetch_to_mounts(ctx, registry, image, tag, path, insecure):
+    if not hasattr(os, 'setxattr'):
+        print('Sorry, your OS module implementation lacks setxattr')
+        sys.exit(1)
+    if not hasattr(os, 'mknod'):
+        print('Sorry, your OS module implementation lacks mknod')
+        sys.exit(1)
+
+    d = output_mounts.MountWriter(image, tag, path)
+    _fetch(registry, image, tag, d, ctx.obj['OS'], ctx.obj['ARCHITECTURE'],
+           ctx.obj['VARIANT'], secure=(not insecure))
+    d.write_bundle()
+
+
+cli.add_command(fetch_to_mounts)
+
+
 @click.command()
 @click.argument('path')
 @click.argument('image')

occystrap/output_mounts.py

@@ -0,0 +1,155 @@
+import json
+import logging
+import os
+import stat
+import tarfile
+
+from occystrap import common
+from occystrap import constants
+from occystrap import util
+
+
+LOG = logging.getLogger(__name__)
+LOG.setLevel(logging.INFO)
+
+
+class MountWriter(object):
+    def __init__(self, image, tag, image_path):
+        self.image = image
+        self.tag = tag
+        self.image_path = image_path
+
+        self.tar_manifest = [{
+            'Layers': [],
+            'RepoTags': ['%s:%s' % (self.image.split('/')[-1], self.tag)]
+        }]
+
+        self.bundle = {}
+
+        if not os.path.exists(self.image_path):
+            os.makedirs(self.image_path)
+
+    def _manifest_filename(self):
+        return 'manifest'
+
+    def fetch_callback(self, digest):
+        layer_file_in_dir = os.path.join(self.image_path, digest, 'layer.tar')
+        LOG.info('Layer file is %s' % layer_file_in_dir)
+        return not os.path.exists(layer_file_in_dir)
+
+    def process_image_element(self, element_type, name, data):
+        if element_type == constants.CONFIG_FILE:
+            with open(os.path.join(self.image_path, name), 'wb') as f:
+                d = json.loads(data.read())
+                f.write(json.dumps(d, indent=4, sort_keys=True).encode('ascii'))
+            self.tar_manifest[0]['Config'] = name
+
+        elif element_type == constants.IMAGE_LAYER:
+            layer_dir = os.path.join(self.image_path, name)
+            if not os.path.exists(layer_dir):
+                os.makedirs(layer_dir)
+
+            layer_file = os.path.join(name, 'layer.tar')
+            self.tar_manifest[0]['Layers'].append(layer_file)
+
+            layer_file_in_dir = os.path.join(self.image_path, layer_file)
+            if os.path.exists(layer_file_in_dir):
+                LOG.info('Skipping layer already in output directory')
+            else:
+                with open(layer_file_in_dir, 'wb') as f:
+                    d = data.read(102400)
+                    while d:
+                        f.write(d)
+                        d = data.read(102400)
+
+                layer_dir_in_dir = os.path.join(self.image_path, name, 'layer')
+                os.makedirs(layer_dir_in_dir)
+                with tarfile.open(layer_file_in_dir) as layer:
+                    for mem in layer.getmembers():
+                        dirname, filename = os.path.split(mem.name)
+
+                        # Some light reading on how this works...
+                        # https://www.madebymikal.com/interpreting-whiteout-files-in-docker-image-layers/
+                        # https://github.com/opencontainers/image-spec/blob/main/layer.md#opaque-whiteout
+                        if filename == '.wh..wh..opq':
+                            # A deleted directory, but only for layers below
+                            # this one.
+                            os.setxattr(os.path.join(layer_dir_in_dir, dirname),
+                                        'trusted.overlay.opaque', b'y')
+
+                        elif filename.startswith('.wh.'):
+                            # A single deleted element, which might not be a
+                            # file.
+                            os.mknod(os.path.join(layer_dir_in_dir,
+                                     mem.name[4:]),
+                                     mode=stat.S_IFCHR, device=0)
+
+                        else:
+                            path = mem.name
+                            layer.extract(path, path=layer_dir_in_dir)
+
+    def finalize(self):
+        manifest_filename = self._manifest_filename() + '.json'
+        manifest_path = os.path.join(self.image_path, manifest_filename)
+        with open(manifest_path, 'wb') as f:
+            f.write(json.dumps(self.tar_manifest, indent=4,
+                               sort_keys=True).encode('ascii'))
+
+        c = {}
+        catalog_path = os.path.join(self.image_path, 'catalog.json')
+        if os.path.exists(catalog_path):
+            with open(catalog_path, 'r') as f:
+                c = json.loads(f.read())
+
+        c.setdefault(self.image, {})
+        c[self.image][self.tag] = manifest_filename
+        with open(catalog_path, 'w') as f:
+            f.write(json.dumps(c, indent=4, sort_keys=True))
+
+    def write_bundle(self, container_template=constants.RUNC_SPEC_TEMPLATE,
+                     container_values=None):
+        if not container_values:
+            container_values = {}
+
+        rootfs_path = os.path.join(self.image_path, 'rootfs')
+        if not os.path.exists(rootfs_path):
+            os.makedirs(rootfs_path)
+        LOG.info('Writing image bundle to %s' % rootfs_path)
+
+        working_path = os.path.join(self.image_path, 'working')
+        if not os.path.exists(working_path):
+            os.makedirs(working_path)
+
+        delta_path = os.path.join(self.image_path, 'delta')
+        if not os.path.exists(delta_path):
+            os.makedirs(delta_path)
+
+        # The newest layer is listed first in the mount command
+        layer_dirs = []
+        self.tar_manifest[0]['Layers'].reverse()
+        for layer in self.tar_manifest[0]['Layers']:
+            layer_dirs.append(os.path.join(
+                self.image_path, layer.replace('.tar', '')))
+
+        # Extract the rootfs as overlay mounts
+        util.execute('mount -t overlay overlay -o lowerdir=%(layers)s,'
+                     'upperdir=%(upper)s,workdir=%(working)s %(rootfs)s'
+                     % {
+                         'layers': ':'.join(layer_dirs),
+                         'upper': delta_path,
+                         'working': working_path,
+                         'rootfs': rootfs_path
+                     })
+
+        # Rename the container configuration to a well known location. This is
+        # not part of the OCI specification, but is convenient for now.
+        container_config_filename = os.path.join(self.image_path,
+                                                 'container-config.json')
+        runtime_config_filename = os.path.join(self.image_path, 'config.json')
+        os.rename(os.path.join(self.image_path, self.tar_manifest[0]['Config']),
+                  container_config_filename)
+
+        common.write_container_config(container_config_filename,
+                                      runtime_config_filename,
+                                      container_template=container_template,
+                                      container_values=container_values)

occystrap/output_ocibundle.py

@@ -2,12 +2,12 @@
 # all of the data that a directory output does, and they place the data they
 # do contain into different locations within the directory structure.
 
-import json
 import logging
 import os
 import shutil
 
 from occystrap.constants import RUNC_SPEC_TEMPLATE
+from occystrap import common
 from occystrap.output_directory import DirWriter
 
 
@@ -43,28 +43,11 @@ def write_bundle(self, container_template=RUNC_SPEC_TEMPLATE,
         # not part of the OCI specification, but is convenient for now.
         container_config_filename = os.path.join(self.image_path,
                                                  'container-config.json')
+        runtime_config_filename = os.path.join(self.image_path, 'config.json')
         os.rename(os.path.join(self.image_path, self.tar_manifest[0]['Config']),
                   container_config_filename)
 
-        # Read the container config
-        with open(container_config_filename) as f:
-            image_conf = json.loads(f.read())
-
-        # Write a runc specification for the container
-        container_conf = json.loads(container_template)
-
-        container_conf['process']['terminal'] = True
-        cwd = image_conf['config']['WorkingDir']
-        if cwd == '':
-            cwd = '/'
-        container_conf['process']['cwd'] = cwd
-        container_conf['process']['args'] = image_conf['config']['Cmd']
-
-        # terminal = false means "pass through existing file descriptors"
-        container_conf['process']['terminal'] = False
-
-        container_conf['hostname'] = container_values.get(
-            'hostname', 'occystrap')
-
-        with open(os.path.join(self.image_path, 'config.json'), 'w') as f:
-            f.write(json.dumps(container_conf, indent=4, sort_keys=True))
+        common.write_container_config(container_config_filename,
+                                      runtime_config_filename,
+                                      container_template=container_template,
+                                      container_values=container_values)

occystrap/util.py

@@ -1,5 +1,6 @@
 import json
 import logging
+from oslo_concurrency import processutils
 from pbr.version import VersionInfo
 import requests
 
@@ -74,3 +75,10 @@ def request_url(method, url, headers=None, data=None, stream=False):
         raise APIException(
             'API request failed', method, url, r.status_code, r.text, r.headers)
     return r
+
+
+def execute(command, check_exit_code=[0], env_variables=None,
+            cwd=None):
+    return processutils.execute(
+        command, check_exit_code=check_exit_code,
+        env_variables=env_variables, shell=True, cwd=cwd)