From f1b1a0cd6d9124e8c2aed24081e2494035f90820 Mon Sep 17 00:00:00 2001 From: Michael Still Date: Mon, 11 Jul 2022 16:26:13 +1000 Subject: [PATCH 1/2] Add overlay mounts as an output renderer. --- deploy/occystrap_ci/tests/test_whiteout.py | 37 ++++- occystrap/common.py | 40 ++++++ occystrap/main.py | 27 ++++ occystrap/output_mounts.py | 155 +++++++++++++++++++++ occystrap/output_ocibundle.py | 29 +--- occystrap/util.py | 8 ++ requirements.txt | 1 + 7 files changed, 273 insertions(+), 24 deletions(-) create mode 100644 occystrap/common.py create mode 100644 occystrap/output_mounts.py diff --git a/deploy/occystrap_ci/tests/test_whiteout.py b/deploy/occystrap_ci/tests/test_whiteout.py index 07d4c6e..e86bda4 100644 --- a/deploy/occystrap_ci/tests/test_whiteout.py +++ b/deploy/occystrap_ci/tests/test_whiteout.py @@ -1,12 +1,14 @@ import logging import os +from oslo_concurrency import processutils import tempfile import testtools from occystrap import docker_registry from occystrap import output_ocibundle +from occystrap import output_mounts logging.basicConfig(level=logging.INFO, format='%(message)s') @@ -14,7 +16,7 @@ class WhiteoutsTestCase(testtools.TestCase): - def test_whiteouts(self): + def test_whiteouts_ocibundle(self): image = 'occystrap_deletion_layers' tag = 'latest' @@ -43,3 +45,36 @@ def test_whiteouts(self): self.assertFalse( os.path.exists(os.path.join(tempdir, 'rootfs/anotherdirectory/.wh..wh..opq'))) + + def test_whiteouts_mounts(self): + image = 'occystrap_deletion_layers' + tag = 'latest' + + with tempfile.TemporaryDirectory() as tempdir: + oci = output_mounts.MountWriter(image, tag, tempdir) + img = docker_registry.Image( + 'localhost:5000', image, tag, 'linux', 'amd64', '', + secure=False) + for image_element in img.fetch(fetch_callback=oci.fetch_callback): + oci.process_image_element(*image_element) + oci.finalize() + oci.write_bundle() + + self.assertTrue( + os.path.exists(os.path.join(tempdir, 'rootfs'))) + self.assertFalse( + os.path.exists(os.path.join(tempdir, 'rootfs/file'))) + self.assertFalse( + os.path.exists(os.path.join(tempdir, 'rootfs/.wh.file'))) + self.assertFalse( + os.path.exists(os.path.join(tempdir, 'rootfs/directory'))) + self.assertTrue( + os.path.exists(os.path.join(tempdir, 'rootfs/anotherfile'))) + self.assertTrue( + os.path.exists(os.path.join(tempdir, 'rootfs/anotherdirectory'))) + self.assertFalse( + os.path.exists(os.path.join(tempdir, + 'rootfs/anotherdirectory/.wh..wh..opq'))) + + processutils.execute('umount %s' % os.path.join(tempdir, 'rootfs'), + shell=True) diff --git a/occystrap/common.py b/occystrap/common.py new file mode 100644 index 0000000..d007c91 --- /dev/null +++ b/occystrap/common.py @@ -0,0 +1,40 @@ +import json + +from occystrap.constants import RUNC_SPEC_TEMPLATE + + +def write_container_config(container_config_filename, runtime_config_filename, + container_template=RUNC_SPEC_TEMPLATE, + container_values=None): + if not container_values: + container_values = {} + + # Read the container config + with open(container_config_filename) as f: + image_conf = json.loads(f.read()) + + # Write a runc specification for the container + container_conf = json.loads(container_template) + + container_conf['process']['terminal'] = True + cwd = image_conf['config']['WorkingDir'] + if cwd == '': + cwd = '/' + container_conf['process']['cwd'] = cwd + + entrypoint = image_conf['config'].get('Entrypoint', []) + if not entrypoint: + entrypoint = [] + cmd = image_conf['config'].get('Cmd', []) + if cmd: + entrypoint.extend(cmd) + container_conf['process']['args'] = entrypoint + + # terminal = false means "pass through existing file descriptors" + container_conf['process']['terminal'] = False + + container_conf['hostname'] = container_values.get( + 'hostname', 'occystrap') + + with open(runtime_config_filename, 'w') as f: + f.write(json.dumps(container_conf, indent=4, sort_keys=True)) diff --git a/occystrap/main.py b/occystrap/main.py index b4ca001..ba66ad3 100644 --- a/occystrap/main.py +++ b/occystrap/main.py @@ -1,8 +1,11 @@ import click import logging +import os +import sys from occystrap import docker_registry from occystrap import output_directory +from occystrap import output_mounts from occystrap import output_ocibundle from occystrap import output_tarfile @@ -94,6 +97,30 @@ def fetch_to_tarfile(ctx, registry, image, tag, tarfile, insecure): cli.add_command(fetch_to_tarfile) +@click.command() +@click.argument('registry') +@click.argument('image') +@click.argument('tag') +@click.argument('path') +@click.option('--insecure', is_flag=True, default=False) +@click.pass_context +def fetch_to_mounts(ctx, registry, image, tag, path, insecure): + if not hasattr(os, 'setxattr'): + print('Sorry, your OS module implementation lacks setxattr') + sys.exit(1) + if not hasattr(os, 'mknod'): + print('Sorry, your OS module implementation lacks mknod') + sys.exit(1) + + d = output_mounts.MountWriter(image, tag, path) + _fetch(registry, image, tag, d, ctx.obj['OS'], ctx.obj['ARCHITECTURE'], + ctx.obj['VARIANT'], secure=(not insecure)) + d.write_bundle() + + +cli.add_command(fetch_to_mounts) + + @click.command() @click.argument('path') @click.argument('image') diff --git a/occystrap/output_mounts.py b/occystrap/output_mounts.py new file mode 100644 index 0000000..a2bab4f --- /dev/null +++ b/occystrap/output_mounts.py @@ -0,0 +1,155 @@ +import json +import logging +import os +import stat +import tarfile + +from occystrap import common +from occystrap import constants +from occystrap import util + + +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.INFO) + + +class MountWriter(object): + def __init__(self, image, tag, image_path): + self.image = image + self.tag = tag + self.image_path = image_path + + self.tar_manifest = [{ + 'Layers': [], + 'RepoTags': ['%s:%s' % (self.image.split('/')[-1], self.tag)] + }] + + self.bundle = {} + + if not os.path.exists(self.image_path): + os.makedirs(self.image_path) + + def _manifest_filename(self): + return 'manifest' + + def fetch_callback(self, digest): + layer_file_in_dir = os.path.join(self.image_path, digest, 'layer.tar') + LOG.info('Layer file is %s' % layer_file_in_dir) + return not os.path.exists(layer_file_in_dir) + + def process_image_element(self, element_type, name, data): + if element_type == constants.CONFIG_FILE: + with open(os.path.join(self.image_path, name), 'wb') as f: + d = json.loads(data.read()) + f.write(json.dumps(d, indent=4, sort_keys=True).encode('ascii')) + self.tar_manifest[0]['Config'] = name + + elif element_type == constants.IMAGE_LAYER: + layer_dir = os.path.join(self.image_path, name) + if not os.path.exists(layer_dir): + os.makedirs(layer_dir) + + layer_file = os.path.join(name, 'layer.tar') + self.tar_manifest[0]['Layers'].append(layer_file) + + layer_file_in_dir = os.path.join(self.image_path, layer_file) + if os.path.exists(layer_file_in_dir): + LOG.info('Skipping layer already in output directory') + else: + with open(layer_file_in_dir, 'wb') as f: + d = data.read(102400) + while d: + f.write(d) + d = data.read(102400) + + layer_dir_in_dir = os.path.join(self.image_path, name, 'layer') + os.makedirs(layer_dir_in_dir) + with tarfile.open(layer_file_in_dir) as layer: + for mem in layer.getmembers(): + dirname, filename = os.path.split(mem.name) + + # Some light reading on how this works... + # https://www.madebymikal.com/interpreting-whiteout-files-in-docker-image-layers/ + # https://github.com/opencontainers/image-spec/blob/main/layer.md#opaque-whiteout + if filename == '.wh..wh..opq': + # A deleted directory, but only for layers below + # this one. + os.setxattr(os.path.join(layer_dir_in_dir, dirname), + 'trusted.overlay.opaque', b'y') + + elif filename.startswith('.wh.'): + # A single deleted element, which might not be a + # file. + os.mknod(os.path.join(layer_dir_in_dir, + mem.name[4:]), + mode=stat.S_IFCHR, device=0) + + else: + path = mem.name + layer.extract(path, path=layer_dir_in_dir) + + def finalize(self): + manifest_filename = self._manifest_filename() + '.json' + manifest_path = os.path.join(self.image_path, manifest_filename) + with open(manifest_path, 'wb') as f: + f.write(json.dumps(self.tar_manifest, indent=4, + sort_keys=True).encode('ascii')) + + c = {} + catalog_path = os.path.join(self.image_path, 'catalog.json') + if os.path.exists(catalog_path): + with open(catalog_path, 'r') as f: + c = json.loads(f.read()) + + c.setdefault(self.image, {}) + c[self.image][self.tag] = manifest_filename + with open(catalog_path, 'w') as f: + f.write(json.dumps(c, indent=4, sort_keys=True)) + + def write_bundle(self, container_template=constants.RUNC_SPEC_TEMPLATE, + container_values=None): + if not container_values: + container_values = {} + + rootfs_path = os.path.join(self.image_path, 'rootfs') + if not os.path.exists(rootfs_path): + os.makedirs(rootfs_path) + LOG.info('Writing image bundle to %s' % rootfs_path) + + working_path = os.path.join(self.image_path, 'working') + if not os.path.exists(working_path): + os.makedirs(working_path) + + delta_path = os.path.join(self.image_path, 'delta') + if not os.path.exists(delta_path): + os.makedirs(delta_path) + + # The newest layer is listed first in the mount command + layer_dirs = [] + self.tar_manifest[0]['Layers'].reverse() + for layer in self.tar_manifest[0]['Layers']: + layer_dirs.append(os.path.join( + self.image_path, layer.replace('.tar', ''))) + + # Extract the rootfs as overlay mounts + util.execute('mount -t overlay overlay -o lowerdir=%(layers)s,' + 'upperdir=%(upper)s,workdir=%(working)s %(rootfs)s' + % { + 'layers': ':'.join(layer_dirs), + 'upper': delta_path, + 'working': working_path, + 'rootfs': rootfs_path + }) + + # Rename the container configuration to a well known location. This is + # not part of the OCI specification, but is convenient for now. + container_config_filename = os.path.join(self.image_path, + 'container-config.json') + runtime_config_filename = os.path.join(self.image_path, 'config.json') + os.rename(os.path.join(self.image_path, self.tar_manifest[0]['Config']), + container_config_filename) + + common.write_container_config(container_config_filename, + runtime_config_filename, + container_template=container_template, + container_values=container_values) diff --git a/occystrap/output_ocibundle.py b/occystrap/output_ocibundle.py index 982a006..98a2523 100644 --- a/occystrap/output_ocibundle.py +++ b/occystrap/output_ocibundle.py @@ -2,12 +2,12 @@ # all of the data that a directory output does, and they place the data they # do contain into different locations within the directory structure. -import json import logging import os import shutil from occystrap.constants import RUNC_SPEC_TEMPLATE +from occystrap import common from occystrap.output_directory import DirWriter @@ -43,28 +43,11 @@ def write_bundle(self, container_template=RUNC_SPEC_TEMPLATE, # not part of the OCI specification, but is convenient for now. container_config_filename = os.path.join(self.image_path, 'container-config.json') + runtime_config_filename = os.path.join(self.image_path, 'config.json') os.rename(os.path.join(self.image_path, self.tar_manifest[0]['Config']), container_config_filename) - # Read the container config - with open(container_config_filename) as f: - image_conf = json.loads(f.read()) - - # Write a runc specification for the container - container_conf = json.loads(container_template) - - container_conf['process']['terminal'] = True - cwd = image_conf['config']['WorkingDir'] - if cwd == '': - cwd = '/' - container_conf['process']['cwd'] = cwd - container_conf['process']['args'] = image_conf['config']['Cmd'] - - # terminal = false means "pass through existing file descriptors" - container_conf['process']['terminal'] = False - - container_conf['hostname'] = container_values.get( - 'hostname', 'occystrap') - - with open(os.path.join(self.image_path, 'config.json'), 'w') as f: - f.write(json.dumps(container_conf, indent=4, sort_keys=True)) + common.write_container_config(container_config_filename, + runtime_config_filename, + container_template=container_template, + container_values=container_values) diff --git a/occystrap/util.py b/occystrap/util.py index df5c30d..d564135 100644 --- a/occystrap/util.py +++ b/occystrap/util.py @@ -1,5 +1,6 @@ import json import logging +from oslo_concurrency import processutils from pbr.version import VersionInfo import requests @@ -74,3 +75,10 @@ def request_url(method, url, headers=None, data=None, stream=False): raise APIException( 'API request failed', method, url, r.status_code, r.text, r.headers) return r + + +def execute(command, check_exit_code=[0], env_variables=None, + cwd=None): + return processutils.execute( + command, check_exit_code=check_exit_code, + env_variables=env_variables, shell=True, cwd=cwd) diff --git a/requirements.txt b/requirements.txt index bf35faf..b87eff4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ click>=7.1.1 # bsd pbr # apache2 requests # apache2 prettytable # bsd +oslo.concurrency # apache2 \ No newline at end of file From 46ee9e553d0dff94eb8ed108eaa4a15f2715f5b7 Mon Sep 17 00:00:00 2001 From: Michael Still Date: Tue, 10 Jan 2023 17:54:24 +1100 Subject: [PATCH 2/2] Convert to SF shared libraries. --- .github/workflows/functional-tests.yml | 7 +++++-- occystrap/main.py | 5 ++--- requirements.txt | 11 ++++++----- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml index 255389d..67f6c07 100644 --- a/.github/workflows/functional-tests.yml +++ b/.github/workflows/functional-tests.yml @@ -56,7 +56,7 @@ jobs: cd /srv/github/_work/occystrap/occystrap/occystrap rm -f dist/* python3 setup.py sdist bdist_wheel - pip3 install dist/occystrap*.whl + sudo pip3 install dist/occystrap*.whl - name: Run a local docker registry to talk to, and populate it with test data run: | @@ -75,4 +75,7 @@ jobs: cd /srv/github/_work/occystrap/occystrap/occystrap/deploy sudo pip3 install -r requirements.txt sudo pip3 install -r test-requirements.txt - stestr run --concurrency=5 + + # This needs to run as root because some of the tests require + # escalated permissions. + sudo stestr run --concurrency=5 diff --git a/occystrap/main.py b/occystrap/main.py index ba66ad3..33a5d46 100644 --- a/occystrap/main.py +++ b/occystrap/main.py @@ -1,6 +1,7 @@ import click import logging import os +from shakenfist_utilities import logs import sys from occystrap import docker_registry @@ -9,10 +10,8 @@ from occystrap import output_ocibundle from occystrap import output_tarfile -logging.basicConfig(level=logging.INFO) -LOG = logging.getLogger(__name__) -LOG.setLevel(logging.INFO) +LOG = logs.setup_console(__name__) @click.group() diff --git a/requirements.txt b/requirements.txt index b87eff4..c31060a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ -click>=7.1.1 # bsd -pbr # apache2 -requests # apache2 -prettytable # bsd -oslo.concurrency # apache2 \ No newline at end of file +click>=7.1.1 # bsd +pbr # apache2 +requests # apache2 +prettytable # bsd +oslo.concurrency # apache2 +shakenfist-utilities # apache2 \ No newline at end of file