[merged] verify: validate layers of system image stored on disk

.gitignore

@@ -5,5 +5,4 @@ build
 *.log
 htmlcov
 .coverage
-
 tests/test-images/Dockerfile.secret

Atomic/syscontainers.py

@@ -10,6 +10,7 @@
 import stat
 import subprocess
 import time
+import gzip
 from .client import AtomicDocker
 from ctypes import cdll, CDLL
 from dateutil.parser import parse as dateparse
@@ -31,6 +32,7 @@
     DEVNULL = open(os.devnull, 'wb')
 
 ATOMIC_LIBEXEC = os.environ.get('ATOMIC_LIBEXEC', '/usr/libexec/atomic')
+ATOMIC_VAR = '/var/lib/containers/atomic'
 
 OSTREE_OCIIMAGE_PREFIX = "ociimage/"
 SYSTEMD_UNIT_FILES_DEST = "/etc/systemd/system"
@@ -96,7 +98,7 @@ def _checkout_layer(self, repo, rootfs_fd, rootfs, rev):
                 user = ["--user-mode"]
             else:
                 user = []
-            util.check_call(["ostree", "--repo=%s" % self._get_ostree_repo_location(),
+            util.check_call(["ostree", "--repo=%s" % self.get_ostree_repo_location(),
                              "checkout",
                              "--union"] +
                             user +
@@ -367,9 +369,9 @@ def _get_system_checkout_path(self):
             home = os.path.expanduser("~")
             return "%s/.containers/atomic" % home
         else:
-            return "/var/lib/containers/atomic"
+            return ATOMIC_VAR
 
-    def _get_ostree_repo_location(self):
+    def get_ostree_repo_location(self):
         if self.user:
             home = os.path.expanduser("~")
             return "%s/.containers/repo" % home
@@ -382,7 +384,7 @@ def _get_ostree_repo(self):
         if not OSTREE_PRESENT:
             return None
 
-        repo_location = self._get_ostree_repo_location()
+        repo_location = self.get_ostree_repo_location()
         repo = OSTree.Repo.new(Gio.File.new_for_path(repo_location))
 
         # If the repository doesn't exist at the specified location, create it
@@ -766,10 +768,11 @@ def _check_system_oci_image(self, repo, img, upgrade):
         missing_layers = []
         for i in layers:
             layer = i.replace("sha256:", "")
-            if not repo.resolve_rev("%s%s" % (OSTREE_OCIIMAGE_PREFIX, layer), True)[1]:
+            has_layer = repo.resolve_rev("%s%s" % (OSTREE_OCIIMAGE_PREFIX, layer), True)[1]
+            has_gomtree_manifest = os.path.isfile(os.path.join(ATOMIC_VAR, "gomtree-manifests/%s.mtree" % layer))
+            if not has_layer or not has_gomtree_manifest:
                 missing_layers.append(layer)
                 util.write_out("Missing layer %s" % layer)
-
         layers_dir = None
         try:
             layers_to_import = {}
@@ -780,9 +783,10 @@ def _check_system_oci_image(self, repo, img, upgrade):
                         if f.endswith(".tar"):
                             layer_file = os.path.join(root, f)
                             layer = f.replace(".tar", "")
-                            if layer in missing_layers:
+                            if not repo.resolve_rev("%s%s" % (OSTREE_OCIIMAGE_PREFIX, layer), True)[1]:
                                 layers_to_import[layer] = layer_file
-
+                            if not os.path.isfile(os.path.join(ATOMIC_VAR, "gomtree-manifests/%s.mtree" % layer)):
+                                SystemContainers._generate_validation_manifest(layer,layer_file)
             SystemContainers._import_layers_into_ostree(repo, imagebranch, manifest, layers_to_import)
         finally:
             if layers_dir:
@@ -858,3 +862,35 @@ def _pull_dockertar_layers(self, repo, imagebranch, temp_dir, input_layers):
         for k, v in layers.items():
             layers_to_import[layers_map[k]] = v
         SystemContainers._import_layers_into_ostree(repo, imagebranch, manifest, layers_to_import)
+
+    @staticmethod
+    def _generate_validation_manifest(layer,tar,root=os.path.join(ATOMIC_VAR, "gomtree-manifests")):
+        """
+        Creates a gomtree validation manifest using a tar archive associated with
+        it's layer. Defaults to placing the manifest in a folder in the ATOMIC_VAR folder.
+        :param layer: layer digest
+        :param tar: may or may not be gzip compressed
+        :return: None
+        """
+        if not os.path.exists(root):
+            os.makedirs(root)
+        manifestname = os.path.join(root, "%s.mtree" % layer)
+        if os.path.isfile(manifestname):
+            util.write_out("validation manifest for layer %s already created" % layer)
+        else:
+            gomtree = util.generate_validation_manifest(img_tar=tar,keywords="type,uid,gid,mode,size,sha256digest")
+            if gomtree.return_code != 0:
+                decompressed = tar.replace(".tar",".tar.gz")
+                os.rename(tar,decompressed)
+                with gzip.open(decompressed, "rb") as g:
+                    _, tmptar = tempfile.mkstemp()
+                    with open(tmptar, "wb") as t:
+                        t.write(g.read())
+                        gomtree = util.generate_validation_manifest(img_tar=tmptar,keywords="type,uid,gid,mode,size,sha256digest")
+                    os.remove(tmptar)
+                os.rename(decompressed, tar.replace(".tar.gz",".tar"))
+            if gomtree.return_code == 0:
+                with open(manifestname, "w", 0) as m:
+                    m.write(gomtree.stdout)
+            else:
+                util.write_out(gomtree.stderr)

Atomic/util.py

@@ -24,6 +24,7 @@
                                      ['return_code', 'stdout', 'stderr'])
 ATOMIC_CONF = os.environ.get('ATOMIC_CONF', '/etc/atomic.conf')
 ATOMIC_CONFD = os.environ.get('ATOMIC_CONFD', '/etc/atomic.d/')
+ATOMIC_LIBEXEC = os.environ.get('ATOMIC_LIBEXEC', '/usr/libexec/atomic')
 
 def check_if_python2():
     if int(sys.version_info[0]) < 3:
@@ -296,7 +297,6 @@ def skopeo_layers(image, args=None, layers=None):
     finally:
         if not success:
             shutil.rmtree(temp_dir)
-
     return temp_dir
 
 
@@ -435,3 +435,39 @@ def find_remote_image(client, image):
 
 def is_user_mode():
     return os.geteuid() != 0
+
+def generate_validation_manifest(img_rootfs=None, img_tar=None, keywords=""):
+    """
+    Executes the gomtree validation manifest creation command
+    :param img_rootfs: path to directory
+    :param img_tar: path to tar file (or tgz file)
+    :param keywords: use only the keywords specified to create the manifest
+    :return: output of gomtree validation manifest creation
+    """
+    if img_rootfs == None and img_tar == None:
+        write_out("no source for gomtree to generate a manifest from")
+    if img_rootfs:
+        cmd = [ATOMIC_LIBEXEC + '/gomtree','-c','-p',img_rootfs]
+    elif img_tar:
+        cmd = [ATOMIC_LIBEXEC + '/gomtree','-c','-T',img_tar]
+    if keywords:
+        cmd += ['-k',keywords]
+    return subp(cmd)
+
+def validate_manifest(spec, img_rootfs=None, img_tar=None, keywords=""):
+    """
+    Executes the gomtree validation manife  st validation command
+    :param img_rootfs: path to directory
+    :param img_tar: path to tar file (or tgz file)
+    :param keywords: use only the keywords specified to validate the manifest
+    :return: output of gomtree validation manifest validation
+    """
+    if img_rootfs == None and img_tar == None:
+        write_out("no source for gomtree to validate a manifest")
+    if img_rootfs:
+        cmd = [ATOMIC_LIBEXEC + '/gomtree', '-p', img_rootfs, '-f', spec]
+    elif img_tar:
+        cmd = [ATOMIC_LIBEXEC + '/gomtree','-T',img_tar, '-f', spec]
+    if keywords:
+        cmd += ['-k',keywords]
+    return subp(cmd)

Atomic/verify.py

@@ -5,9 +5,14 @@
 from operator import itemgetter
 from .atomic import AtomicError
 from .syscontainers import SystemContainers
+from .mount import Mount
+import shutil
 import itertools
+import tempfile
 import subprocess
 
+ATOMIC_VAR = '/var/lib/containers/atomic'
+
 class Verify(Atomic):
     def __init__(self):
         super(Verify, self).__init__()
@@ -16,6 +21,10 @@ def __init__(self):
     def verify_system_image(self):
         manifest = self.syscontainers.get_manifest(self.image)
         layers = SystemContainers.get_layers_from_manifest(manifest)
+
+        if hasattr(self.args,"validate") and self.args.validate:
+            self.validate_system_image_manifests(layers)
+
         remote = True
         try:
             remote_manifest = self.syscontainers.get_manifest(self.image, remote=True)
@@ -88,6 +97,12 @@ def fix_layers(layers):
             except AtomicError:
                 self._no_such_image()
 
+        if hasattr(self.args,"generate") and self.args.generate:
+            self.generate_docker_validation_manifest()
+
+        elif hasattr(self.args,"validate") and self.args.validate:
+            self.validate_docker_image_manifest()
+
         layers = fix_layers(self.get_layers())
         if self.debug:
             for l in layers:
@@ -343,6 +358,98 @@ def assemble_nvr(self, image, image_name=None):
         else:
             return nvr
 
+    def validate_system_image_manifests(self,layers):
+        """
+        Validate a system image's layers against the the associated validation manifests
+        created from those image layers on atomic pull.
+        :param layers: list of the names of the layers to validate
+        :return: None
+        """
+        missing_manifests = []
+        for layer in layers:
+            layer = layer.replace("sha256:","")
+            manifestpath = self.get_gomtree_manifest(layer)
+            if not manifestpath:
+                missing_manifests.append(layer)
+                continue
+            tmp_dir = tempfile.mkdtemp()
+            ref = os.path.join("ociimage", layer)
+            cmd = ['ostree','checkout','--union','--repo=%s' % self.syscontainers.get_ostree_repo_location(),ref,tmp_dir]
+            r = util.subp(cmd)
+            if r.return_code != 0:
+                util.write_err(r.stderr)
+                continue
+            r = util.validate_manifest(manifestpath, img_rootfs=tmp_dir,keywords="type,uid,gid,mode,size,sha256digest")
+            if r.return_code != 0:
+                util.write_out("modifications in layer %s layer:\n" % layer)
+                if r.return_code > 1:
+                    util.write_err(r.stderr)
+                else:
+                    util.write_err(r.stdout)
+            shutil.rmtree(tmp_dir)
+        if len(missing_manifests):
+            util.write_out("validation manifests for the following layers do not exist:\n")
+            for layer in missing_manifests:
+                util.write_out("\t%s" % layer)
+            util.write_out("\n")
+            util.write_out("perform an `atomic pull \"%s\"` if you want to generate these manifests\n" % self.image)
+
+    def generate_docker_validation_manifest(self):
+        """
+        Generates a gomtree validation manifest for a non-system image and stores it in
+        ATOMIC_VAR
+        :param:
+        :return: None
+        """
+        iid = self._is_image(self.image)
+        if os.path.exists(os.path.join(ATOMIC_VAR,"gomtree-manifests/%s.mtree" % iid)):
+            return
+        if not os.path.exists(os.path.join(ATOMIC_VAR,"gomtree-manifests")):
+            os.makedirs(os.path.join(ATOMIC_VAR,"gomtree-manifests"))
+        manifestname = os.path.join(ATOMIC_VAR, "gomtree-manifests/%s.mtree" % iid)
+        tmpdir = tempfile.mkdtemp()
+        m = Mount()
+        m.args = []
+        m.image = self.image
+        m.mountpoint = tmpdir
+        m.mount()
+        r = util.generate_validation_manifest(img_rootfs=tmpdir, keywords="type,uid,gid,mode,size,sha256digest")
+        m.unmount()
+        with open(manifestname,"w",0) as f:
+            f.write(r.stdout)
+        shutil.rmtree(tmpdir)
+
+    def validate_docker_image_manifest(self):
+        """
+        Validates a docker image by mounting the image on a rootfs and validate that
+        rootfs against the manifests that were created. Note that it won't be validated
+        layer by layer.
+        :param:
+        :return: None
+        """
+        iid = self._is_image(self.image)
+        if not os.path.exists(os.path.join(ATOMIC_VAR,"gomtree-manifests/%s.mtree" % iid)):
+            return
+        manifestname = os.path.join(ATOMIC_VAR, "gomtree-manifests/%s.mtree" % iid)
+        tmpdir = tempfile.mkdtemp()
+        m = Mount()
+        m.args = []
+        m.image = self.image
+        m.mountpoint = tmpdir
+        m.mount()
+        r = util.validate_manifest(manifestname, img_rootfs=tmpdir, keywords="type,uid,gid,mode,size,sha256digest")
+        m.unmount()
+        if r.return_code != 0:
+            util.write_err(r.stdout)
+        shutil.rmtree(tmpdir)
+
+    @staticmethod
+    def get_gomtree_manifest(layer, root=os.path.join(ATOMIC_VAR, "gomtree-manifests")):
+        manifestpath = os.path.join(root,"%s.mtree" % layer)
+        if os.path.isfile(manifestpath):
+            return manifestpath
+        return None
+
     @staticmethod
     def get_local_version(name, layers):
         for layer in layers:

Makefile

@@ -15,7 +15,6 @@ VERSION=$(shell $(PYTHON) setup.py --version)
 all: python-build docs pylint-check dockertar-sha256-helper
 
 .PHONY: test-python3-pylint
-test-python3-pylint: 
 	$(PYTHON3_PYLINT) --disable=all --enable=E --enable=W --additional-builtins=_ *.py atomic Atomic tests/unit/*.py -d=no-absolute-import,print-statement,no-absolute-import,bad-builtin
 
 .PHONY: test check
@@ -62,7 +61,7 @@ install-only:
 	ln -fs ../share/atomic/atomic $(DESTDIR)/usr/bin/atomic
 
 	install -d -m 0755 $(DESTDIR)/usr/libexec/atomic
-	install -m 0755 dockertar-sha256-helper migrate.sh gotar $(DESTDIR)/usr/libexec/atomic
+	install -m 0755 dockertar-sha256-helper migrate.sh gotar gomtree $(DESTDIR)/usr/libexec/atomic
 
 	[ -d $(SYSCONFDIR) ] || mkdir -p $(SYSCONFDIR)
 	install -m 644 atomic.sysconfig $(SYSCONFDIR)/atomic
@@ -86,4 +85,3 @@ install: all install-only
 .PHONY: install-openscap
 install-openscap:
 	install -m 644 atomic.d/openscap $(DESTDIR)/etc/atomic.d
-

atomic

@@ -66,7 +66,6 @@ class HelpByDefaultArgumentParser(argparse.ArgumentParser):
     def print_usage(self, message="too few arguments"):
         self.prog = " ".join(sys.argv)
         self.error(message)
-
 
 # Code for python2 copied from Adrian Sampson hack
 # https://gist.github.com/sampsyo/471779
@@ -683,6 +682,12 @@ def create_parser(help_text):
     verifyp.add_argument("-v", "--verbose", default=False,
                           action="store_true",
                           help=_("Report status of each layer"))
+    verifyp.add_argument("-V", "--validate", default=True,
+                                action="store_false",
+                                help=_("disable validating system images"))
+    verifyp.add_argument("-g", "--generate", default=False,
+                                action="store_true",
+                                help=_("generate gomtree manifest of image and store it"))
     return parser
 
 if __name__ == '__main__':

docs/atomic-verify.1.md

@@ -13,7 +13,9 @@ IMAGE
 # DESCRIPTION
 **atomic verify** checks whether there is a newer image available and scans
 through all layers to see if any of the layers, which are base images themselves, have a new version available.
-If the tool finds an out of date image, it will report as such.
+If the tool finds an out of date image, it will report as such. If the image is a system image, it will
+also look through the layers and validate each layer to determine if it has been tampered with and output
+details of these changes (if at all).
 
 If the image or any of its layers are pulled from a repository, it will attempt to check the repository
 to see if there is a new image and capture any of its relevant information like version (where applicable).
@@ -29,6 +31,9 @@ the version information.
 **-v** **--verbose**
   Will output the status of each base image that makes up the image being verified.
 
+**-g** **--generate**
+  Generates a gomtree validation manifest when user specifies a non-system image
+
 # EXAMPLES
 Verify the Red Hat rsyslog image
 
@@ -43,9 +48,20 @@ Verify the Red Hat rsyslog image and show status of each image layer
      redhat/rhel7-7.1-24            redhat/rhel7-7.1-24
 
      * = version difference
+Verify a system image
+
+    # sudo atomic verify busybox
+    validation output for layer a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4:
+
+	(no changes detected)
+
+    validation output for layer 8ddc19f16526912237dd8af81971d5e4dd0587907234be2b83e249518d5b673f:
+
+    "etc/shadow": keyword "size": expected 243; got 268
+    "etc/shadow": keyword "sha256digest": expected 22d9cee21ee808c52af44ac...; got 7a07ac69054c2a3533569874c2...
+
 
 # HISTORY
 May 2015, Originally compiled by Daniel Walsh (dwalsh at redhat dot com)
 
 Nov 2015, Updated for remote inspect by Brent Baude (bbaude at redhat dot com)
-