Rework MountPoints detection (#110)

docker_explorer/container.py

@@ -18,19 +18,15 @@
 
 import collections
 import json
+import logging
 import os
 import subprocess
 
 from docker_explorer import errors
 from docker_explorer import storage
 from docker_explorer import utils
 
-# Ugly Py2/Py3 compat code.
-# Undo in 2020+
-try:
-  input = raw_input  # pylint: disable=redefined-builtin
-except NameError:
-  pass
+logger = logging.getLogger('docker-explorer')
 
 
 def GetAllContainersIDs(docker_root_directory):
@@ -119,7 +115,6 @@ def __init__(self, docker_directory, container_id, docker_version=2):
     with open(container_info_json_path) as container_info_json_file:
       container_info_dict = json.load(container_info_json_file)
 
-
     if container_info_dict is None:
       raise errors.BadContainerException(
           'Could not load container configuration file {0}'.format(
@@ -145,11 +140,11 @@ def __init__(self, docker_directory, container_id, docker_version=2):
       raise errors.BadContainerException(
           '{0} container config file lacks Driver key'.format(
               container_info_json_path))
-
-    self._SetStorage(self.storage_name)
     self.upper_dir = None
     self.volumes = container_info_dict.get('Volumes', None)
 
+    self._SetStorage(self.storage_name)
+
     if self.docker_version == 2:
       c_path = os.path.join(
           self.docker_directory, 'image', self.storage_name, 'layerdb',
@@ -165,7 +160,6 @@ def __init__(self, docker_directory, container_id, docker_version=2):
 
     self.log_path = container_info_dict.get('LogPath', None)
 
-
   def GetLayerSize(self, layer_id):
     """Returns the size of the layer.
 
@@ -267,6 +261,59 @@ def GetHistory(self, show_empty_layers=False):
 
     return result_dict
 
+  def GetMountpoints(self):
+    """Returns the mount points & volumes for a container.
+
+    Returns:
+      list((str, str)): list of mount points (source_path, destination_path).
+    """
+    mount_points = list()
+
+    if self.docker_version == 1:
+      if self.volumes:
+        for source, destination in self.volumes.items():
+          # Stripping leading '/' for easier joining later.
+          source_path = source.lstrip(os.path.sep)
+          destination_path = destination.lstrip(os.path.sep)
+          mount_points.append((source_path, destination_path))
+
+    elif self.docker_version == 2:
+      if self.mount_points:
+        for dst_mount_ihp, storage_info in self.mount_points.items():
+          src_mount_ihp = None
+          if 'Type' not in storage_info:
+            # Let's do some guesswork
+            if 'Source' in storage_info:
+              storage_info['Type'] = 'volume'
+            else:
+              storage_info['Type'] = 'bind'
+
+          if storage_info.get('Type') == 'bind':
+            src_mount_ihp = storage_info['Source']
+
+          elif storage_info.get('Type') == 'volume':
+            volume_driver = storage_info.get('Driver')
+            if storage_info.get('Driver') != 'local':
+              logger.warning(
+                  'Unsupported driver "{0:s}" for volume "{1:s}"'.format(
+                      volume_driver, dst_mount_ihp))
+              continue
+            volume_name = storage_info['Name']
+            src_mount_ihp = os.path.join('volumes', volume_name, '_data')
+
+          else:
+            logger.warning(
+                'Unsupported storage type "{0!s}" for Volume "{1:s}"'.format(
+                    storage_info.get('Type'), dst_mount_ihp))
+            continue
+
+          # Removing leading path separator, otherwise os.path.join is behaving
+          # 'smartly' (read: 'terribly').
+          src_mount = src_mount_ihp.lstrip(os.path.sep)
+          dst_mount = dst_mount_ihp.lstrip(os.path.sep)
+          mount_points.append((src_mount, dst_mount))
+    return mount_points
+
   def _SetStorage(self, storage_name):
     """Sets the storage_object attribute.
 

docker_explorer/explorer.py

@@ -202,15 +202,14 @@ def GetContainersJson(self, only_running=False):
       if container_object.mount_id:
         container_json['mount_id'] = container_object.mount_id
 
-      if container_object.mount_points:
-        container_json['mount_points'] = {}
-        for mount_point, details in container_object.mount_points.items():
-          d = collections.OrderedDict()
-          d['type'] = details.get('Type')
-          d['mount_point'] = mount_point
-          d['source'] = details.get('Source')
-          d['RW'] = details.get('RW')
-          container_json['mount_points'][mount_point] = d
+      mount_points = container_object.GetMountpoints()
+      for source, destination in mount_points:
+        mountpoint_dict = collections.OrderedDict()
+        mountpoint_dict['source'] = os.path.join(
+            self.docker_directory, source)
+        mountpoint_dict['destination'] = os.path.join(
+            os.path.sep, destination)
+        container_json.setdefault('mount_points', []).append(mountpoint_dict)
 
       if container_object.upper_dir:
         container_json['upper_dir'] = container_object.upper_dir

docker_explorer/storage.py

@@ -16,13 +16,11 @@
 
 from __future__ import unicode_literals
 
-import logging
 import os
 
 import docker_explorer
 from docker_explorer import errors
 
-logger = logging.getLogger('docker-explorer')
 
 class BaseStorage:
   """This class provides tools to list and access containers metadata.
@@ -67,48 +65,33 @@ def MakeMountCommands(self, container_object, mount_dir):
     """
     raise NotImplementedError('Please implement MakeMountCommands()')
 
-  def _MakeExtraVolumeCommands(self, container_object, mount_dir):
+  def _MakeVolumeMountCommands(self, container_object, mount_dir):
     """Generates the shell command to mount external Volumes if present.
 
     Args:
       container_object (Container): the container object.
       mount_dir (str): the destination mount_point.
 
     Returns:
-      list(str): a list of extra commands, or the empty list if no volume is to
-        be mounted.
+      list(list(str)): a list of extra commands, or the empty list if no volume
+        is to be mounted. Commands are list(str).
     """
     extra_commands = []
+    mount_points = container_object.GetMountpoints()
     if self.docker_version == 1:
       # 'Volumes'
-      container_volumes = container_object.volumes
-      if container_volumes:
-        for mountpoint, storage in container_volumes.items():
-          mountpoint_ihp = mountpoint.lstrip(os.path.sep)
-          storage_ihp = storage.lstrip(os.path.sep)
-          storage_path = os.path.join(self.root_directory, storage_ihp)
-          volume_mountpoint = os.path.join(mount_dir, mountpoint_ihp)
-          extra_commands.append(
-              ['/bin/mount', '--bind', '-o', 'ro', storage_path,
-               volume_mountpoint]
-          )
+      for source, destination in mount_points:
+        storage_path = os.path.join(self.root_directory, source)
+        extra_commands.append(
+            ['/bin/mount', '--bind', '-o', 'ro', storage_path, destination]
+        )
     elif self.docker_version == 2:
-      # 'MountPoints'
-      container_mount_points = container_object.mount_points
-      if container_mount_points:
-        for _, storage_info in container_mount_points.items():
-          src_mount_ihp = storage_info['Source']
-          dst_mount_ihp = storage_info['Destination']
-          src_mount = src_mount_ihp.lstrip(os.path.sep)
-          dst_mount = dst_mount_ihp.lstrip(os.path.sep)
-          if not src_mount:
-            volume_name = storage_info['Name']
-            src_mount = os.path.join('docker', 'volumes', volume_name, '_data')
-          storage_path = os.path.join(self.root_directory, src_mount)
-          volume_mountpoint = os.path.join(mount_dir, dst_mount)
-          extra_commands.append(
-              ['/bin/mount', '--bind', '-o', 'ro', storage_path,
-               volume_mountpoint])
+      for source, destination in mount_points:
+        storage_path = os.path.join(self.root_directory, source)
+        volume_mountpoint = os.path.join(mount_dir, destination)
+        extra_commands.append(
+            ['/bin/mount', '--bind', '-o', 'ro', storage_path,
+             volume_mountpoint])
 
     return extra_commands
 
@@ -126,8 +109,8 @@ def MakeMountCommands(self, container_object, mount_dir):
       mount_dir (str): the path to the target mount point.
 
     Returns:
-      list: a list commands that needs to be run to mount the container's view
-        of the file system.
+      list(list(str)): a list commands that needs to be run to mount the
+        container's view of the file system. Commands to run are list(str).
     """
 
     mount_id = container_object.mount_id
@@ -159,7 +142,8 @@ def MakeMountCommands(self, container_object, mount_dir):
              'ro,remount,append:{0:s}=ro+wh'.format(mountpoint_path), 'none',
              mount_dir])
 
-    commands.extend(self._MakeExtraVolumeCommands(container_object, mount_dir))
+    # Adding the commands to mount any extra declared Volumes and Mounts
+    commands.extend(self._MakeVolumeMountCommands(container_object, mount_dir))
 
     return commands
 
@@ -192,8 +176,8 @@ def MakeMountCommands(self, container_object, mount_dir):
       mount_dir (str): the path to the target mount point.
 
     Returns:
-      list: a list commands that needs to be run to mount the container's view
-        of the file system.
+      list(list(str)): a list commands that needs to be run to mount the
+        container's view of the file system. Commands to run are list(str).
     """
     mount_id_path = os.path.join(
         self.docker_directory, self.STORAGE_METHOD, container_object.mount_id)
@@ -202,10 +186,13 @@ def MakeMountCommands(self, container_object, mount_dir):
       lower_dir = self._BuildLowerLayers(lower_fd.read().strip())
     upper_dir = os.path.join(mount_id_path, self.UPPERDIR_NAME)
 
-    cmd = [
+    commands = [[
         '/bin/mount', '-t', 'overlay', 'overlay', '-o',
-        'ro,lowerdir={0:s}:{1:s}'.format(upper_dir, lower_dir), mount_dir]
-    return [cmd]
+        'ro,lowerdir={0:s}:{1:s}'.format(upper_dir, lower_dir), mount_dir]]
+
+    # Adding the commands to mount any extra declared Volumes and Mounts
+    commands.extend(self._MakeVolumeMountCommands(container_object, mount_dir))
+    return commands
 
 
 class Overlay2Storage(OverlayStorage):

tests.py

@@ -226,21 +226,23 @@ def testGetRunningContainersList(self):
 
   def testGetContainersJson(self):
     """Tests the GetContainersJson function on a AuFS storage."""
+    self.maxDiff = None
     result = self.explorer_object.GetContainersJson(only_running=True)
 
     mount_point = collections.OrderedDict()
-    mount_point['type'] = None
-    mount_point['mount_point'] = '/var/jenkins_home'
-    mount_point['source'] = ''
-    mount_point['RW'] = True
+    mount_point['source'] = (
+        'test_data/docker/volumes/'
+        '28297de547b5473a9aff90aaab45ed108ebf019981b40c3c35c226f54c13ac0d/_data'
+    )
+    mount_point['destination'] = '/var/jenkins_home'
 
     expected = collections.OrderedDict()
     expected['image_name'] = 'busybox'
     expected['container_id'] = '7b02fb3e8a665a63e32b909af5babb7d6ba0b64e10003b2d9534c7d5f2af8966'
     expected['image_id'] = '7968321274dc6b6171697c33df7815310468e694ac5be0ec03ff053bb135e768'
     expected['start_date'] = '2017-02-13T16:45:05.785658'
     expected['mount_id'] = 'b16a494082bba0091e572b58ff80af1b7b5d28737a3eedbe01e73cd7f4e01d23'
-    expected['mount_points'] = {'/var/jenkins_home': mount_point}
+    expected['mount_points'] = [mount_point]
     expected['log_path'] = '/tmp/docker/containers/7b02fb3e8a665a63e32b909af5babb7d6ba0b64e10003b2d9534c7d5f2af8966/7b02fb3e8a665a63e32b909af5babb7d6ba0b64e10003b2d9534c7d5f2af8966-json.log'
 
     self.assertEqual([expected], result)
@@ -276,6 +278,7 @@ def testGetRepositoriesString(self):
 
   def testMakeMountCommands(self):
     """Tests the BaseStorage.MakeMountCommands function on a AuFS storage."""
+    self.maxDiff = None
     container_obj = self.explorer_object.GetContainer(
         '7b02fb3e8a665a63e32b909af5babb7d6ba0b64e10003b2d9534c7d5f2af8966')
     commands = container_obj.storage_object.MakeMountCommands(
@@ -296,7 +299,7 @@ def testMakeMountCommands(self):
             'd1c54c46d331de21587a16397e8bd95bdbb1015e1a04797c76de128107da83ae'
             '=ro+wh none /mnt'),
         (
-            '/bin/mount --bind -o ro {0:s}/docker/volumes/'
+            '/bin/mount --bind -o ro {0:s}/volumes/'
             '28297de547b5473a9aff90aaab45ed108ebf019981b40c3c35c226f54c13ac0d/'
             '_data /mnt/var/jenkins_home').format(os.path.abspath('test_data'))
     ]
@@ -875,8 +878,8 @@ def testDownloadDockerFile(self):
     expected_dockerfile = (
         '# Pseudo Dockerfile\n'
         '# Generated by de.py ({0:s})\n\n'
-        'COPY file:f77490f70ce51da25bd21bfc30cb5e1a24b2b65eb37d4af0c327ddc24f09'
-        '86a6 in / \n'
+        'COPY file:7bf12aab75c3867a023fe3b8bd6d113d43a4fcc415f3cc27cbcf0fff37b6'
+        '5a02 in / \n'
         'CMD ["/hello"]'.format(de_version))
     with tempfile.TemporaryDirectory() as tmp_dir:
       self.dl_object._output_directory = tmp_dir
@@ -885,6 +888,62 @@ def testDownloadDockerFile(self):
         self.assertEqual(expected_dockerfile, f.read())
 
 
+class TestDEVolumes(unittest.TestCase):
+  """Tests various volumes/bind mounts."""
+
+  @classmethod
+  def setUpClass(cls):
+    """Internal method to set up the TestCase on a specific storage."""
+    cls.driver = 'overlay2'
+    cls.docker_directory_path = os.path.join('test_data', 'docker')
+    if not os.path.isdir(cls.docker_directory_path):
+      docker_tar = os.path.join('test_data', 'vols.v2.tgz')
+      tar = tarfile.open(docker_tar, 'r:gz')
+      tar.extractall('test_data')
+      tar.close()
+    cls.explorer_object = explorer.Explorer()
+    cls.explorer_object.SetDockerDirectory(cls.docker_directory_path)
+
+    cls.driver_class = storage.Overlay2Storage
+    cls.storage_version = 2
+
+  @classmethod
+  def tearDownClass(cls):
+    shutil.rmtree(cls.docker_directory_path)
+
+  def testGenerateBindMountPoints(self):
+    """Tests generating command to mount 'bind' MountPoints."""
+    self.maxDiff = None
+    de_object = de.DockerExplorerTool()
+    de_object._explorer = self.explorer_object
+    container_obj = de_object._explorer.GetContainer(
+        '8b6e90cc742bd63f6acb7ecd40ddadb4e5dee27d8db2b739963f7cd2c7bcff4a')
+
+    commands = container_obj.storage_object._MakeVolumeMountCommands(
+        container_obj, '/mnt')
+    commands = [' '.join(x) for x in commands]
+    expected_commands = [
+        ('/bin/mount --bind -o ro {0:s}/opt/vols/bind'
+         ' /mnt/opt').format(os.path.abspath('test_data'))]
+    self.assertEqual(expected_commands, commands)
+
+  def testGenerateVolumesMountpoints(self):
+    """Tests generating command to mount 'volumes' MountPoints."""
+    self.maxDiff = None
+    de_object = de.DockerExplorerTool()
+    de_object._explorer = self.explorer_object
+    container_obj = de_object._explorer.GetContainer(
+        '712909b5ab80d8785841f12e361c218a2faf5365f9ed525f2a0d6b6590ba89cb')
+
+    commands = container_obj.storage_object._MakeVolumeMountCommands(
+        container_obj, '/mnt')
+    commands = [' '.join(x) for x in commands]
+    expected_commands = [
+        ('/bin/mount --bind -o ro {0:s}/volumes/'
+         'f5479c534bbc6e2b9861973c2fbb4863ff5b7b5843c098d7fb1a027fe730a4dc/'
+         '_data /mnt/opt/vols/volume').format(os.path.abspath('test_data'))]
+    self.assertEqual(expected_commands, commands)
+
 del DockerTestCase
 
 if __name__ == '__main__':