Enumerate Docker containers as new Evidences to be processed (google#465

)

* init

* typo header

* yapf

* comments

* comment

* check PATH in docker worker too

* add tests

* yapf

* comments

turbinia/evidence.py

@@ -22,6 +22,7 @@
 
 from turbinia import config
 from turbinia import TurbiniaException
+from turbinia.processors import docker
 from turbinia.processors import mount_local
 from turbinia.processors import archive
 
@@ -609,3 +610,40 @@ def __init__(self, module_list=None, profile=None, *args, **kwargs):
 class BinaryExtraction(CompressedDirectory):
   """Binaries extracted from evidence."""
   pass
+
+
+class DockerContainer(Evidence):
+  """Evidence object for a DockerContainer filesystem.
+
+  Attributes:
+    container_id(str): The ID of the container to mount.
+    _container_fs_path(str): Full path to where the container filesystem will
+      be mounted.
+    _docker_root_directory(str): Full path to the docker root directory.
+  """
+
+  # ABSOLUTELY NO LEADING / HERE
+  DEFAULT_DOCKER_DIRECTORY_PATH = 'var/lib/docker'
+
+  def __init__(self, container_id=None, *args, **kwargs):
+    """Initialization for Docker Container."""
+    super(DockerContainer, self).__init__(*args, **kwargs)
+    self.container_id = container_id
+    self._container_fs_path = None
+    self._docker_root_directory = None
+
+    self.context_dependent = True
+
+  def _preprocess(self, _):
+
+    self._docker_root_directory = os.path.join(
+        self.parent_evidence.mount_path, self.DEFAULT_DOCKER_DIRECTORY_PATH)
+    # Mounting the container's filesystem
+    self._container_fs_path = docker.PreprocessMountDockerFS(
+        self._docker_root_directory, self.container_id)
+    self.mount_path = self._container_fs_path
+    self.local_path = self.mount_path
+
+  def _postprocess(self):
+    # Unmount the container's filesystem
+    mount_local.PostprocessUnmountPath(self._container_fs_path)

turbinia/jobs/__init__.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 """Turbinia jobs."""
 from turbinia.jobs import finalize_request
+from turbinia.jobs import docker
 from turbinia.jobs import grep
 from turbinia.jobs import hadoop
 from turbinia.jobs import http_access_logs

turbinia/jobs/docker.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Job to execute docker-explorer, and generate new Evidence per container"""
+
+from __future__ import unicode_literals
+
+from turbinia.evidence import DockerContainer
+from turbinia.evidence import GoogleCloudDisk
+from turbinia.evidence import GoogleCloudDiskRawEmbedded
+from turbinia.evidence import RawDisk
+from turbinia.jobs import interface
+from turbinia.jobs import manager
+from turbinia.workers.docker import DockerContainersEnumerationTask
+
+
+class DockerContainersEnumerationJob(interface.TurbiniaJob):
+  """Use docker-explorer to list all containers in a Linux docker environment"""
+
+  # Types of evidence that this Job will process.
+  evidence_input = [GoogleCloudDisk, GoogleCloudDiskRawEmbedded, RawDisk]
+  evidence_output = [DockerContainer]
+
+  NAME = 'DockerContainersEnumerationJob'
+
+  def create_tasks(self, evidence):
+    """Create task.
+
+    Args:
+      evidence: List of evidence object to process
+
+    Returns:
+        A list of tasks to schedule.
+    """
+    tasks = [DockerContainersEnumerationTask() for _ in evidence]
+    return tasks
+
+
+manager.JobsManager.RegisterJob(DockerContainersEnumerationJob)

turbinia/jobs/hadoop.py

@@ -16,6 +16,7 @@
 
 from __future__ import unicode_literals
 
+from turbinia.evidence import DockerContainer
 from turbinia.evidence import GoogleCloudDisk
 from turbinia.evidence import GoogleCloudDiskRawEmbedded
 from turbinia.evidence import RawDisk
@@ -28,7 +29,9 @@
 class HadoopAnalysisJob(interface.TurbiniaJob):
   """Analyzes Hadoop AppRoot files."""
 
-  evidence_input = [GoogleCloudDisk, GoogleCloudDiskRawEmbedded, RawDisk]
+  evidence_input = [
+      DockerContainer, GoogleCloudDisk, GoogleCloudDiskRawEmbedded, RawDisk
+  ]
   evidence_output = [ReportText]
 
   NAME = 'HadoopAnalysisJob'

turbinia/jobs/http_access_logs.py

@@ -18,6 +18,7 @@
 from turbinia.workers import artifact
 
 from turbinia.evidence import Directory
+from turbinia.evidence import DockerContainer
 from turbinia.evidence import RawDisk
 from turbinia.evidence import GoogleCloudDisk
 from turbinia.evidence import GoogleCloudDiskRawEmbedded
@@ -36,7 +37,8 @@ class HTTPAccessLogExtractionJob(interface.TurbiniaJob):
   """HTTP Access log extraction job."""
 
   evidence_input = [
-      Directory, RawDisk, GoogleCloudDisk, GoogleCloudDiskRawEmbedded
+      Directory, DockerContainer, RawDisk, GoogleCloudDisk,
+      GoogleCloudDiskRawEmbedded
   ]
 
   evidence_output = [ExportedFileArtifact]

turbinia/jobs/jenkins.py

@@ -17,6 +17,7 @@
 from __future__ import unicode_literals
 
 from turbinia.evidence import Directory
+from turbinia.evidence import DockerContainer
 from turbinia.evidence import RawDisk
 from turbinia.evidence import GoogleCloudDisk
 from turbinia.evidence import GoogleCloudDiskRawEmbedded
@@ -30,7 +31,8 @@ class JenkinsAnalysisJob(interface.TurbiniaJob):
   """Jenkins analysis job."""
 
   evidence_input = [
-      Directory, RawDisk, GoogleCloudDisk, GoogleCloudDiskRawEmbedded
+      Directory, DockerContainer, RawDisk, GoogleCloudDisk,
+      GoogleCloudDiskRawEmbedded
   ]
   evidence_output = [ReportText]
 

turbinia/jobs/sshd.py

@@ -19,6 +19,7 @@
 from turbinia.workers import artifact
 from turbinia.workers import sshd
 from turbinia.evidence import Directory
+from turbinia.evidence import DockerContainer
 from turbinia.evidence import GoogleCloudDisk
 from turbinia.evidence import GoogleCloudDiskRawEmbedded
 from turbinia.evidence import ExportedFileArtifact
@@ -33,7 +34,8 @@ class SSHDExtractionJob(interface.TurbiniaJob):
 
   # The types of evidence that this Job will process
   evidence_input = [
-      Directory, RawDisk, GoogleCloudDisk, GoogleCloudDiskRawEmbedded
+      Directory, DockerContainer, RawDisk, GoogleCloudDisk,
+      GoogleCloudDiskRawEmbedded
   ]
 
   evidence_output = [ExportedFileArtifact]

turbinia/jobs/tomcat.py

@@ -17,6 +17,7 @@
 from turbinia.workers import artifact
 from turbinia.workers import tomcat
 from turbinia.evidence import Directory
+from turbinia.evidence import DockerContainer
 from turbinia.evidence import GoogleCloudDisk
 from turbinia.evidence import GoogleCloudDiskRawEmbedded
 from turbinia.evidence import ExportedFileArtifact
@@ -31,7 +32,8 @@ class TomcatExtractionJob(interface.TurbiniaJob):
 
   # The types of evidence that this Job will process
   evidence_input = [
-      Directory, RawDisk, GoogleCloudDisk, GoogleCloudDiskRawEmbedded
+      Directory, DockerContainer, RawDisk, GoogleCloudDisk,
+      GoogleCloudDiskRawEmbedded
   ]
 
   evidence_output = [ExportedFileArtifact]

turbinia/output_manager.py

@@ -170,8 +170,9 @@ def save_evidence(self, evidence_, result):
       evidence_.local_path = local_path
     evidence_.saved_path = path
     evidence_.saved_path_type = path_type
-    log.info(
-        'Saved copyable evidence data to {0!s}'.format(evidence_.saved_path))
+    if evidence_.saved_path:
+      log.info(
+          'Saved copyable evidence data to {0:s}'.format(evidence_.saved_path))
     return evidence_
 
   def save_local_file(self, file_, result):

turbinia/processors/docker.py

@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Evidence processor to mount local Docker containers."""
+
+from __future__ import unicode_literals
+
+import logging
+import os
+import subprocess
+import tempfile
+
+from turbinia import config
+from turbinia import TurbiniaException
+
+log = logging.getLogger('turbinia')
+
+
+def PreprocessMountDockerFS(docker_dir, container_id):
+  """Mounts a Docker container Filesystem locally.
+
+  We use subprocess to run the DockerExplorer script, instead of using the
+  Python module, because we need to make sure all DockerExplorer code runs
+  as root.
+
+  Args:
+    docker_dir(str): the root Docker directory.
+    container_id(str): the complete ID of the container.
+
+  Returns:
+    The path to the mounted container file system, as a string.
+
+  Raises:
+    TurbiniaException: if there was an error trying to mount the filesystem.
+  """
+  # Most of the code is copied from PreprocessMountDisk
+  # Only the mount command changes
+  config.LoadConfig()
+  mount_prefix = config.MOUNT_DIR_PREFIX
+
+  if os.path.exists(mount_prefix) and not os.path.isdir(mount_prefix):
+    raise TurbiniaException(
+        'Mount dir {0:s} exists, but is not a directory'.format(mount_prefix))
+  if not os.path.exists(mount_prefix):
+    log.info('Creating local mount parent directory {0:s}'.format(mount_prefix))
+    try:
+      os.makedirs(mount_prefix)
+    except OSError as e:
+      raise TurbiniaException(
+          'Could not create mount directory {0:s}: {1!s}'.format(
+              mount_prefix, e))
+
+  container_mount_path = tempfile.mkdtemp(prefix='turbinia', dir=mount_prefix)
+
+  log.info(
+      'Using docker_explorer to mount container {0:s} on {1:s}'.format(
+          container_id, container_mount_path))
+  de_binary = None
+  for path in os.environ['PATH'].split(os.pathsep):
+    tentative_path = os.path.join(path, 'de.py')
+    if os.path.exists(tentative_path):
+      de_binary = tentative_path
+      break
+
+  if not de_binary:
+    raise TurbiniaException('Could not find docker-explorer script: de.py')
+
+  # TODO(aarontp): Remove hard-coded sudo in commands:
+  # https://github.com/google/turbinia/issues/73
+  mount_cmd = [
+      'sudo', de_binary, '-r', docker_dir, 'mount', container_id,
+      container_mount_path
+  ]
+  log.info('Running: {0:s}'.format(' '.join(mount_cmd)))
+
+  try:
+    subprocess.check_call(mount_cmd)
+  except subprocess.CalledProcessError as e:
+    raise TurbiniaException(
+        'Could not mount container {0:s}: {1!s}'.format(container_id, e))
+
+  return container_mount_path