Support Linux kernels > 4.13. (#326)

On more recent linux kernels task_struct layout is randomized using
randomized_struct_fields_start/randomized_struct_fields_end
macros. The effectively creates an anonymous struct embedded within
task_struct.

This change implements code to merge the definition of the anonymous
struct into task_struct. This allows us to unrandomize the struct
layout transparently.

README.md

@@ -17,7 +17,7 @@ images:
 - Microsoft Windows 7 Service Pack 0 and 1
 - Microsoft Windows 8 and 8.1
 - Microsoft Windows 10
-- Linux Kernels 2.6.24 to 4.4.
+- Linux Kernels 2.6.24 to most recent.
 - OSX 10.7-10.12.x.
 
 Rekall also provides a complete memory sample acquisition capability for all

rekall-core/rekall/obj.py

@@ -59,6 +59,7 @@
 
 if six.PY3:
     unicode = str
+    long = int
 
 
 class ProfileLog(object):
@@ -663,8 +664,13 @@ def cdecl(self):
         return self.obj_name
 
     def __repr__(self):
-        return " [{0}:{1}]: 0x{2:08x}".format(self.obj_type, self.obj_name,
-                                              int(self.v()))
+        value = self.v()
+        if isinstance(value, (int, long)):
+            return " [{0}:{1}]: 0x{2:08x}".format(
+                self.obj_type, self.obj_name, value)
+        else:
+            return " [{0}:{1}]: '{2}'".format(
+                self.obj_type, self.obj_name, utils.SmartUnicode(value))
 
 
 class Bool(NativeType):

rekall-core/rekall/plugins/addrspaces/elfcore.py

@@ -196,7 +196,6 @@ def __init__(self, **kwargs):
         super(KCoreAddressSpace, self).__init__(**kwargs)
 
         # This is a live address space.
-        self.as_assert(self.base == None, 'Must be first Address Space')
         self.volatile = True
         self.mapped_files = {}
 
@@ -240,25 +239,20 @@ def __init__(self, **kwargs):
         self.runs.clear()
 
         # At this point, we think this is a valid, usable kcore file.
-        # RHEL, however, disabled read access to /proc/kcore past the ELF
+        # RHEL, however, disables read access to /proc/kcore past the ELF
         # headers and the file size reflects this. /proc/kcore usually has a
         # size of at least 64TB (46bits of physical address space in x64).
         # We use the file size to detect cases where kcore will be unusable.
-
         if getattr(self.base, "fhandle", None):
             try:
-                statinfo = os.fstat(self.base.fhandle.fileno())
-                if statinfo.st_size < 2**46:
-                    # We raise a TypeError and not an ASAssertionError because
-                    # we need it to be catchable when rekall is used as a
-                    # library (i.e GRR). ASAssertionErrors are swallowed by
-                    # the address space selection algorithm.
-                    raise RuntimeError(
-                        "This kcore file is too small (%d bytes) and likely "
-                        "invalid for memory analysis. You may want to use pmem "
-                        "instead." % statinfo.st_size)
-            except(IOError, AttributeError):
-                pass
+                size = os.fstat(self.base.fhandle.fileno()).st_size
+            except IOError:
+                size = 0
+
+            self.as_assert(size > 2**45,
+                           "This kcore file is too small (%d bytes) and likely "
+                           "invalid for memory analysis. You may want to use pmem "
+                           "instead." % size)
 
         for x in runs:
             self.add_run(*x)

rekall-core/rekall/plugins/linux/common.py

@@ -131,9 +131,11 @@ def parse_data(self, data):
 
     def _OpenLiveSymbolsFile(self, physical_address_space):
         """Opens the live symbols file to parse."""
-        return physical_address_space.get_file_address_space(
+        file_as = physical_address_space.get_file_address_space(
             self.KALLSYMS_FILE)
 
+        return file_as
+
 
 class AbstractLinuxCommandPlugin(plugin.PhysicalASMixin,
                                  plugin.TypedProfileCommand,
@@ -211,8 +213,12 @@ def dtb_hits(self):
                 self.profile.get_constant("swapper_pg_dir", is_address=True))
 
         else:
-            yield self.profile.phys_addr(
-                self.profile.get_constant("init_level4_pgt", is_address=True))
+            pgt_virt_addres = (
+                self.profile.get_constant("init_level4_pgt", is_address=True) or
+                # This was renamed https://patchwork.kernel.org/patch/9667543/
+                self.profile.get_constant("init_top_pgt", is_address=True))
+
+            yield self.profile.phys_addr(pgt_virt_addres)
 
     def render(self, renderer):
         renderer.table_header([("DTB", "dtv", "[addrpad]"),
@@ -433,7 +439,6 @@ def calculate(self):
         seen = set()
         task_head = self.session.profile.get_constant_object(
             "init_task", "task_struct")
-
         for task in task_head.tasks:
             if task.obj_offset not in seen:
                 seen.add(task.obj_offset)

rekall-core/rekall/plugins/overlays/linux/linux.py

@@ -840,6 +840,7 @@ def __sub__(self, other):
         return self + ts
 
     def normalized_timespec(self):
+
         """Normalizes a timespec's secs and nsecs.
 
         Based on set_normalized_timespec:
@@ -1033,6 +1034,30 @@ def Initialize(cls, profile):
         elif profile.metadata("arch") == "AMD64":
             basic.ProfileLP64.Initialize(profile)
 
+        task_struct_def = profile.vtypes.get("task_struct")
+        if task_struct_def is None:
+            raise RuntimeError("Profile does not have task_struct")
+
+        # Recent kernel versions randomize task_struct memory layout
+        # using the macros randomized_struct_fields_start and
+        # randomized_struct_fields_end. These macros create an
+        # anonymous struct within task_struct which contains most of
+        # the fields. Rekall does not currently automatically access
+        # embedded anonymous structs, instead requiring to access via
+        # the "u1" member. The following code merges the anonymous
+        # struct into task_struct so it can be access transparently.
+
+        # http://elixir.free-electrons.com/linux/v4.13/source/include/linux/sched.h#L534
+        if "pid" not in task_struct_def[1]:
+            # Merge the anonymous union into task_struct.
+            union_offset, union_name = task_struct_def[1]["u1"]
+            size, union = profile.vtypes.get(union_name[0])
+            for field, field_definition in union.items():
+                field_definition[0] += union_offset
+                task_struct_def[1][field] = field_definition
+
+            profile.vtypes["task_struct"] = task_struct_def
+
     def GetImageBase(self):
         if self.image_base is None:
             self.image_base = obj.Pointer.integer_to_address(
@@ -1253,6 +1278,7 @@ class Linux32(Linux):
 
 
 class Linux64(Linux):
+
     @classmethod
     def Initialize(cls, profile):
         profile.set_metadata("arch", "AMD64")

rekall-core/setup.py

@@ -68,7 +68,7 @@ def find_data_files(source):
     'python-dateutil==2.6.1',
     'pytsk3==20170802',
     'pytz==2017.3',
-    'rekall-capstone==3.0.5.post1',
+    'rekall-capstone==3.0.5.post2',
     "rekall-efilter >= 1.6, < 1.7",
 
     # Should match exactly the version of this package.