Add support for Volatility's Intermediate Symbol Format (JSON) (#834)

.gitignore

@@ -16,6 +16,7 @@ examples/interrupt-event-example
 examples/step-event-example
 examples/va-pages
 examples/xen-emulate-response
+examples/breakpoint-emulate-example
 libtool
 libvmi.pc
 tests/check_libvmi
@@ -26,6 +27,34 @@ tools/linux-offset-finder/.tmp_versions/
 tools/linux-offset-finder/findoffsets.ko
 tools/linux-offset-finder/findoffsets.mod.c
 tools/linux-offset-finder/modules.order
-*.o
-*.lo
+libvmi/config/grammar.c
+libvmi/config/grammar.h
+libvmi/config/lexicon.c
+**/*.o
+**/*.lo
+**/*.Plo
+**/*.Po
+**/.dirstamp
 build
+autom4te.cache/*
+Makefile
+Makefile.in
+**/*.a
+**/*.la
+**/*.lai
+**/*.so*
+**/*.m4
+ltmain.sh
+missing
+stamp-h1
+ylwrap
+compile
+config.guess
+config.h
+config.log
+config.status
+config.sub
+configure
+depcomp
+install-sh
+**/.libs/*

.travis.yml

@@ -282,6 +282,32 @@ matrix:
         - ./configure --disable-rekall-profiles --disable-examples
         - make
 
+#
+# disable volatility ist
+#
+    - env:
+        - TEST="disable volatility ist (cmake)"
+      install:
+        - sudo apt-get update
+        - sudo apt-get -q -y install bison flex libjson-c-dev libvirt-dev libxen-dev libfuse-dev
+      script:
+        - mkdir build && cd build
+        - cmake -DVOLATILITY_IST=OFF -DBUILD_EXAMPLES=OFF ..
+        - make
+
+#
+# disable rekall profiles
+#
+    - env:
+        - TEST="disable volatility ist (autotools)"
+      install:
+        - sudo apt-get update
+        - sudo apt-get -q -y install bison flex libjson-c-dev libvirt-dev libxen-dev libfuse-dev
+      script:
+        - autoreconf -vif
+        - ./configure --disable-volatility-ist --disable-examples
+        - make
+
 #
 # disable safety checks
 #

CMakeLists.txt

@@ -69,7 +69,8 @@ option(ENABLE_VMIFS "Enable vmifs: maps memory to a file through FUSE" ON)
 option(ENABLE_CONFIGFILE "Enable config file" ON)
 option(ENABLE_ADDRESS_CACHE "Enable address cache" ON)
 option(ENABLE_PAGE_CACHE "Enable page cache" ON)
-option(REKALL_PROFILES "Use Rekall's JSON profiles" ON)
+option(REKALL_PROFILES "Support Rekall's JSON profiles" ON)
+option(VOLATILITY_IST "Support Volatility's JSON ISTs" ON)
 option(ENABLE_SAFETY_CHECKS "Enable API safety checks" ON)
 option(ENABLE_TESTING "Build libvmi test suite" OFF)
 option(BUILD_EXAMPLES "Build the examples" ON)
@@ -81,6 +82,7 @@ option(ENV_DEBUG "Toggle the debug output via LIBVMI_DEBUG environment variable"
 
 # default values
 set(MAX_PAGE_CACHE_SIZE "512")
+set(ENABLE_JSON_PROFILES "Support JSON profiles" ON)
 
 # enable hardening if requested
 if (HARDENING)
@@ -144,7 +146,8 @@ add_feature_info(ENABLE_VMIFS ENABLE_VMIFS "Enable vmifs: maps memory to a file
 add_feature_info(ENABLE_CONFIGFILE ENABLE_CONFIGFILE "Enable config file")
 add_feature_info(ENABLE_ADDRESS_CACHE ENABLE_ADDRESS_CACHE "Enable address cache")
 add_feature_info(ENABLE_PAGE_CACHE ENABLE_PAGE_CACHE "Enable page cache")
-add_feature_info(REKALL_PROFILES REKALL_PROFILES "Enable Rekall's JSON profiles")
+add_feature_info(REKALL_PROFILES REKALL_PROFILES "Support Rekall's JSON profiles")
+add_feature_info(VOLATILITY_IST VOLATILITY_IST "Support Volatility's JSON ISTs")
 add_feature_info(ENABLE_SAFETY_CHECKS ENABLE_SAFETY_CHECKS "Enable API safety checks")
 add_feature_info(ENABLE_TESTING ENABLE_TESTING "Enable LibVMI test suite")
 add_feature_info(BUILD_EXAMPLES BUILD_EXAMPLES "Build the examples")

Makefile.am

@@ -7,7 +7,6 @@ h_private = \
     libvmi/private.h \
     libvmi/debug.h \
     libvmi/glib_compat.h \
-    libvmi/rekall.h \
     libvmi/arch/arch_interface.h \
     libvmi/arch/intel.h \
     libvmi/arch/amd64.h \
@@ -43,8 +42,18 @@ if ENABLE_ADDRESS_CACHE
     c_sources   += libvmi/cache.c
 endif
 
+if ENABLE_JSON_PROFILES
+    h_private   += libvmi/json_profiles/json_profiles.h
+endif
+
+if ENABLE_VOLATILITY_IST
+    c_sources   += libvmi/json_profiles/volatility_ist.c
+    h_private   += libvmi/json_profiles/volatility_ist.h
+endif
+
 if ENABLE_REKALL_PROFILES
-    c_sources   += libvmi/rekall.c
+    c_sources   += libvmi/json_profiles/rekall.c
+    h_private   += libvmi/json_profiles/rekall.h
 endif
 
 drivers =

README.rst

@@ -171,8 +171,24 @@ File / Snapshot Support
 If you would like LibVMI to work on physical memory snapshots saved to
 a file, then you don't need any special setup.
 
+Volatility3 Intermediate Symbol Table (IST) Format
+------------------------------
+LibVMI supports the use of Volatility3's IST JSONs for introspecting Windows and Linux. By using
+these jsons, LibVMI is able to bypass the use if the in-memory KdDebuggerData (KDBG)
+structure normally used by memory forensics tools and thus allows introspecting domains
+where this structure is either corrupted, or encoded (like in the case of Windows 8 x64).
+However, these ISTs have to be created for each kernel version, and therefore if an
+update is made to the kernel, the JSON file has to be re-generated, thus it's a bit less stable
+as the standard LibVMI configuration entries.
+
+Volatility3 is available at https://github.com/volatilityfoundation/volatility3.
+
+To read about how to generate the IST see: https://volatility3.readthedocs.io/en/latest/symbol-tables.html.
+
 Rekall profiles
 ------------------------------
+Note: Rekall is no longer maintained. Support for Rekall profiles will be deprecated.
+
 LibVMI also supports the use of Rekall profiles for introspecting Windows and Linux. By using
 Rekall profiles, LibVMI is able to bypass the use if the in-memory KdDebuggerData (KDBG)
 structure normally used by memory forensics tools and thus allows introspecting domains

configure.ac

@@ -130,26 +130,32 @@ AC_ARG_ENABLE([vmifs],
       [enable_vmifs=$enableval],
       [enable_vmifs=yes])
 
-AC_ARG_ENABLE([address_cache],
+AC_ARG_ENABLE([address-cache],
       [AS_HELP_STRING([--disable-address-cache],
          [Disable caching addresses (v2p, pid, etc) @<:@no@:>@])],
       [enable_address_cache=$enableval],
       [enable_address_cache=yes])
 AM_CONDITIONAL([ENABLE_ADDRESS_CACHE], [test x"$enable_address_cache" = "xyes"])
 
-AC_ARG_ENABLE([page_cache],
+AC_ARG_ENABLE([page-cache],
       [AS_HELP_STRING([--disable-page-cache],
          [Disable caching pages @<:@no@:>@])],
       [enable_page_cache=$enableval],
       [enable_page_cache=512])
 AM_CONDITIONAL([ENABLE_PAGE_CACHE], [test x"$enable_page_cache" = "xyes"])
 
-AC_ARG_ENABLE([rekall_profiles],
+AC_ARG_ENABLE([rekall-profiles],
       [AS_HELP_STRING([--disable-rekall-profiles],
-         [Disable using Rekall's JSON profiles @<:@no@:>@])],
+         [Disable support for Rekall's JSON profiles @<:@no@:>@])],
       [rekall="$enableval"],
       [rekall="yes"])
 
+AC_ARG_ENABLE([volatility-ist],
+      [AS_HELP_STRING([--disable-volatility-ist],
+         [Disable support for Volatility's JSON intermediate symbol table @<:@no@:>@])],
+      [volatility_ist="$enableval"],
+      [volatility_ist="yes"])
+
 AC_ARG_ENABLE([config-file],
       [AS_HELP_STRING([--disable-config-file],
          [Disable using LibVMI config files @<:@no@:>@])],
@@ -269,16 +275,13 @@ AC_SUBST([GLIB_LIBS])
 [fi]
 
 have_vmifs='yes'
-vmifs_space='       '
 [if test "$enable_vmifs" = "yes"]
 [then]
     PKG_CHECK_MODULES([FUSE], [fuse >= 2.2], [missing="no"], [missing="yes"])
     [if test x"$missing" = "xyes"]
     [then]
         AC_DEFINE([ENABLE_VMIFS], [0], [Define to 1 to build VMIFS.])
-        have_vmifs='FUSE library missing (libfuse-dev)'
         enable_vmifs='no'
-	    vmifs_space=' '
     [else]
         AC_DEFINE([ENABLE_VMIFS], [1], [Define to 1 to build VMIFS.])
         AC_SUBST([FUSE_CFLAGS])
@@ -316,19 +319,29 @@ AM_CONDITIONAL([VMIFS], [test x"$enable_vmifs" = xyes])
 [fi]
 
 missing='no'
-[if test "$rekall" = "yes"]
+[if test "$rekall" = "yes" || test "$volatility_ist" = "yes"]
 [then]
+    AC_CHECK_HEADERS([json-c/json.h])
     AC_CHECK_LIB(json-c, json_object_get_int64,, [missing="yes"])
     [if test x"$missing" = "xno"]
     [then]
-        AC_CHECK_HEADERS([json-c/json.h])
-        AC_DEFINE([REKALL_PROFILES], [1], [Defined to 1 when working JSON-C library was found to parse Rekall profiles.])
+        AM_CONDITIONAL([ENABLE_JSON_PROFILES], [test "xyes" = "xyes"])
+        AC_DEFINE([ENABLE_JSON_PROFILES], [1], [Defined to 1 when support to JSON profiles is available])
+        [if test "$rekall" = "yes"]
+        [then]
+            AC_DEFINE([REKALL_PROFILES], [1], [Defined to 1 when support to Rekall profiles should be compiled])
+        [fi]
+        [if test "$volatility_ist" = "yes"]
+        [then]
+            AC_DEFINE([VOLATILITY_IST], [1], [Defined to 1 when support to Volatility IST should be compiled])
+        [fi]
     [else]
         rekall='Disabled, missing libjson-c-dev.'
-        [echo "No working JSON-C library found (libjson-c-dev), Rekall system profiles won't be supported."]
+        [echo "No working JSON-C library found (libjson-c-dev), Rekall profiles and Volatility IST won't be supported."]
     [fi]
 [fi]
 AM_CONDITIONAL([ENABLE_REKALL_PROFILES], [test x"$rekall" = "xyes"])
+AM_CONDITIONAL([ENABLE_VOLATILITY_IST], [test x"$volatility_ist" = "xyes"])
 
 [if test x"$enable_address_cache" = "xyes"]
 [then]
@@ -365,25 +378,32 @@ Host system type: $host
 Build system type: $build
 Installation prefix: $prefix
 
-Feature         | Option
+Hypervisor      | Option
 ----------------|---------------------------
 Xen Support     | --enable-xen=$enable_xen
 KVM Support     | --enable-kvm=$enable_kvm
 File Support    | --enable-file=$enable_file
 Bareflank       | --enable-bareflank=$enable_bareflank
-Rekall profiles | --enable-rekall-profiles=$rekall
 ----------------|---------------------------
 
 OS              | Option
 ----------------|---------------------------
 Windows         | --enable-windows=$enable_windows
 Linux           | --enable-linux=$enable_linux
 FreeBSD         | --enable-freebsd=$enable_freebsd
+----------------|---------------------------
+
+Configuration   | Option
+----------------|---------------------------
+libvmi.conf     | --enable-config-file=$configfile
+Rekall profiles | --enable-rekall-profiles=$rekall
+Volatility IST  | --enable-volatility-ist=$volatility_ist
+----------------|---------------------------
 
-Tools           | Option                    | Reason
-----------------|---------------------------|----------------------------
+Tools           | Option
+----------------|---------------------------
 Examples        | --enable-examples=$enable_examples
-VMIFS           | --enable-vmifs=$enable_vmifs$vmifs_space  | $have_vmifs
+VMIFS           | --enable-vmifs=$enable_vmifs
 
 If everything is correct, you can now run 'make' and (optionally)
 'make install'.  Otherwise, you can run './configure' again.

libvmi/CMakeLists.txt

@@ -167,15 +167,23 @@ if (ENABLE_ADDRESS_CACHE)
     target_sources(vmi_shared PRIVATE cache.c)
 endif ()
 
-if (REKALL_PROFILES)
+if (REKALL_PROFILES OR VOLATILITY_IST)
     find_package(JSON-C)
     set_package_properties(JSON-C PROPERTIES
-        PURPOSE "Dependency for Rekall profiles parsing")
+        PURPOSE "Dependency for Rekall profiles and Volatility IST parsing")
     if (NOT JSON-C_FOUND)
+        set(ENABLE_JSON_PROFILES OFF)
         set(REKALL_PROFILES OFF)
-        message(WARNING "Cannot find JSON: disabling Rekall profiles")
+        set(VOLATILITY_IST OFF)
+        message(WARNING "Cannot find JSON: disabling Rekall profiles and Volatility IST")
     else ()
-        target_sources(vmi_shared PRIVATE rekall.c)
+        set(ENABLE_JSON_PROFILES ON)
+        if (REKALL_PROFILES)
+            target_sources(vmi_shared PRIVATE json_profiles/rekall.c)
+        endif ()
+        if (VOLATILITY_IST)
+            target_sources(vmi_shared PRIVATE json_profiles/volatility_ist.c)
+        endif ()
         target_include_directories(vmi_shared PRIVATE ${JSON-C_INCLUDE_DIRS})
         target_link_libraries(vmi_shared PRIVATE ${JSON-C_LIBRARIES})
         list(APPEND VMI_PUBLIC_DEPS ${JSON-C_LIBRARIES})

libvmi/accessors.c

@@ -374,22 +374,6 @@ vmi_get_name(
     }
 }
 
-const char *
-vmi_get_rekall_path(
-    vmi_instance_t vmi)
-{
-#ifdef ENABLE_SAFETY_CHECKS
-    if (!vmi)
-        return NULL;
-#endif
-
-#ifdef REKALL_PROFILES
-    return vmi->rekall_profile;
-#else
-    return NULL;
-#endif
-}
-
 uint64_t
 vmi_get_vmid(
     vmi_instance_t vmi)
@@ -858,3 +842,54 @@ vmi_get_freebsd_sysmap(
     return freebsd_instance->sysmap;
 
 }
+
+const char *
+vmi_get_rekall_path(
+    vmi_instance_t vmi)
+{
+#ifdef ENABLE_SAFETY_CHECKS
+    if (!vmi)
+        return NULL;
+#endif
+
+#ifdef ENABLE_JSON_PROFILES
+    return vmi->json_profile_path;
+#else
+    return NULL;
+#endif
+}
+
+const char *
+vmi_get_os_profile_path(
+    vmi_instance_t vmi)
+{
+#ifdef ENABLE_SAFETY_CHECKS
+    if (!vmi)
+        return NULL;
+#endif
+
+#ifdef ENABLE_JSON_PROFILES
+    if ( vmi->json_profile_path )
+        return vmi->json_profile_path;
+#endif
+
+#ifdef ENABLE_SAFETY_CHECKS
+    if (!vmi->os_data)
+        return NULL;
+#endif
+
+    switch (vmi->os_type) {
+        case VMI_OS_LINUX: {
+            linux_instance_t linux_instance = vmi->os_data;
+            return linux_instance->sysmap;
+        }
+        case VMI_OS_FREEBSD: {
+            freebsd_instance_t freebsd_instance = vmi->os_data;
+            return freebsd_instance->sysmap;
+        }
+        default:
+            break;
+    };
+
+    return NULL;
+}