Merge branch 'master' of github.com:CENSUS/shadow

cfg/android8_32.cfg

@@ -0,0 +1,48 @@
+[values]
+narenas_total = 0x2
+je_map_bias = 0x3
+je_nhbins = 0x30
+je_map_misc_offset = 0x230
+je_chunksize = 0x80000
+
+[offsets]
+arena_bin_info_t.reg0_offset = 0x34
+arena_bin_t.runcur = 0x18
+arena_chunk_t.map_bits = 0x3c
+arena_chunk_map_bits_t.bits = 0x0
+rtree_t.height = 0x8
+arena_bin_info_t.run_size = 0xc
+arena_bin_info_t.reg_size = 0x0
+pthread_key_data_t.data = 0x4
+arena_t.bins = 0x4d8
+arena_run_t.nfree = 0x4
+arena_run_t.bitmap = 0x8
+extent_node_t.en_addr = 0x4
+arena_chunk_map_misc_t.run = 0xc
+arena_chunk_t.node = 0x0
+rtree_level_t.subtree = 0x0
+tcache_bin_t.lg_fill_div = 0xc
+pthread_internal_t.next = 0x0
+tcache_bin_t.low_water = 0x8
+tcache_bin_t.ncached = 0x10
+rtree_level_t.bits = 0x4
+tcache_bin_t.avail = 0x14
+extent_node_t.en_arena = 0x0
+pthread_internal_t.tid = 0x8
+arena_bin_info_t.nregs = 0x10
+tcache_t.tbins = 0x20
+pthread_internal_t.key_data = 0x74
+rtree_t.levels = 0x14
+
+[sizes]
+pthread_key_data_t = 0x8
+arena_run_t = 0x4c
+arena_chunk_map_bits_t = 0x4
+rtree_level_t = 0xc
+arena_bin_info_t = 0x38
+rtree_t = 0x2c
+arena_bin_t = 0x68
+arena_chunk_map_misc_t = 0x58
+pthread_internal_t = 0x688
+tcache_bin_t = 0x18
+

cfg/android8_64.cfg

@@ -0,0 +1,48 @@
+[values]
+narenas_total = 0x2
+je_map_bias = 0xd
+je_nhbins = 0x2d
+je_map_misc_offset = 0x1010
+je_chunksize = 0x200000
+
+[offsets]
+arena_bin_info_t.reg0_offset = 0x38
+arena_bin_t.runcur = 0x50
+arena_chunk_t.map_bits = 0x78
+arena_chunk_map_bits_t.bits = 0x0
+rtree_t.height = 0x10
+arena_bin_info_t.run_size = 0x18
+arena_bin_info_t.reg_size = 0x0
+pthread_key_data_t.data = 0x8
+arena_t.bins = 0x980
+arena_run_t.nfree = 0x4
+arena_run_t.bitmap = 0x8
+extent_node_t.en_addr = 0x8
+arena_chunk_map_misc_t.run = 0x18
+arena_chunk_t.node = 0x0
+rtree_level_t.subtree = 0x0
+tcache_bin_t.lg_fill_div = 0xc
+pthread_internal_t.next = 0x0
+tcache_bin_t.low_water = 0x8
+tcache_bin_t.ncached = 0x10
+rtree_level_t.bits = 0x8
+tcache_bin_t.avail = 0x18
+extent_node_t.en_arena = 0x0
+pthread_internal_t.tid = 0x10
+arena_bin_info_t.nregs = 0x20
+tcache_t.tbins = 0x28
+pthread_internal_t.key_data = 0xe0
+rtree_t.levels = 0x28
+
+[sizes]
+pthread_key_data_t = 0x10
+arena_run_t = 0x48
+arena_chunk_map_bits_t = 0x8
+rtree_level_t = 0x10
+arena_bin_info_t = 0x40
+rtree_t = 0x68
+arena_bin_t = 0xa8
+arena_chunk_map_misc_t = 0x60
+pthread_internal_t = 0xb08
+tcache_bin_t = 0x20
+

docs/android_heap.md

@@ -41,7 +41,7 @@ You can view the heap's chunks by using the "jechunks" command:
 Chunks in Android have the following sizes:
 
 |          |  32-bit |   64-bit |
-|----------+---------+----------|
+|-------------------------------|
 | Android6 | 0x40000 |  0x40000 |
 | Android7 | 0x80000 | 0x200000 |
 
@@ -110,15 +110,15 @@ a small run is associated with exactly one region size class.
 
 That means that you can think of a small run as an array of regions:
 
-|----------|
-| region 0 |
-|----------|
-| region 1 |
-|----------|
-| ...      |
-|----------|
-| region N |
-|----------|
+    |----------|
+    | region 0 |
+    |----------|
+    | region 1 |
+    |----------|
+    | ...      |
+    |----------|
+    | region N |
+    |----------|
 
 You can view the layout of a small run by using the "jerun" command:
 

docs/android_quickstart.md

@@ -29,7 +29,7 @@ This is what we assume you have:
         1. git clone http://android.googlesource.com/toolchain/gdb
         2. cd ./gdb/gdb-7.11
         3. mkdir build64; cd build64
-        4. ../configure --program-prefix=aarch64-eabi-linux- --target=aarch64-eabi-linux --disable-werror
+        4. ../configure --program-prefix=aarch64-eabi-linux- --target=aarch64-eabi-linux --disable-werror --with-python
         5. make
         6. make install
 

shadow.py

@@ -41,7 +41,7 @@
     import gdb_engine as dbg
     dbg_engine = 'gdb'
     storage_path = '/tmp/shadow'
-    android_version = '7'
+    android_version = '8'
 except ImportError:
     try:
         import pykd
@@ -64,7 +64,7 @@
             import lldb_engine as dbg
             dbg_engine = 'lldb'
             storage_path = '/tmp/shadow'
-            android_version = '7'
+            android_version = '8'
         except ImportError:
             pass
 
@@ -176,14 +176,12 @@ def has_symbols():
         return False
 
 
-
 debug_log_f = None
 debug_log = lambda x: None
 def _debug_log(s):
     debug_log_f.write(s + "\n")
 
 
-
 # parse functions
 def parse(read_content_preview, config_path, do_debug_log=False):
     global jeheap
@@ -192,7 +190,6 @@ def parse(read_content_preview, config_path, do_debug_log=False):
     global storage_path
     global android_version
 
-
     if do_debug_log:
         debug_log_p = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                            "debug.log")
@@ -203,31 +200,50 @@ def parse(read_content_preview, config_path, do_debug_log=False):
 
     if config_path:
         print('[shadow] parsing configuration...')
-        if 'android7' in config_path:
+        if 'android8' in config_path:
+            android_version = '8'
+        elif 'android7' in config_path:
             android_version = '7'
         elif 'android6' in config_path:
             android_version = '6'
         update_dbg_cache_from_config(config_path)
     else:
         if is_standalone_variant() and not has_symbols():
             print("[shadow] Detecting Android version...")
-            chunksize = int_from_sym(['je_chunksize', 'chunksize'])
+            chunksize = int_from_sym(["je_chunksize", "chunksize"])
+            map_misc_offset = int_from_sym(["je_map_misc_offset"])
 
             shadow_path = os.path.dirname(os.path.realpath(__file__))
             cfg_path = os.path.join(shadow_path, "cfg")
 
-            # android 7 32bit
+            # android 7/8 32bit
             if chunksize == 0x80000:
-                android_version = '7'
-                cfg_path = os.path.join(cfg_path, "android7_32.cfg")
-                print("[shadow] Using Android 7 32 bit configuration.")
-                print("         (%s)" % cfg_path)
-            # android 7 64bit
+                # android 8 32 bit
+                if map_misc_offset == 0x230:
+                    android_version = '8'
+                    cfg_path = os.path.join(cfg_path, "android8_32.cfg")
+                    print("[shadow] Using Android 8 32 bit configuration.")
+                    print("         (%s)" % cfg_path)
+                # android 8 64 bit
+                elif map_misc_offset == 0x228:
+                    android_version = '7'
+                    cfg_path = os.path.join(cfg_path, "android7_32.cfg")
+                    print("[shadow] Using Android 7 32 bit configuration.")
+                    print("         (%s)" % cfg_path)
+            # android 7/8 64bit
             elif chunksize == 0x200000:
-                android_version = '7'
-                cfg_path = os.path.join(cfg_path, "android7_64.cfg")
-                print("[shadow] Using Android 7 64 bit configuration.")
-                print("         (%s)" % cfg_path)
+                # android 8 64bit
+                if map_misc_offset == 0x1010:
+                    android_version = '8'
+                    cfg_path = os.path.join(cfg_path, "android8_64.cfg")
+                    print("[shadow] Using Android 8 64 bit configuration.")
+                    print("         (%s)" % cfg_path)
+                # android 7 64bit
+                elif map_misc_offset == 0x1008:
+                    android_version = '7'
+                    cfg_path = os.path.join(cfg_path, "android7_64.cfg")
+                    print("[shadow] Using Android 7 64 bit configuration.")
+                    print("         (%s)" % cfg_path)
             # android 6 32bit
             elif chunksize == 0x40000 and dbg.get_dword_size() == 4:
                 android_version = '6'
@@ -242,7 +258,7 @@ def parse(read_content_preview, config_path, do_debug_log=False):
                 print("         (%s)" % cfg_path)
             else:
                 print("[shadow] Could not detect Android version, try to use"
-                      "a configuration file.")
+                      " a configuration file.")
                 return
             update_dbg_cache_from_config(cfg_path)
 
@@ -264,10 +280,10 @@ def parse(read_content_preview, config_path, do_debug_log=False):
         store_jeheap(path)
 
     # write current config
-    # p = os.path.join(storage_path, 'shadow.cfg')
-    # if not os.path.isdir(storage_path):
-    #     os.makedirs(storage_path)
-    # generate_config(p)
+    p = os.path.join(storage_path, 'shadow.cfg')
+    if not os.path.isdir(storage_path):
+        os.makedirs(storage_path)
+    generate_config(p)
 
     print('[shadow] structures parsed')
     print_timestamp()
@@ -311,12 +327,12 @@ def parse_general(jeheap):
             jeheap.nbins = jeheap.ntbins + jeheap.nsbins + jeheap.nqbins
 
     # third attempt
-    if dbg_engine == 'gdb':
-        try:
-            jeheap.nbins = int(dbg.execute('p __mallinfo_nbins()').split()[2])
-        except:
-            # print("[shadow] Using hardcoded number of bins.")
-            pass
+    # if dbg_engine == 'gdb':
+    #     try:
+    #         jeheap.nbins = int(dbg.execute('p __mallinfo_nbins()').split()[2])
+    #     except:
+    #         # print("[shadow] Using hardcoded number of bins.")
+    #         pass
 
     # fourth attempt - hardcoded values
     if not jeheap.nbins:
@@ -593,7 +609,7 @@ def parse_all_runs(jeheap, read_content_preview):
 
                     if android_version == '6':
                         offset = mapelm & ~flags_mask
-                    elif android_version == '7':
+                    elif android_version == '7' or android_version == '8':
                         # offset = (mapelm >> 13) << 12
                         offset = (mapelm & ~0x1FFF) >> 1
 
@@ -624,7 +640,7 @@ def parse_all_runs(jeheap, read_content_preview):
                     # decode the bin index
                     if android_version == '6':
                         binind = (mapelm & 0xFF0) >> 4
-                    elif android_version== '7':
+                    elif android_version== '7' or android_version == '8':
                         binind = (mapelm & 0x1FE0) >> 5
                     debug_log("      binind = 0x%x" % binind)
 
@@ -678,7 +694,7 @@ def parse_all_runs(jeheap, read_content_preview):
 
                     if android_version == '6':
                         size = mapelm & ~flags_mask
-                    elif android_version == '7':
+                    elif android_version == '7' or android_version == '8':
                         size = (mapelm & ~0x1FFF) >> 1
                     debug_log("      size = 0x%x" % size)
 
@@ -1046,7 +1062,7 @@ def parse_tcache(addr, mem, tid):
 
         stack_size = ncached_max * dword_size
 
-        if android_version == '7':
+        if android_version == '7' or android_version == '8':
             avail_off = avail - addr - (ncached_max * dword_size)
             stack_mem = mem[avail_off:avail_off+stack_size]
             stack = []
@@ -1120,7 +1136,7 @@ def pageind(ptr):
     # print('[shadow] mapbits success')
     if android_version == '6':
         binind = (mapbits & 0xFF0) >> 4
-    elif android_version== '7':
+    elif android_version == '7' or android_version == '8':
         binind = (mapbits & 0x1FE0) >> 5
 
     # print('[shadow] fake binind = 0x%x' % binind)
@@ -1714,7 +1730,6 @@ def dump_regions(size_class):
     print(ascii_table(table))
 
 
-
 def dump_run(addr, view_maps=False):
     global jeheap
 
@@ -1795,7 +1810,7 @@ def dump_run(addr, view_maps=False):
     else:
         table = [("*", "status", "address", "preview")]
 
-    data_fmt_str = "%" + ("0%d" % (jeheap.dword_size * 2)) + "x" # damn :(
+    data_fmt_str = "%" + ("0%d" % (jeheap.dword_size * 2)) + "x"
     for region in run.regions:
         if region.is_free:
             status = "free"