Merge pull request #254 from fireeye/feature/shellcode

Analyze shellcode with FLOSS

doc/usage.md

@@ -17,15 +17,17 @@ Here's a summary of the command line flags and options you
  can provide to FLOSS to modify its behavior.
 
 
-### Extract obfuscated strings (default mode)
+### Extract static, obfuscated, and stack strings (default mode)
 
 The default mode for FLOSS is to extract the following string types from an executable file:
 - static ASCII and UTF16LE strings
 - obfuscated strings
 - stackstrings
 
-By default FLOSS uses a minimum string length of four.
+See the section on [Shellcode analysis](#shellcode) below on how to analyze raw binary files
+containing shellcode.
 
+By default FLOSS uses a minimum string length of four.
 
     floss.exe malware.bin
 
@@ -46,6 +48,7 @@ Analogous, you can disable the extraction of obfuscated strings or stackstrings.
     floss.exe --no-decoded-strings malware.bin
     floss.exe --no-stack-strings malware.bin
 
+
 ### Quiet mode (`-q`)
 
 You can supress the formatting of FLOSS output by providing
@@ -73,7 +76,6 @@ Supplying a larger minimum length reduces the chances
  however, FLOSS may then pass over short legitimate
  human-readable strings
 
-
     floss.exe -n 10 malware.bin
     floss.exe --minimum-length=10 malware.bin
 
@@ -146,6 +148,7 @@ in x64dbg, use the `--x64dbg` switch.
 
     floss.exe --x64dbg=myx64dbgdatabase malware.bin
 
+
 ### Verbose and debug modes (`-v`/`-d`)
 
 If FLOSS seems to encounter any issues, try re-running the program
@@ -158,7 +161,6 @@ This provides additional context if FLOSS encounters an
 The verbose mode enables a moderate amount of logging output,
  while the debug mode enables a large amount of logging output.
 
-
      floss.exe -v malware.bin
      floss.exe --verbose malware.bin
 
@@ -194,3 +196,23 @@ Manipulating the plugin list may be useful during the development
 
     floss.exe -p XORPlugin,ShiftPlugin malware.bin
     floss.exe --plugins=XORPlugin,ShiftPlugin malware.bin
+
+
+## <a name="shellcode"></a>Shellcode analysis options
+
+Malicious shellcode often times contains obfuscated strings and/or stackstrings.
+FLOSS can analyze raw binary files containing shellcode via the `-s` switch. All
+options mentioned above can also be applied when analyzing shellcode.
+
+    floss.exe -s malware.bin
+
+If you want to specify a base address for the shellcode, use the the `-b` or
+`--shellcode_base` switch.
+
+    floss.exe -s malware.bin -b 0x1000000
+
+You can specify an entry point for the shellcode with the `-e` or `--shellcode_ep`
+option. Although vivisect does a good job identifying code, providing an entry point
+might improve code analysis.
+
+    floss.exe -s malware.bin -b 0x1000000 -e 0x100

floss/main.py

@@ -125,6 +125,15 @@ def make_parser():
     parser.add_option("--save-workspace", dest="save_workspace",
                       help="save vivisect .viv workspace file in current directory", action="store_true")
 
+    shellcode_group = OptionGroup(parser, "Shellcode options", "Analyze raw binary file containing shellcode")
+    shellcode_group.add_option("-s", "--shellcode", dest="is_shellcode", help="analyze shellcode",
+                      action="store_true")
+    shellcode_group.add_option("-e", "--shellcode_ep", dest="shellcode_entry_point",
+                      help="shellcode entry point", type="string")
+    shellcode_group.add_option("-b", "--shellcode_base", dest="shellcode_base",
+                      help="shellcode base offset", type="string")
+    parser.add_option_group(shellcode_group)
+
     extraction_group = OptionGroup(parser, "Extraction options", "Specify which string types FLOSS shows from a file, "
                                                                  "by default all types are shown")
     extraction_group.add_option("--no-static-strings", dest="no_static_strings", action="store_true",
@@ -687,10 +696,13 @@ def main(argv=None):
     sample_file_path = parse_sample_file_path(parser, args)
     min_length = parse_min_length_option(options.min_length)
 
-    if not is_workspace_file(sample_file_path):
-        with open(sample_file_path, "rb") as f:
-            magic = f.read(2)
+    # expert profile settings
+    if options.expert:
+        options.save_workspace = True
+        options.group_functions = True
+        options.quiet = False
 
+    if not is_workspace_file(sample_file_path):
         if not options.no_static_strings and not options.functions:
             floss_logger.info("Extracting static strings...")
             print_static_strings(sample_file_path, min_length=min_length, quiet=options.quiet)
@@ -699,29 +711,49 @@ def main(argv=None):
             # we are done
             return 0
 
-        if magic not in SUPPORTED_FILE_MAGIC:
-            floss_logger.error("FLOSS currently supports the following formats for string decoding and stackstrings: PE")
-            return 1
+    if options.is_shellcode:
+        shellcode_entry_point = 0
+        if options.shellcode_entry_point:
+            shellcode_entry_point = int(options.shellcode_entry_point, 0x10)
 
-        if os.path.getsize(sample_file_path) > MAX_FILE_SIZE:
-            floss_logger.error("FLOSS cannot extract obfuscated strings from files larger than %d bytes" % (MAX_FILE_SIZE))
-            return 1
+        shellcode_base = 0
+        if options.shellcode_base:
+            shellcode_base = int(options.shellcode_base, 0x10)
 
-        floss_logger.info("Generating vivisect workspace...")
+        try:
+            floss_logger.info("Generating vivisect workspace for shellcode, base: 0x%x, entry point: 0x%x...", shellcode_base,
+                              shellcode_entry_point)
+            with open(sample_file_path, "rb") as f:
+                shellcode_data = f.read()
+            vw = viv_utils.getShellcodeWorkspace(shellcode_data, "i386", shellcode_base, shellcode_entry_point,
+                                                 options.save_workspace, sample_file_path)
+        except Exception, e:
+            floss_logger.error("Vivisect failed to load the input file: {0}".format(e.message),
+                               exc_info=options.verbose)
+            return 1
     else:
-        floss_logger.info("Loading existing vivisect workspace...")
+        if not is_workspace_file(sample_file_path):
+            with open(sample_file_path, "rb") as f:
+                magic = f.read(2)
 
-    # expert profile settings
-    if options.expert:
-        options.save_workspace = True
-        options.group_functions = True
-        options.quiet = False
+            if magic not in SUPPORTED_FILE_MAGIC:
+                floss_logger.error("FLOSS currently supports the following formats for string decoding and stackstrings: PE\n"
+                                   "You can analyze shellcode using the -s switch. See the help (-h) for more information.")
+                return 1
 
-    try:
-        vw = viv_utils.getWorkspace(sample_file_path, should_save=options.save_workspace)
-    except Exception, e:
-        floss_logger.error("Vivisect failed to load the input file: {0}".format(e.message), exc_info=options.verbose)
-        return 1
+            if os.path.getsize(sample_file_path) > MAX_FILE_SIZE:
+                floss_logger.error("FLOSS cannot extract obfuscated strings from files larger than %d bytes" % (MAX_FILE_SIZE))
+                return 1
+
+            floss_logger.info("Generating vivisect workspace...")
+        else:
+            floss_logger.info("Loading existing vivisect workspace...")
+
+        try:
+            vw = viv_utils.getWorkspace(sample_file_path, should_save=options.save_workspace)
+        except Exception, e:
+            floss_logger.error("Vivisect failed to load the input file: {0}".format(e.message), exc_info=options.verbose)
+            return 1
 
     selected_functions = select_functions(vw, options.functions)
     floss_logger.debug("Selected the following functions: %s", ", ".join(map(hex, selected_functions)))

floss/utils.py

@@ -26,3 +26,4 @@ def removeStackMemory(emu):
             emu.stack_map_base = None
             return
     raise Exception  # STACK_MEM_NAME not in memory map
+

tests/conftest.py

@@ -4,17 +4,15 @@
 
 import viv_utils
 
-import footer
 import floss.main as floss_main
 import floss.identification_manager as im
 import floss.stackstrings as stackstrings
 
 
-def extract_strings(sample_path):
+def extract_strings(vw):
     """
-    Deobfuscate strings from sample_path
+    Deobfuscate strings from vivisect workspace
     """
-    vw = viv_utils.getWorkspace(sample_path)
     function_index = viv_utils.InstructionFunctionIndex(vw)
     decoding_functions_candidates = identify_decoding_functions(vw)
     decoded_strings = floss_main.decode_strings(vw, function_index, decoding_functions_candidates)
@@ -90,27 +88,28 @@ def __init__(self, path, platform, arch, filename, spec):
         self.filename = filename
 
     def _test_strings(self, test_path):
-        if footer.has_footer(test_path):
-            expected_strings = set(footer.read_footer(test_path)["all"])
-        else:
-            expected_strings = set(self.spec["Decoded strings"])
-
+        expected_strings = set(self.spec["Decoded strings"])
         if not expected_strings:
             return
 
-        found_strings = set(extract_strings(test_path))
+        test_shellcode = self.spec.get("Test shellcode")
+        if test_shellcode:
+            with open(test_path, "rb") as f:
+                shellcode_data = f.read()
+            vw = viv_utils.getShellcodeWorkspace(shellcode_data)  # TODO provide arch from test.yml
+            found_strings = set(extract_strings(vw))
+        else:
+            vw = viv_utils.getWorkspace(test_path)
+            found_strings = set(extract_strings(vw))
 
         if not (expected_strings <= found_strings):
             raise FLOSSStringsNotExtracted(expected_strings, found_strings)
 
     def _test_detection(self, test_path):
-        if footer.has_footer(test_path):
-            expected_functions = set(footer.read_footer(test_path).keys()) - set("all")
-        else:
-            try:
-                expected_functions = set(self.spec["Decoding routines"][self.platform][self.arch])
-            except KeyError:
-                expected_functions = set([])
+        try:
+            expected_functions = set(self.spec["Decoding routines"][self.platform][self.arch])
+        except KeyError:
+            expected_functions = set([])
 
         if not expected_functions:
             return
@@ -144,10 +143,8 @@ def repr_failure(self, excinfo):
         if isinstance(excinfo.value, FLOSSStringsNotExtracted):
             expected = excinfo.value.expected
             got = excinfo.value.got
-            missing = map(str, expected - got)
             return "\n".join([
                 "FLOSS extraction failed:",
                 "   expected: %s" % str(expected),
                 "   got: %s" % str(got),
-                "   missing expected strings:\n\t%s" % "\n\t".join(missing),
             ])

tests/src/shellcode-stackstrings/test.yml

@@ -0,0 +1,21 @@
+Test Name: shellcode-stackstrings
+Test Purpose: real world shellcode sample using stackstrings
+Added on: '2016-10-29 19:57:33'
+Test shellcode: 'True'
+Decoded strings:
+- '%d/%02d/%02d %02d:%02d:%02d'  # ----------- stackstrings
+- Result is empty!
+- Checksum error!
+- Incompatible module received!
+- Invalid parameter!
+- Failed to load module!
+- Activation is required!
+- Module is not found!
+- Invalid data received!
+Output Files:
+  Windows:
+    x86: bin/shellcode-stackstrings.bin
+Decoding routines:
+  Windows:
+    x86: ''
+FLOSS running time: 0.463015 seconds