Add dictionary support for a binary load of directories of files.

Raw binary files are supported by the AFL dictionaries, and matter
most for the cases where we're dealing with binary chunks that would
otherwise be tedious to insert into the token file.

pythonfuzz/corpus.py

@@ -347,6 +347,23 @@ def mutate(self, res):
         return res
 
 
+@register_mutator
+class MutatorDictionaryWordInsert(Mutator):
+    name = 'Insert a word at a random position'
+    types = set(['text', 'dictionary'])
+
+    def mutate(self, res):
+        word = self.corpus._dict.get_word()
+        if not word:
+            return None
+        pos = self._rand(len(res) + 1)
+        for _ in word:
+            res.append(0)
+        self.copy(res, res, pos, pos+len(word))
+        for k in range(len(word)):
+            res[pos+k] = ord(word[k])
+
+
 class CorpusError(Exception):
     pass
 
@@ -355,7 +372,9 @@ class Corpus(object):
 
     def __init__(self, dirs=None, max_input_size=4096, mutators_filter=None, dict_path=None):
         self._inputs = []
-        self._dict = dictionary.Dictionary(dict_path)
+        self._dict = dictionary.Dictionary()
+        if dict_path:
+            self._dict.load(dict_path)
         self._max_input_size = max_input_size
         self._dirs = dirs if dirs else []
         for i, path in enumerate(dirs):
@@ -460,6 +479,7 @@ def mutate(self, buf):
         res = buf[:]
         nm = self._rand_exp()
         for i in range(nm):
+
             # Select a mutator from those we can apply
             # We'll try up to 20 times, but if we don't find a
             # suitable mutator after that, we'll just give up.
@@ -470,7 +490,8 @@ def mutate(self, buf):
                 newres = mutator.mutate(res)
                 if newres is not None:
                     break
-            res = newres
+            if newres is not None:
+                res = newres
 
         if len(res) > self._max_input_size:
             res = res[:self._max_input_size]

pythonfuzz/dictionary.py

@@ -16,16 +16,39 @@
 class Dictionary:
     line_re = re.compile('"(.+)"$')
 
-    def __init__(self, dict_path=None):
+    def __init__(self):
+        self._dict = list()
+
+    def load(self, dict_path):
+        if os.path.isfile(dict_path):
+            self.load_file(dict_path)
+        else:
+            self.load_directory(dict_path)
+
+    def load_directory(self, dict_path):
+        """
+        Read a directory of files, which are loaded raw.
+        """
+        for bin_file in os.listdir(dict_path):
+            filename = os.path.join(dict_path, bin_file)
+            if os.path.isfile(filename):
+                with open(filename, 'rb') as fh:
+                    self._dict.append(fh.read())
+
+    def load_file(self, dict_path):
+        """
+        Read a dictionary file containing tokens.
+        """
+        # Token names are discarded, as per the AFL documentation
+
         if not dict_path or not os.path.exists(dict_path):
-            self._dict = list()
             return
 
         _dict = set()
         with open(dict_path) as f:
             for line in f:
                 line = line.lstrip()
-                if line.startswith('#'):
+                if not line or line.startswith('#'):
                     continue
                 word = self.line_re.search(line)
                 if word:

pythonfuzz/main.py

@@ -20,6 +20,7 @@ def __call__(self, *args, **kwargs):
         parser.add_argument('--dict', type=str, help='dictionary file')
         parser.add_argument('--close-fd-mask', type=int, default=0, help='Indicate output streams to close at startup')
         parser.add_argument('--runs', type=int, default=-1, help='Number of individual test runs, -1 (the default) to run indefinitely.')
+        parser.add_argument('--help-mutators', action='store_true', help='Display help on the mutators')
         parser.add_argument('--mutator-filter', type=str, default=None, help='Filter for mutator types to use; prefix with ! to disable')
         parser.add_argument('--timeout', type=int, default=30,
                             help='If input takes longer then this timeout the process is treated as failure case')