Tidy up, and add harness for examples

examples/htmlparser/fuzz.py

@@ -1,4 +1,8 @@
-from html.parser import HTMLParser
+try:
+    from html.parser import HTMLParser
+except ImportError:
+    from HTMLParser import HTMLParser
+
 from pythonfuzz.main import PythonFuzz
 
 

examples/run_all_examples.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python
+"""
+Run all the examples and collect the timings and results.
+"""
+
+import argparse
+import os.path
+import re
+import subprocess
+import sys
+import time
+
+here = os.path.dirname(__file__)
+
+
+class Result(object):
+    """
+    Use the output from the tool to collect the information about its execution.
+
+    Very sensitive to the format of the output as to whether it collects information or not.
+    """
+    coverage_re = re.compile('cov: (\d+)')
+    corpus_re = re.compile('corp: (\d+)')
+    speed_re = re.compile('exec/s: (\d+)')
+    memory_re = re.compile('rss: (\d+\.\d+)')
+    count_re = re.compile('^#(\d+)')
+    count2_re = re.compile('^did (\d+) runs, stopping now')
+    exception_re = re.compile('^Exception: (.*)')
+    failfile_re = re.compile('^sample written to (.*)')
+
+    def __init__(self):
+        self.coverage = None
+        self.corpus = None
+        self.speed = None
+        self.memory = None
+        self.count = None
+        self.time_start = None
+        self.time_end = None
+        self.fail_file = None
+        self.exception = None
+        self.lines = []
+        self.rc = None
+
+    def record_start(self):
+        self.time_start = time.time()
+
+    def record_end(self):
+        self.time_end = time.time()
+
+    @property
+    def time_duration(self):
+        """
+        Number of seconds the execution took, or None if not known
+        """
+        if self.time_start and self.time_end:
+            return self.time_end - self.time_start
+        if self.time_start:
+            return time.time() - self.time_start
+
+        return None
+
+    def process_output(self, line):
+        match = self.coverage_re.search(line)
+        if match:
+            self.coverage = int(match.group(1))
+
+        match = self.corpus_re.search(line)
+        if match:
+            self.corpus = int(match.group(1))
+
+        match = self.speed_re.search(line)
+        if match:
+            self.speed = int(match.group(1))
+
+        match = self.memory_re.search(line)
+        if match:
+            self.memory = float(match.group(1))
+
+        match = self.count_re.search(line) or self.count2_re.search(line)
+        if match:
+            self.count = int(match.group(1))
+
+        match = self.exception_re.search(line)
+        if match:
+            self.exception = match.group(1)
+
+        match = self.failfile_re.search(line)
+        if match:
+            self.fail_file = match.group(1)
+
+        self.lines.append(line)
+
+    def show(self, show_lines=False, indent=''):
+        """
+        Show the status of this result.
+        """
+        print("{}Executions       : {}".format(indent, self.count))
+        print("{}Corpus           : {}".format(indent, self.corpus))
+        print("{}Coverage         : {}".format(indent, self.coverage))
+        print("{}Final speed      : {}/s".format(indent, self.speed))
+        if self.memory:
+            print("{}Memory           : {:.2f} MB".format(indent, self.memory))
+        print("{}Runtime          : {:.2f} s".format(indent, self.time_duration))
+        if self.time_duration and self.count:
+            print("{}Overall speed    : {:.2f}/s".format(indent, self.count / self.time_duration))
+        print("{}Return code      : {}".format(indent, self.rc))
+        if self.exception:
+            print("{}Exception        : {}".format(indent, self.exception))
+        if self.fail_file:
+            print("{}Failed filename  : {}".format(indent, self.fail_file))
+
+        if show_lines or self.rc:
+            print("{}Lines:".format(indent))
+            for line in self.lines:
+                print("{}  {}".format(indent, line.strip('\n')))
+
+
+class Example(object):
+
+    def __init__(self, name, path, script):
+        self.name = name
+        self.path = path
+        self.script = script
+
+    def run(self, python='python', runs=100, log='/dev/null'):
+        """
+        Run the example script, capturing the output and maybe processing it.
+        """
+        cmd = [python, self.script, '--runs', str(runs)]
+
+        result = Result()
+        with open(log, 'w') as log_fh:
+            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+            result.record_start()
+            for line in proc.stdout:
+                line = line.decode('utf-8', 'replace')
+                result.process_output(line)
+            result.record_end()
+
+            proc.wait()
+            result.rc = proc.returncode
+
+        return result
+
+
+def find_examples():
+    examples = []
+    print("Looking in {} for examples".format(here))
+    for obj in os.listdir(here):
+        path = os.path.join(here, obj)
+        if os.path.isdir(path):
+            # At a later date we might actually provide multiple fuzz's per example.
+            fuzz = os.path.join(path, 'fuzz.py')
+            if os.path.isfile(fuzz):
+                examples.append(Example(obj, path, fuzz))
+
+    return examples
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Exercise the example code')
+    parser.add_argument('--runs', type=int, default=1000, help='Number of individual test runs.')
+    parser.add_argument('--keep', action='store_true', help='Keep the crash/timeout files')
+
+    args = parser.parse_args()
+
+    # We remember whether the example itself failed (not the underlying module being fuzz'd)
+    # so that we can fail this run.
+    any_failed = False
+    for example in find_examples():
+        print("Example: {}".format(example.name))
+        result = example.run(python=sys.executable, runs=args.runs)
+        result.show(indent='  ')
+        if not args.keep:
+            if result.fail_file:
+                os.remove(result.fail_file)
+        if result.rc != 0:
+            any_failed = True
+
+    sys.exit(1 if any_failed else 0)
+
+
+if __name__ == '__main__':
+    main()

examples/xml/fuzz.py

@@ -1,12 +1,19 @@
 import xml.etree.ElementTree as ET
 from xml.etree.ElementTree import ParseError
+import sys
+
 from pythonfuzz.main import PythonFuzz
 
 
 @PythonFuzz
 def fuzz(buf):
     try:
-        string = buf.decode("utf-8")
+        # In Python 2, the ElementTree only consumes bytes, not unicode strings,
+        # so we need to supply the correct format depending on the python version.
+        if sys.version_info[0] == 2:
+            string = bytes(buf)
+        else:
+            string = buf.decode("utf-8")
         ET.fromstring(string)
     except (UnicodeDecodeError, ParseError):
         pass

examples/zipfile/fuzz.py

@@ -0,0 +1,23 @@
+import io
+import zipfile
+from pythonfuzz.main import PythonFuzz
+
+try:
+    allowed_exceptions = (zipfile.BadZipFile, zipfile.LargeZipFile)
+except AttributeError:
+    # In Python2, one of these had an inconsistent capitalisation
+    allowed_exceptions = (zipfile.BadZipfile, zipfile.LargeZipFile)
+
+
+@PythonFuzz
+def fuzz(buf):
+    f = io.BytesIO(buf)
+    try:
+        z = zipfile.ZipFile(f)
+        z.testzip()
+    except :
+        pass
+
+
+if __name__ == '__main__':
+    fuzz()