Add a harness to run all the example fuzzers.

The example fuzzers provide a useful way to see how you might use
the pythonfuzz package. However, they're no use if they don't work.
Adding a tool that is able to run them all, and report on their
status will ensure that these do not get broken in the future.

The tool is able to be configured with the number of runs that it
expects to execute, and whether it will keep any crash or timeout
files that are generated. At the end of the run of each example,
it will report the collected information from the run. This should
be useful for checking the performance in the future, as well.

If any of the examples fails to function - ie it reports an error
itself, then the runner will exit with a non-0 return code. This
will allow it to be used as a gate for the examples being functional.

examples/run_all_examples.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python
+"""
+Run all the examples and collect the timings and results.
+"""
+
+import argparse
+import os.path
+import re
+import subprocess
+import sys
+import time
+
+here = os.path.dirname(__file__)
+
+
+class Result(object):
+    """
+    Use the output from the tool to collect the information about its execution.
+
+    Very sensitive to the format of the output as to whether it collects information or not.
+    """
+    coverage_re = re.compile('cov: (\d+)')
+    corpus_re = re.compile('corp: (\d+)')
+    speed_re = re.compile('exec/s: (\d+)')
+    memory_re = re.compile('rss: (\d+\.\d+)')
+    count_re = re.compile('^#(\d+)')
+    count2_re = re.compile('^did (\d+) runs, stopping now')
+    exception_re = re.compile('^Exception: (.*)')
+    failfile_re = re.compile('^sample written to (.*)')
+
+    def __init__(self):
+        self.coverage = None
+        self.corpus = None
+        self.speed = None
+        self.memory = None
+        self.count = None
+        self.time_start = None
+        self.time_end = None
+        self.fail_file = None
+        self.exception = None
+        self.lines = []
+        self.rc = None
+
+    def record_start(self):
+        self.time_start = time.time()
+
+    def record_end(self):
+        self.time_end = time.time()
+
+    @property
+    def time_duration(self):
+        """
+        Number of seconds the execution took, or None if not known
+        """
+        if self.time_start and self.time_end:
+            return self.time_end - self.time_start
+        if self.time_start:
+            return time.time() - self.time_start
+
+        return None
+
+    def process_output(self, line):
+        match = self.coverage_re.search(line)
+        if match:
+            self.coverage = int(match.group(1))
+
+        match = self.corpus_re.search(line)
+        if match:
+            self.corpus = int(match.group(1))
+
+        match = self.speed_re.search(line)
+        if match:
+            self.speed = int(match.group(1))
+
+        match = self.memory_re.search(line)
+        if match:
+            self.memory = float(match.group(1))
+
+        match = self.count_re.search(line) or self.count2_re.search(line)
+        if match:
+            self.count = int(match.group(1))
+
+        match = self.exception_re.search(line)
+        if match:
+            self.exception = match.group(1)
+
+        match = self.failfile_re.search(line)
+        if match:
+            self.fail_file = match.group(1)
+
+        self.lines.append(line)
+
+    def show(self, show_lines=False, indent=''):
+        """
+        Show the status of this result.
+        """
+        print("{}Executions       : {}".format(indent, self.count))
+        print("{}Corpus           : {}".format(indent, self.corpus))
+        print("{}Coverage         : {}".format(indent, self.coverage))
+        print("{}Final speed      : {}/s".format(indent, self.speed))
+        if self.memory:
+            print("{}Memory           : {:.2f} MB".format(indent, self.memory))
+        print("{}Runtime          : {:.2f} s".format(indent, self.time_duration))
+        if self.time_duration and self.count:
+            print("{}Overall speed    : {:.2f}/s".format(indent, self.count / self.time_duration))
+        print("{}Return code      : {}".format(indent, self.rc))
+        if self.exception:
+            print("{}Exception        : {}".format(indent, self.exception))
+        if self.fail_file:
+            print("{}Failed filename  : {}".format(indent, self.fail_file))
+
+        if show_lines or self.rc:
+            print("{}Lines:".format(indent))
+            for line in self.lines:
+                print("{}  {}".format(indent, line.strip('\n')))
+
+
+class Example(object):
+
+    def __init__(self, name, path, script):
+        self.name = name
+        self.path = path
+        self.script = script
+
+    def run(self, python='python', runs=100, log='/dev/null'):
+        """
+        Run the example script, capturing the output and maybe processing it.
+        """
+        cmd = [python, self.script, '--runs', str(runs)]
+
+        result = Result()
+        with open(log, 'w') as log_fh:
+            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+            result.record_start()
+            for line in proc.stdout:
+                line = line.decode('utf-8', 'replace')
+                result.process_output(line)
+            result.record_end()
+
+            proc.wait()
+            result.rc = proc.returncode
+
+        return result
+
+
+def find_examples():
+    examples = []
+    print("Looking in {} for examples".format(here))
+    for obj in os.listdir(here):
+        path = os.path.join(here, obj)
+        if os.path.isdir(path):
+            # At a later date we might actually provide multiple fuzz's per example.
+            fuzz = os.path.join(path, 'fuzz.py')
+            if os.path.isfile(fuzz):
+                examples.append(Example(obj, path, fuzz))
+
+    return examples
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Exercise the example code')
+    parser.add_argument('--runs', type=int, default=1000, help='Number of individual test runs.')
+    parser.add_argument('--keep', action='store_true', help='Keep the crash/timeout files')
+
+    args = parser.parse_args()
+
+    # We remember whether the example itself failed (not the underlying module being fuzz'd)
+    # so that we can fail this run.
+    any_failed = False
+    for example in find_examples():
+        print("Example: {}".format(example.name))
+        result = example.run(runs=args.runs)
+        result.show(indent='  ')
+        if not args.keep:
+            if result.fail_file:
+                os.remove(result.fail_file)
+        if result.rc != 0:
+            any_failed = True
+
+    sys.exit(1 if any_failed else 0)
+
+
+if __name__ == '__main__':
+    main()