Add unit tests to check the behaviour of mutators matches expectations.

The mutators weren't being tested so we only had to assume that they
were doing the right thing. In my own tests I saw a lot of input that
had NUL bytes in it, so I'm pretty sure that some of the mutations were
not doing the right thing. With these tests, the copy method was found
to have poorly named parameters.

The pattern it's using is copy(a, b, posa, posb, lena, lenb) (where lena
and lenb can be omitted). The 'a' parameter is where the update will
take place, and the 'b' parameter is where the copy originates. However,
the first parameter was called 'src' and the second parameter 'dst',
which is the precise opposite of the expectation. The naming of the
parameters was kept consistent (a named as src, despite being the
destination) with the other named parameters and within the function.

This has been corrected, and with the correct naming, it became obvious
that the insert, remove and duplicate functions were not working as
intended.

It is unclear what the difference is intended to be for Duplicate and
Copy bytes - I have 'fixed' Duplicate, but this means that it now works
identically to Copy, so it's not clear to me what's meant to be done
there.

pythonfuzz/corpus.py

@@ -49,22 +49,27 @@ def _rand(n):
             return 0
         return random.randint(0, n-1)
 
-    @staticmethod
-    def _choose_len(n):
-        x = Corpus._rand(100)
+    @classmethod
+    def _choose_len(cls, n):
+        x = cls._rand(100)
         if x < 90:
-            return Corpus._rand(min(8, n)) + 1
+            return cls._rand(min(8, n)) + 1
         elif x < 99:
-            return Corpus._rand(min(32, n)) + 1
+            return cls._rand(min(32, n)) + 1
         else:
-            return Corpus._rand(n) + 1
+            return cls._rand(n) + 1
 
     @staticmethod
-    def copy(src, dst, start_source, start_dst, end_source=None, end_dst=None):
-        end_source = len(src) if end_source is None else end_source
+    def copy(dst, src, start_dst, start_src, end_dst=None, end_src=None):
+        """
+        Copy of content from one slice of a source object to a destination object.
+
+        dst and src may be the same object.
+        """
+        end_src = len(src) if end_src is None else end_src
         end_dst = len(dst) if end_dst is None else end_dst
-        byte_to_copy = min(end_source-start_source, end_dst-start_dst)
-        src[start_source:start_source+byte_to_copy] = dst[start_dst:start_dst+byte_to_copy]
+        byte_to_copy = min(end_src-start_src, end_dst-start_dst)
+        dst[start_dst:start_dst+byte_to_copy] = src[start_src:start_src+byte_to_copy]
 
     def mutate(self, res):
         """
@@ -87,9 +92,10 @@ def mutate(self, res):
             return None
 
         pos0 = self._rand(len(res))
-        pos1 = pos0 + self._choose_len(len(res) - pos0)
-        self.copy(res, res, pos1, pos0)
-        return res[:len(res) - (pos1-pos0)]
+        num_to_remove = self._choose_len(len(res) - pos0)
+        pos1 = pos0 + num_to_remove
+        self.copy(res, res, pos0, pos1)
+        return res[:len(res) - num_to_remove]
 
 
 @register_mutator
@@ -102,7 +108,7 @@ def mutate(self, res):
         n = self._choose_len(10)
         for k in range(n):
             res.append(0)
-        self.copy(res, res, pos, pos+n)
+        self.copy(res, res, pos+n, pos)
         for k in range(n):
             res[pos+k] = self._rand(256)
         return res
@@ -121,18 +127,18 @@ def mutate(self, res):
         while src == dst:
             dst = self._rand(len(res))
         n = self._choose_len(len(res) - src)
-        tmp = bytearray(n)
-        self.copy(res, tmp, src, 0)
+        tmp = bytearray(res[src:src+n])
         for k in range(n):
             res.append(0)
-        self.copy(res, res, dst, dst+n)
+        self.copy(res, res, dst+n, dst)
         for k in range(n):
             res[dst+k] = tmp[k]
         return res
 
 
 @register_mutator
 class MutatorCopyBytes(Mutator):
+    # FIXME: Check how this diffs from DuplicateBytes
     name = 'Copy a range of bytes'
     types = set(['byte', 'copy'])
 
@@ -170,6 +176,7 @@ def mutate(self, res):
         if len(res) == 0:
             return None
         pos = self._rand(len(res))
+        # We use rand(255) + 1 so that there is no `^ 0` applied to the byte; it always changes.
         res[pos] ^= self._rand(255) + 1
         return res
 
@@ -492,6 +499,7 @@ def generate_input(self):
     def mutate(self, buf):
         res = buf[:]
         nm = self._rand_exp()
+        #print("Start with {}".format(res))
         for i in range(nm):
 
             # Select a mutator from those we can apply
@@ -501,6 +509,7 @@ def mutate(self, buf):
                 x = self._rand(len(self.mutators))
                 mutator = self.mutators[x]
 
+                #print("Mutate with {}".format(mutator.__class__.__name__))
                 newres = mutator.mutate(res)
                 if newres is not None:
                     break

tests/unittest_mutators.py

@@ -0,0 +1,198 @@
+import unittest
+
+try:
+    from unittest.mock import patch
+except ImportError:
+    # Python 2 backport of mock
+    from mock import patch
+
+import pythonfuzz.corpus as corpus
+
+
+class FakeCorpus(object):
+    pass
+
+
+class BaseTestMutators(unittest.TestCase):
+    """
+    Test that the mutators objects are doing what we want them to do.
+    """
+    # Subclasses should set this - 'mutator' will be created as part of setup.
+    mutator_class = None
+
+    def setUp(self):
+        self.corpus = FakeCorpus()
+        self.patch_rand = patch('pythonfuzz.corpus.Mutator._rand')
+        self.mock_rand = self.patch_rand.start()
+        self.mock_rand.side_effect = []
+        # Update the side effects in your subclass
+
+        self.addCleanup(self.patch_rand.stop)
+
+        self.mutator = self.mutator_class(self.corpus)
+
+
+class TestMutatorRemoveRange(BaseTestMutators):
+    mutator_class = corpus.MutatorRemoveRange
+
+    def test01_empty(self):
+        # You cannot remove values from an empty input
+        res = self.mutator.mutate(bytearray(b''))
+        self.assertIsNone(res)
+
+    def test02_remove_section(self):
+        # Check that it removes a sensible range
+
+        # Check that removing at the 2nd position, removing 4 characters leaves the right string.
+        self.mock_rand.side_effect = [2, 0, 3]
+
+        res = self.mutator.mutate(bytearray(b'1234567890'))
+        self.assertEqual(res, bytearray(b'127890'))
+
+
+class TestMutatorInsertBytes(BaseTestMutators):
+    mutator_class = corpus.MutatorInsertBytes
+
+    def test02_insert_bytes(self):
+        # Check that it inserts sensibly
+
+        # Check that inserting at the 2nd position, adding 4 characters gives us the right string
+        self.mock_rand.side_effect = [2, 0, 3, 65, 66, 67, 68]
+
+        res = self.mutator.mutate(bytearray(b'123456789'))
+        self.assertEqual(res, bytearray(b'12ABCD3456789'))
+
+
+class TestMutatorDuplicateBytes(BaseTestMutators):
+    mutator_class = corpus.MutatorDuplicateBytes
+
+    def test01_empty(self):
+        # Cannot work with an empty input
+        res = self.mutator.mutate(bytearray(b''))
+        self.assertIsNone(res)
+
+    def test02_duplicate_bytes(self):
+        # Check that it duplicates
+
+        # Duplicate from offset 2 to offset 5, length 2
+        self.mock_rand.side_effect = [2, 5, 0, 1]
+
+        res = self.mutator.mutate(bytearray(b'123456789'))
+        self.assertEqual(res, bytearray(b'12345346789'))
+
+
+class TestMutatorCopyBytes(BaseTestMutators):
+    mutator_class = corpus.MutatorDuplicateBytes
+
+    def test01_empty(self):
+        # Cannot work with an empty input
+        res = self.mutator.mutate(bytearray(b''))
+        self.assertIsNone(res)
+
+    def test02_duplicate_bytes(self):
+        # Check that it duplicates
+
+        # Duplicate from offset 2 to offset 5, length 2
+        self.mock_rand.side_effect = [2, 5, 0, 1]
+
+        res = self.mutator.mutate(bytearray(b'123456789'))
+        self.assertEqual(res, bytearray(b'12345346789'))
+
+
+class TestMutatorBitFlip(BaseTestMutators):
+    mutator_class = corpus.MutatorBitFlip
+
+    def test01_empty(self):
+        # Cannot work with an empty input
+        res = self.mutator.mutate(bytearray(b''))
+        self.assertIsNone(res)
+
+    def test02_flip_bit(self):
+        # Check that it flips
+
+        # At offset 4, flip bit 3
+        self.mock_rand.side_effect = [4, 3]
+
+        res = self.mutator.mutate(bytearray(b'123456789'))
+        self.assertEqual(res, bytearray(b'1234=6789'))
+
+
+class TestMutatorRandomiseByte(BaseTestMutators):
+    mutator_class = corpus.MutatorRandomiseByte
+
+    def test01_empty(self):
+        # Cannot work with an empty input
+        res = self.mutator.mutate(bytearray(b''))
+        self.assertIsNone(res)
+
+    def test02_randomise_byte(self):
+        # Check that it changes a byte
+
+        # At offset 4, EOR with 65+1
+        self.mock_rand.side_effect = [4, 65]
+
+        res = self.mutator.mutate(bytearray(b'123456789'))
+        self.assertEqual(res, bytearray(b'1234w6789'))
+
+
+class TestMutatorSwapBytes(BaseTestMutators):
+    mutator_class = corpus.MutatorSwapBytes
+
+    def test01_empty(self):
+        # Cannot work with an empty input
+        res = self.mutator.mutate(bytearray(b''))
+        self.assertIsNone(res)
+
+    def test02_swap_bytes(self):
+        # Check that it swaps bytes
+
+        # Swap bytes at 1 and 6
+        self.mock_rand.side_effect = [1, 6]
+
+        res = self.mutator.mutate(bytearray(b'123456789'))
+        self.assertEqual(res, bytearray(b'173456289'))
+
+
+class TestMutatorAddSubByte(BaseTestMutators):
+    mutator_class = corpus.MutatorAddSubByte
+
+    def test01_empty(self):
+        # Cannot work with an empty input
+        res = self.mutator.mutate(bytearray(b''))
+        self.assertIsNone(res)
+
+    def test02_add_bytes(self):
+        # Check that it adds/subs
+        # FIXME: Not yet implemented - uses a randomised bit for the add/sub
+        pass
+
+# FIXME: Also not implemented AddSubShort, AddSubLong, AddSubLongLong
+# FIXME: Not yet implemented ReplaceByte, ReplaceShort, ReplaceLong
+
+
+class TestMutatorReplaceDigit(BaseTestMutators):
+    mutator_class = corpus.MutatorReplaceDigit
+
+    def test01_empty(self):
+        # Cannot work with an empty input
+        res = self.mutator.mutate(bytearray(b''))
+        self.assertIsNone(res)
+
+    def test02_no_digits(self):
+        # Cannot work with a string that has no digits
+        res = self.mutator.mutate(bytearray(b'wibble'))
+        self.assertIsNone(res)
+
+    def test03_replace_digit(self):
+        # Check that it replaces a digit
+        self.mock_rand.side_effect = [0, 5]
+
+        res = self.mutator.mutate(bytearray(b'there are 4 lights'))
+        self.assertEqual(res, bytearray(b'there are 5 lights'))
+
+
+# FIXME: Not yet implemented: Dictionary insert, Dictionary Append
+
+
+if __name__ == '__main__':
+    unittest.main()