Add mutator unittests

Makefile

@@ -24,7 +24,8 @@ ACTIVATE = source venv/${PYTHON_TOOL}/bin/activate
 
 .PHONY: tests venv
 
-TEST_MODULES = ${patsubst tests/%.py,%,$(wildcard tests/test_*.py)}
+UNITTEST_MODULES = ${patsubst tests/%.py,%,$(wildcard tests/unittest_*.py)}
+INTTEST_MODULES = ${patsubst tests/%.py,%,$(wildcard tests/test_*.py)}
 
 ifeq (${NOCOLOUR},)
 COL_NOTICE = "\\e[35m"
@@ -51,14 +52,17 @@ test_level_system: test_level_integration systemtests
 # Unit tests test individual parse of a small unit.
 unittests: test_testable
 	${NOTICE} "Running unit tests"
-	${GOOD} "Unit tests passed (we don't have any yet)"
+	@# Note: We cd into the tests directory, so that we are testing the installed version, not
+	@# 		the version in the repository.
+	${ACTIVATE} && cd tests && python -munittest -v ${UNITTEST_MODULES}
+	${GOOD} "Unit tests passed"
 
 # Integration tests check the integration of those units.
 integrationtests: test_testable
 	${NOTICE} "Running integration tests"
 	@# Note: We cd into the tests directory, so that we are testing the installed version, not
 	@# 		the version in the repository.
-	${ACTIVATE} && cd tests && python -munittest -v ${TEST_MODULES}
+	${ACTIVATE} && cd tests && python -munittest -v ${INTTEST_MODULES}
 	${GOOD} "Integration tests passed"
 
 # System tests check that the way that a user might use it works.

examples/run_all_examples.py

@@ -1,6 +1,11 @@
 #!/usr/bin/env python
 """
 Run all the examples and collect the timings and results.
+
+SUT:    Invocation
+Area:   Examples run
+Class:  Functional
+Type:   System test
 """
 
 import argparse

pythonfuzz/corpus.py

@@ -49,22 +49,27 @@ def _rand(n):
             return 0
         return random.randint(0, n-1)
 
-    @staticmethod
-    def _choose_len(n):
-        x = Corpus._rand(100)
+    @classmethod
+    def _choose_len(cls, n):
+        x = cls._rand(100)
         if x < 90:
-            return Corpus._rand(min(8, n)) + 1
+            return cls._rand(min(8, n)) + 1
         elif x < 99:
-            return Corpus._rand(min(32, n)) + 1
+            return cls._rand(min(32, n)) + 1
         else:
-            return Corpus._rand(n) + 1
+            return cls._rand(n) + 1
 
     @staticmethod
-    def copy(src, dst, start_source, start_dst, end_source=None, end_dst=None):
-        end_source = len(src) if end_source is None else end_source
+    def copy(dst, src, start_dst, start_src, end_dst=None, end_src=None):
+        """
+        Copy of content from one slice of a source object to a destination object.
+
+        dst and src may be the same object.
+        """
+        end_src = len(src) if end_src is None else end_src
         end_dst = len(dst) if end_dst is None else end_dst
-        byte_to_copy = min(end_source-start_source, end_dst-start_dst)
-        src[start_source:start_source+byte_to_copy] = dst[start_dst:start_dst+byte_to_copy]
+        byte_to_copy = min(end_src-start_src, end_dst-start_dst)
+        dst[start_dst:start_dst+byte_to_copy] = src[start_src:start_src+byte_to_copy]
 
     def mutate(self, res):
         """
@@ -87,9 +92,10 @@ def mutate(self, res):
             return None
 
         pos0 = self._rand(len(res))
-        pos1 = pos0 + self._choose_len(len(res) - pos0)
-        self.copy(res, res, pos1, pos0)
-        return res[:len(res) - (pos1-pos0)]
+        num_to_remove = self._choose_len(len(res) - pos0)
+        pos1 = pos0 + num_to_remove
+        self.copy(res, res, pos0, pos1)
+        return res[:len(res) - num_to_remove]
 
 
 @register_mutator
@@ -102,7 +108,7 @@ def mutate(self, res):
         n = self._choose_len(10)
         for k in range(n):
             res.append(0)
-        self.copy(res, res, pos, pos+n)
+        self.copy(res, res, pos+n, pos)
         for k in range(n):
             res[pos+k] = self._rand(256)
         return res
@@ -121,18 +127,18 @@ def mutate(self, res):
         while src == dst:
             dst = self._rand(len(res))
         n = self._choose_len(len(res) - src)
-        tmp = bytearray(n)
-        self.copy(res, tmp, src, 0)
+        tmp = bytearray(res[src:src+n])
         for k in range(n):
             res.append(0)
-        self.copy(res, res, dst, dst+n)
+        self.copy(res, res, dst+n, dst)
         for k in range(n):
             res[dst+k] = tmp[k]
         return res
 
 
 @register_mutator
 class MutatorCopyBytes(Mutator):
+    # FIXME: Check how this diffs from DuplicateBytes
     name = 'Copy a range of bytes'
     types = set(['byte', 'copy'])
 
@@ -170,6 +176,7 @@ def mutate(self, res):
         if len(res) == 0:
             return None
         pos = self._rand(len(res))
+        # We use rand(255) + 1 so that there is no `^ 0` applied to the byte; it always changes.
         res[pos] ^= self._rand(255) + 1
         return res
 
@@ -492,6 +499,7 @@ def generate_input(self):
     def mutate(self, buf):
         res = buf[:]
         nm = self._rand_exp()
+        #print("Start with {}".format(res))
         for i in range(nm):
 
             # Select a mutator from those we can apply
@@ -501,6 +509,7 @@ def mutate(self, buf):
                 x = self._rand(len(self.mutators))
                 mutator = self.mutators[x]
 
+                #print("Mutate with {}".format(mutator.__class__.__name__))
                 newres = mutator.mutate(res)
                 if newres is not None:
                     break

tests/test_crash.py

@@ -1,3 +1,12 @@
+"""
+Test the fuzzing terminates when a fault is found.
+
+SUT:    Fuzzer
+Area:   Fault finding
+Class:  Functional
+Type:   Integration test
+"""
+
 import io
 import os
 import unittest

tests/test_nocrash.py

@@ -1,3 +1,12 @@
+"""
+Test the fuzzing terminates when no faults found, at a run limit.
+
+SUT:    Fuzzer
+Area:   Non-fault operation
+Class:  Functional
+Type:   Integration test
+"""
+
 import unittest
 
 try:

tests/unittest_mutators.py

@@ -0,0 +1,207 @@
+"""
+Test the mutators operate as desired.
+
+SUT:    Corpus
+Area:   Mutators
+Class:  Functional
+Type:   Unit test
+"""
+
+import unittest
+
+try:
+    from unittest.mock import patch
+except ImportError:
+    # Python 2 backport of mock
+    from mock import patch
+
+import pythonfuzz.corpus as corpus
+
+
+class FakeCorpus(object):
+    pass
+
+
+class BaseTestMutators(unittest.TestCase):
+    """
+    Test that the mutators objects are doing what we want them to do.
+    """
+    # Subclasses should set this - 'mutator' will be created as part of setup.
+    mutator_class = None
+
+    def setUp(self):
+        self.corpus = FakeCorpus()
+        self.patch_rand = patch('pythonfuzz.corpus.Mutator._rand')
+        self.mock_rand = self.patch_rand.start()
+        self.mock_rand.side_effect = []
+        # Update the side effects in your subclass
+
+        self.addCleanup(self.patch_rand.stop)
+
+        self.mutator = self.mutator_class(self.corpus)
+
+
+class TestMutatorRemoveRange(BaseTestMutators):
+    mutator_class = corpus.MutatorRemoveRange
+
+    def test01_empty(self):
+        # You cannot remove values from an empty input
+        res = self.mutator.mutate(bytearray(b''))
+        self.assertIsNone(res)
+
+    def test02_remove_section(self):
+        # Check that it removes a sensible range
+
+        # Check that removing at the 2nd position, removing 4 characters leaves the right string.
+        self.mock_rand.side_effect = [2, 0, 3]
+
+        res = self.mutator.mutate(bytearray(b'1234567890'))
+        self.assertEqual(res, bytearray(b'127890'))
+
+
+class TestMutatorInsertBytes(BaseTestMutators):
+    mutator_class = corpus.MutatorInsertBytes
+
+    def test02_insert_bytes(self):
+        # Check that it inserts sensibly
+
+        # Check that inserting at the 2nd position, adding 4 characters gives us the right string
+        self.mock_rand.side_effect = [2, 0, 3, 65, 66, 67, 68]
+
+        res = self.mutator.mutate(bytearray(b'123456789'))
+        self.assertEqual(res, bytearray(b'12ABCD3456789'))
+
+
+class TestMutatorDuplicateBytes(BaseTestMutators):
+    mutator_class = corpus.MutatorDuplicateBytes
+
+    def test01_empty(self):
+        # Cannot work with an empty input
+        res = self.mutator.mutate(bytearray(b''))
+        self.assertIsNone(res)
+
+    def test02_duplicate_bytes(self):
+        # Check that it duplicates
+
+        # Duplicate from offset 2 to offset 5, length 2
+        self.mock_rand.side_effect = [2, 5, 0, 1]
+
+        res = self.mutator.mutate(bytearray(b'123456789'))
+        self.assertEqual(res, bytearray(b'12345346789'))
+
+
+class TestMutatorCopyBytes(BaseTestMutators):
+    mutator_class = corpus.MutatorDuplicateBytes
+
+    def test01_empty(self):
+        # Cannot work with an empty input
+        res = self.mutator.mutate(bytearray(b''))
+        self.assertIsNone(res)
+
+    def test02_duplicate_bytes(self):
+        # Check that it duplicates
+
+        # Duplicate from offset 2 to offset 5, length 2
+        self.mock_rand.side_effect = [2, 5, 0, 1]
+
+        res = self.mutator.mutate(bytearray(b'123456789'))
+        self.assertEqual(res, bytearray(b'12345346789'))
+
+
+class TestMutatorBitFlip(BaseTestMutators):
+    mutator_class = corpus.MutatorBitFlip
+
+    def test01_empty(self):
+        # Cannot work with an empty input
+        res = self.mutator.mutate(bytearray(b''))
+        self.assertIsNone(res)
+
+    def test02_flip_bit(self):
+        # Check that it flips
+
+        # At offset 4, flip bit 3
+        self.mock_rand.side_effect = [4, 3]
+
+        res = self.mutator.mutate(bytearray(b'123456789'))
+        self.assertEqual(res, bytearray(b'1234=6789'))
+
+
+class TestMutatorRandomiseByte(BaseTestMutators):
+    mutator_class = corpus.MutatorRandomiseByte
+
+    def test01_empty(self):
+        # Cannot work with an empty input
+        res = self.mutator.mutate(bytearray(b''))
+        self.assertIsNone(res)
+
+    def test02_randomise_byte(self):
+        # Check that it changes a byte
+
+        # At offset 4, EOR with 65+1
+        self.mock_rand.side_effect = [4, 65]
+
+        res = self.mutator.mutate(bytearray(b'123456789'))
+        self.assertEqual(res, bytearray(b'1234w6789'))
+
+
+class TestMutatorSwapBytes(BaseTestMutators):
+    mutator_class = corpus.MutatorSwapBytes
+
+    def test01_empty(self):
+        # Cannot work with an empty input
+        res = self.mutator.mutate(bytearray(b''))
+        self.assertIsNone(res)
+
+    def test02_swap_bytes(self):
+        # Check that it swaps bytes
+
+        # Swap bytes at 1 and 6
+        self.mock_rand.side_effect = [1, 6]
+
+        res = self.mutator.mutate(bytearray(b'123456789'))
+        self.assertEqual(res, bytearray(b'173456289'))
+
+
+class TestMutatorAddSubByte(BaseTestMutators):
+    mutator_class = corpus.MutatorAddSubByte
+
+    def test01_empty(self):
+        # Cannot work with an empty input
+        res = self.mutator.mutate(bytearray(b''))
+        self.assertIsNone(res)
+
+    def test02_add_bytes(self):
+        # Check that it adds/subs
+        # FIXME: Not yet implemented - uses a randomised bit for the add/sub
+        pass
+
+# FIXME: Also not implemented AddSubShort, AddSubLong, AddSubLongLong
+# FIXME: Not yet implemented ReplaceByte, ReplaceShort, ReplaceLong
+
+
+class TestMutatorReplaceDigit(BaseTestMutators):
+    mutator_class = corpus.MutatorReplaceDigit
+
+    def test01_empty(self):
+        # Cannot work with an empty input
+        res = self.mutator.mutate(bytearray(b''))
+        self.assertIsNone(res)
+
+    def test02_no_digits(self):
+        # Cannot work with a string that has no digits
+        res = self.mutator.mutate(bytearray(b'wibble'))
+        self.assertIsNone(res)
+
+    def test03_replace_digit(self):
+        # Check that it replaces a digit
+        self.mock_rand.side_effect = [0, 5]
+
+        res = self.mutator.mutate(bytearray(b'there are 4 lights'))
+        self.assertEqual(res, bytearray(b'there are 5 lights'))
+
+
+# FIXME: Not yet implemented: Dictionary insert, Dictionary Append
+
+
+if __name__ == '__main__':
+    unittest.main()