From a55f3b3e676f6a4cea892631e0441e826b2defb8 Mon Sep 17 00:00:00 2001 From: Philipp Hossner Date: Wed, 6 Sep 2017 23:00:19 +0200 Subject: [PATCH 01/24] Externalize test runner --- mutpy/controller.py | 21 +++---------- mutpy/test_runner.py | 75 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 17 deletions(-) create mode 100644 mutpy/test_runner.py diff --git a/mutpy/controller.py b/mutpy/controller.py index 960c688..96686c0 100644 --- a/mutpy/controller.py +++ b/mutpy/controller.py @@ -2,6 +2,7 @@ import sys import unittest from mutpy import views, utils, coverage +from mutpy.test_runner import UnittestTestRunner class TestsFailAtOriginal(Exception): @@ -191,23 +192,9 @@ def add_skip(test): @utils.TimeRegister def run_tests_with_mutant(self, total_duration, mutant_module, mutations, coverage_result): - suite = self.create_test_suite(mutant_module) - if coverage_result: - self.mark_not_covered_tests_as_skip(mutations, coverage_result, suite) - timer = utils.Timer() - result = self.run_mutation_test_runner(suite, total_duration) - timer.stop() - self.update_score_and_notify_views(result, timer.duration) - - def run_mutation_test_runner(self, suite, total_duration): - live_time = self.timeout_factor * (total_duration if total_duration > 1 else 1) - test_runner_class = utils.get_mutation_test_runner_class() - test_runner = test_runner_class(suite=suite) - with self.stdout_manager: - test_runner.start() - result = test_runner.get_result(live_time) - test_runner.terminate() - return result + runner = UnittestTestRunner(self.test_loader, self.timeout_factor,self.stdout_manager, self.init_modules) + result, duration = runner.run_tests_with_mutant(total_duration, mutant_module, mutations, coverage_result) + self.update_score_and_notify_views(result, duration) def update_score_and_notify_views(self, result, mutant_duration): if not result: diff --git a/mutpy/test_runner.py b/mutpy/test_runner.py new file mode 100644 index 0000000..08240e9 --- /dev/null +++ b/mutpy/test_runner.py @@ -0,0 +1,75 @@ +import sys +import unittest + +from mutpy import utils + + +class UnittestTestRunner: + def __init__(self, test_loader, timeout_factor, stdout_manager, init_modules): + self.test_loader = test_loader + self.timeout_factor = timeout_factor + self.stdout_manager = stdout_manager + self.init_modules = init_modules + + def create_test_suite(self, mutant_module): + suite = unittest.TestSuite() + utils.InjectImporter(mutant_module).install() + self.remove_loaded_modules() + for test_module, target_test in self.test_loader.load(): + suite.addTests(self.get_test_suite(test_module, target_test)) + utils.InjectImporter.uninstall() + return suite + + @utils.TimeRegister + def run_tests_with_mutant(self, total_duration, mutant_module, mutations, coverage_result): + suite = self.create_test_suite(mutant_module) + if coverage_result: + self.mark_not_covered_tests_as_skip(mutations, coverage_result, suite) + timer = utils.Timer() + result = self.run_mutation_test_runner(suite, total_duration) + timer.stop() + return result, timer.duration + + def run_mutation_test_runner(self, suite, total_duration): + live_time = self.timeout_factor * (total_duration if total_duration > 1 else 1) + test_runner_class = utils.get_mutation_test_runner_class() + test_runner = test_runner_class(suite=suite) + with self.stdout_manager: + test_runner.start() + result = test_runner.get_result(live_time) + test_runner.terminate() + return result + + def get_test_suite(self, test_module, target_test): + if target_test: + return unittest.TestLoader().loadTestsFromName(target_test, test_module) + else: + return unittest.TestLoader().loadTestsFromModule(test_module) + + def store_init_modules(self): + test_runner_class = utils.get_mutation_test_runner_class() + test_runner = test_runner_class(suite=unittest.TestSuite()) + test_runner.start() + self.init_modules = list(sys.modules.keys()) + + def remove_loaded_modules(self): + for module in list(sys.modules.keys()): + if module not in self.init_modules: + del sys.modules[module] + + def mark_not_covered_tests_as_skip(self, mutations, coverage_result, suite): + mutated_nodes = {mutation.node.marker for mutation in mutations} + + def iter_tests(tests): + try: + for test in tests: + iter_tests(test) + except TypeError: + add_skip(tests) + + def add_skip(test): + if mutated_nodes.isdisjoint(coverage_result.test_covered_nodes[repr(test)]): + test_method = getattr(test, test._testMethodName) + setattr(test, test._testMethodName, unittest.skip('not covered')(test_method)) + + iter_tests(suite) From aeae72550821d3814495e88a48eeaa57f019a6ba Mon Sep 17 00:00:00 2001 From: Philipp Hossner Date: Wed, 6 Sep 2017 23:07:24 +0200 Subject: [PATCH 02/24] Improve readability in HOM tests --- mutpy/test/test_controller.py | 262 ++++++++++++++++++---------------- 1 file changed, 135 insertions(+), 127 deletions(-) diff --git a/mutpy/test/test_controller.py b/mutpy/test/test_controller.py index 6522e16..a006105 100644 --- a/mutpy/test/test_controller.py +++ b/mutpy/test/test_controller.py @@ -1,12 +1,12 @@ import ast -import unittest -import types import sys +import types +import unittest + from mutpy import controller, operators, utils, codegen class MutationScoreTest(unittest.TestCase): - def setUp(self): self.score = controller.MutationScore() @@ -49,7 +49,6 @@ def test_update_coverage(self): class MockModulesLoader: - def __init__(self, name, source): self.names = [name] self.source = source @@ -67,13 +66,11 @@ def get_source(self): class MockMutationController(controller.MutationController): - def create_target_ast(self, target_module): return utils.create_ast(self.target_loader.get_source()) class MutationScoreStoreView: - def end(self, score, *args): self.score = score @@ -112,159 +109,175 @@ def test_run(self): self.assertEqual(score.survived_mutants, 1) -class FirstToLastHOMStrategyTest(unittest.TestCase): +class BaseHOMStrategyTest(unittest.TestCase): - def test_generate(self): - mutations = [ - operators.Mutation(operator=operators.ArithmeticOperatorReplacement, node=ast.Sub(children=[])), - operators.Mutation(operator=operators.ArithmeticOperatorReplacement, node=ast.Sub(children=[])), - operators.Mutation(operator=operators.ArithmeticOperatorReplacement, node=ast.Sub(children=[])), + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.TWO_AOR_MUTATIONS_ON_SUBTRACTION = [ + cls.aor_mutation_on_subtraction(), + cls.aor_mutation_on_subtraction(), + ] + cls.THREE_AOR_MUTATIONS_ON_SUBTRACTION = cls.TWO_AOR_MUTATIONS_ON_SUBTRACTION + [ + cls.aor_mutation_on_subtraction() ] - hom_strategy = controller.FirstToLastHOMStrategy(order=2) - changes_to_apply = list(hom_strategy.generate(mutations)) + @staticmethod + def aor_mutation(node): + return operators.Mutation(operator=operators.ArithmeticOperatorReplacement, node=node) + + @staticmethod + def asr_mutation(node): + return operators.Mutation(operator=operators.AssignmentOperatorReplacement, node=node) + + @staticmethod + def crp_mutation(node): + return operators.Mutation(operator=operators.ConstantReplacement, node=node) + + @staticmethod + def aor_mutation_on_subtraction(): + return operators.Mutation(operator=operators.ArithmeticOperatorReplacement, node=ast.Sub(children=[])) + + @staticmethod + def apply_strategy_to_mutations(hom_strategy_cls, mutations, order, hom_kwargs=None): + if hom_kwargs is None: + hom_kwargs = {} + hom_strategy = hom_strategy_cls(order=order, **hom_kwargs) + return list(hom_strategy.generate(mutations)) + + def apply_strategy_to_mutations_with_order_2(self, hom_strategy_cls, mutations, hom_kwargs=None): + if hom_kwargs is None: + hom_kwargs = {} + return self.apply_strategy_to_mutations(hom_strategy_cls, mutations, 2, hom_kwargs=hom_kwargs) + + def assert_num_changesets(self, changes, num_changes): + self.assertEqual(len(changes), num_changes) + + def assert_num_changeset_entries(self, changes, changeset_index, num_entries): + self.assertEqual(len(changes[changeset_index]), num_entries) + + def assert_mutation_in_changeset_at_position_equals(self, changes, changeset_index, change_index, mutation): + self.assertEqual(changes[changeset_index][change_index], mutation) + + +class FirstToLastHOMStrategyTest(BaseHOMStrategyTest): + def test_generate(self): + mutations = self.THREE_AOR_MUTATIONS_ON_SUBTRACTION + + changes_to_apply = self.apply_strategy_to_mutations_with_order_2(controller.FirstToLastHOMStrategy, mutations) - self.assertEqual(len(changes_to_apply), 2) - self.assertEqual(len(changes_to_apply[0]), 2) - self.assertEqual(changes_to_apply[0][0], mutations[0]) - self.assertEqual(changes_to_apply[0][1], mutations[2]) - self.assertEqual(len(changes_to_apply[1]), 1) - self.assertEqual(changes_to_apply[1][0], mutations[1]) + self.assert_num_changesets(changes_to_apply, 2) + self.assert_num_changeset_entries(changes_to_apply, 0, 2) + self.assert_mutation_in_changeset_at_position_equals(changes_to_apply, 0, 0, mutations[0]) + self.assert_mutation_in_changeset_at_position_equals(changes_to_apply, 0, 1, mutations[2]) + self.assert_num_changeset_entries(changes_to_apply, 1, 1) + self.assert_mutation_in_changeset_at_position_equals(changes_to_apply, 1, 0, mutations[1]) def test_generate_if_same_node(self): node = ast.Sub() mutations = [ - operators.Mutation(operator=operators.ArithmeticOperatorReplacement, node=node), - operators.Mutation(operator=operators.ArithmeticOperatorReplacement, node=node), + self.aor_mutation(node=node), + self.aor_mutation(node=node), ] - hom_strategy = controller.FirstToLastHOMStrategy(order=2) - changes_to_apply = list(hom_strategy.generate(mutations)) + changes_to_apply = self.apply_strategy_to_mutations_with_order_2(controller.FirstToLastHOMStrategy, mutations) - self.assertEqual(len(changes_to_apply), 2) - self.assertEqual(len(changes_to_apply[0]), 1) - self.assertEqual(changes_to_apply[0][0], mutations[0]) - self.assertEqual(len(changes_to_apply[1]), 1) - self.assertEqual(changes_to_apply[1][0], mutations[1]) + self.assert_num_changesets(changes_to_apply, 2) + self.assert_num_changeset_entries(changes_to_apply, 0, 1) + self.assert_mutation_in_changeset_at_position_equals(changes_to_apply, 0, 0, mutations[0]) + self.assert_num_changeset_entries(changes_to_apply, 1, 1) + self.assert_mutation_in_changeset_at_position_equals(changes_to_apply, 1, 0, mutations[1]) def test_generate_if_node_child(self): node = ast.Sub(children=[]) mutations = [ - operators.Mutation(operator=operators.ArithmeticOperatorReplacement, node=ast.UnaryOp(children=[node])), - operators.Mutation(operator=operators.ArithmeticOperatorReplacement, node=node), + self.aor_mutation(node=ast.UnaryOp(children=[node])), + self.aor_mutation(node=node), ] - hom_strategy = controller.FirstToLastHOMStrategy(order=2) - changes_to_apply = list(hom_strategy.generate(mutations)) + changes_to_apply = self.apply_strategy_to_mutations_with_order_2(controller.FirstToLastHOMStrategy, mutations) - self.assertEqual(len(changes_to_apply), 2) - self.assertEqual(len(changes_to_apply[0]), 1) - self.assertEqual(changes_to_apply[0][0], mutations[0]) - self.assertEqual(len(changes_to_apply[1]), 1) - self.assertEqual(changes_to_apply[1][0], mutations[1]) + self.assert_num_changesets(changes_to_apply, 2) + self.assert_num_changeset_entries(changes_to_apply, 0, 1) + self.assert_mutation_in_changeset_at_position_equals(changes_to_apply, 0, 0, mutations[0]) + self.assert_num_changeset_entries(changes_to_apply, 1, 1) + self.assert_mutation_in_changeset_at_position_equals(changes_to_apply, 1, 0, mutations[1]) -class EachChoiceHOMStrategyTest(unittest.TestCase): - +class EachChoiceHOMStrategyTest(BaseHOMStrategyTest): def test_generate(self): - mutations = [ - operators.Mutation(operator=operators.ArithmeticOperatorReplacement, node=ast.Sub(children=[])), - operators.Mutation(operator=operators.ArithmeticOperatorReplacement, node=ast.Sub(children=[])), - operators.Mutation(operator=operators.ArithmeticOperatorReplacement, node=ast.Sub(children=[])), - ] - hom_strategy = controller.EachChoiceHOMStrategy(order=2) + mutations = self.THREE_AOR_MUTATIONS_ON_SUBTRACTION - changes_to_apply = list(hom_strategy.generate(mutations)) + changes_to_apply = self.apply_strategy_to_mutations_with_order_2(controller.EachChoiceHOMStrategy, mutations) - self.assertEqual(len(changes_to_apply), 2) - self.assertEqual(len(changes_to_apply[0]), 2) - self.assertEqual(changes_to_apply[0][0], mutations[0]) - self.assertEqual(changes_to_apply[0][1], mutations[1]) - self.assertEqual(len(changes_to_apply[1]), 1) - self.assertEqual(changes_to_apply[1][0], mutations[2]) + self.assert_num_changesets(changes_to_apply, 2) + self.assert_num_changeset_entries(changes_to_apply, 0, 2) + self.assert_mutation_in_changeset_at_position_equals(changes_to_apply, 0, 0, mutations[0]) + self.assert_mutation_in_changeset_at_position_equals(changes_to_apply, 0, 1, mutations[1]) + self.assert_num_changeset_entries(changes_to_apply, 1, 1) + self.assert_mutation_in_changeset_at_position_equals(changes_to_apply, 1, 0, mutations[2]) -class BetweenOperatorsHOMStrategyTest(unittest.TestCase): - +class BetweenOperatorsHOMStrategyTest(BaseHOMStrategyTest): def test_generate_if_one_operator(self): - mutations = [ - operators.Mutation(operator=operators.ArithmeticOperatorReplacement, node=ast.Sub(children=[])), - operators.Mutation(operator=operators.ArithmeticOperatorReplacement, node=ast.Sub(children=[])), - ] - hom_strategy = controller.BetweenOperatorsHOMStrategy(order=2) + mutations = self.TWO_AOR_MUTATIONS_ON_SUBTRACTION - changes_to_apply = list(hom_strategy.generate(mutations)) - - self.assertEqual(len(changes_to_apply), 2) - self.assertEqual(len(changes_to_apply[0]), 1) - self.assertEqual(changes_to_apply[0][0], mutations[0]) - self.assertEqual(len(changes_to_apply[1]), 1) - self.assertEqual(changes_to_apply[1][0], mutations[1]) + changes_to_apply = self.apply_strategy_to_mutations_with_order_2(controller.BetweenOperatorsHOMStrategy, + mutations) + self.assert_num_changesets(changes_to_apply, 2) + self.assert_num_changeset_entries(changes_to_apply, 0, 1) + self.assert_mutation_in_changeset_at_position_equals(changes_to_apply, 0, 0, mutations[0]) + self.assert_num_changeset_entries(changes_to_apply, 1, 1) + self.assert_mutation_in_changeset_at_position_equals(changes_to_apply, 1, 0, mutations[1]) def test_generate_if_two_operators(self): - mutations = [ - operators.Mutation(operator=operators.ArithmeticOperatorReplacement, node=ast.Sub(children=[])), - operators.Mutation(operator=operators.ArithmeticOperatorReplacement, node=ast.Sub(children=[])), - operators.Mutation(operator=operators.AssignmentOperatorReplacement, node=ast.Sub(children=[])), - ] - hom_strategy = controller.BetweenOperatorsHOMStrategy(order=2) - - changes_to_apply = list(hom_strategy.generate(mutations)) - - self.assertEqual(len(changes_to_apply), 2) - self.assertEqual(len(changes_to_apply[0]), 2) - self.assertEqual(changes_to_apply[0][0], mutations[0]) - self.assertEqual(changes_to_apply[0][1], mutations[2]) - self.assertEqual(len(changes_to_apply[1]), 2) - self.assertEqual(changes_to_apply[1][0], mutations[1]) - self.assertEqual(changes_to_apply[1][1], mutations[2]) + mutations = self.TWO_AOR_MUTATIONS_ON_SUBTRACTION + [self.asr_mutation(node=ast.Sub(children=[]))] + + changes_to_apply = self.apply_strategy_to_mutations_with_order_2(controller.BetweenOperatorsHOMStrategy, + mutations) + self.assert_num_changesets(changes_to_apply, 2) + self.assert_num_changeset_entries(changes_to_apply, 0, 2) + self.assert_mutation_in_changeset_at_position_equals(changes_to_apply, 0, 0, mutations[0]) + self.assert_mutation_in_changeset_at_position_equals(changes_to_apply, 0, 1, mutations[2]) + self.assert_num_changeset_entries(changes_to_apply, 1, 2) + self.assert_mutation_in_changeset_at_position_equals(changes_to_apply, 1, 0, mutations[1]) + self.assert_mutation_in_changeset_at_position_equals(changes_to_apply, 1, 1, mutations[2]) def test_generate_if_three_operators(self): - mutations = [ - operators.Mutation(operator=operators.ArithmeticOperatorReplacement, node=ast.Sub(children=[])), - operators.Mutation(operator=operators.ArithmeticOperatorReplacement, node=ast.Sub(children=[])), - operators.Mutation(operator=operators.AssignmentOperatorReplacement, node=ast.Sub(children=[])), - operators.Mutation(operator=operators.ConstantReplacement, node=ast.Sub(children=[])), + mutations = self.TWO_AOR_MUTATIONS_ON_SUBTRACTION + [ + self.asr_mutation(node=ast.Sub(children=[])), + self.crp_mutation(node=ast.Sub(children=[])), ] - hom_strategy = controller.BetweenOperatorsHOMStrategy(order=2) - changes_to_apply = list(hom_strategy.generate(mutations)) + changes_to_apply = self.apply_strategy_to_mutations_with_order_2(controller.BetweenOperatorsHOMStrategy, + mutations) + self.assert_num_changesets(changes_to_apply, 2) + self.assert_num_changeset_entries(changes_to_apply, 0, 2) + self.assert_mutation_in_changeset_at_position_equals(changes_to_apply, 0, 0, mutations[0]) + self.assert_mutation_in_changeset_at_position_equals(changes_to_apply, 0, 1, mutations[2]) + self.assert_num_changeset_entries(changes_to_apply, 1, 2) + self.assert_mutation_in_changeset_at_position_equals(changes_to_apply, 1, 0, mutations[1]) + self.assert_mutation_in_changeset_at_position_equals(changes_to_apply, 1, 1, mutations[3]) - self.assertEqual(len(changes_to_apply), 2) - self.assertEqual(len(changes_to_apply[0]), 2) - self.assertEqual(changes_to_apply[0][0], mutations[0]) - self.assertEqual(changes_to_apply[0][1], mutations[2]) - self.assertEqual(len(changes_to_apply[1]), 2) - self.assertEqual(changes_to_apply[1][0], mutations[1]) - self.assertEqual(changes_to_apply[1][1], mutations[3]) - - -class RandomHOMStrategyTest(unittest.TestCase): +class RandomHOMStrategyTest(BaseHOMStrategyTest): def test_generate(self): - mutations = [ - operators.Mutation(operator=operators.ArithmeticOperatorReplacement, node=ast.Sub(children=[])), - operators.Mutation(operator=operators.ArithmeticOperatorReplacement, node=ast.Sub(children=[])), - operators.Mutation(operator=operators.ArithmeticOperatorReplacement, node=ast.Sub(children=[])), - ] + mutations = self.THREE_AOR_MUTATIONS_ON_SUBTRACTION def shuffler(mutations): mutations.reverse() - hom_strategy = controller.RandomHOMStrategy(order=2, shuffler=shuffler) - - changes_to_apply = list(hom_strategy.generate(mutations)) - - self.assertEqual(len(changes_to_apply), 2) - self.assertEqual(len(changes_to_apply[0]), 2) - self.assertEqual(changes_to_apply[0][0], mutations[2]) - self.assertEqual(changes_to_apply[0][1], mutations[1]) - self.assertEqual(len(changes_to_apply[1]), 1) - self.assertEqual(changes_to_apply[1][0], mutations[0]) + changes_to_apply = self.apply_strategy_to_mutations_with_order_2(controller.RandomHOMStrategy, mutations, + hom_kwargs={'shuffler': shuffler}) + self.assert_num_changesets(changes_to_apply, 2) + self.assert_num_changeset_entries(changes_to_apply, 0, 2) + self.assert_mutation_in_changeset_at_position_equals(changes_to_apply, 0, 0, mutations[2]) + self.assert_mutation_in_changeset_at_position_equals(changes_to_apply, 0, 1, mutations[1]) + self.assert_num_changeset_entries(changes_to_apply, 1, 1) + self.assert_mutation_in_changeset_at_position_equals(changes_to_apply, 1, 0, mutations[0]) class FirstOrderMutatorTest(unittest.TestCase): - def test_first_order_mutation(self): mutator = controller.FirstOrderMutator( operators=[operators.ArithmeticOperatorReplacement, operators.AssignmentOperatorReplacement], @@ -272,20 +285,17 @@ def test_first_order_mutation(self): target_ast = utils.create_ast('x += y + z') for number, (mutations, mutant) in enumerate(mutator.mutate(target_ast)): + self.assertIn(number, [0, 1]) + self.assertEqual(len(mutations), 1) if number == 0: self.assertEqual('x += y - z', codegen.to_source(mutant)) - self.assertEqual(len(mutations), 1) elif number == 1: self.assertEqual('x -= y + z', codegen.to_source(mutant)) - self.assertEqual(len(mutations), 1) - else: - self.fail() self.assertEqual(codegen.to_source(target_ast), 'x += y + z') class HighOrderMutatorTest(unittest.TestCase): - def test_second_order_mutation(self): mutator = controller.HighOrderMutator( operators=[operators.ArithmeticOperatorReplacement, operators.AssignmentOperatorReplacement], @@ -293,11 +303,9 @@ def test_second_order_mutation(self): target_ast = utils.create_ast('x += y + z') for number, (mutations, mutant) in enumerate(mutator.mutate(target_ast)): - if number == 0: - self.assertEqual('x -= y - z', codegen.to_source(mutant)) - self.assertEqual(len(mutations), 2) - else: - self.fail() + self.assertEqual(number, 0) + self.assertEqual('x -= y - z', codegen.to_source(mutant)) + self.assertEqual(len(mutations), 2) self.assertEqual(codegen.to_source(target_ast), 'x += y + z') @@ -331,4 +339,4 @@ def test_second_order_mutation_with_multiple_visitors(self): self.assertEqual("x = ''", codegen.to_source(mutant)) self.assertEqual(len(mutations), 1) self.assertEqual(number, 1) - self.assertEqual(codegen.to_source(target_ast), "x = 'test'") + self.assertEqual(codegen.to_source(target_ast), "x = 'test'") \ No newline at end of file From 89c20262085cb67302eabeedaf767ca6140f6f83 Mon Sep 17 00:00:00 2001 From: Philipp Hossner Date: Fri, 27 Oct 2017 02:23:22 +0200 Subject: [PATCH 03/24] Add tox and split up requirements --- .travis.yml | 4 ++-- MANIFEST.in | 2 +- requirements/development.txt | 3 +++ requirements.txt => requirements/production.txt | 0 setup.py | 2 +- tox.ini | 12 ++++++++++++ 6 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 requirements/development.txt rename requirements.txt => requirements/production.txt (100%) create mode 100644 tox.ini diff --git a/.travis.yml b/.travis.yml index 63cf185..6c6b30a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,9 +5,9 @@ python: - 3.5 - 3.6 install: - - pip install -r requirements.txt + - pip install tox-travis - pip install coveralls script: - - coverage run --source=mutpy -m unittest discover -s mutpy/test + - tox after_success: coveralls diff --git a/MANIFEST.in b/MANIFEST.in index a4b62ec..0e70e4a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,2 @@ -include requirements.txt +include requirements/* include mutpy/templates/* \ No newline at end of file diff --git a/requirements/development.txt b/requirements/development.txt new file mode 100644 index 0000000..4d01459 --- /dev/null +++ b/requirements/development.txt @@ -0,0 +1,3 @@ +-r production.txt + +coverage \ No newline at end of file diff --git a/requirements.txt b/requirements/production.txt similarity index 100% rename from requirements.txt rename to requirements/production.txt diff --git a/setup.py b/setup.py index eab0507..236f844 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ print('MutPy requires Python 3.3 or newer!') sys.exit() -with open('requirements.txt') as f: +with open('requirements/production.txt') as f: requirements = f.read().splitlines() with open('README.rst') as f: diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..bb73696 --- /dev/null +++ b/tox.ini @@ -0,0 +1,12 @@ +[tox] +envlist = + coverage-erase + test-py{33,34,35,36} + coverage-report +[testenv] +deps= + -rrequirements/development.txt +commands = + coverage-erase: coverage erase + test: coverage run --source=mutpy -m unittest discover -s mutpy/test + coverage-report: coverage report \ No newline at end of file From ddc85dffbf61f2ccbff1974015f045fe7cc5cd39 Mon Sep 17 00:00:00 2001 From: Philipp Hossner Date: Wed, 20 Dec 2017 16:44:29 +0100 Subject: [PATCH 04/24] Implement PyTest runner --- mutpy/commandline.py | 19 +- mutpy/controller.py | 86 +-- mutpy/coverage.py | 23 +- mutpy/operators.py | 730 -------------------------- mutpy/operators/__init__.py | 46 ++ mutpy/operators/arithmetic.py | 79 +++ mutpy/operators/base.py | 146 ++++++ mutpy/operators/decorator.py | 50 ++ mutpy/operators/exception.py | 21 + mutpy/operators/inheritance.py | 185 +++++++ mutpy/operators/logical.py | 91 ++++ mutpy/operators/loop.py | 45 ++ mutpy/operators/misc.py | 97 ++++ mutpy/test/test_controller.py | 22 +- mutpy/test/test_coverage.py | 5 +- mutpy/test/test_runners.py | 118 +++++ mutpy/test/test_utils.py | 5 +- mutpy/test/utils.py | 49 ++ mutpy/test_runner.py | 75 --- mutpy/test_runners/__init__.py | 11 + mutpy/test_runners/base.py | 225 ++++++++ mutpy/test_runners/pytest_runner.py | 114 ++++ mutpy/test_runners/unittest_runner.py | 127 +++++ mutpy/utils.py | 140 +++-- mutpy/views.py | 17 +- setup.py | 6 +- tox.ini | 5 +- 27 files changed, 1529 insertions(+), 1008 deletions(-) delete mode 100644 mutpy/operators.py create mode 100644 mutpy/operators/__init__.py create mode 100644 mutpy/operators/arithmetic.py create mode 100644 mutpy/operators/base.py create mode 100644 mutpy/operators/decorator.py create mode 100644 mutpy/operators/exception.py create mode 100644 mutpy/operators/inheritance.py create mode 100644 mutpy/operators/logical.py create mode 100644 mutpy/operators/loop.py create mode 100644 mutpy/operators/misc.py create mode 100644 mutpy/test/test_runners.py create mode 100644 mutpy/test/utils.py delete mode 100644 mutpy/test_runner.py create mode 100644 mutpy/test_runners/__init__.py create mode 100644 mutpy/test_runners/base.py create mode 100644 mutpy/test_runners/pytest_runner.py create mode 100644 mutpy/test_runners/unittest_runner.py diff --git a/mutpy/commandline.py b/mutpy/commandline.py index 8b83d67..d5c39de 100644 --- a/mutpy/commandline.py +++ b/mutpy/commandline.py @@ -1,7 +1,8 @@ import argparse import sys -from mutpy import controller, views, operators, utils + from mutpy import __version__ as version +from mutpy import controller, views, operators, utils def main(argv): @@ -17,6 +18,8 @@ def build_parser(): parser.add_argument('--target', '-t', type=str, nargs='+', help='target module or package to mutate') parser.add_argument('--unit-test', '-u', type=str, nargs='+', help='test class, test method, module or package with unit tests') + parser.add_argument('--runner', type=str, choices=['unittest', 'pytest'], metavar='RUNNER', + help='test runner') parser.add_argument('--report', '-r', type=str, help='generate YAML report', metavar='REPORT_FILE') parser.add_argument('--report-html', type=str, help='generate HTML report', metavar='DIR_NAME') parser.add_argument('--timeout-factor', '-f', type=float, default=DEF_TIMEOUT_FACTOR, @@ -27,7 +30,7 @@ def build_parser(): parser.add_argument('--colored-output', '-c', action='store_true', help='try print colored output') parser.add_argument('--disable-stdout', '-d', action='store_true', help='try disable stdout during mutation ' - '(this option can damage your tests if you interact with sys.stdout)') + '(this option can damage your tests if you interact with sys.stdout)') parser.add_argument('--experimental-operators', '-e', action='store_true', help='use experimental operators') parser.add_argument('--operator', '-o', type=str, nargs='+', help='use only selected operators', metavar='OPERATOR') @@ -62,11 +65,13 @@ def run_mutpy(parser): def build_controller(cfg): + runner_cls = get_runner_cls(cfg.runner) built_views = build_views(cfg) mutant_generator = build_mutator(cfg) target_loader = utils.ModulesLoader(cfg.target, cfg.path) test_loader = utils.ModulesLoader(cfg.unit_test, cfg.path) return controller.MutationController( + runner_cls=runner_cls, target_loader=target_loader, test_loader=test_loader, views=built_views, @@ -78,6 +83,16 @@ def build_controller(cfg): ) +def get_runner_cls(runner): + if runner == 'unittest': + from mutpy.test_runners import UnittestTestRunner + return UnittestTestRunner + elif runner == 'pytest': + from mutpy.test_runners import PytestTestRunner + return PytestTestRunner + raise ValueError('Unknown runner: {0}'.format(runner)) + + def build_mutator(cfg): operators_set = set() diff --git a/mutpy/controller.py b/mutpy/controller.py index 96686c0..cbda802 100644 --- a/mutpy/controller.py +++ b/mutpy/controller.py @@ -1,8 +1,8 @@ import random import sys -import unittest -from mutpy import views, utils, coverage -from mutpy.test_runner import UnittestTestRunner +import time + +from mutpy import views, utils class TestsFailAtOriginal(Exception): @@ -48,7 +48,7 @@ def all_mutants(self): class MutationController(views.ViewNotifier): - def __init__(self, target_loader, test_loader, views, mutant_generator, + def __init__(self, runner_cls, target_loader, test_loader, views, mutant_generator, timeout_factor=5, disable_stdout=False, mutate_covered=False, mutation_number=None): super().__init__(views) self.target_loader = target_loader @@ -56,9 +56,8 @@ def __init__(self, target_loader, test_loader, views, mutant_generator, self.mutant_generator = mutant_generator self.timeout_factor = timeout_factor self.stdout_manager = utils.StdoutManager(disable_stdout) - self.mutate_covered = mutate_covered self.mutation_number = mutation_number - self.store_init_modules() + self.runner = runner_cls(self.test_loader, self.timeout_factor, self.stdout_manager, mutate_covered) def run(self): self.notify_initialize(self.target_loader.names, self.test_loader.names) @@ -93,34 +92,22 @@ def load_and_check_tests(self): total_duration = 0 for test_module, target_test in self.test_loader.load(): result, duration = self.run_test(test_module, target_test) - if result.wasSuccessful(): + if result.was_successful(): test_modules.append((test_module, target_test, duration)) else: raise TestsFailAtOriginal(result) - number_of_tests += result.testsRun + number_of_tests += result.tests_run() total_duration += duration return test_modules, total_duration, number_of_tests def run_test(self, test_module, target_test): - suite = self.get_test_suite(test_module, target_test) - result = unittest.TestResult() - timer = utils.Timer() - with self.stdout_manager: - suite.run(result) - return result, timer.stop() - - def get_test_suite(self, test_module, target_test): - if target_test: - return unittest.TestLoader().loadTestsFromName(target_test, test_module) - else: - return unittest.TestLoader().loadTestsFromModule(test_module) + return self.runner.run_test(test_module, target_test) @utils.TimeRegister def mutate_module(self, target_module, to_mutate, total_duration): target_ast = self.create_target_ast(target_module) coverage_injector, coverage_result = self.inject_coverage(target_ast, target_module) - if coverage_injector: self.score.update_coverage(*coverage_injector.get_result()) for mutations, mutant_ast in self.mutant_generator.mutate(target_ast, to_mutate, coverage_injector, @@ -137,15 +124,7 @@ def mutate_module(self, target_module, to_mutate, total_duration): self.score.inc_incompetent() def inject_coverage(self, target_ast, target_module): - if not self.mutate_covered: - return None, None - coverage_injector = coverage.CoverageInjector() - coverage_module = coverage_injector.inject(target_ast, target_module.__name__) - suite = self.create_test_suite(coverage_module) - coverage_result = coverage.CoverageTestResult(coverage_injector=coverage_injector) - with self.stdout_manager: - suite.run(coverage_result) - return coverage_injector, coverage_result + return self.runner.inject_coverage(target_ast, target_module) @utils.TimeRegister def create_target_ast(self, target_module): @@ -164,36 +143,8 @@ def create_mutant_module(self, target_module, mutant_ast): self.notify_incompetent(0, exception, tests_run=0) return None - def create_test_suite(self, mutant_module): - suite = unittest.TestSuite() - utils.InjectImporter(mutant_module).install() - self.remove_loaded_modules() - for test_module, target_test in self.test_loader.load(): - suite.addTests(self.get_test_suite(test_module, target_test)) - utils.InjectImporter.uninstall() - return suite - - def mark_not_covered_tests_as_skip(self, mutations, coverage_result, suite): - mutated_nodes = {mutation.node.marker for mutation in mutations} - - def iter_tests(tests): - try: - for test in tests: - iter_tests(test) - except TypeError: - add_skip(tests) - - def add_skip(test): - if mutated_nodes.isdisjoint(coverage_result.test_covered_nodes[repr(test)]): - test_method = getattr(test, test._testMethodName) - setattr(test, test._testMethodName, unittest.skip('not covered')(test_method)) - - iter_tests(suite) - - @utils.TimeRegister def run_tests_with_mutant(self, total_duration, mutant_module, mutations, coverage_result): - runner = UnittestTestRunner(self.test_loader, self.timeout_factor,self.stdout_manager, self.init_modules) - result, duration = runner.run_tests_with_mutant(total_duration, mutant_module, mutations, coverage_result) + result, duration = self.runner.run_tests_with_mutant(total_duration, mutant_module, mutations, coverage_result) self.update_score_and_notify_views(result, duration) def update_score_and_notify_views(self, result, mutant_duration): @@ -222,17 +173,6 @@ def update_killed_mutant(self, result, duration): self.notify_killed(duration, result.killer, result.exception_traceback, result.tests_run) self.score.inc_killed() - def store_init_modules(self): - test_runner_class = utils.get_mutation_test_runner_class() - test_runner = test_runner_class(suite=unittest.TestSuite()) - test_runner.start() - self.init_modules = list(sys.modules.keys()) - - def remove_loaded_modules(self): - for module in list(sys.modules.keys()): - if module not in self.init_modules: - del sys.modules[module] - class HOMStrategy: @@ -243,9 +183,9 @@ def remove_bad_mutations(self, mutations_to_apply, available_mutations, allow_sa for mutation_to_apply in mutations_to_apply: for available_mutation in available_mutations[:]: if mutation_to_apply.node == available_mutation.node or \ - mutation_to_apply.node in available_mutation.node.children or \ - available_mutation.node in mutation_to_apply.node.children or \ - (not allow_same_operators and mutation_to_apply.operator == available_mutation.operator): + mutation_to_apply.node in available_mutation.node.children or \ + available_mutation.node in mutation_to_apply.node.children or \ + (not allow_same_operators and mutation_to_apply.operator == available_mutation.operator): available_mutations.remove(available_mutation) diff --git a/mutpy/coverage.py b/mutpy/coverage.py index d04f71f..191c972 100644 --- a/mutpy/coverage.py +++ b/mutpy/coverage.py @@ -1,6 +1,6 @@ import ast import copy -import unittest + from mutpy import utils COVERAGE_SET_NAME = '__covered_nodes__' @@ -79,7 +79,6 @@ def get_markers_from_body_node(self, node): class CoverageNodeTransformerPython32(AbstractCoverageNodeTransformer): - __python_version__ = (3, 2) @classmethod @@ -111,7 +110,6 @@ def get_coverable_nodes(cls): class CoverageNodeTransformerPython33(AbstractCoverageNodeTransformer): - __python_version__ = (3, 3) @classmethod @@ -170,22 +168,3 @@ def is_covered(self, child_node): def get_result(self): return len(self.covered_nodes), self.marker_transformer.last_marker - - -class CoverageTestResult(unittest.TestResult): - - def __init__(self, *args, coverage_injector=None, **kwargs): - super().__init__(*args, **kwargs) - self.coverage_injector = coverage_injector - self.always_covered_nodes = coverage_injector.covered_nodes.copy() - self.test_covered_nodes = {} - - def startTest(self, test): - super().startTest(test) - self.covered_nodes = self.coverage_injector.covered_nodes.copy() - self.coverage_injector.covered_nodes.clear() - - def stopTest(self, test): - super().stopTest(test) - self.test_covered_nodes[repr(test)] = self.coverage_injector.covered_nodes.copy() | self.always_covered_nodes - self.coverage_injector.covered_nodes.update(self.covered_nodes) diff --git a/mutpy/operators.py b/mutpy/operators.py deleted file mode 100644 index 0f7d55e..0000000 --- a/mutpy/operators.py +++ /dev/null @@ -1,730 +0,0 @@ -import ast -import copy -import functools -import re - -from mutpy import utils - - -class MutationResign(Exception): - pass - - -class Mutation: - def __init__(self, operator, node, visitor=None): - self.operator = operator - self.node = node - self.visitor = visitor - - -def copy_node(mutate): - def f(self, node): - copied_node = copy.deepcopy(node, memo={ - id(node.parent): node.parent, - }) - return mutate(self, copied_node) - - return f - - -class MutationOperator: - def mutate(self, node, to_mutate=None, sampler=None, coverage_injector=None, module=None, only_mutation=None): - self.to_mutate = to_mutate - self.sampler = sampler - self.only_mutation = only_mutation - self.coverage_injector = coverage_injector - self.module = module - for new_node in self.visit(node): - yield Mutation(operator=self.__class__, node=self.current_node, visitor=self.visitor), new_node - - def visit(self, node): - if self.has_notmutate(node) or (self.coverage_injector and not self.coverage_injector.is_covered(node)): - return - if self.only_mutation and self.only_mutation.node != node and self.only_mutation.node not in node.children: - return - self.fix_lineno(node) - visitors = self.find_visitors(node) - if visitors: - for visitor in visitors: - try: - if self.sampler and not self.sampler.is_mutation_time(): - raise MutationResign - if self.only_mutation and \ - (self.only_mutation.node != node or self.only_mutation.visitor != visitor.__name__): - raise MutationResign - new_node = visitor(node) - self.visitor = visitor.__name__ - self.current_node = node - self.fix_node_internals(node, new_node) - ast.fix_missing_locations(new_node) - yield new_node - except MutationResign: - pass - finally: - for new_node in self.generic_visit(node): - yield new_node - else: - for new_node in self.generic_visit(node): - yield new_node - - def generic_visit(self, node): - for field, old_value in ast.iter_fields(node): - if isinstance(old_value, list): - generator = self.generic_visit_list(old_value) - elif isinstance(old_value, ast.AST): - generator = self.generic_visit_real_node(node, field, old_value) - else: - generator = [] - - for _ in generator: - yield node - - def generic_visit_list(self, old_value): - old_values_copy = old_value[:] - for position, value in enumerate(old_values_copy): - if isinstance(value, ast.AST): - for new_value in self.visit(value): - if not isinstance(new_value, ast.AST): - old_value[position:position + 1] = new_value - elif value is None: - del old_value[position] - else: - old_value[position] = new_value - - yield - old_value[:] = old_values_copy - - def generic_visit_real_node(self, node, field, old_value): - for new_node in self.visit(old_value): - if new_node is None: - delattr(node, field) - else: - setattr(node, field, new_node) - yield - setattr(node, field, old_value) - - def has_notmutate(self, node): - try: - for decorator in node.decorator_list: - if decorator.id == utils.notmutate.__name__: - return True - return False - except AttributeError: - return False - - def fix_lineno(self, node): - if not hasattr(node, 'lineno') and getattr(node, 'parent', None) is not None and hasattr(node.parent, 'lineno'): - node.lineno = node.parent.lineno - - def fix_node_internals(self, old_node, new_node): - if not hasattr(new_node, 'parent'): - new_node.children = old_node.children - new_node.parent = old_node.parent - if hasattr(old_node, 'marker'): - new_node.marker = old_node.marker - - def find_visitors(self, node): - method_prefix = 'mutate_' + node.__class__.__name__ - return self.getattrs_like(method_prefix) - - def getattrs_like(ob, attr_like): - pattern = re.compile(attr_like + "($|(_\w+)+$)") - return [getattr(ob, attr) for attr in dir(ob) if pattern.match(attr)] - - @classmethod - def name(cls): - return ''.join([c for c in cls.__name__ if str.isupper(c)]) - - @classmethod - def long_name(cls): - return ' '.join(map(str.lower, (re.split('([A-Z][a-z]*)', cls.__name__)[1::2]))) - - -class AbstractUnaryOperatorDeletion(MutationOperator): - def mutate_UnaryOp(self, node): - if isinstance(node.op, self.get_operator_type()): - return node.operand - raise MutationResign() - - -class ArithmeticOperatorDeletion(AbstractUnaryOperatorDeletion): - def get_operator_type(self): - return ast.UAdd, ast.USub - - -class AbstractArithmeticOperatorReplacement(MutationOperator): - def should_mutate(self, node): - raise NotImplementedError() - - def mutate_Add(self, node): - if self.should_mutate(node): - return ast.Sub() - raise MutationResign() - - def mutate_Sub(self, node): - if self.should_mutate(node): - return ast.Add() - raise MutationResign() - - def mutate_Mult_to_Div(self, node): - if self.should_mutate(node): - return ast.Div() - raise MutationResign() - - def mutate_Mult_to_FloorDiv(self, node): - if self.should_mutate(node): - return ast.FloorDiv() - raise MutationResign() - - def mutate_Mult_to_Pow(self, node): - if self.should_mutate(node): - return ast.Pow() - raise MutationResign() - - def mutate_Div_to_Mult(self, node): - if self.should_mutate(node): - return ast.Mult() - raise MutationResign() - - def mutate_Div_to_FloorDiv(self, node): - if self.should_mutate(node): - return ast.FloorDiv() - raise MutationResign() - - def mutate_FloorDiv_to_Div(self, node): - if self.should_mutate(node): - return ast.Div() - raise MutationResign() - - def mutate_FloorDiv_to_Mult(self, node): - if self.should_mutate(node): - return ast.Mult() - raise MutationResign() - - def mutate_Mod(self, node): - if self.should_mutate(node): - return ast.Mult() - raise MutationResign() - - def mutate_Pow(self, node): - if self.should_mutate(node): - return ast.Mult() - raise MutationResign() - - -class ArithmeticOperatorReplacement(AbstractArithmeticOperatorReplacement): - def should_mutate(self, node): - return not isinstance(node.parent, ast.AugAssign) - - def mutate_USub(self, node): - return ast.UAdd() - - def mutate_UAdd(self, node): - return ast.USub() - - -class AssignmentOperatorReplacement(AbstractArithmeticOperatorReplacement): - def should_mutate(self, node): - return isinstance(node.parent, ast.AugAssign) - - @classmethod - def name(cls): - return 'ASR' - - -class BreakContinueReplacement(MutationOperator): - def mutate_Break(self, node): - return ast.Continue() - - def mutate_Continue(self, node): - return ast.Break() - - -class ConditionalOperatorDeletion(AbstractUnaryOperatorDeletion): - def get_operator_type(self): - return ast.Not - - def mutate_NotIn(self, node): - return ast.In() - - -class ConditionalOperatorInsertion(MutationOperator): - def negate_test(self, node): - not_node = ast.UnaryOp(op=ast.Not(), operand=node.test) - node.test = not_node - return node - - @copy_node - def mutate_While(self, node): - return self.negate_test(node) - - @copy_node - def mutate_If(self, node): - return self.negate_test(node) - - def mutate_In(self, node): - return ast.NotIn() - - -class ConstantReplacement(MutationOperator): - FIRST_CONST_STRING = 'mutpy' - SECOND_CONST_STRING = 'python' - - def mutate_Num(self, node): - return ast.Num(n=node.n + 1) - - def mutate_Str(self, node): - if utils.is_docstring(node): - raise MutationResign() - - if node.s != self.FIRST_CONST_STRING: - return ast.Str(s=self.FIRST_CONST_STRING) - else: - return ast.Str(s=self.SECOND_CONST_STRING) - - def mutate_Str_empty(self, node): - if not node.s or utils.is_docstring(node): - raise MutationResign() - - return ast.Str(s='') - - @classmethod - def name(cls): - return 'CRP' - - -class DecoratorDeletion(MutationOperator): - @copy_node - def mutate_FunctionDef(self, node): - if node.decorator_list: - node.decorator_list = [] - return node - else: - raise MutationResign() - - @classmethod - def name(cls): - return 'DDL' - - -class ExceptionHandlerDeletion(MutationOperator): - def mutate_ExceptHandler(self, node): - if node.body and isinstance(node.body[0], ast.Raise): - raise MutationResign() - return ast.ExceptHandler(type=node.type, name=node.name, body=[ast.Raise()]) - - -class ExceptionSwallowing(MutationOperator): - def mutate_ExceptHandler(self, node): - if len(node.body) == 1 and isinstance(node.body[0], ast.Pass): - raise MutationResign() - return ast.ExceptHandler(type=node.type, name=node.name, body=[ast.Pass()]) - - @classmethod - def name(cls): - return 'EXS' - - -class AbstractOverriddenElementModification(MutationOperator): - def is_overridden(self, node, name=None): - if not isinstance(node.parent, ast.ClassDef): - raise MutationResign() - if not name: - name = node.name - parent = node.parent - parent_names = [] - while parent: - if not isinstance(parent, ast.Module): - parent_names.append(parent.name) - if not isinstance(parent, ast.ClassDef) and not isinstance(parent, ast.Module): - raise MutationResign() - parent = parent.parent - getattr_rec = lambda obj, attr: functools.reduce(getattr, attr, obj) - try: - klass = getattr_rec(self.module, reversed(parent_names)) - except AttributeError: - raise MutationResign() - for base_klass in type.mro(klass)[1:-1]: - if hasattr(base_klass, name): - return True - return False - - -class HidingVariableDeletion(AbstractOverriddenElementModification): - def mutate_Assign(self, node): - if len(node.targets) > 1: - raise MutationResign() - if isinstance(node.targets[0], ast.Name) and self.is_overridden(node, name=node.targets[0].id): - return ast.Pass() - elif isinstance(node.targets[0], ast.Tuple) and isinstance(node.value, ast.Tuple): - return self.mutate_unpack(node) - else: - raise MutationResign() - - def mutate_unpack(self, node): - target = node.targets[0] - value = node.value - new_targets = [] - new_values = [] - for target_element, value_element in zip(target.elts, value.elts): - if not self.is_overridden(node, getattr(target_element, 'id', None)): - new_targets.append(target_element) - new_values.append(value_element) - if len(new_targets) == len(target.elts): - raise MutationResign() - if not new_targets: - return ast.Pass() - elif len(new_targets) == 1: - node.targets = new_targets - node.value = new_values[0] - return node - else: - target.elts = new_targets - value.elts = new_values - return node - - @classmethod - def name(cls): - return 'IHD' - - -class LogicalConnectorReplacement(MutationOperator): - def mutate_And(self, node): - return ast.Or() - - def mutate_Or(self, node): - return ast.And() - - -class LogicalOperatorDeletion(AbstractUnaryOperatorDeletion): - def get_operator_type(self): - return ast.Invert - - -class LogicalOperatorReplacement(MutationOperator): - def mutate_BitAnd(self, node): - return ast.BitOr() - - def mutate_BitOr(self, node): - return ast.BitAnd() - - def mutate_BitXor(self, node): - return ast.BitAnd() - - def mutate_LShift(self, node): - return ast.RShift() - - def mutate_RShift(self, node): - return ast.LShift() - - -class AbstractSuperCallingModification(MutationOperator): - def is_super_call(self, node, stmt): - return isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Call) and \ - isinstance(stmt.value.func, ast.Attribute) and isinstance(stmt.value.func.value, ast.Call) and \ - isinstance(stmt.value.func.value.func, ast.Name) and stmt.value.func.value.func.id == 'super' and \ - stmt.value.func.attr == node.name - - def should_mutate(self, node): - return isinstance(node.parent, ast.ClassDef) - - def get_super_call(self, node): - for index, stmt in enumerate(node.body): - if self.is_super_call(node, stmt): - break - else: - return None, None - return index, stmt - - -class OverriddenMethodCallingPositionChange(AbstractSuperCallingModification): - def should_mutate(self, node): - return super().should_mutate(node) and len(node.body) > 1 - - @copy_node - def mutate_FunctionDef(self, node): - if not self.should_mutate(node): - raise MutationResign() - index, stmt = self.get_super_call(node) - if index is None: - raise MutationResign() - super_call = node.body[index] - del node.body[index] - if index == 0: - node.body.append(super_call) - else: - node.body.insert(0, super_call) - return node - - @classmethod - def name(cls): - return 'IOP' - - -class OverridingMethodDeletion(AbstractOverriddenElementModification): - def mutate_FunctionDef(self, node): - if self.is_overridden(node): - return ast.Pass() - raise MutationResign() - - @classmethod - def name(cls): - return 'IOD' - - -class RelationalOperatorReplacement(MutationOperator): - def mutate_Lt(self, node): - return ast.Gt() - - def mutate_Lt_to_LtE(self, node): - return ast.LtE() - - def mutate_Gt(self, node): - return ast.Lt() - - def mutate_Gt_to_GtE(self, node): - return ast.GtE() - - def mutate_LtE(self, node): - return ast.GtE() - - def mutate_LtE_to_Lt(self, node): - return ast.Lt() - - def mutate_GtE(self, node): - return ast.LtE() - - def mutate_GtE_to_Gt(self, node): - return ast.Gt() - - def mutate_Eq(self, node): - return ast.NotEq() - - def mutate_NotEq(self, node): - return ast.Eq() - - -class SliceIndexRemove(MutationOperator): - def mutate_Slice_remove_lower(self, node): - if not node.lower: - raise MutationResign() - - return ast.Slice(lower=None, upper=node.upper, step=node.step) - - def mutate_Slice_remove_upper(self, node): - if not node.upper: - raise MutationResign() - - return ast.Slice(lower=node.lower, upper=None, step=node.step) - - def mutate_Slice_remove_step(self, node): - if not node.step: - raise MutationResign() - - return ast.Slice(lower=node.lower, upper=node.upper, step=None) - - -class SuperCallingDeletion(AbstractSuperCallingModification): - @copy_node - def mutate_FunctionDef(self, node): - if not self.should_mutate(node): - raise MutationResign() - index, _ = self.get_super_call(node) - if index is None: - raise MutationResign() - node.body[index] = ast.Pass() - return node - - -class SuperCallingInsertPython27(AbstractSuperCallingModification, AbstractOverriddenElementModification): - __python_version__ = (2, 7) - - def should_mutate(self, node): - return super().should_mutate(node) and self.is_overridden(node) - - @copy_node - def mutate_FunctionDef(self, node): - if not self.should_mutate(node): - raise MutationResign() - index, stmt = self.get_super_call(node) - if index is not None: - raise MutationResign() - node.body.insert(0, self.create_super_call(node)) - return node - - def create_super_call(self, node): - super_call = utils.create_ast('super().{}()'.format(node.name)).body[0] - for arg in node.args.args[1:-len(node.args.defaults) or None]: - super_call.value.args.append(ast.Name(id=arg.arg, ctx=ast.Load())) - for arg, default in zip(node.args.args[-len(node.args.defaults):], node.args.defaults): - super_call.value.keywords.append(ast.keyword(arg=arg.arg, value=default)) - for arg, default in zip(node.args.kwonlyargs, node.args.kw_defaults): - super_call.value.keywords.append(ast.keyword(arg=arg.arg, value=default)) - if node.args.vararg: - self.add_vararg_to_super_call(super_call, node.args.vararg) - if node.args.kwarg: - self.add_kwarg_to_super_call(super_call, node.args.kwarg) - return super_call - - @staticmethod - def add_kwarg_to_super_call(super_call, kwarg): - super_call.value.kwargs = ast.Name(id=kwarg, ctx=ast.Load()) - - @staticmethod - def add_vararg_to_super_call(super_call, vararg): - super_call.value.starargs = ast.Name(id=vararg, ctx=ast.Load()) - - -class SuperCallingInsertPython35(SuperCallingInsertPython27): - __python_version__ = (3, 5) - - @staticmethod - def add_kwarg_to_super_call(super_call, kwarg): - super_call.value.keywords.append(ast.keyword(arg=None, value=ast.Name(id=kwarg.arg, ctx=ast.Load()))) - - @staticmethod - def add_vararg_to_super_call(super_call, vararg): - super_call.value.args.append(ast.Starred(ctx=ast.Load(), value=ast.Name(id=vararg.arg, ctx=ast.Load()))) - - -SuperCallingInsert = utils.get_by_python_version([ - SuperCallingInsertPython27, - SuperCallingInsertPython35, -]) - - -class AbstractMethodDecoratorInsertionMutationOperator(MutationOperator): - @copy_node - def mutate_FunctionDef(self, node): - if not isinstance(node.parent, ast.ClassDef): - raise MutationResign() - for decorator in node.decorator_list: - if isinstance(decorator, ast.Call): - decorator_name = decorator.func.id - elif isinstance(decorator, ast.Attribute): - decorator_name = decorator.value.id - else: - decorator_name = decorator.id - if decorator_name == self.get_decorator_name(): - raise MutationResign() - - decorator = ast.Name(id=self.get_decorator_name(), ctx=ast.Load()) - node.decorator_list.append(decorator) - return node - - def get_decorator_name(self): - raise NotImplementedError() - - -class ClassmethodDecoratorInsertion(AbstractMethodDecoratorInsertionMutationOperator): - def get_decorator_name(self): - return 'classmethod' - - -class OneIterationLoop(MutationOperator): - def one_iteration(self, node): - node.body.append(ast.Break()) - return node - - @copy_node - def mutate_For(self, node): - return self.one_iteration(node) - - @copy_node - def mutate_While(self, node): - return self.one_iteration(node) - - -class ReverseIterationLoop(MutationOperator): - @copy_node - def mutate_For(self, node): - old_iter = node.iter - node.iter = ast.Call( - func=ast.Name(id=reversed.__name__, ctx=ast.Load()), - args=[old_iter], - keywords=[], - starargs=None, - kwargs=None, - ) - return node - - -class SelfVariableDeletion(MutationOperator): - def mutate_Attribute(self, node): - try: - if node.value.id == 'self': - return ast.Name(id=node.attr, ctx=ast.Load()) - else: - raise MutationResign() - except AttributeError: - raise MutationResign() - - -class StatementDeletion(MutationOperator): - def mutate_Assign(self, node): - return ast.Pass() - - def mutate_Return(self, node): - return ast.Pass() - - def mutate_Expr(self, node): - if utils.is_docstring(node.value): - raise MutationResign() - return ast.Pass() - - @classmethod - def name(cls): - return 'SDL' - - -class StaticmethodDecoratorInsertion(AbstractMethodDecoratorInsertionMutationOperator): - def get_decorator_name(self): - return 'staticmethod' - - -class ZeroIterationLoop(MutationOperator): - def zero_iteration(self, node): - node.body = [ast.Break()] - return node - - @copy_node - def mutate_For(self, node): - return self.zero_iteration(node) - - @copy_node - def mutate_While(self, node): - return self.zero_iteration(node) - - -standard_operators = { - ArithmeticOperatorDeletion, - ArithmeticOperatorReplacement, - AssignmentOperatorReplacement, - BreakContinueReplacement, - ConditionalOperatorDeletion, - ConditionalOperatorInsertion, - ConstantReplacement, - DecoratorDeletion, - ExceptionHandlerDeletion, - ExceptionSwallowing, - HidingVariableDeletion, - LogicalConnectorReplacement, - LogicalOperatorDeletion, - LogicalOperatorReplacement, - OverriddenMethodCallingPositionChange, - OverridingMethodDeletion, - RelationalOperatorReplacement, - SliceIndexRemove, - SuperCallingDeletion, - SuperCallingInsert, -} - -experimental_operators = { - ClassmethodDecoratorInsertion, - OneIterationLoop, - ReverseIterationLoop, - SelfVariableDeletion, - StatementDeletion, - StaticmethodDecoratorInsertion, - ZeroIterationLoop, -} diff --git a/mutpy/operators/__init__.py b/mutpy/operators/__init__.py new file mode 100644 index 0000000..5dfde79 --- /dev/null +++ b/mutpy/operators/__init__.py @@ -0,0 +1,46 @@ +from .arithmetic import * +from .base import * +from .decorator import * +from .exception import * +from .inheritance import * +from .logical import * +from .loop import * +from .misc import * + +SuperCallingInsert = utils.get_by_python_version([ + SuperCallingInsertPython27, + SuperCallingInsertPython35, +]) + +standard_operators = { + ArithmeticOperatorDeletion, + ArithmeticOperatorReplacement, + AssignmentOperatorReplacement, + BreakContinueReplacement, + ConditionalOperatorDeletion, + ConditionalOperatorInsertion, + ConstantReplacement, + DecoratorDeletion, + ExceptionHandlerDeletion, + ExceptionSwallowing, + HidingVariableDeletion, + LogicalConnectorReplacement, + LogicalOperatorDeletion, + LogicalOperatorReplacement, + OverriddenMethodCallingPositionChange, + OverridingMethodDeletion, + RelationalOperatorReplacement, + SliceIndexRemove, + SuperCallingDeletion, + SuperCallingInsert, +} + +experimental_operators = { + ClassmethodDecoratorInsertion, + OneIterationLoop, + ReverseIterationLoop, + SelfVariableDeletion, + StatementDeletion, + StaticmethodDecoratorInsertion, + ZeroIterationLoop, +} diff --git a/mutpy/operators/arithmetic.py b/mutpy/operators/arithmetic.py new file mode 100644 index 0000000..633c0a9 --- /dev/null +++ b/mutpy/operators/arithmetic.py @@ -0,0 +1,79 @@ +import ast + +from mutpy.operators.base import MutationResign, MutationOperator, AbstractUnaryOperatorDeletion + + +class ArithmeticOperatorDeletion(AbstractUnaryOperatorDeletion): + def get_operator_type(self): + return ast.UAdd, ast.USub + + +class AbstractArithmeticOperatorReplacement(MutationOperator): + def should_mutate(self, node): + raise NotImplementedError() + + def mutate_Add(self, node): + if self.should_mutate(node): + return ast.Sub() + raise MutationResign() + + def mutate_Sub(self, node): + if self.should_mutate(node): + return ast.Add() + raise MutationResign() + + def mutate_Mult_to_Div(self, node): + if self.should_mutate(node): + return ast.Div() + raise MutationResign() + + def mutate_Mult_to_FloorDiv(self, node): + if self.should_mutate(node): + return ast.FloorDiv() + raise MutationResign() + + def mutate_Mult_to_Pow(self, node): + if self.should_mutate(node): + return ast.Pow() + raise MutationResign() + + def mutate_Div_to_Mult(self, node): + if self.should_mutate(node): + return ast.Mult() + raise MutationResign() + + def mutate_Div_to_FloorDiv(self, node): + if self.should_mutate(node): + return ast.FloorDiv() + raise MutationResign() + + def mutate_FloorDiv_to_Div(self, node): + if self.should_mutate(node): + return ast.Div() + raise MutationResign() + + def mutate_FloorDiv_to_Mult(self, node): + if self.should_mutate(node): + return ast.Mult() + raise MutationResign() + + def mutate_Mod(self, node): + if self.should_mutate(node): + return ast.Mult() + raise MutationResign() + + def mutate_Pow(self, node): + if self.should_mutate(node): + return ast.Mult() + raise MutationResign() + + +class ArithmeticOperatorReplacement(AbstractArithmeticOperatorReplacement): + def should_mutate(self, node): + return not isinstance(node.parent, ast.AugAssign) + + def mutate_USub(self, node): + return ast.UAdd() + + def mutate_UAdd(self, node): + return ast.USub() diff --git a/mutpy/operators/base.py b/mutpy/operators/base.py new file mode 100644 index 0000000..ff9db61 --- /dev/null +++ b/mutpy/operators/base.py @@ -0,0 +1,146 @@ +import ast +import copy +import re + +from mutpy import utils + + +class MutationResign(Exception): + pass + + +class Mutation: + def __init__(self, operator, node, visitor=None): + self.operator = operator + self.node = node + self.visitor = visitor + + +def copy_node(mutate): + def f(self, node): + copied_node = copy.deepcopy(node, memo={ + id(node.parent): node.parent, + }) + return mutate(self, copied_node) + + return f + + +class MutationOperator: + def mutate(self, node, to_mutate=None, sampler=None, coverage_injector=None, module=None, only_mutation=None): + self.to_mutate = to_mutate + self.sampler = sampler + self.only_mutation = only_mutation + self.coverage_injector = coverage_injector + self.module = module + for new_node in self.visit(node): + yield Mutation(operator=self.__class__, node=self.current_node, visitor=self.visitor), new_node + + def visit(self, node): + if self.has_notmutate(node) or (self.coverage_injector and not self.coverage_injector.is_covered(node)): + return + if self.only_mutation and self.only_mutation.node != node and self.only_mutation.node not in node.children: + return + self.fix_lineno(node) + visitors = self.find_visitors(node) + if visitors: + for visitor in visitors: + try: + if self.sampler and not self.sampler.is_mutation_time(): + raise MutationResign + if self.only_mutation and \ + (self.only_mutation.node != node or self.only_mutation.visitor != visitor.__name__): + raise MutationResign + new_node = visitor(node) + self.visitor = visitor.__name__ + self.current_node = node + self.fix_node_internals(node, new_node) + ast.fix_missing_locations(new_node) + yield new_node + except MutationResign: + pass + finally: + for new_node in self.generic_visit(node): + yield new_node + else: + for new_node in self.generic_visit(node): + yield new_node + + def generic_visit(self, node): + for field, old_value in ast.iter_fields(node): + if isinstance(old_value, list): + generator = self.generic_visit_list(old_value) + elif isinstance(old_value, ast.AST): + generator = self.generic_visit_real_node(node, field, old_value) + else: + generator = [] + + for _ in generator: + yield node + + def generic_visit_list(self, old_value): + old_values_copy = old_value[:] + for position, value in enumerate(old_values_copy): + if isinstance(value, ast.AST): + for new_value in self.visit(value): + if not isinstance(new_value, ast.AST): + old_value[position:position + 1] = new_value + elif value is None: + del old_value[position] + else: + old_value[position] = new_value + + yield + old_value[:] = old_values_copy + + def generic_visit_real_node(self, node, field, old_value): + for new_node in self.visit(old_value): + if new_node is None: + delattr(node, field) + else: + setattr(node, field, new_node) + yield + setattr(node, field, old_value) + + def has_notmutate(self, node): + try: + for decorator in node.decorator_list: + if decorator.id == utils.notmutate.__name__: + return True + return False + except AttributeError: + return False + + def fix_lineno(self, node): + if not hasattr(node, 'lineno') and getattr(node, 'parent', None) is not None and hasattr(node.parent, 'lineno'): + node.lineno = node.parent.lineno + + def fix_node_internals(self, old_node, new_node): + if not hasattr(new_node, 'parent'): + new_node.children = old_node.children + new_node.parent = old_node.parent + if hasattr(old_node, 'marker'): + new_node.marker = old_node.marker + + def find_visitors(self, node): + method_prefix = 'mutate_' + node.__class__.__name__ + return self.getattrs_like(method_prefix) + + def getattrs_like(ob, attr_like): + pattern = re.compile(attr_like + "($|(_\w+)+$)") + return [getattr(ob, attr) for attr in dir(ob) if pattern.match(attr)] + + @classmethod + def name(cls): + return ''.join([c for c in cls.__name__ if str.isupper(c)]) + + @classmethod + def long_name(cls): + return ' '.join(map(str.lower, (re.split('([A-Z][a-z]*)', cls.__name__)[1::2]))) + + +class AbstractUnaryOperatorDeletion(MutationOperator): + def mutate_UnaryOp(self, node): + if isinstance(node.op, self.get_operator_type()): + return node.operand + raise MutationResign() diff --git a/mutpy/operators/decorator.py b/mutpy/operators/decorator.py new file mode 100644 index 0000000..53e0d39 --- /dev/null +++ b/mutpy/operators/decorator.py @@ -0,0 +1,50 @@ +import ast + +from mutpy.operators.base import MutationOperator, copy_node, MutationResign + + +class DecoratorDeletion(MutationOperator): + @copy_node + def mutate_FunctionDef(self, node): + if node.decorator_list: + node.decorator_list = [] + return node + else: + raise MutationResign() + + @classmethod + def name(cls): + return 'DDL' + + +class AbstractMethodDecoratorInsertionMutationOperator(MutationOperator): + @copy_node + def mutate_FunctionDef(self, node): + if not isinstance(node.parent, ast.ClassDef): + raise MutationResign() + for decorator in node.decorator_list: + if isinstance(decorator, ast.Call): + decorator_name = decorator.func.id + elif isinstance(decorator, ast.Attribute): + decorator_name = decorator.value.id + else: + decorator_name = decorator.id + if decorator_name == self.get_decorator_name(): + raise MutationResign() + + decorator = ast.Name(id=self.get_decorator_name(), ctx=ast.Load()) + node.decorator_list.append(decorator) + return node + + def get_decorator_name(self): + raise NotImplementedError() + + +class ClassmethodDecoratorInsertion(AbstractMethodDecoratorInsertionMutationOperator): + def get_decorator_name(self): + return 'classmethod' + + +class StaticmethodDecoratorInsertion(AbstractMethodDecoratorInsertionMutationOperator): + def get_decorator_name(self): + return 'staticmethod' diff --git a/mutpy/operators/exception.py b/mutpy/operators/exception.py new file mode 100644 index 0000000..0f46afb --- /dev/null +++ b/mutpy/operators/exception.py @@ -0,0 +1,21 @@ +import ast + +from mutpy.operators.base import MutationOperator, MutationResign + + +class ExceptionHandlerDeletion(MutationOperator): + def mutate_ExceptHandler(self, node): + if node.body and isinstance(node.body[0], ast.Raise): + raise MutationResign() + return ast.ExceptHandler(type=node.type, name=node.name, body=[ast.Raise()]) + + +class ExceptionSwallowing(MutationOperator): + def mutate_ExceptHandler(self, node): + if len(node.body) == 1 and isinstance(node.body[0], ast.Pass): + raise MutationResign() + return ast.ExceptHandler(type=node.type, name=node.name, body=[ast.Pass()]) + + @classmethod + def name(cls): + return 'EXS' diff --git a/mutpy/operators/inheritance.py b/mutpy/operators/inheritance.py new file mode 100644 index 0000000..36e5c5a --- /dev/null +++ b/mutpy/operators/inheritance.py @@ -0,0 +1,185 @@ +import ast +import functools + +from mutpy import utils +from mutpy.operators.base import MutationResign, MutationOperator, copy_node + + +class AbstractOverriddenElementModification(MutationOperator): + def is_overridden(self, node, name=None): + if not isinstance(node.parent, ast.ClassDef): + raise MutationResign() + if not name: + name = node.name + parent = node.parent + parent_names = [] + while parent: + if not isinstance(parent, ast.Module): + parent_names.append(parent.name) + if not isinstance(parent, ast.ClassDef) and not isinstance(parent, ast.Module): + raise MutationResign() + parent = parent.parent + getattr_rec = lambda obj, attr: functools.reduce(getattr, attr, obj) + try: + klass = getattr_rec(self.module, reversed(parent_names)) + except AttributeError: + raise MutationResign() + for base_klass in type.mro(klass)[1:-1]: + if hasattr(base_klass, name): + return True + return False + + +class HidingVariableDeletion(AbstractOverriddenElementModification): + def mutate_Assign(self, node): + if len(node.targets) > 1: + raise MutationResign() + if isinstance(node.targets[0], ast.Name) and self.is_overridden(node, name=node.targets[0].id): + return ast.Pass() + elif isinstance(node.targets[0], ast.Tuple) and isinstance(node.value, ast.Tuple): + return self.mutate_unpack(node) + else: + raise MutationResign() + + def mutate_unpack(self, node): + target = node.targets[0] + value = node.value + new_targets = [] + new_values = [] + for target_element, value_element in zip(target.elts, value.elts): + if not self.is_overridden(node, getattr(target_element, 'id', None)): + new_targets.append(target_element) + new_values.append(value_element) + if len(new_targets) == len(target.elts): + raise MutationResign() + if not new_targets: + return ast.Pass() + elif len(new_targets) == 1: + node.targets = new_targets + node.value = new_values[0] + return node + else: + target.elts = new_targets + value.elts = new_values + return node + + @classmethod + def name(cls): + return 'IHD' + + +class AbstractSuperCallingModification(MutationOperator): + def is_super_call(self, node, stmt): + return isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Call) and \ + isinstance(stmt.value.func, ast.Attribute) and isinstance(stmt.value.func.value, ast.Call) and \ + isinstance(stmt.value.func.value.func, ast.Name) and stmt.value.func.value.func.id == 'super' and \ + stmt.value.func.attr == node.name + + def should_mutate(self, node): + return isinstance(node.parent, ast.ClassDef) + + def get_super_call(self, node): + for index, stmt in enumerate(node.body): + if self.is_super_call(node, stmt): + break + else: + return None, None + return index, stmt + + +class OverriddenMethodCallingPositionChange(AbstractSuperCallingModification): + def should_mutate(self, node): + return super().should_mutate(node) and len(node.body) > 1 + + @copy_node + def mutate_FunctionDef(self, node): + if not self.should_mutate(node): + raise MutationResign() + index, stmt = self.get_super_call(node) + if index is None: + raise MutationResign() + super_call = node.body[index] + del node.body[index] + if index == 0: + node.body.append(super_call) + else: + node.body.insert(0, super_call) + return node + + @classmethod + def name(cls): + return 'IOP' + + +class OverridingMethodDeletion(AbstractOverriddenElementModification): + def mutate_FunctionDef(self, node): + if self.is_overridden(node): + return ast.Pass() + raise MutationResign() + + @classmethod + def name(cls): + return 'IOD' + + +class SuperCallingDeletion(AbstractSuperCallingModification): + @copy_node + def mutate_FunctionDef(self, node): + if not self.should_mutate(node): + raise MutationResign() + index, _ = self.get_super_call(node) + if index is None: + raise MutationResign() + node.body[index] = ast.Pass() + return node + + +class SuperCallingInsertPython27(AbstractSuperCallingModification, AbstractOverriddenElementModification): + __python_version__ = (2, 7) + + def should_mutate(self, node): + return super().should_mutate(node) and self.is_overridden(node) + + @copy_node + def mutate_FunctionDef(self, node): + if not self.should_mutate(node): + raise MutationResign() + index, stmt = self.get_super_call(node) + if index is not None: + raise MutationResign() + node.body.insert(0, self.create_super_call(node)) + return node + + def create_super_call(self, node): + super_call = utils.create_ast('super().{}()'.format(node.name)).body[0] + for arg in node.args.args[1:-len(node.args.defaults) or None]: + super_call.value.args.append(ast.Name(id=arg.arg, ctx=ast.Load())) + for arg, default in zip(node.args.args[-len(node.args.defaults):], node.args.defaults): + super_call.value.keywords.append(ast.keyword(arg=arg.arg, value=default)) + for arg, default in zip(node.args.kwonlyargs, node.args.kw_defaults): + super_call.value.keywords.append(ast.keyword(arg=arg.arg, value=default)) + if node.args.vararg: + self.add_vararg_to_super_call(super_call, node.args.vararg) + if node.args.kwarg: + self.add_kwarg_to_super_call(super_call, node.args.kwarg) + return super_call + + @staticmethod + def add_kwarg_to_super_call(super_call, kwarg): + super_call.value.kwargs = ast.Name(id=kwarg, ctx=ast.Load()) + + @staticmethod + def add_vararg_to_super_call(super_call, vararg): + super_call.value.starargs = ast.Name(id=vararg, ctx=ast.Load()) + + +class SuperCallingInsertPython35(SuperCallingInsertPython27): + __python_version__ = (3, 5) + + @staticmethod + def add_kwarg_to_super_call(super_call, kwarg): + super_call.value.keywords.append(ast.keyword(arg=None, value=ast.Name(id=kwarg.arg, ctx=ast.Load()))) + + @staticmethod + def add_vararg_to_super_call(super_call, vararg): + super_call.value.args.append(ast.Starred(ctx=ast.Load(), value=ast.Name(id=vararg.arg, ctx=ast.Load()))) diff --git a/mutpy/operators/logical.py b/mutpy/operators/logical.py new file mode 100644 index 0000000..205f0e5 --- /dev/null +++ b/mutpy/operators/logical.py @@ -0,0 +1,91 @@ +import ast + +from mutpy.operators.base import MutationOperator, AbstractUnaryOperatorDeletion, copy_node + + +class ConditionalOperatorDeletion(AbstractUnaryOperatorDeletion): + def get_operator_type(self): + return ast.Not + + def mutate_NotIn(self, node): + return ast.In() + + +class ConditionalOperatorInsertion(MutationOperator): + def negate_test(self, node): + not_node = ast.UnaryOp(op=ast.Not(), operand=node.test) + node.test = not_node + return node + + @copy_node + def mutate_While(self, node): + return self.negate_test(node) + + @copy_node + def mutate_If(self, node): + return self.negate_test(node) + + def mutate_In(self, node): + return ast.NotIn() + + +class LogicalConnectorReplacement(MutationOperator): + def mutate_And(self, node): + return ast.Or() + + def mutate_Or(self, node): + return ast.And() + + +class LogicalOperatorDeletion(AbstractUnaryOperatorDeletion): + def get_operator_type(self): + return ast.Invert + + +class LogicalOperatorReplacement(MutationOperator): + def mutate_BitAnd(self, node): + return ast.BitOr() + + def mutate_BitOr(self, node): + return ast.BitAnd() + + def mutate_BitXor(self, node): + return ast.BitAnd() + + def mutate_LShift(self, node): + return ast.RShift() + + def mutate_RShift(self, node): + return ast.LShift() + + +class RelationalOperatorReplacement(MutationOperator): + def mutate_Lt(self, node): + return ast.Gt() + + def mutate_Lt_to_LtE(self, node): + return ast.LtE() + + def mutate_Gt(self, node): + return ast.Lt() + + def mutate_Gt_to_GtE(self, node): + return ast.GtE() + + def mutate_LtE(self, node): + return ast.GtE() + + def mutate_LtE_to_Lt(self, node): + return ast.Lt() + + def mutate_GtE(self, node): + return ast.LtE() + + def mutate_GtE_to_Gt(self, node): + return ast.Gt() + + def mutate_Eq(self, node): + return ast.NotEq() + + def mutate_NotEq(self, node): + return ast.Eq() diff --git a/mutpy/operators/loop.py b/mutpy/operators/loop.py new file mode 100644 index 0000000..e743308 --- /dev/null +++ b/mutpy/operators/loop.py @@ -0,0 +1,45 @@ +import ast + +from mutpy.operators import copy_node, MutationOperator + + +class OneIterationLoop(MutationOperator): + def one_iteration(self, node): + node.body.append(ast.Break()) + return node + + @copy_node + def mutate_For(self, node): + return self.one_iteration(node) + + @copy_node + def mutate_While(self, node): + return self.one_iteration(node) + + +class ReverseIterationLoop(MutationOperator): + @copy_node + def mutate_For(self, node): + old_iter = node.iter + node.iter = ast.Call( + func=ast.Name(id=reversed.__name__, ctx=ast.Load()), + args=[old_iter], + keywords=[], + starargs=None, + kwargs=None, + ) + return node + + +class ZeroIterationLoop(MutationOperator): + def zero_iteration(self, node): + node.body = [ast.Break()] + return node + + @copy_node + def mutate_For(self, node): + return self.zero_iteration(node) + + @copy_node + def mutate_While(self, node): + return self.zero_iteration(node) diff --git a/mutpy/operators/misc.py b/mutpy/operators/misc.py new file mode 100644 index 0000000..d321f97 --- /dev/null +++ b/mutpy/operators/misc.py @@ -0,0 +1,97 @@ +import ast + +from mutpy import utils +from mutpy.operators.arithmetic import AbstractArithmeticOperatorReplacement +from mutpy.operators.base import MutationOperator, MutationResign + + +class AssignmentOperatorReplacement(AbstractArithmeticOperatorReplacement): + def should_mutate(self, node): + return isinstance(node.parent, ast.AugAssign) + + @classmethod + def name(cls): + return 'ASR' + + +class BreakContinueReplacement(MutationOperator): + def mutate_Break(self, node): + return ast.Continue() + + def mutate_Continue(self, node): + return ast.Break() + + +class ConstantReplacement(MutationOperator): + FIRST_CONST_STRING = 'mutpy' + SECOND_CONST_STRING = 'python' + + def mutate_Num(self, node): + return ast.Num(n=node.n + 1) + + def mutate_Str(self, node): + if utils.is_docstring(node): + raise MutationResign() + + if node.s != self.FIRST_CONST_STRING: + return ast.Str(s=self.FIRST_CONST_STRING) + else: + return ast.Str(s=self.SECOND_CONST_STRING) + + def mutate_Str_empty(self, node): + if not node.s or utils.is_docstring(node): + raise MutationResign() + + return ast.Str(s='') + + @classmethod + def name(cls): + return 'CRP' + + +class SliceIndexRemove(MutationOperator): + def mutate_Slice_remove_lower(self, node): + if not node.lower: + raise MutationResign() + + return ast.Slice(lower=None, upper=node.upper, step=node.step) + + def mutate_Slice_remove_upper(self, node): + if not node.upper: + raise MutationResign() + + return ast.Slice(lower=node.lower, upper=None, step=node.step) + + def mutate_Slice_remove_step(self, node): + if not node.step: + raise MutationResign() + + return ast.Slice(lower=node.lower, upper=node.upper, step=None) + + +class SelfVariableDeletion(MutationOperator): + def mutate_Attribute(self, node): + try: + if node.value.id == 'self': + return ast.Name(id=node.attr, ctx=ast.Load()) + else: + raise MutationResign() + except AttributeError: + raise MutationResign() + + +class StatementDeletion(MutationOperator): + def mutate_Assign(self, node): + return ast.Pass() + + def mutate_Return(self, node): + return ast.Pass() + + def mutate_Expr(self, node): + if utils.is_docstring(node.value): + raise MutationResign() + return ast.Pass() + + @classmethod + def name(cls): + return 'SDL' diff --git a/mutpy/test/test_controller.py b/mutpy/test/test_controller.py index a006105..5d6013b 100644 --- a/mutpy/test/test_controller.py +++ b/mutpy/test/test_controller.py @@ -4,6 +4,8 @@ import unittest from mutpy import controller, operators, utils, codegen +from mutpy.test.utils import MockModulesLoader +from mutpy.test_runners import UnittestTestRunner class MutationScoreTest(unittest.TestCase): @@ -48,23 +50,6 @@ def test_update_coverage(self): self.assertEqual(self.score.all_nodes, 1) -class MockModulesLoader: - def __init__(self, name, source): - self.names = [name] - self.source = source - self.module = types.ModuleType(name) - self.module.__file__ = '' - self.load() - - def load(self, *args, **kwargs): - exec(self.source, self.module.__dict__) - sys.modules[self.names[0]] = self.module - return [(self.module, None)] - - def get_source(self): - return self.source - - class MockMutationController(controller.MutationController): def create_target_ast(self, target_module): return utils.create_ast(self.target_loader.get_source()) @@ -93,6 +78,7 @@ def setUp(self): self.score_view = MutationScoreStoreView() mutator = controller.FirstOrderMutator([operators.ArithmeticOperatorReplacement], percentage=100) self.mutation_controller = MockMutationController( + runner_cls=UnittestTestRunner, target_loader=target_loader, test_loader=test_loader, views=[self.score_view], @@ -339,4 +325,4 @@ def test_second_order_mutation_with_multiple_visitors(self): self.assertEqual("x = ''", codegen.to_source(mutant)) self.assertEqual(len(mutations), 1) self.assertEqual(number, 1) - self.assertEqual(codegen.to_source(target_ast), "x = 'test'") \ No newline at end of file + self.assertEqual(codegen.to_source(target_ast), "x = 'test'") diff --git a/mutpy/test/test_coverage.py b/mutpy/test/test_coverage.py index 02be87a..f8ffbca 100644 --- a/mutpy/test/test_coverage.py +++ b/mutpy/test/test_coverage.py @@ -1,6 +1,7 @@ import unittest from mutpy import coverage, utils +from mutpy.test_runners.unittest_runner import UnittestCoverageResult class MarkerNodeTransformerTest(unittest.TestCase): @@ -189,7 +190,7 @@ def test_for_coverage(self): self.assert_not_covered([for_body_el]) -class CoverageTestResultTest(unittest.TestCase): +class UnittestCoverageResultTest(unittest.TestCase): def test_run(self): coverage_injector = coverage.CoverageInjector() @@ -204,7 +205,7 @@ def test_x(self): def test_y(self): pass - result = coverage.CoverageTestResult(coverage_injector=coverage_injector) + result = UnittestCoverageResult(coverage_injector=coverage_injector) suite = unittest.TestSuite() test_x = ATest(methodName='test_x') suite.addTest(test_x) diff --git a/mutpy/test/test_runners.py b/mutpy/test/test_runners.py new file mode 100644 index 0000000..f21c4d0 --- /dev/null +++ b/mutpy/test/test_runners.py @@ -0,0 +1,118 @@ +import unittest + +from mutpy import utils +from mutpy.test.utils import FileMockModulesLoader +from mutpy.test_runners import UnittestTestRunner, PytestTestRunner + +TARGET_MUL_SRC = 'def mul(x): return x * x' +TARGET_MUL_TYPEERROR_SRC = 'def mul(x): return x * "a"' + + +class BaseTestCases: + """Wrapper class around abstract test superclasses to prevent unittest from running them directly.""" + + class BaseTestRunnerTest(unittest.TestCase): + TEST_RUNNER_CLS = None + TEST_SRC_SUCCESS = None + TEST_SRC_FAIL = None + TEST_SRC_SKIP = None + + def setUp(self): + if None in [self.TEST_RUNNER_CLS, self.TEST_SRC_SUCCESS, self.TEST_SRC_FAIL, self.TEST_SRC_SKIP]: + self.fail('Subclasses must override TEST_RUNNER_CLS, TEST_SRC_SUCCESS, TEST_SRC_FAIL and TEST_SRC_SKIP') + + def run_test(self, target_src, test_src): + with FileMockModulesLoader('target', target_src) as target_loader, \ + FileMockModulesLoader('test', test_src) as test_loader: + target_loader.load() + runner = self.TEST_RUNNER_CLS(test_loader, 5, utils.StdoutManager(True), False) + test_module, target_test = test_loader.load()[0] + result, time = runner.run_test(test_module, target_test) + return result + + def test_run_test_success(self): + result = self.run_test(TARGET_MUL_SRC, self.TEST_SRC_SUCCESS) + self.assertTrue(result.was_successful()) + self.assertFalse(result.is_incompetent()) + self.assertTrue(result.is_survived()) + self.assertIsNone(result.get_killer()) + self.assertIsNone(result.get_exception_traceback()) + self.assertIsNone(result.get_exception()) + self.assertEqual(1, result.tests_run()) + self.assertEqual(0, result.tests_skipped()) + self.assertEqual(1, len(result.passed)) + self.assertEqual(0, len(result.failed)) + + def test_run_test_fail(self): + result = self.run_test(TARGET_MUL_SRC, self.TEST_SRC_FAIL) + self.assertFalse(result.was_successful()) + self.assertFalse(result.is_incompetent()) + self.assertFalse(result.is_survived()) + self.assertIn('test_mul', result.get_killer()) + self.assertIsNotNone(result.get_exception_traceback()) + self.assertIsNone(result.get_exception()) + self.assertEqual(1, result.tests_run()) + self.assertEqual(0, result.tests_skipped()) + self.assertEqual(0, len(result.passed)) + self.assertEqual(1, len(result.failed)) + + def test_run_test_skip(self): + result = self.run_test(TARGET_MUL_SRC, self.TEST_SRC_SKIP) + self.assertTrue(result.was_successful()) + self.assertFalse(result.is_incompetent()) + self.assertTrue(result.is_survived()) + self.assertIsNone(result.get_killer()) + self.assertIsNone(result.get_exception_traceback()) + self.assertIsNone(result.get_exception()) + self.assertEqual(0, result.tests_run()) + self.assertEqual(1, result.tests_skipped()) + self.assertEqual(0, len(result.passed)) + self.assertEqual(0, len(result.failed)) + + +class UnittestTestRunnerTest(BaseTestCases.BaseTestRunnerTest): + TEST_RUNNER_CLS = UnittestTestRunner + TEST_SRC_SUCCESS = utils.f(""" + import target + from unittest import TestCase + class MulTest(TestCase): + def test_mul(self): + self.assertEqual(target.mul(2), 4) + """) + TEST_SRC_FAIL = utils.f(""" + import target + from unittest import TestCase + class MulTest(TestCase): + def test_mul(self): + self.assertEqual(target.mul(2), 5) + """) + + TEST_SRC_SKIP = (utils.f(""" + import target + from unittest import TestCase, skip + class MulTest(TestCase): + @skip("test skipping") + def test_skipped(self): + pass + """)) + + +class PytestTestRunnerTest(BaseTestCases.BaseTestRunnerTest): + TEST_RUNNER_CLS = PytestTestRunner + TEST_SRC_SUCCESS = utils.f(""" + import target + def test_mul(): + assert target.mul(2) == 4 + """) + TEST_SRC_FAIL = utils.f(""" + import target + def test_mul(): + assert target.mul(2) == 5 + """) + TEST_SRC_SKIP = utils.f(""" + import target + import pytest + @pytest.mark.skip(reason="test skipping") + def test_mul(): + assert target.mul(2) == 4 + """) diff --git a/mutpy/test/test_utils.py b/mutpy/test/test_utils.py index 9b24f02..09261c4 100644 --- a/mutpy/test/test_utils.py +++ b/mutpy/test/test_utils.py @@ -39,7 +39,10 @@ def setUp(self): self.loader = utils.ModulesLoader(None, ModulesLoaderTest.tmp) def test_load_file(self): - self.assertRaises(NotImplementedError, lambda: self.loader.load_single('sample.py')) + module, to_mutate = self.loader.load_single('a/b/c/sample.py')[0] + + self.assert_module(module, 'sample', 'a/b/c/sample.py', ['X']) + self.assertIsNone(to_mutate) def test_load_module(self): module, to_mutate = self.loader.load_single('a.b.c.sample')[0] diff --git a/mutpy/test/utils.py b/mutpy/test/utils.py new file mode 100644 index 0000000..45483bd --- /dev/null +++ b/mutpy/test/utils.py @@ -0,0 +1,49 @@ +import os +import sys +import tempfile +import types + + +class MockModulesLoader: + def __init__(self, name, source): + self.names = [name] + self.source = source + self.module = types.ModuleType(name) + self.module.__file__ = '' + self.load() + + def load(self, *args, **kwargs): + exec(self.source, self.module.__dict__) + sys.modules[self.names[0]] = self.module + return [(self.module, None)] + + def get_source(self): + return self.source + + +class FileMockModulesLoader: + """Behaves like MockModulesLoader but creates the module as actual file.""" + + def __init__(self, name, source): + self.names = [name] + self.source = source + self.module = types.ModuleType(name) + + def __enter__(self): + self.module_file = tempfile.NamedTemporaryFile(suffix='.py',delete=False) + self.module_file_path = self.module_file.name + self.module_file.write(self.source.encode()) + self.module_file.close() + self.module.__file__ = self.module_file_path + return self + + def load(self, *args, **kwargs): + exec(self.source, self.module.__dict__) + sys.modules[self.names[0]] = self.module + return [(self.module, None)] + + def get_source(self): + return self.source + + def __exit__(self, exc_type, exc_val, exc_tb): + os.remove(self.module_file_path) diff --git a/mutpy/test_runner.py b/mutpy/test_runner.py deleted file mode 100644 index 08240e9..0000000 --- a/mutpy/test_runner.py +++ /dev/null @@ -1,75 +0,0 @@ -import sys -import unittest - -from mutpy import utils - - -class UnittestTestRunner: - def __init__(self, test_loader, timeout_factor, stdout_manager, init_modules): - self.test_loader = test_loader - self.timeout_factor = timeout_factor - self.stdout_manager = stdout_manager - self.init_modules = init_modules - - def create_test_suite(self, mutant_module): - suite = unittest.TestSuite() - utils.InjectImporter(mutant_module).install() - self.remove_loaded_modules() - for test_module, target_test in self.test_loader.load(): - suite.addTests(self.get_test_suite(test_module, target_test)) - utils.InjectImporter.uninstall() - return suite - - @utils.TimeRegister - def run_tests_with_mutant(self, total_duration, mutant_module, mutations, coverage_result): - suite = self.create_test_suite(mutant_module) - if coverage_result: - self.mark_not_covered_tests_as_skip(mutations, coverage_result, suite) - timer = utils.Timer() - result = self.run_mutation_test_runner(suite, total_duration) - timer.stop() - return result, timer.duration - - def run_mutation_test_runner(self, suite, total_duration): - live_time = self.timeout_factor * (total_duration if total_duration > 1 else 1) - test_runner_class = utils.get_mutation_test_runner_class() - test_runner = test_runner_class(suite=suite) - with self.stdout_manager: - test_runner.start() - result = test_runner.get_result(live_time) - test_runner.terminate() - return result - - def get_test_suite(self, test_module, target_test): - if target_test: - return unittest.TestLoader().loadTestsFromName(target_test, test_module) - else: - return unittest.TestLoader().loadTestsFromModule(test_module) - - def store_init_modules(self): - test_runner_class = utils.get_mutation_test_runner_class() - test_runner = test_runner_class(suite=unittest.TestSuite()) - test_runner.start() - self.init_modules = list(sys.modules.keys()) - - def remove_loaded_modules(self): - for module in list(sys.modules.keys()): - if module not in self.init_modules: - del sys.modules[module] - - def mark_not_covered_tests_as_skip(self, mutations, coverage_result, suite): - mutated_nodes = {mutation.node.marker for mutation in mutations} - - def iter_tests(tests): - try: - for test in tests: - iter_tests(test) - except TypeError: - add_skip(tests) - - def add_skip(test): - if mutated_nodes.isdisjoint(coverage_result.test_covered_nodes[repr(test)]): - test_method = getattr(test, test._testMethodName) - setattr(test, test._testMethodName, unittest.skip('not covered')(test_method)) - - iter_tests(suite) diff --git a/mutpy/test_runners/__init__.py b/mutpy/test_runners/__init__.py new file mode 100644 index 0000000..4ba7206 --- /dev/null +++ b/mutpy/test_runners/__init__.py @@ -0,0 +1,11 @@ +from .unittest_runner import UnittestTestRunner + + +def pytest_installed(): + import importlib + pytest_loader = importlib.find_loader('pytest') + return pytest_loader is not None + + +if pytest_installed(): + from .pytest_runner import PytestTestRunner diff --git a/mutpy/test_runners/base.py b/mutpy/test_runners/base.py new file mode 100644 index 0000000..bbc2be7 --- /dev/null +++ b/mutpy/test_runners/base.py @@ -0,0 +1,225 @@ +import sys +from abc import abstractmethod +from collections import namedtuple + +from mutpy import utils, coverage + + +class BaseTestSuite: + @abstractmethod + def add_tests(self, test_module, target_test): + pass + + @abstractmethod + def skip_test(self, test): + pass + + @abstractmethod + def run(self): + pass + + @abstractmethod + def run_with_coverage(self, coverage_injector=None): + pass + + @abstractmethod + def __iter__(self): + pass + + +class BaseTest: + + @abstractmethod + def __repr__(self): + pass + + +class CoverageTestResult: + + def __init__(self, *args, coverage_injector=None, **kwargs): + super().__init__(*args, **kwargs) + self.coverage_injector = coverage_injector + self.always_covered_nodes = coverage_injector.covered_nodes.copy() + self.test_covered_nodes = {} + + def start_measure_coverage(self): + self.covered_nodes = self.coverage_injector.covered_nodes.copy() + self.coverage_injector.covered_nodes.clear() + + def stop_measure_coverage(self, test): + self.test_covered_nodes[repr(test)] = self.coverage_injector.covered_nodes.copy() | self.always_covered_nodes + self.coverage_injector.covered_nodes.update(self.covered_nodes) + + +SerializableMutationTestResult = namedtuple( + 'SerializableMutationTestResult', [ + 'is_incompetent', + 'is_survived', + 'killer', + 'exception_traceback', + 'exception', + 'tests_run', + ] +) + + +class MutationTestResult: + def __init__(self, *args, coverage_injector=None, **kwargs): + super(MutationTestResult, self).__init__(*args, **kwargs) + self.coverage_injector = coverage_injector + self.passed = [] + self.failed = [] + self.type_error = None + self.skipped = [] + + def was_successful(self): + return len(self.failed) == 0 + + def is_incompetent(self): + return bool(self.type_error) + + def is_survived(self): + return self.was_successful() + + def _get_killer(self): + if self.failed: + return self.failed[0] + + def get_killer(self): + killer = self._get_killer() + if killer: + return killer.name + + def get_exception_traceback(self): + killer = self._get_killer() + if killer: + return killer.long_message + + def get_exception(self): + return self.type_error + + def tests_run(self): + return len(self.passed) + len(self.failed) + + def tests_skipped(self): + return len(self.skipped) + + def serialize(self): + return SerializableMutationTestResult( + self.is_incompetent(), + self.is_survived(), + str(self.get_killer()), + str(self.get_exception_traceback()), + self.get_exception(), + self.tests_run() - self.tests_skipped(), + ) + + def set_type_error(self, err): + self.type_error = err + + def add_passed(self, name): + self.passed.append(TestInfo(name)) + + def add_skipped(self, name): + self.skipped.append(TestInfo(name)) + + def add_failed(self, name, short_message, long_message): + self.failed.append(TestFailure(name, short_message, long_message)) + + +class TestInfo: + def __init__(self, name): + self.name = name + + +class TestFailure(TestInfo): + def __init__(self, name, short_message, long_message): + super().__init__(name) + self.short_message = short_message + self.long_message = long_message + + +class BaseTestRunner: + test_suite_cls = None + + def __init__(self, test_loader, timeout_factor, stdout_manager, mutate_covered): + self.test_loader = test_loader + self.timeout_factor = timeout_factor + self.stdout_manager = stdout_manager + self.mutate_covered = mutate_covered + self.init_modules = self.find_init_modules() + + def create_empty_test_suite(self): + return self.test_suite_cls() + + def create_test_suite(self, mutant_module): + if not issubclass(self.test_suite_cls, BaseTestSuite): + raise ValueError('{0} is not a subclass of {1}'.format(self.test_suite_cls, BaseTestSuite)) + suite = self.create_empty_test_suite() + utils.InjectImporter(mutant_module).install() + self.remove_loaded_modules() + for test_module, target_test in self.test_loader.load(): + suite.add_tests(test_module, target_test) + utils.InjectImporter.uninstall() + return suite + + @utils.TimeRegister + def run_tests_with_mutant(self, total_duration, mutant_module, mutations, coverage_result): + suite = self.create_test_suite(mutant_module) + if coverage_result: + self.mark_not_covered_tests_as_skip(mutations, coverage_result, suite) + timer = utils.Timer() + result = self.run_mutation_test_runner(suite, total_duration) + timer.stop() + return result, timer.duration + + def run_mutation_test_runner(self, suite, total_duration): + live_time = self.timeout_factor * (total_duration if total_duration > 1 else 1) + test_runner_class = utils.get_mutation_test_runner_class() + test_runner = test_runner_class(suite=suite) + with self.stdout_manager: + test_runner.start() + result = test_runner.get_result(live_time) + test_runner.terminate() + return result + + def inject_coverage(self, target_ast, target_module): + if not self.mutate_covered: + return None, None + coverage_injector = coverage.CoverageInjector() + coverage_module = coverage_injector.inject(target_ast, target_module.__name__) + suite = self.create_test_suite(coverage_module) + with self.stdout_manager: + coverage_result = suite.run_with_coverage(coverage_injector=coverage_injector) + return coverage_injector, coverage_result + + def run_test(self, test_module, target_test): + suite = self.create_empty_test_suite() + suite.add_tests(test_module, target_test) + timer = utils.Timer() + with self.stdout_manager: + result = suite.run() + return result, timer.stop() + + def find_init_modules(self): + #test_runner_class = utils.get_mutation_test_runner_class() + #test_runner = test_runner_class(suite=self.create_empty_test_suite()) + #test_runner.start() + #suite = self.create_empty_test_suite() + #suite.run() + return list(sys.modules.keys()) + + def remove_loaded_modules(self): + for module in list(sys.modules.keys()): + if module not in self.init_modules: + pass + #del sys.modules[module] + + def mark_not_covered_tests_as_skip(self, mutations, coverage_result, suite): + mutated_nodes = {mutation.node.marker for mutation in mutations} + + for test in suite: + test_id = repr(test) + if test_id in coverage_result.test_covered_nodes and mutated_nodes.isdisjoint( + coverage_result.test_covered_nodes[test_id]): + suite.skip_test(test) diff --git a/mutpy/test_runners/pytest_runner.py b/mutpy/test_runners/pytest_runner.py new file mode 100644 index 0000000..1051810 --- /dev/null +++ b/mutpy/test_runners/pytest_runner.py @@ -0,0 +1,114 @@ +from pprint import pprint + +import pytest + +from mutpy.test_runners.base import BaseTestSuite, BaseTestRunner, MutationTestResult, CoverageTestResult, BaseTest + + +class PytestMutpyPlugin: + + def __init__(self, skipped_tests): + self.skipped_tests = skipped_tests + self.mutation_test_result = MutationTestResult() + + def has_failed_before(self, nodeid): + return next((test for test in self.mutation_test_result.failed if test.name == nodeid), None) is not None + + def has_been_skipped_before(self, nodeid): + return next((test for test in self.mutation_test_result.skipped if test.name == nodeid), None) is not None + + def pytest_collection_modifyitems(self, items): + for item in items: + if item.nodeid in self.skipped_tests: + item.add_marker(pytest.mark.skip) + + def pytest_runtest_logreport(self, report): + if report.skipped: + self.mutation_test_result.add_skipped(report.nodeid) + elif report.failed and not self.has_failed_before(report.nodeid): + if 'TypeError' in report.longrepr.reprcrash.message: + self.mutation_test_result.set_type_error(self._recreate_type_error(report.longrepr.reprcrash.message)) + else: + if not hasattr(report, 'longreprtext'): + with open("Output.txt", "w") as text_file: + text_file.write(report.nodeid+' '+vars(report)) + self.mutation_test_result.add_failed(report.nodeid, report.longrepr.reprcrash.message.splitlines()[0], + report.longreprtext) + elif report.passed and report.when == 'teardown' and not self.has_failed_before(report.nodeid) \ + and not self.has_been_skipped_before(report.nodeid): + self.mutation_test_result.add_passed(report.nodeid) + + @staticmethod + def _recreate_type_error(string): + _, message = string.split(":", maxsplit=1) + return TypeError(message.lstrip()) + + +class PytestMutpyCoveragePlugin: + + def __init__(self, coverage_injector): + self.current_test = None + self.coverage_result = CoverageTestResult(coverage_injector=coverage_injector) + + def pytest_runtest_setup(self, item): + self.coverage_result.start_measure_coverage() + self.current_test = item + + def pytest_runtest_teardown(self, nextitem): + self.coverage_result.stop_measure_coverage(PytestTest(self.current_test)) + self.current_test = None + + +class PytestMutpyTestDiscoveryPlugin: + def __init__(self): + self.tests = [] + + def pytest_collection_modifyitems(self, items): + for item in items: + self.tests.append(item) + + +class PytestTestSuite(BaseTestSuite): + def __init__(self): + self.tests = set() + self.skipped_tests = set() + + def add_tests(self, test_module, target_test): + if target_test: + self.tests.add('{0}::{1}'.format(test_module.__file__, target_test)) + elif hasattr(test_module, '__file__'): + self.tests.add(test_module.__file__) + else: + self.tests.add(test_module.__name__) + + def skip_test(self, test): + self.skipped_tests.add(test.internal_test_obj.nodeid) + + def run(self): + mutpy_plugin = PytestMutpyPlugin(skipped_tests=self.skipped_tests) + pytest.main(list(self.tests) + ['-x', '-p', 'no:terminal'], plugins=[mutpy_plugin]) + return mutpy_plugin.mutation_test_result + + def run_with_coverage(self, coverage_injector=None): + mutpy_plugin = PytestMutpyCoveragePlugin(coverage_injector=coverage_injector) + pytest.main(list(self.tests) + ['-p', 'no:terminal'], plugins=[mutpy_plugin]) + return mutpy_plugin.coverage_result + + def __iter__(self): + mutpy_plugin = PytestMutpyTestDiscoveryPlugin() + pytest.main(list(self.tests) + ['--collect-only', '-p', 'no:terminal'], plugins=[mutpy_plugin]) + for test in mutpy_plugin.tests: + yield PytestTest(test) + + +class PytestTest(BaseTest): + + def __repr__(self): + return self.internal_test_obj.nodeid + + def __init__(self, internal_test_obj): + self.internal_test_obj = internal_test_obj + + +class PytestTestRunner(BaseTestRunner): + test_suite_cls = PytestTestSuite diff --git a/mutpy/test_runners/unittest_runner.py b/mutpy/test_runners/unittest_runner.py new file mode 100644 index 0000000..2d0081f --- /dev/null +++ b/mutpy/test_runners/unittest_runner.py @@ -0,0 +1,127 @@ +import unittest + +from mutpy.test_runners.base import CoverageTestResult, BaseTestSuite, BaseTestRunner, MutationTestResult, BaseTest + + +class UnittestMutationTestResult(unittest.TestResult): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.type_error = None + self.failfast = True + self.mutation_test_result = MutationTestResult() + + def addSuccess(self, test): + super().addSuccess(test) + self._add_success(test) + + def addExpectedFailure(self, test, err): + super().addExpectedFailure(test, err) + self._add_success(test) + + def addUnexpectedSuccess(self, test): + super().addUnexpectedSuccess(test) + self._add_latest_unexpected_success() + + def addFailure(self, test, err): + super().addFailure(test, err) + self._add_latest_failure() + + def addSkip(self, test, reason): + super().addSkip(test, reason) + self._add_latest_skip() + + def addError(self, test, err): + if err[0] == TypeError: + self.mutation_test_result.set_type_error(err) + else: + super(UnittestMutationTestResult, self).addError(test, err) + self._add_latest_error() + + def _add_success(self, test): + self.mutation_test_result.add_passed(str(test)) + + def _add_latest_failure(self): + failure = self.failures[-1] + self.mutation_test_result.add_failed(str(failure[0]), self._get_short_message(failure[1]), failure[1]) + + def _add_latest_error(self): + failure = self.errors[-1] + self.mutation_test_result.add_failed(str(failure[0]), self._get_short_message(failure[1]), failure[1]) + + def _add_latest_unexpected_success(self): + failure = self.unexpectedSuccesses[-1] + self.mutation_test_result.add_failed(str(failure[0]), 'Unexpected success') + + def _add_latest_skip(self): + skip = self.skipped[-1] + self.mutation_test_result.add_skipped(str(skip)) + + @staticmethod + def _get_short_message(traceback): + return traceback.split("\n")[-2] + + +class UnittestCoverageResult(CoverageTestResult, unittest.TestResult): + + def startTest(self, test): + super().startTest(test) + self.start_measure_coverage() + + def stopTest(self, test): + super().stopTest(test) + self.stop_measure_coverage(UnittestTest(test)) + + +class UnittestTestSuite(BaseTestSuite): + + def __init__(self): + self.suite = unittest.TestSuite() + + def add_tests(self, test_module, target_test): + self.suite.addTests(self.load_tests(test_module, target_test)) + + def skip_test(self, test): + test_method = getattr(test.internal_test_obj, test.internal_test_obj._testMethodName) + setattr(test.internal_test_obj, test.internal_test_obj._testMethodName, + unittest.skip('not covered')(test_method)) + + def run(self): + result = UnittestMutationTestResult() + self.suite.run(result) + return result.mutation_test_result + + def run_with_coverage(self, coverage_injector=None): + coverage_result = UnittestCoverageResult(coverage_injector=coverage_injector) + self.suite.run(coverage_result) + return coverage_result + + def load_tests(self, test_module, target_test): + if target_test: + return unittest.TestLoader().loadTestsFromName(target_test, test_module) + else: + return unittest.TestLoader().loadTestsFromModule(test_module) + + def iter_tests(self, tests): + try: + for test in tests: + self.iter_tests(test) + except TypeError: + yield tests + + def __iter__(self): + for test in self.iter_tests(self.suite): + yield UnittestTest(test) + + +class UnittestTest(BaseTest): + + def __repr__(self): + return repr(self.internal_test_obj) + + def __init__(self, internal_test_obj): + self.internal_test_obj = internal_test_obj + + +class UnittestTestRunner(BaseTestRunner): + test_suite_cls = UnittestTestSuite diff --git a/mutpy/utils.py b/mutpy/utils.py index 1a24d89..77e3259 100644 --- a/mutpy/utils.py +++ b/mutpy/utils.py @@ -9,9 +9,8 @@ import sys import time import types -import unittest from _pyio import StringIO -from collections import defaultdict, namedtuple +from collections import defaultdict from multiprocessing import Process, Queue from queue import Empty from threading import Thread @@ -41,7 +40,8 @@ def __str__(self): class ModulesLoader: def __init__(self, names, path): self.names = names - sys.path.insert(0, path or '.') + self.path = path or '.' + self.ensure_in_path(self.path) def load(self, without_modules=None): results = [] @@ -53,42 +53,83 @@ def load(self, without_modules=None): yield module, to_mutate def load_single(self, name): - if self.is_file(name): - return self.load_file(name) - elif self.is_package(name): + full_path = self.get_full_path(name) + if os.path.exists(full_path): + if self.is_file(full_path): + return self.load_file(full_path) + elif self.is_directory(full_path): + return self.load_directory(full_path) + if self.is_package(name): return self.load_package(name) else: return self.load_module(name) + def get_full_path(self, name): + if os.path.isabs(name): + return name + return os.path.abspath(os.path.join(self.path, name)) + @staticmethod def is_file(name): - return name.endswith('.py') + return os.path.isfile(name) + + @staticmethod + def is_directory(name): + return os.path.exists(name) and os.path.isdir(name) @staticmethod def is_package(name): try: module = importlib.import_module(name) - return module.__file__.endswith('__init__.py') + return hasattr(module, '__file__') and module.__file__.endswith('__init__.py') except ImportError: return False finally: sys.path_importer_cache.clear() def load_file(self, name): - raise NotImplementedError('File loading is not supported!') + if name.endswith('.py'): + dirname = os.path.dirname(name) + self.ensure_in_path(dirname) + module_name = self.get_filename_without_extension(name) + return self.load_module(module_name) + + def ensure_in_path(self, directory): + if directory not in sys.path: + sys.path.insert(0, directory) + + @staticmethod + def get_filename_without_extension(path): + return os.path.basename(os.path.splitext(path)[0]) @staticmethod def load_package(name): + result = [] try: package = importlib.import_module(name) - result = [] for _, module_name, ispkg in pkgutil.walk_packages(package.__path__, package.__name__ + '.'): if not ispkg: - module = importlib.import_module(module_name) - result.append((module, None)) + try: + module = importlib.import_module(module_name) + result.append((module, None)) + except ImportError as _: + pass + except ImportError as _: + pass + return result + + def load_directory(self, name): + if os.path.isfile(os.path.join(name, '__init__.py')): + parent_dir = self._get_parent_directory(name) + self.ensure_in_path(parent_dir) + return self.load_package(os.path.basename(name)) + else: + result = [] + for file in os.listdir(name): + modules = self.load_single(os.path.join(name, file)) + if modules: + result += modules return result - except ImportError as error: - raise ModulesLoaderException(name, error) def load_module(self, name): module, remainder_path, last_exception = self._split_by_module_and_remainder(name) @@ -96,6 +137,11 @@ def load_module(self, name): raise ModulesLoaderException(name, last_exception) return [(module, '.'.join(remainder_path) if remainder_path else None)] + @staticmethod + def _get_parent_directory(name): + parent_dir = os.path.abspath(os.path.join(name, os.pardir)) + return parent_dir + @staticmethod def _split_by_module_and_remainder(name): """Takes a path string and returns the contained module and the remaining path after it. @@ -160,71 +206,16 @@ def uninstall(cls): class StdoutManager: def __init__(self, disable=True): self.disable = disable + self.original_stdout = None def __enter__(self): if self.disable: + self.original_stdout = sys.stdout sys.stdout = StringIO() def __exit__(self, type, value, traceback): - sys.stdout = sys.__stdout__ - - -SerializableMutationTestResult = namedtuple( - 'SerializableMutationTestResult', [ - 'is_incompetent', - 'is_survived', - 'killer', - 'exception_traceback', - 'exception', - 'tests_run', - ] -) - - -class MutationTestResult(unittest.TestResult): - def __init__(self, *args, coverage_injector=None, **kwargs): - super(MutationTestResult, self).__init__(*args, **kwargs) - self.type_error = None - self.failfast = True - self.coverage_injector = coverage_injector - - def addError(self, test, err): - if err[0] == TypeError: - self.type_error = err - else: - super(MutationTestResult, self).addError(test, err) - - def is_incompetent(self): - return bool(self.type_error) - - def is_survived(self): - return self.wasSuccessful() - - def get_killer(self): - if self.failures: - return self.failures[0][0] - elif self.errors: - return self.errors[0][0] - - def get_exception_traceback(self): - if self.failures: - return self.failures[0][1] - elif self.errors: - return self.errors[0][1] - - def get_exception(self): - if self.type_error: - return self.type_error[1] - - def serialize(self): - return SerializableMutationTestResult( - self.is_incompetent(), - self.is_survived(), - str(self.get_killer()), - str(self.get_exception_traceback()), - self.get_exception(), - self.testsRun - len(self.skipped), - ) + if self.disable: + sys.stdout = self.original_stdout class Timer: @@ -282,8 +273,7 @@ def __init__(self, suite): self.suite = suite def run(self): - result = MutationTestResult() - self.suite.run(result) + result = self.suite.run() self.set_result(result) diff --git a/mutpy/views.py b/mutpy/views.py index 99980e6..3a088d8 100644 --- a/mutpy/views.py +++ b/mutpy/views.py @@ -1,8 +1,10 @@ +import datetime import os import traceback -import datetime -import yaml + import jinja2 +import yaml + from mutpy import codegen, termcolor, utils @@ -112,11 +114,9 @@ def passed(self, tests, number_of_tests): def original_tests_fail(self, result): self.level_print(self.decorate('Tests failed:', 'red', attrs=['bold'])) - for error in result.errors: - self.level_print('error in {} - {} '.format(error[0], error[1].split("\n")[-2]), 2) - for fail in result.failures: - self.level_print('fail in {} - {}'.format(fail[0], fail[1].split("\n")[-2]), 2) + for fail in result.failed: + self.level_print('fail in {} - {}'.format(fail.name, fail.short_message), 2) def mutation(self, number, mutations, module, mutant): for mutation in mutations: @@ -132,7 +132,8 @@ def mutation(self, number, mutations, module, mutant): def cant_load(self, name, exception): self.level_print(self.decorate('Can\'t load module: ', 'red', attrs=['bold']) + '{} ({}: {})'.format(name, - exception.__class__.__name__, exception)) + exception.__class__.__name__, + exception)) def print_code(self, mutant, lineno): mutant_src = codegen.to_source(mutant) @@ -141,7 +142,7 @@ def print_code(self, mutant, lineno): lineno = min(lineno, len(src_lines)) src_lines[lineno - 1] = self.decorate('~' + src_lines[lineno - 1][1:], 'yellow') snippet = src_lines[max(0, lineno - 5):min(len(src_lines), lineno + 5)] - print("\n{}\n".format('-'*80) + "\n".join(snippet) + "\n{}".format('-'*80)) + print("\n{}\n".format('-' * 80) + "\n".join(snippet) + "\n{}".format('-' * 80)) def killed(self, time, killer, *args, **kwargs): self.level_print(self.time_format(time) + ' ' + self.decorate('killed', 'green') + ' by ' + str(killer), diff --git a/setup.py b/setup.py index 236f844..8246943 100644 --- a/setup.py +++ b/setup.py @@ -2,9 +2,10 @@ import sys -import mutpy from setuptools import setup +import mutpy + if sys.hexversion < 0x3030000: print('MutPy requires Python 3.3 or newer!') sys.exit() @@ -27,6 +28,9 @@ packages=['mutpy'], scripts=['bin/mut.py'], install_requires=requirements, + extras_require={ + 'pytest': ["pytest>=3.0"] + }, test_suite='mutpy.test', classifiers=[ 'Programming Language :: Python :: 3.3', diff --git a/tox.ini b/tox.ini index bb73696..a7b2d49 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,10 @@ envlist = coverage-report [testenv] deps= - -rrequirements/development.txt + tox + coverage +extras= + pytest commands = coverage-erase: coverage erase test: coverage run --source=mutpy -m unittest discover -s mutpy/test From 885a38ecacfcb2ceb986271a1d6194a1bef1c28d Mon Sep 17 00:00:00 2001 From: Philipp Hossner Date: Thu, 30 Aug 2018 02:56:21 +0200 Subject: [PATCH 05/24] Add python_requires to setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 8246943..e35099e 100644 --- a/setup.py +++ b/setup.py @@ -19,6 +19,7 @@ setup( name='MutPy', version=mutpy.__version__, + python_requires='>=3.3', description='Mutation testing tool for Python 3.x source code.', long_description=long_description, author='Konrad Hałas', From 00080955f2e623d64bb3c974a5c160d3b1f201a3 Mon Sep 17 00:00:00 2001 From: Philipp Hossner Date: Mon, 3 Sep 2018 12:58:08 +0200 Subject: [PATCH 06/24] Add mutpy.operators to packages in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e35099e..e33625b 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ author_email='halas.konrad@gmail.com', url='https://github.com/mutpy/mutpy', download_url='https://github.com/mutpy/mutpy', - packages=['mutpy'], + packages=['mutpy', 'mutpy.operators'], scripts=['bin/mut.py'], install_requires=requirements, extras_require={ From 3f54379098e4e95364f4e2c51cc49ec3d45e4551 Mon Sep 17 00:00:00 2001 From: Andreas Mueller Date: Mon, 10 Sep 2018 17:21:44 -0400 Subject: [PATCH 07/24] Typo in readme I think this is what was meant? --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index e715923..042161b 100644 --- a/README.rst +++ b/README.rst @@ -5,7 +5,7 @@ MutPy MutPy is a mutation testing tool for Python 3.3+ source code. MutPy supports standard unittest module, generates YAML/HTML reports and has -colorful output. It's apply mutation on AST level. You could boost your +colorful output. It applies mutation on AST level. You could boost your mutation testing process with high order mutations (HOM) and code coverage analysis. From 596a61fcea3bfa60620f93495ac3b570c44a9eaf Mon Sep 17 00:00:00 2001 From: Philipp Hossner Date: Thu, 30 Aug 2018 01:03:58 +0200 Subject: [PATCH 08/24] Add templates to package_data in setup.py (fixes #25) --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index e33625b..5960a15 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ url='https://github.com/mutpy/mutpy', download_url='https://github.com/mutpy/mutpy', packages=['mutpy', 'mutpy.operators'], + package_data={'mutpy': ['templates/*.html']}, scripts=['bin/mut.py'], install_requires=requirements, extras_require={ From ebedaefefa2b5dc49199219599b11000b265f230 Mon Sep 17 00:00:00 2001 From: Philipp Hossner Date: Thu, 30 Aug 2018 02:36:29 +0200 Subject: [PATCH 09/24] Fix tox issue with python 3.3 --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 6c6b30a..36eb29e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,8 @@ python: - 3.5 - 3.6 install: + # python 3.3 compatibility + - if [[ $TRAVIS_PYTHON_VERSION == 3.3 ]]; then pip install virtualenv==15.1.0 setuptools==39.2.0; fi - pip install tox-travis - pip install coveralls script: From 2999eb11c9498ea5e97bf3777b5735719eb26bd0 Mon Sep 17 00:00:00 2001 From: konradhalas Date: Thu, 27 Sep 2018 10:14:25 +0200 Subject: [PATCH 10/24] Add deploy settings to Travis CI configuration --- .travis.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.travis.yml b/.travis.yml index 36eb29e..547e5b8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,3 +13,12 @@ script: - tox after_success: coveralls +deploy: + provider: pypi + user: khalas + password: + secure: pj92Jas0WUVO3lCi6WfYyMgfCVKFpPJkYNr2C0zs3DPWw4VlV78OBXQnauXe0kePDIMIU/4Ye+Mm4u49M8aAbrPgxUaZGL6wJffKgD+DuD6HQooRdSn930tZHUVRAPosQAOFQpD3PwCj+GcVJJs8RuhJpThtLsArPMYmLM0DrDA1pSY/kwJzCnfJSfV5HeydmVDYTg3insDZ3w/+wfjQSlvtuUTo4nsie9MMtj/it7ESy/EBtHL6NDfISPrx/G7O30vGe818YCUxD0inn2ZoU5u8xJxDFnoltB5YRsS2IP6dk6SsUWtA87aqBwjwiYnOVd0X++OJPYCcC5YIyNwcUPK44QJw9nV1+guRAtrQn8R+W4T6JLvu+P547yHHIjUt5h+is1Z7fVC8dzN+Zwxjw6ilI9jNzGuneO7hlsa5PJNdyvK2ELVCH2x7gQN7BDm1BpP7B1QdLbXTP0vByMVcDWoCyZpaVKoIqAU989Zxm/cUvyCg4JKst3IdcgTMGhZelbvx3WMbrEBj/vcQlWQX/F9jbiYF/LP3/6e8U7koJzceh4jhMLO8scG8JnQGp+nfFwLm4XmIU+ziW4F5+pgb3rg+L7PqakpWZ9+HRVqU7hw4kPPB5IByTzpKf5s7osB7ZsKcBZDIJscSQqWDeYITGKQ5cHLe3lxX67oVoIaRW0o= + on: + tags: true + distributions: sdist bdist_wheel + skip_existing: true From 092638f2a5dd5e1a0a20dc9f5f25cfeea8df8288 Mon Sep 17 00:00:00 2001 From: Philipp Hossner Date: Thu, 8 Nov 2018 22:01:57 +0100 Subject: [PATCH 11/24] Fix line numbers of mutants and remove obsolete parentheses in tests --- mutpy/operators/base.py | 11 +++++++++++ mutpy/operators/decorator.py | 8 ++++++-- mutpy/operators/exception.py | 16 ++++++++++++---- mutpy/operators/inheritance.py | 9 ++++++++- mutpy/operators/loop.py | 4 ++-- mutpy/test/test_controller.py | 4 ++-- mutpy/test/test_operators.py | 34 +++++++++++++++++----------------- 7 files changed, 58 insertions(+), 28 deletions(-) diff --git a/mutpy/operators/base.py b/mutpy/operators/base.py index ff9db61..630b323 100644 --- a/mutpy/operators/base.py +++ b/mutpy/operators/base.py @@ -119,6 +119,8 @@ def fix_node_internals(self, old_node, new_node): if not hasattr(new_node, 'parent'): new_node.children = old_node.children new_node.parent = old_node.parent + if not hasattr(new_node, 'lineno') and hasattr(old_node, 'lineno'): + new_node.lineno = old_node.lineno if hasattr(old_node, 'marker'): new_node.marker = old_node.marker @@ -130,6 +132,15 @@ def getattrs_like(ob, attr_like): pattern = re.compile(attr_like + "($|(_\w+)+$)") return [getattr(ob, attr) for attr in dir(ob) if pattern.match(attr)] + def set_lineno(self, node, lineno): + for n in ast.walk(node): + if hasattr(n, 'lineno'): + n.lineno = lineno + + def shift_lines(self, nodes, shift_by=1): + for node in nodes: + ast.increment_lineno(node, shift_by) + @classmethod def name(cls): return ''.join([c for c in cls.__name__ if str.isupper(c)]) diff --git a/mutpy/operators/decorator.py b/mutpy/operators/decorator.py index 53e0d39..ccab9a6 100644 --- a/mutpy/operators/decorator.py +++ b/mutpy/operators/decorator.py @@ -31,8 +31,12 @@ def mutate_FunctionDef(self, node): decorator_name = decorator.id if decorator_name == self.get_decorator_name(): raise MutationResign() - - decorator = ast.Name(id=self.get_decorator_name(), ctx=ast.Load()) + if node.decorator_list: + lineno = node.decorator_list[-1].lineno + else: + lineno = node.lineno + decorator = ast.Name(id=self.get_decorator_name(), ctx=ast.Load(), lineno=lineno) + self.shift_lines(node.body, 1) node.decorator_list.append(decorator) return node diff --git a/mutpy/operators/exception.py b/mutpy/operators/exception.py index 0f46afb..743030a 100644 --- a/mutpy/operators/exception.py +++ b/mutpy/operators/exception.py @@ -3,18 +3,26 @@ from mutpy.operators.base import MutationOperator, MutationResign -class ExceptionHandlerDeletion(MutationOperator): +class BaseExceptionHandlerOperator(MutationOperator): + + @staticmethod + def _replace_exception_body(exception_node, body): + return ast.ExceptHandler(type=exception_node.type, name=exception_node.name, lineno=exception_node.lineno, + body=body) + + +class ExceptionHandlerDeletion(BaseExceptionHandlerOperator): def mutate_ExceptHandler(self, node): if node.body and isinstance(node.body[0], ast.Raise): raise MutationResign() - return ast.ExceptHandler(type=node.type, name=node.name, body=[ast.Raise()]) + return self._replace_exception_body(node, [ast.Raise(lineno=node.body[0].lineno)]) -class ExceptionSwallowing(MutationOperator): +class ExceptionSwallowing(BaseExceptionHandlerOperator): def mutate_ExceptHandler(self, node): if len(node.body) == 1 and isinstance(node.body[0], ast.Pass): raise MutationResign() - return ast.ExceptHandler(type=node.type, name=node.name, body=[ast.Pass()]) + return self._replace_exception_body(node, [ast.Pass(lineno=node.body[0].lineno)]) @classmethod def name(cls): diff --git a/mutpy/operators/inheritance.py b/mutpy/operators/inheritance.py index 36e5c5a..a8db819 100644 --- a/mutpy/operators/inheritance.py +++ b/mutpy/operators/inheritance.py @@ -101,8 +101,12 @@ def mutate_FunctionDef(self, node): super_call = node.body[index] del node.body[index] if index == 0: + self.set_lineno(super_call, node.body[-1].lineno) + self.shift_lines(node.body, -1) node.body.append(super_call) else: + self.set_lineno(super_call, node.body[0].lineno) + self.shift_lines(node.body, 1) node.body.insert(0, super_call) return node @@ -130,7 +134,7 @@ def mutate_FunctionDef(self, node): index, _ = self.get_super_call(node) if index is None: raise MutationResign() - node.body[index] = ast.Pass() + node.body[index] = ast.Pass(lineno=node.body[index].lineno) return node @@ -148,8 +152,10 @@ def mutate_FunctionDef(self, node): if index is not None: raise MutationResign() node.body.insert(0, self.create_super_call(node)) + self.shift_lines(node.body[1:], 1) return node + @copy_node def create_super_call(self, node): super_call = utils.create_ast('super().{}()'.format(node.name)).body[0] for arg in node.args.args[1:-len(node.args.defaults) or None]: @@ -162,6 +168,7 @@ def create_super_call(self, node): self.add_vararg_to_super_call(super_call, node.args.vararg) if node.args.kwarg: self.add_kwarg_to_super_call(super_call, node.args.kwarg) + self.set_lineno(super_call, node.body[0].lineno) return super_call @staticmethod diff --git a/mutpy/operators/loop.py b/mutpy/operators/loop.py index e743308..29327a0 100644 --- a/mutpy/operators/loop.py +++ b/mutpy/operators/loop.py @@ -5,7 +5,7 @@ class OneIterationLoop(MutationOperator): def one_iteration(self, node): - node.body.append(ast.Break()) + node.body.append(ast.Break(lineno=node.body[-1].lineno + 1)) return node @copy_node @@ -33,7 +33,7 @@ def mutate_For(self, node): class ZeroIterationLoop(MutationOperator): def zero_iteration(self, node): - node.body = [ast.Break()] + node.body = [ast.Break(lineno=node.body[0].lineno)] return node @copy_node diff --git a/mutpy/test/test_controller.py b/mutpy/test/test_controller.py index 5d6013b..36924d1 100644 --- a/mutpy/test/test_controller.py +++ b/mutpy/test/test_controller.py @@ -306,10 +306,10 @@ def test_second_order_mutation_with_same_node_as_target(self): self.assertEqual('a', codegen.to_source(mutant)) self.assertEqual(len(mutations), 1) elif number == 1: - self.assertEqual('(+a)', codegen.to_source(mutant)) + self.assertEqual('+a', codegen.to_source(mutant)) self.assertEqual(len(mutations), 1) self.assertEqual(number, 1) - self.assertEqual(codegen.to_source(target_ast), '(-a)') + self.assertEqual(codegen.to_source(target_ast), '-a') def test_second_order_mutation_with_multiple_visitors(self): mutator = controller.HighOrderMutator( diff --git a/mutpy/test/test_operators.py b/mutpy/test/test_operators.py index 55b8779..3069a89 100644 --- a/mutpy/test/test_operators.py +++ b/mutpy/test/test_operators.py @@ -1,7 +1,7 @@ -import unittest import ast -from mutpy import operators, codegen, coverage, utils +import unittest +from mutpy import operators, codegen, coverage, utils EOL = '\n' INDENT = ' ' * 4 @@ -9,7 +9,6 @@ class MutationOperatorTest(unittest.TestCase): - class PassIdOperator(operators.MutationOperator): def mutate_Pass(self, node): @@ -20,7 +19,6 @@ def setUp(self): self.target_ast = utils.create_ast(PASS) def test_generate_all_mutations_if_always_sampler(self): - class AlwaysSampler: def is_mutation_time(self): @@ -31,7 +29,6 @@ def is_mutation_time(self): self.assertEqual(len(mutations), 1) def test_no_mutations_if_never_sampler(self): - class NeverSampler: def is_mutation_time(self): @@ -67,8 +64,11 @@ def assert_mutation(self, original, mutants, lines=None, operator=None, with_cov self.assertIn(mutant_code, mutants, msg) mutants.remove(mutant_code) self.assert_location(mutatnt) - if not lines is None: - self.assert_mutation_lineo(mutation.node.lineno, lines) + if lines is not None: + if not hasattr(mutation.node, 'lineno'): + self.assert_mutation_lineo(mutation.node.parent.lineno, lines) + else: + self.assert_mutation_lineo(mutation.node.lineno, lines) self.assertListEqual(mutants, [], 'did not generate all mutants') @@ -94,7 +94,7 @@ def setUpClass(cls): cls.op = operators.ConstantReplacement() def test_numbers_increment(self): - self.assert_mutation('2 + 3 - 99', ['3 + 3 - 99', '2 + 4 - 99', '2 + 3 - 100']) + self.assert_mutation('2 + 3 - 99', ['(3 + 3) - 99', '(2 + 4) - 99', '(2 + 3) - 100']) def test_string_replacement(self): self.assert_mutation( @@ -149,7 +149,7 @@ def setUpClass(cls): cls.op = operators.ArithmeticOperatorReplacement() def test_add_to_sub_replacement(self): - self.assert_mutation('x + y + z', ['x - y + z', 'x + y - z']) + self.assert_mutation('x + y + z', ['(x - y) + z', '(x + y) - z']) def test_sub_to_add_replacement(self): self.assert_mutation('x - y', ['x + y']) @@ -180,10 +180,10 @@ def test_not_mutate_augmented_assign(self): self.assert_mutation('x += y', []) def test_usub(self): - self.assert_mutation('(-x)', ['(+x)']) + self.assert_mutation('(-x)', ['+x']) def test_uadd(self): - self.assert_mutation('(+x)', ['(-x)']) + self.assert_mutation('(+x)', ['-x']) class AssignmentOperatorReplacementTest(OperatorTestCase): @@ -299,17 +299,17 @@ def setUpClass(cls): cls.op = operators.ConditionalOperatorInsertion() def test_negate_while_condition(self): - self.assert_mutation("while x:\n pass", ["while (not x):\n pass"]) + self.assert_mutation("while x:\n pass", ["while not x:\n pass"]) def test_negate_if_condition(self): - self.assert_mutation('if x:\n pass', ['if (not x):\n pass']) + self.assert_mutation('if x:\n pass', ['if not x:\n pass']) def test_negate_if_and_elif_condition(self): self.assert_mutation( 'if x:' + EOL + INDENT + 'pass' + EOL + 'elif y:' + EOL + INDENT + 'pass', [ - 'if (not x):' + EOL + INDENT + 'pass' + EOL + 'elif y:' + EOL + INDENT + 'pass', - 'if x:' + EOL + INDENT + 'pass' + EOL + 'elif (not y):' + EOL + INDENT + 'pass', + 'if not x:' + EOL + INDENT + 'pass' + EOL + 'elif y:' + EOL + INDENT + 'pass', + 'if x:' + EOL + INDENT + 'pass' + EOL + 'elif not y:' + EOL + INDENT + 'pass', ], lines=[1, 3], ) @@ -440,7 +440,7 @@ def test_not_covered_assign_node(self): def test_not_covered_if_node(self): self.assert_mutation('if False:' + EOL + INDENT + 'if False:' + EOL + 2 * INDENT + PASS, - ['if (not False):' + EOL + INDENT + 'if False:' + EOL + 2 * INDENT + PASS], + ['if not False:' + EOL + INDENT + 'if False:' + EOL + 2 * INDENT + PASS], operator=operators.ConditionalOperatorInsertion(), with_coverage=True) @@ -452,7 +452,7 @@ def test_not_covered_expr_node(self): def test_not_covered_while_node(self): self.assert_mutation('while False:' + EOL + INDENT + 'while False:' + EOL + 2 * INDENT + PASS, - ['while (not False):' + EOL + INDENT + 'while False:' + EOL + 2 * INDENT + PASS], + ['while not False:' + EOL + INDENT + 'while False:' + EOL + 2 * INDENT + PASS], operator=operators.ConditionalOperatorInsertion(), with_coverage=True) From a606b50d3682253191c894206f5faa613d2a6b68 Mon Sep 17 00:00:00 2001 From: Philipp Hossner Date: Thu, 22 Nov 2018 21:36:28 +0100 Subject: [PATCH 12/24] Mutation loader now skips C extensions (fixes #36) --- mutpy/utils.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/mutpy/utils.py b/mutpy/utils.py index 77e3259..ac41bbb 100644 --- a/mutpy/utils.py +++ b/mutpy/utils.py @@ -2,6 +2,7 @@ import copy import ctypes import importlib +import inspect import os import pkgutil import random @@ -11,6 +12,12 @@ import types from _pyio import StringIO from collections import defaultdict + +if sys.version_info >= (3, 5): + from importlib._bootstrap_external import EXTENSION_SUFFIXES, ExtensionFileLoader +else: + from importlib._bootstrap import ExtensionFileLoader, EXTENSION_SUFFIXES + from multiprocessing import Process, Queue from queue import Empty from threading import Thread @@ -43,13 +50,14 @@ def __init__(self, names, path): self.path = path or '.' self.ensure_in_path(self.path) - def load(self, without_modules=None): + def load(self, without_modules=None, exclude_c_extensions=True): results = [] without_modules = without_modules or [] for name in self.names: results += self.load_single(name) for module, to_mutate in results: - if module not in without_modules: + # yield only if module is not explicitly excluded and only source modules (.py) if demanded + if module not in without_modules and not (exclude_c_extensions and self._is_c_extension(module)): yield module, to_mutate def load_single(self, name): @@ -172,6 +180,14 @@ def _module_has_member(module, member_path): return False return True + @staticmethod + def _is_c_extension(module): + if isinstance(getattr(module, '__loader__', None), ExtensionFileLoader): + return True + module_filename = inspect.getfile(module) + module_filetype = os.path.splitext(module_filename)[1] + return module_filetype in EXTENSION_SUFFIXES + class InjectImporter: def __init__(self, module): From d4bee2c70be3d7781e2431f8d6392ae5c7dfd9a6 Mon Sep 17 00:00:00 2001 From: Philipp Hossner Date: Fri, 23 Nov 2018 12:10:49 +0100 Subject: [PATCH 13/24] Fix wrong acronym of the SCI mutation operation (fixes #39) --- mutpy/operators/inheritance.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mutpy/operators/inheritance.py b/mutpy/operators/inheritance.py index a8db819..362b25d 100644 --- a/mutpy/operators/inheritance.py +++ b/mutpy/operators/inheritance.py @@ -179,6 +179,10 @@ def add_kwarg_to_super_call(super_call, kwarg): def add_vararg_to_super_call(super_call, vararg): super_call.value.starargs = ast.Name(id=vararg, ctx=ast.Load()) + @classmethod + def name(cls): + return 'SCI' + class SuperCallingInsertPython35(SuperCallingInsertPython27): __python_version__ = (3, 5) From d0fb222ce4bf6d3801950cc200eabb2eca0a6c29 Mon Sep 17 00:00:00 2001 From: Philipp Hossner Date: Fri, 23 Nov 2018 13:24:29 +0100 Subject: [PATCH 14/24] Fix broken module unloading before testing mutants --- mutpy/test_runners/base.py | 14 ++++++-------- mutpy/test_runners/pytest_runner.py | 9 ++------- mutpy/views.py | 2 ++ 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/mutpy/test_runners/base.py b/mutpy/test_runners/base.py index bbc2be7..fff2a3c 100644 --- a/mutpy/test_runners/base.py +++ b/mutpy/test_runners/base.py @@ -73,7 +73,7 @@ def __init__(self, *args, coverage_injector=None, **kwargs): self.skipped = [] def was_successful(self): - return len(self.failed) == 0 + return len(self.failed) == 0 and not self.is_incompetent() def is_incompetent(self): return bool(self.type_error) @@ -202,18 +202,16 @@ def run_test(self, test_module, target_test): return result, timer.stop() def find_init_modules(self): - #test_runner_class = utils.get_mutation_test_runner_class() - #test_runner = test_runner_class(suite=self.create_empty_test_suite()) - #test_runner.start() - #suite = self.create_empty_test_suite() - #suite.run() + test_runner_class = utils.get_mutation_test_runner_class() + test_runner = test_runner_class(suite=self.create_empty_test_suite()) + test_runner.start() + test_runner.terminate() return list(sys.modules.keys()) def remove_loaded_modules(self): for module in list(sys.modules.keys()): if module not in self.init_modules: - pass - #del sys.modules[module] + del sys.modules[module] def mark_not_covered_tests_as_skip(self, mutations, coverage_result, suite): mutated_nodes = {mutation.node.marker for mutation in mutations} diff --git a/mutpy/test_runners/pytest_runner.py b/mutpy/test_runners/pytest_runner.py index 1051810..cb95d83 100644 --- a/mutpy/test_runners/pytest_runner.py +++ b/mutpy/test_runners/pytest_runner.py @@ -27,22 +27,17 @@ def pytest_runtest_logreport(self, report): self.mutation_test_result.add_skipped(report.nodeid) elif report.failed and not self.has_failed_before(report.nodeid): if 'TypeError' in report.longrepr.reprcrash.message: - self.mutation_test_result.set_type_error(self._recreate_type_error(report.longrepr.reprcrash.message)) + self.mutation_test_result.set_type_error(TypeError(str(report.longrepr.reprcrash))) else: if not hasattr(report, 'longreprtext'): with open("Output.txt", "w") as text_file: - text_file.write(report.nodeid+' '+vars(report)) + text_file.write(report.nodeid + ' ' + vars(report)) self.mutation_test_result.add_failed(report.nodeid, report.longrepr.reprcrash.message.splitlines()[0], report.longreprtext) elif report.passed and report.when == 'teardown' and not self.has_failed_before(report.nodeid) \ and not self.has_been_skipped_before(report.nodeid): self.mutation_test_result.add_passed(report.nodeid) - @staticmethod - def _recreate_type_error(string): - _, message = string.split(":", maxsplit=1) - return TypeError(message.lstrip()) - class PytestMutpyCoveragePlugin: diff --git a/mutpy/views.py b/mutpy/views.py index 3a088d8..f1cf1aa 100644 --- a/mutpy/views.py +++ b/mutpy/views.py @@ -117,6 +117,8 @@ def original_tests_fail(self, result): for fail in result.failed: self.level_print('fail in {} - {}'.format(fail.name, fail.short_message), 2) + if result.is_incompetent(): + self.level_print(str(result.get_exception()), 2) def mutation(self, number, mutations, module, mutant): for mutation in mutations: From 40316454bd710a1f18b0413743df5ce8266ed3d2 Mon Sep 17 00:00:00 2001 From: Philipp Hossner Date: Fri, 23 Nov 2018 17:03:48 +0100 Subject: [PATCH 15/24] Raise descriptive exception when using the pytest test runner, but pytest is not installed --- mutpy/test_runners/__init__.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/mutpy/test_runners/__init__.py b/mutpy/test_runners/__init__.py index 4ba7206..e4df30a 100644 --- a/mutpy/test_runners/__init__.py +++ b/mutpy/test_runners/__init__.py @@ -7,5 +7,17 @@ def pytest_installed(): return pytest_loader is not None +class TestRunnerNotInstalledException(Exception): + pass + + +def __pytest_not_installed(*args, **kwargs): + raise TestRunnerNotInstalledException( + 'Pytest is not installed. Please run "pip install pytest" to resolve this issue.' + ) + + if pytest_installed(): from .pytest_runner import PytestTestRunner +else: + PytestTestRunner = __pytest_not_installed From 6915691a09f1ede3d81c7d5d2e93f2ed7fdbef26 Mon Sep 17 00:00:00 2001 From: Philipp Hossner Date: Fri, 23 Nov 2018 17:06:12 +0100 Subject: [PATCH 16/24] Make the unittest test runner the default runner --- mutpy/commandline.py | 4 ++-- mutpy/test_runners/pytest_runner.py | 8 ++++---- requirements/development.txt | 3 ++- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/mutpy/commandline.py b/mutpy/commandline.py index d5c39de..5d6d957 100644 --- a/mutpy/commandline.py +++ b/mutpy/commandline.py @@ -18,8 +18,8 @@ def build_parser(): parser.add_argument('--target', '-t', type=str, nargs='+', help='target module or package to mutate') parser.add_argument('--unit-test', '-u', type=str, nargs='+', help='test class, test method, module or package with unit tests') - parser.add_argument('--runner', type=str, choices=['unittest', 'pytest'], metavar='RUNNER', - help='test runner') + parser.add_argument('--runner', type=str, choices=['unittest', 'pytest'], default='unittest', + metavar='RUNNER', help='test runner') parser.add_argument('--report', '-r', type=str, help='generate YAML report', metavar='REPORT_FILE') parser.add_argument('--report-html', type=str, help='generate HTML report', metavar='DIR_NAME') parser.add_argument('--timeout-factor', '-f', type=float, default=DEF_TIMEOUT_FACTOR, diff --git a/mutpy/test_runners/pytest_runner.py b/mutpy/test_runners/pytest_runner.py index cb95d83..2fa1d10 100644 --- a/mutpy/test_runners/pytest_runner.py +++ b/mutpy/test_runners/pytest_runner.py @@ -1,6 +1,5 @@ -from pprint import pprint - import pytest +from _pytest.config import default_plugins from mutpy.test_runners.base import BaseTestSuite, BaseTestRunner, MutationTestResult, CoverageTestResult, BaseTest @@ -81,7 +80,7 @@ def skip_test(self, test): def run(self): mutpy_plugin = PytestMutpyPlugin(skipped_tests=self.skipped_tests) - pytest.main(list(self.tests) + ['-x', '-p', 'no:terminal'], plugins=[mutpy_plugin]) + pytest.main(args=list(self.tests) + ['-x', '-p', 'no:terminal'], plugins=[*default_plugins, mutpy_plugin]) return mutpy_plugin.mutation_test_result def run_with_coverage(self, coverage_injector=None): @@ -91,7 +90,8 @@ def run_with_coverage(self, coverage_injector=None): def __iter__(self): mutpy_plugin = PytestMutpyTestDiscoveryPlugin() - pytest.main(list(self.tests) + ['--collect-only', '-p', 'no:terminal'], plugins=[mutpy_plugin]) + pytest.main(args=list(self.tests) + ['--collect-only', '-p', 'no:terminal'], + plugins=[*default_plugins, mutpy_plugin]) for test in mutpy_plugin.tests: yield PytestTest(test) diff --git a/requirements/development.txt b/requirements/development.txt index 4d01459..ee4dd56 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -1,3 +1,4 @@ -r production.txt -coverage \ No newline at end of file +coverage +pytest \ No newline at end of file From ddf482a5620147ade1bf820759e8272e2e80ed50 Mon Sep 17 00:00:00 2001 From: Philipp Hossner Date: Fri, 23 Nov 2018 15:14:12 +0100 Subject: [PATCH 17/24] -m flag now prints mutation diff instead of just the mutation (fixes #38,#40) --- mutpy/controller.py | 2 +- mutpy/test/test_views.py | 41 +++++++++++++++++++++++++++++++++++++++- mutpy/views.py | 28 +++++++++++++++++++-------- 3 files changed, 61 insertions(+), 10 deletions(-) diff --git a/mutpy/controller.py b/mutpy/controller.py index cbda802..45b74c7 100644 --- a/mutpy/controller.py +++ b/mutpy/controller.py @@ -116,7 +116,7 @@ def mutate_module(self, target_module, to_mutate, total_duration): if self.mutation_number and self.mutation_number != mutation_number: self.score.inc_incompetent() continue - self.notify_mutation(mutation_number, mutations, target_module.__name__, mutant_ast) + self.notify_mutation(mutation_number, mutations, target_module, mutant_ast) mutant_module = self.create_mutant_module(target_module, mutant_ast) if mutant_module: self.run_tests_with_mutant(total_duration, mutant_module, mutations, coverage_result) diff --git a/mutpy/test/test_views.py b/mutpy/test/test_views.py index 09e11c9..0a40777 100644 --- a/mutpy/test/test_views.py +++ b/mutpy/test/test_views.py @@ -1,10 +1,25 @@ +import sys import unittest +from contextlib import contextmanager +from io import StringIO -from mutpy.views import QuietTextView +from mutpy import utils +from mutpy.views import QuietTextView, TextView COLOR_RED = 'red' +@contextmanager +def captured_output(): + new_out, new_err = StringIO(), StringIO() + old_out, old_err = sys.stdout, sys.stderr + try: + sys.stdout, sys.stderr = new_out, new_err + yield sys.stdout, sys.stderr + finally: + sys.stdout, sys.stderr = old_out, old_err + + class QuietTextViewTest(unittest.TestCase): @staticmethod def get_quiet_text_view(colored_output=False): @@ -19,3 +34,27 @@ def test_decorate_with_color(self): colored_text = text_view.decorate(text, color=COLOR_RED) # then self.assertEqual(expected_colored_text, colored_text) + + +class TextViewTest(unittest.TestCase): + SEPARATOR = '--------------------------------------------------------------------------------' + EOL = "\n" + + @staticmethod + def get_text_view(colored_output=False, show_mutants=False): + return TextView(colored_output=colored_output, show_mutants=show_mutants) + + def test_print_code(self): + # given + text_view = self.get_text_view(show_mutants=True) + original = utils.create_ast('x = x + 1') + mutant = utils.create_ast('x = x - 1') + # when + with captured_output() as (out, err): + text_view.print_code(mutant, original) + # then + output = out.getvalue().strip() + self.assertEqual( + self.SEPARATOR + self.EOL + '- 1: x = x + 1' + self.EOL + '+ 1: x = x - 1' + self.EOL + self.SEPARATOR, + output + ) diff --git a/mutpy/views.py b/mutpy/views.py index f1cf1aa..e3ad03b 100644 --- a/mutpy/views.py +++ b/mutpy/views.py @@ -1,6 +1,9 @@ +import ast import datetime +import inspect import os import traceback +from difflib import unified_diff import jinja2 import yaml @@ -123,28 +126,37 @@ def original_tests_fail(self, result): def mutation(self, number, mutations, module, mutant): for mutation in mutations: self.level_print( - '[#{:>4}] {:<3} {}:{:<3}: '.format(number, mutation.operator.name(), module, mutation.node.lineno), + '[#{:>4}] {:<3} {}: '.format(number, mutation.operator.name(), module.__name__), ended=False, level=2, ) if mutation != mutations[-1]: print() if self.show_mutants: - self.print_code(mutant, mutation.node.lineno) + self.print_code(mutant, ast.parse(inspect.getsource(module))) def cant_load(self, name, exception): self.level_print(self.decorate('Can\'t load module: ', 'red', attrs=['bold']) + '{} ({}: {})'.format(name, exception.__class__.__name__, exception)) - def print_code(self, mutant, lineno): + def print_code(self, mutant, original): mutant_src = codegen.to_source(mutant) mutant_src = codegen.add_line_numbers(mutant_src) - src_lines = mutant_src.split("\n") - lineno = min(lineno, len(src_lines)) - src_lines[lineno - 1] = self.decorate('~' + src_lines[lineno - 1][1:], 'yellow') - snippet = src_lines[max(0, lineno - 5):min(len(src_lines), lineno + 5)] - print("\n{}\n".format('-' * 80) + "\n".join(snippet) + "\n{}".format('-' * 80)) + original_src = codegen.to_source(original) + original_src = codegen.add_line_numbers(original_src) + self._print_diff(mutant_src, original_src) + + def _print_diff(self, mutant_src, original_src): + diff = self._create_diff(mutant_src, original_src) + diff = [line for line in diff if not line.startswith(('---', '+++', '@@'))] + diff = [self.decorate(line, 'blue') if line.startswith('- ') else line for line in diff] + diff = [self.decorate(line, 'green') if line.startswith('+ ') else line for line in diff] + print("\n{}\n".format('-' * 80) + "\n".join(diff) + "\n{}".format('-' * 80)) + + @staticmethod + def _create_diff(mutant_src, original_src): + return list(unified_diff(original_src.split('\n'), mutant_src.split('\n'), n=4, lineterm='')) def killed(self, time, killer, *args, **kwargs): self.level_print(self.time_format(time) + ' ' + self.decorate('killed', 'green') + ' by ' + str(killer), From 018c6eedd65ea1f5ec34d707e78a33b21f0a5faa Mon Sep 17 00:00:00 2001 From: Philipp Hossner Date: Tue, 16 Apr 2019 19:56:12 +0200 Subject: [PATCH 18/24] Update README --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index 042161b..87e534e 100644 --- a/README.rst +++ b/README.rst @@ -169,6 +169,7 @@ List of all arguments with which you can run MutPy: - ``-u UNIT_TEST [UNIT_TEST ...]``, ``--unit-test UNIT_TEST [UNIT_TEST ...]`` - test class, test method, module or package with unit tests, +- ``--runner RUNNER`` - currently supported are: unittest (default), pytest (experimental) - ``-m``, ``--show-mutants`` - show mutants source code, - ``-r REPORT_FILE``, ``--report REPORT_FILE`` - generate YAML report, - ``--report-html DIR_NAME`` - generate HTML report, @@ -240,6 +241,7 @@ Supported Test Runners Currently the following test runners are supported by MutPy: - `unittest `_ +- `pytest `_ License ------- From 169d562240217d597245522c7df7fd104a812f26 Mon Sep 17 00:00:00 2001 From: Philipp Hossner Date: Tue, 16 Apr 2019 19:57:11 +0200 Subject: [PATCH 19/24] Bump version to 0.6.0 --- mutpy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mutpy/__init__.py b/mutpy/__init__.py index 93b60a1..ef7eb44 100644 --- a/mutpy/__init__.py +++ b/mutpy/__init__.py @@ -1 +1 @@ -__version__ = '0.5.1' +__version__ = '0.6.0' From d97d71a7bb06d62bcc494388cb9678987115fdd3 Mon Sep 17 00:00:00 2001 From: Philipp Hossner Date: Tue, 16 Apr 2019 19:57:11 +0200 Subject: [PATCH 20/24] Remove pytest from dev requirements to avoid problems with older python versions --- mutpy/__init__.py | 2 +- requirements/development.txt | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/mutpy/__init__.py b/mutpy/__init__.py index 93b60a1..ef7eb44 100644 --- a/mutpy/__init__.py +++ b/mutpy/__init__.py @@ -1 +1 @@ -__version__ = '0.5.1' +__version__ = '0.6.0' diff --git a/requirements/development.txt b/requirements/development.txt index ee4dd56..4d01459 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -1,4 +1,3 @@ -r production.txt -coverage -pytest \ No newline at end of file +coverage \ No newline at end of file From 6c2f298bba02bcaf10e89815bc440afca73b26c6 Mon Sep 17 00:00:00 2001 From: Philipp Hossner Date: Tue, 16 Apr 2019 20:16:45 +0200 Subject: [PATCH 21/24] Add mutpy.test_runners to packages in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5960a15..d67fc28 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ author_email='halas.konrad@gmail.com', url='https://github.com/mutpy/mutpy', download_url='https://github.com/mutpy/mutpy', - packages=['mutpy', 'mutpy.operators'], + packages=['mutpy', 'mutpy.operators', 'mutpy.test_runners'], package_data={'mutpy': ['templates/*.html']}, scripts=['bin/mut.py'], install_requires=requirements, From 2a7e766e3e98a6875bee66cfffe8e7e7234c4d2f Mon Sep 17 00:00:00 2001 From: Philipp Hossner Date: Tue, 16 Apr 2019 20:18:01 +0200 Subject: [PATCH 22/24] Fix test issue on Python <=3.4 --- mutpy/test_runners/pytest_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mutpy/test_runners/pytest_runner.py b/mutpy/test_runners/pytest_runner.py index 2fa1d10..6f1cdcd 100644 --- a/mutpy/test_runners/pytest_runner.py +++ b/mutpy/test_runners/pytest_runner.py @@ -80,12 +80,12 @@ def skip_test(self, test): def run(self): mutpy_plugin = PytestMutpyPlugin(skipped_tests=self.skipped_tests) - pytest.main(args=list(self.tests) + ['-x', '-p', 'no:terminal'], plugins=[*default_plugins, mutpy_plugin]) + pytest.main(args=list(self.tests) + ['-x', '-p', 'no:terminal'], plugins=list(default_plugins) + [mutpy_plugin]) return mutpy_plugin.mutation_test_result def run_with_coverage(self, coverage_injector=None): mutpy_plugin = PytestMutpyCoveragePlugin(coverage_injector=coverage_injector) - pytest.main(list(self.tests) + ['-p', 'no:terminal'], plugins=[mutpy_plugin]) + pytest.main(list(self.tests) + ['-p', 'no:terminal'], plugins=list(default_plugins) + [mutpy_plugin]) return mutpy_plugin.coverage_result def __iter__(self): From 22f6ff86cea3a56e9548b503f03d3e2a19ca9027 Mon Sep 17 00:00:00 2001 From: Philipp Hossner Date: Tue, 16 Apr 2019 20:22:48 +0200 Subject: [PATCH 23/24] Fix dependencies for Python 3.3 --- requirements/production.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements/production.txt b/requirements/production.txt index 1b739eb..f7a8bf9 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -1,4 +1,5 @@ -PyYAML>=3.1 +PyYAML<=3.11,>=3.10; python_version <= '3.3' +PyYAML>=3.10; python_version > '3.3' Jinja2>=2.7.1 termcolor>=1.0.0 astmonkey>=0.2.2 \ No newline at end of file From 41764c96eeea218c8b21af5fb8bd7b1ed4fc712b Mon Sep 17 00:00:00 2001 From: Philipp Hossner Date: Tue, 16 Apr 2019 20:30:22 +0200 Subject: [PATCH 24/24] Fix test issue on Python <=3.4 --- mutpy/test_runners/pytest_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mutpy/test_runners/pytest_runner.py b/mutpy/test_runners/pytest_runner.py index 6f1cdcd..d0f017a 100644 --- a/mutpy/test_runners/pytest_runner.py +++ b/mutpy/test_runners/pytest_runner.py @@ -91,7 +91,7 @@ def run_with_coverage(self, coverage_injector=None): def __iter__(self): mutpy_plugin = PytestMutpyTestDiscoveryPlugin() pytest.main(args=list(self.tests) + ['--collect-only', '-p', 'no:terminal'], - plugins=[*default_plugins, mutpy_plugin]) + plugins=list(default_plugins) + [mutpy_plugin]) for test in mutpy_plugin.tests: yield PytestTest(test)