From 39a06a39100d4063c8718659797f65d09d62dabf Mon Sep 17 00:00:00 2001 From: Marko Ristin Date: Sun, 4 Oct 2020 15:42:25 +0200 Subject: [PATCH] compared against `deal` This change compares the features and the performance of the icontract with the latest version of the `deal` library. --- README.rst | 54 +++++---- .../compare_invariant.py | 15 ++- .../compare_postcondition.py | 35 ++++-- .../compare_precondition.py | 37 +++--- mypy.ini | 2 + setup.py | 3 +- tests/test_others.py | 105 ++++++++++++++++++ 7 files changed, 200 insertions(+), 51 deletions(-) rename benchmarks/{against_dpcontracts => against_others}/compare_invariant.py (90%) rename benchmarks/{against_dpcontracts => against_others}/compare_postcondition.py (69%) rename benchmarks/{against_dpcontracts => against_others}/compare_precondition.py (70%) create mode 100644 tests/test_others.py diff --git a/README.rst b/README.rst index 62fdfc3..836a957 100644 --- a/README.rst +++ b/README.rst @@ -24,13 +24,14 @@ redundant condition descriptions ( *e.g.*, `contracts `_, `covenant `_, +`deal `_, `dpcontracts `_, `pyadbc `_ and `pcd `_). This library was strongly inspired by them, but we go two steps further. -First, our violation message on contract breach are much more informatinve. The message includes the source code of the +First, our violation message on contract breach are much more informative. The message includes the source code of the contract condition as well as variable values at the time of the breach. This promotes don't-repeat-yourself principle (`DRY `_) and spare the programmer the tedious task of repeating the message that was already written in code. @@ -42,8 +43,9 @@ To the best of our knowledge, there is currently no other Python library that su correct way. In the long run, we hope that design-by-contract will be adopted and integrated in the language. Consider this library -a work-around till that happens. An ongoing discussion on how to bring design-by-contract into Python language can -be followed on `python-ideas mailing list `_. +a work-around till that happens. You might be also interested in the archived discussion on how to bring +design-by-contract into Python language on +`python-ideas mailing list `_. Usage ===== @@ -782,26 +784,26 @@ relevance. If there is enough feedback from the users, we will of course conside Benchmarks ========== -We run benchmarks against dpcontracts as part of our continuous integration. +We run benchmarks against `deal` and `dpcontracts` libraries as part of our continuous integration. The bodies of the constructors and functions were intentionally left simple so that you can better estimate **overhead** of the contracts in absolute terms rather than relative. This means that the code without contracts will run extremely fast (nanoseconds) in the benchmarks which might make the contracts seem sluggish. However, the methods in the real world usually run -in the order of microseconds and milliseconds, not nanoseconds. Hence, as long as the overhead -of the contract is in the order of microseconds, it is practically acceptable. +in the order of microseconds and milliseconds, not nanoseconds. As long as the overhead +of the contract is in the order of microseconds, it is often practically acceptable. .. Becnhmark report from precommit.py starts. The following scripts were run: -* `benchmarks/against_dpcontracts/compare_invariant.py `_ -* `benchmarks/against_dpcontracts/compare_precondition.py `_ -* `benchmarks/against_dpcontracts/compare_postcondition.py `_ +* `benchmarks/against_others/compare_invariant.py `_ +* `benchmarks/against_others/compare_precondition.py `_ +* `benchmarks/against_others/compare_postcondition.py `_ The benchmarks were executed on Intel(R) Xeon(R) E-2276M CPU @ 2.80GHz. -We used Python 3.8.5, icontract 2.3.4 and dpcontracts 0.6.0. +We used Python 3.8.5, icontract 2.3.5, deal 4.2.0 and dpcontracts 0.6.0. The following tables summarize the results. @@ -810,9 +812,10 @@ Benchmarking invariant at __init__: ========================= ============ ============== ======================= Case Total time Time per run Relative time per run ========================= ============ ============== ======================= -`ClassWithIcontract` 1.43 s 1.43 μs 306% -`ClassWithDpcontracts` 0.47 s 0.47 μs 100% -`ClassWithInlineContract` 0.27 s 0.27 μs 57% +`ClassWithIcontract` 1.74 s 1.74 μs 100% +`ClassWithDpcontracts` 0.55 s 0.55 μs 32% +`ClassWithDeal` 3.26 s 3.26 μs 187% +`ClassWithInlineContract` 0.33 s 0.33 μs 19% ========================= ============ ============== ======================= Benchmarking invariant at a function: @@ -820,9 +823,10 @@ Benchmarking invariant at a function: ========================= ============ ============== ======================= Case Total time Time per run Relative time per run ========================= ============ ============== ======================= -`ClassWithIcontract` 2.00 s 2.00 μs 445% -`ClassWithDpcontracts` 0.45 s 0.45 μs 100% -`ClassWithInlineContract` 0.23 s 0.23 μs 52% +`ClassWithIcontract` 2.48 s 2.48 μs 100% +`ClassWithDpcontracts` 0.56 s 0.56 μs 22% +`ClassWithDeal` 9.76 s 9.76 μs 393% +`ClassWithInlineContract` 0.28 s 0.28 μs 11% ========================= ============ ============== ======================= Benchmarking precondition: @@ -830,9 +834,10 @@ Benchmarking precondition: =============================== ============ ============== ======================= Case Total time Time per run Relative time per run =============================== ============ ============== ======================= -`function_with_icontract` 0.02 s 2.38 μs 5% -`function_with_dpcontracts` 0.51 s 50.89 μs 100% -`function_with_inline_contract` 0.00 s 0.15 μs 0% +`function_with_icontract` 0.03 s 3.17 μs 100% +`function_with_dpcontracts` 0.65 s 64.62 μs 2037% +`function_with_deal` 0.16 s 16.04 μs 506% +`function_with_inline_contract` 0.00 s 0.17 μs 6% =============================== ============ ============== ======================= Benchmarking postcondition: @@ -840,15 +845,20 @@ Benchmarking postcondition: =============================== ============ ============== ======================= Case Total time Time per run Relative time per run =============================== ============ ============== ======================= -`function_with_icontract` 0.02 s 2.48 μs 5% -`function_with_dpcontracts` 0.51 s 50.93 μs 100% -`function_with_inline_contract` 0.00 s 0.15 μs 0% +`function_with_icontract` 0.03 s 3.01 μs 100% +`function_with_dpcontracts` 0.66 s 65.78 μs 2187% +`function_with_deal_post` 0.01 s 1.12 μs 37% +`function_with_deal_ensure` 0.02 s 1.62 μs 54% +`function_with_inline_contract` 0.00 s 0.18 μs 6% =============================== ============ ============== ======================= .. Benchmark report from precommit.py ends. +Note that neither the `dpcontracts` nor the `deal` library support recursion and inheritance of the contracts. +This allows them to use faster enforcement mechanisms and thus gain a speed-up. + We also ran a much more extensive battery of benchmarks on icontract 2.0.7. Unfortunately, it would cost us too much effort to integrate the results in the continous integration. The report is available at: diff --git a/benchmarks/against_dpcontracts/compare_invariant.py b/benchmarks/against_others/compare_invariant.py similarity index 90% rename from benchmarks/against_dpcontracts/compare_invariant.py rename to benchmarks/against_others/compare_invariant.py index be9a6e1..4fe75b6 100644 --- a/benchmarks/against_dpcontracts/compare_invariant.py +++ b/benchmarks/against_others/compare_invariant.py @@ -9,6 +9,7 @@ import timeit from typing import List +import deal import dpcontracts import tabulate @@ -33,6 +34,15 @@ def some_func(self) -> str: return '.'.join(self.parts) +@deal.inv(validator=lambda self: len(self.parts) > 0, message="some dummy invariant") +class ClassWithDeal: + def __init__(self, identifier: str) -> None: + self.parts = identifier.split(".") + + def some_func(self) -> str: + return '.'.join(self.parts) + + class ClassWithInlineContract: def __init__(self, identifier: str) -> None: self.parts = identifier.split(".") @@ -50,6 +60,7 @@ def some_func(self) -> str: clses = [ 'ClassWithIcontract', 'ClassWithDpcontracts', + 'ClassWithDeal', 'ClassWithInlineContract', ] @@ -83,7 +94,7 @@ def measure_invariant_at_init() -> None: '`{}`'.format(cls), '{:.2f} s'.format(duration), '{:.2f} μs'.format(duration * 1000 * 1000 / number), - '{:.0f}%'.format(duration * 100 / durations[1]) + '{:.0f}%'.format(duration * 100 / durations[0]) ]) # yapf: enable @@ -117,7 +128,7 @@ def measure_invariant_at_function() -> None: '`{}`'.format(cls), '{:.2f} s'.format(duration), '{:.2f} μs'.format(duration * 1000 * 1000 / number), - '{:.0f}%'.format(duration * 100 / durations[1]) + '{:.0f}%'.format(duration * 100 / durations[0]) ]) # yapf: enable diff --git a/benchmarks/against_dpcontracts/compare_postcondition.py b/benchmarks/against_others/compare_postcondition.py similarity index 69% rename from benchmarks/against_dpcontracts/compare_postcondition.py rename to benchmarks/against_others/compare_postcondition.py index 9b4ace4..aaa638e 100644 --- a/benchmarks/against_dpcontracts/compare_postcondition.py +++ b/benchmarks/against_others/compare_postcondition.py @@ -10,24 +10,34 @@ import timeit from typing import List -import tabulate - -import icontract +import deal import dpcontracts +import icontract +import tabulate @icontract.ensure(lambda result: result > 0) -def function_with_icontract(someArg: int) -> float: - return math.sqrt(someArg) +def function_with_icontract(some_arg: int) -> float: + return math.sqrt(some_arg) @dpcontracts.ensure("some dummy contract", lambda args, result: result > 0) -def function_with_dpcontracts(someArg: int) -> float: - return math.sqrt(someArg) +def function_with_dpcontracts(some_arg: int) -> float: + return math.sqrt(some_arg) + + +@deal.post(lambda result: result > 0, message="some dummy contract") +def function_with_deal_post(some_arg: int) -> float: + return math.sqrt(some_arg) + + +@deal.ensure(lambda some_arg, result: result > 0, message="some dummy contract") +def function_with_deal_ensure(some_arg: int) -> float: + return math.sqrt(some_arg) -def function_with_inline_contract(someArg: int) -> float: - result = math.sqrt(someArg) +def function_with_inline_contract(some_arg: int) -> float: + result = math.sqrt(some_arg) assert result > 0 return result @@ -44,7 +54,10 @@ def writeln_utf8(text: str) -> None: def measure_functions() -> None: - funcs = ['function_with_icontract', 'function_with_dpcontracts', 'function_with_inline_contract'] + funcs = [ + 'function_with_icontract', 'function_with_dpcontracts', 'function_with_deal_post', 'function_with_deal_ensure', + 'function_with_inline_contract' + ] durations = [0.0] * len(funcs) @@ -62,7 +75,7 @@ def measure_functions() -> None: '`{}`'.format(func), '{:.2f} s'.format(duration), '{:.2f} μs'.format(duration * 1000 * 1000 / number), - '{:.0f}%'.format(duration * 100 / durations[1]) + '{:.0f}%'.format(duration * 100 / durations[0]) ]) # yapf: enable diff --git a/benchmarks/against_dpcontracts/compare_precondition.py b/benchmarks/against_others/compare_precondition.py similarity index 70% rename from benchmarks/against_dpcontracts/compare_precondition.py rename to benchmarks/against_others/compare_precondition.py index e63c5e6..46aa7af 100644 --- a/benchmarks/against_dpcontracts/compare_precondition.py +++ b/benchmarks/against_others/compare_precondition.py @@ -11,29 +11,34 @@ import timeit from typing import List +import deal +import dpcontracts +import icontract import tabulate -import icontract -import dpcontracts + +@icontract.require(lambda some_arg: some_arg > 0) +def function_with_icontract(some_arg: int) -> float: + return math.sqrt(some_arg) -@icontract.require(lambda someArg: someArg > 0) -def function_with_icontract(someArg: int) -> float: - return math.sqrt(someArg) +@dpcontracts.require("some dummy contract", lambda args: args.some_arg > 0) +def function_with_dpcontracts(some_arg: int) -> float: + return math.sqrt(some_arg) -@dpcontracts.require("some dummy contract", lambda args: args.someArg > 0) -def function_with_dpcontracts(someArg: int) -> float: - return math.sqrt(someArg) +@deal.pre(lambda _: _.some_arg > 0) +def function_with_deal(some_arg: int) -> float: + return math.sqrt(some_arg) -def function_with_inline_contract(someArg: int) -> float: - assert (someArg > 0) - return math.sqrt(someArg) +def function_with_inline_contract(some_arg: int) -> float: + assert (some_arg > 0) + return math.sqrt(some_arg) -def function_without_contracts(someArg: int) -> float: - return math.sqrt(someArg) +def function_without_contracts(some_arg: int) -> float: + return math.sqrt(some_arg) def writeln_utf8(text: str) -> None: @@ -48,7 +53,9 @@ def writeln_utf8(text: str) -> None: def measure_functions() -> None: - funcs = ['function_with_icontract', 'function_with_dpcontracts', 'function_with_inline_contract'] + funcs = [ + 'function_with_icontract', 'function_with_dpcontracts', 'function_with_deal', 'function_with_inline_contract' + ] durations = [0.0] * len(funcs) @@ -66,7 +73,7 @@ def measure_functions() -> None: '`{}`'.format(func), '{:.2f} s'.format(duration), '{:.2f} μs'.format(duration * 1000 * 1000 / number), - '{:.0f}%'.format(duration * 100 / durations[1]) + '{:.0f}%'.format(duration * 100 / durations[0]) ]) # yapf: enable diff --git a/mypy.ini b/mypy.ini index 06ddb5b..a7f164c 100644 --- a/mypy.ini +++ b/mypy.ini @@ -3,3 +3,5 @@ [mypy-asttokens] ignore_missing_imports = True +[mypy-dpcontracts] +ignore_missing_imports = True diff --git a/setup.py b/setup.py index 1a797d0..8d02023 100644 --- a/setup.py +++ b/setup.py @@ -5,6 +5,7 @@ https://github.com/pypa/sampleproject """ import os +import sys from setuptools import setup, find_packages @@ -55,7 +56,7 @@ 'tabulate>=0.8.7,<1', 'py-cpuinfo>=5.0.0,<6' # yapf: enable - ], + ] + ['deal==4.1.0'] if sys.version_info >= (3, 8) else [], }, py_modules=['icontract', 'icontract_meta'], package_data={"icontract": ["py.typed"]}) diff --git a/tests/test_others.py b/tests/test_others.py new file mode 100644 index 0000000..4719f41 --- /dev/null +++ b/tests/test_others.py @@ -0,0 +1,105 @@ +# pylint: disable=missing-docstring +# pylint: disable=no-self-use +# pylint: disable=unnecessary-lambda +# pylint: disable=invalid-name +# pylint: disable=broad-except +import sys +import unittest +from typing import List, Optional + +import deal +import dpcontracts + + +class TestDpcontracts(unittest.TestCase): + def test_recursion_unhandled_in_preconditions(self) -> None: + @dpcontracts.require("must another_func", lambda args: another_func()) # type: ignore + @dpcontracts.require("must yet_another_func", lambda args: yet_another_func()) # type: ignore + def some_func() -> bool: + return True + + @dpcontracts.require("must some_func", lambda args: some_func()) # type: ignore + @dpcontracts.require("must yet_yet_another_func", lambda args: yet_yet_another_func()) # type: ignore + def another_func() -> bool: + return True + + def yet_another_func() -> bool: + return True + + def yet_yet_another_func() -> bool: + return True + + cause_err = None # type: Optional[BaseException] + try: + some_func() + except Exception as err: + cause_err = err.__cause__ + + self.assertIsNotNone(cause_err) + self.assertIsInstance(cause_err, RecursionError) + + def test_inheritance_of_postconditions_incorrect(self) -> None: + class A: + @dpcontracts.ensure('dummy contract', lambda args, result: result % 2 == 0) # type: ignore + def some_func(self) -> int: + return 2 + + class B(A): + @dpcontracts.ensure('dummy contract', lambda args, result: result % 3 == 0) # type: ignore + def some_func(self) -> int: + # The result 9 satisfies the postcondition of B.some_func, but not A.some_func. + return 9 + + b = B() + # The correct behavior would be to throw an exception here. + b.some_func() + + +class TestDeal(unittest.TestCase): + @unittest.skipUnless(sys.version_info >= (3, 8), "Test deal only on Python >= 3.8") + def test_recursion_unhandled_in_preconditions(self) -> None: + @deal.pre(lambda _: another_func()) # type: ignore + @deal.pre(lambda _: yet_another_func()) + def some_func() -> bool: + return True + + @deal.pre(lambda _: some_func()) + @deal.pre(lambda _: yet_yet_another_func()) + def another_func() -> bool: + return True + + def yet_another_func() -> bool: + return True + + def yet_yet_another_func() -> bool: + return True + + cause_err = None # type: Optional[BaseException] + try: + some_func() + except Exception as err: + cause_err = err.__cause__ + + self.assertIsNotNone(cause_err) + self.assertIsInstance(cause_err, RecursionError) + + @unittest.skipUnless(sys.version_info >= (3, 8), "Test deal only on Python >= 3.8") + def test_inheritance_of_postconditions_incorrect(self) -> None: + class A: + @deal.post(lambda result: result % 2 == 0) + def some_func(self) -> int: + return 2 + + class B(A): + @deal.post(lambda result: result % 3 == 0) + def some_func(self) -> int: + # The result 9 satisfies the postcondition of B.some_func, but not A.some_func. + return 9 + + b = B() + # The correct behavior would be to throw an exception here. + b.some_func() + + +if __name__ == '__main__': + unittest.main()