From b8384754bccebc1e5e7b165fee2a5345b3890058 Mon Sep 17 00:00:00 2001 From: Andreas Maier Date: Fri, 14 Feb 2020 17:31:28 +0100 Subject: [PATCH 1/2] Refactored garbage_tracked decorator and GarbageTracker formatting Details: * Consolidated the formatting of garbage objects into one method format_garbage() and adjusted the garbage_tracked decorator to use that in combination with an assert statement. Signed-off-by: Andreas Maier --- yagot/_decorators.py | 23 +++++--------- yagot/_garbagetracker.py | 65 +++++++++++----------------------------- 2 files changed, 26 insertions(+), 62 deletions(-) diff --git a/yagot/_decorators.py b/yagot/_decorators.py index 622566d..02a6a39 100644 --- a/yagot/_decorators.py +++ b/yagot/_decorators.py @@ -29,25 +29,18 @@ def garbage_tracked(func): any signature. """ - def wrapper_func(*args, **kwargs): + def garbage_tracked_wrapper(*args, **kwargs): """ - Wrapper function that is invoked instead of the decorated function. - - It puts garbage tracking around the invocation of the decorated function - and reports any garbage objects by raising an AssertionError exception. + Wrapper function for the @garbage_tracked decorator. """ - - tracker = GarbageTracker.get_tracker('_yagot.garbage_tracked') + tracker = GarbageTracker.get_tracker('yagot.garbage_tracked') tracker.enable() tracker.start() - - ret = func(*args, **kwargs) - + ret = func(*args, **kwargs) # The decorated function tracker.stop() - location = "{file}::{func}". \ - format(file=func.__module__, func=func.__name__) - tracker.assert_no_garbage(location) - + location = "{module}::{function}".format( + module=func.__module__, function=func.__name__) + assert not tracker.garbage, tracker.format_garbage(location) return ret - return functools.update_wrapper(wrapper_func, func) + return functools.update_wrapper(garbage_tracked_wrapper, func) diff --git a/yagot/_garbagetracker.py b/yagot/_garbagetracker.py index 4fe686f..2178069 100644 --- a/yagot/_garbagetracker.py +++ b/yagot/_garbagetracker.py @@ -4,7 +4,6 @@ from __future__ import absolute_import, print_function -import sys import types import re import gc @@ -153,62 +152,34 @@ def ignore(self): if self.enabled: self._ignored = True - def print_if_garbage(self, location=None, max=10, stream=sys.stdout): + def format_garbage(self, location=None, max=10): # pylint: disable=redefined-builtin """ - If there were garbage objects found during the last tracking period, - print the garbage objects (up to a maximum number). + Return a formatted multi-line string for all garbage objects detected + during teh tracking period, up to a maximum number. Parameters: - location (string): Location of the tracked code, e.g. as - "module::function". - - max (int): Maximum number of garbage objects to be printed. - - stream: Stream to be printed on. - """ - if self.enabled and self.garbage: - print("\n{num} garbage objects left by {loc}:". - format(num=len(self.garbage), loc=location)) - for i, obj in enumerate(self.garbage): - if i >= max: - print("...", file=stream) - break - obj_str = self._format(obj) - print(obj_str, file=stream) - stream.flush() - - def assert_no_garbage(self, location=None, max=10): - # pylint: disable=redefined-builtin - """ - Assert that there were no garbage objects found during the last - tracking period. Otherwise, raise AssertionError with a message that - describes the garbage objects (up to a maximum number). - - Parameters: - - location (string): Location of the tracked code, e.g. as - "module::function". + location (string): Location of the function that created the garbage + objects, e.g. in the notation "module::function". max (int): Maximum number of garbage objects to be included in the - exception message. - """ - if self.enabled and self.garbage: - ass_str = "{num} garbage objects left by {loc}:\n". \ - format(num=len(self.garbage), loc=location) - for i, obj in enumerate(self.garbage): - # self._generate_objgraph(obj) - if i >= max: - ass_str += "...\n" - break - ass_str += "{}: {}\n".format(i + 1, self._format(obj)) - raise AssertionError(ass_str) + returned string. + """ + ret_str = "\nThere was {num} garbage object(s) caused by function " \ + "{loc}:\n".format(num=len(self.garbage), loc=location) + for i, obj in enumerate(self.garbage): + # self._generate_objgraph(obj) + if i >= max: + ret_str += "\n...\n" + break + ret_str += "\n{}: {}\n".format(i + 1, self.format_obj(obj)) + return ret_str @staticmethod - def _format(obj): + def format_obj(obj): """ - Return a formatted string for the garbage object. + Return a formatted string for a single garbage object. Parameters: From a847921e175deedf3902b62659961be56588b97d Mon Sep 17 00:00:00 2001 From: Andreas Maier Date: Fri, 14 Feb 2020 17:47:31 +0100 Subject: [PATCH 2/2] Initial documentation in README and RTD docs Signed-off-by: Andreas Maier --- README.rst | 155 ++++++++++++++++++++++++++------ docs/index.rst | 31 ++++++- docs/intro.rst | 165 +++++++++++++++++++++++++++++----- examples/test_selfref_dict.py | 12 +++ 4 files changed, 313 insertions(+), 50 deletions(-) create mode 100644 examples/test_selfref_dict.py diff --git a/README.rst b/README.rst index 88e09d6..a21b916 100644 --- a/README.rst +++ b/README.rst @@ -5,10 +5,6 @@ Yagot - Yet Another Garbage Object Tracker for Python :target: https://pypi.python.org/pypi/yagot/ :alt: Version on Pypi -.. # .. image:: https://img.shields.io/pypi/dm/yagot.svg -.. # :target: https://pypi.python.org/pypi/yagot/ -.. # :alt: Pypi downloads - .. image:: https://travis-ci.org/andy-maier/python-yagot.svg?branch=master :target: https://travis-ci.org/andy-maier/python-yagot/branches :alt: Travis test status (master) @@ -29,13 +25,43 @@ Yagot - Yet Another Garbage Object Tracker for Python Overview -------- -TBD +Yagot is Yet Another Garbage Object Tracker for Python. + +It provides a Python decorator named ``garbage_tracked`` which asserts that the +decorated function or method does not create any garbage objects. + +Garbage objects are Python objects that cannot be immediately released when +the object becomes unreachable and are therefore put into the generational +Python garbage collector where more elaborated algorithms are used at a later +point in time to release the objects. + +This may create problems for your Python application for two reasons: + +1. The time delay involved in this approach keeps memory allocated for longer + than necessary, causing increased memory consumption. + +2. There are cases where even the more elaborated algorithms cannot release a + garbage object. If that happens, it is a memory leak that remains until + the Python process ends. + +The ``garbage_tracked`` decorator can be used on any function or method, but +it makes most sense to use it on test functions. It is a signature-preserving +decorator that supports any number of positional and keyword arguments in the +decorated function or method, and any kind of return value(s). + +That decorator can be used with test functions or methods in all test +frameworks, for example `pytest`_, `nose`_, or `unittest`_. + +.. _pytest: https://docs.pytest.org/ +.. _nose: https://nose.readthedocs.io/ +.. _unittest: https://docs.python.org/3/library/unittest.html + Installation ------------ -To install the latest released version of the yagot -package into your active Python environment: +To install the latest released version of the yagot package into your active +Python environment: .. code-block:: bash @@ -43,41 +69,120 @@ package into your active Python environment: This will also install any prerequisite Python packages. -For more details and alternative ways to install, see -`Installation`_. +For more details and alternative ways to install, see `Installation`_. .. _Installation: https://yagot.readthedocs.io/en/stable/intro.html#installation -Documentation -------------- -* `Documentation for latest released version `_ +Quick start +----------- -Change History --------------- +Here is an example of using it with pytest: -* `Change history for latest released version `_ +In ``examples/test_selfref_dict.py``: -Quick Start ------------ +.. code-block:: python -The following simple example script lists the namespaces and the Interop -namespace in a particular WBEM server: + from yagot import garbage_tracked + + @garbage_tracked + def test_selfref_dict(): + + # Dictionary with self-referencing item: + d1 = dict() + d1['self'] = d1 + +Running pytest on this example reveals the garbage object with a test failure +raised by yagot: + +.. code-block:: text + + $ pytest examples -k test_selfref_dict.py + + ===================================== test session starts ====================================== + platform darwin -- Python 2.7.16, pytest-4.6.9, py-1.8.1, pluggy-0.13.1 + rootdir: /Users/maiera/PycharmProjects/python-yagot + plugins: cov-2.8.1 + collected 1 item + + examples/test_selfref_dict.py F [100%] + + =========================================== FAILURES =========================================== + ______________________________________ test_selfref_dict _______________________________________ + + args = (), kwargs = {}, tracker = + ret = None, location = 'test_selfref_dict::test_selfref_dict' + + def garbage_tracked_wrapper(*args, **kwargs): + """ + Wrapper function for the @garbage_tracked decorator. + """ + tracker = GarbageTracker.get_tracker('yagot.garbage_tracked') + tracker.enable() + tracker.start() + ret = func(*args, **kwargs) # The decorated function + tracker.stop() + location = "{module}::{function}".format( + module=func.__module__, function=func.__name__) + > assert not tracker.garbage, tracker.format_garbage(location) + E AssertionError: + E There was 1 garbage object(s) caused by function test_selfref_dict::test_selfref_dict: + E + E 1: object at 0x10e514d70: + E { 'self': } + + yagot/_decorators.py:43: AssertionError + =================================== 1 failed in 0.07 seconds =================================== + +The AssertionError shows that there was one garbage object detected, and +details about that object. In this case, the garbage object is a ``dict`` +object, and we can see that its 'self' item references back to the dict object. + +The failure location and source code shown by pytest is the wrapper function of +the ``garbage_tracked`` decorator, since this is where it is detected. +The decorated function that caused the garbage objects to be created is +reported by pytest as a failing test function, and is also mentioned in the +assertion message using a "module::function" notation. + +Knowing the test function ``test_selfref_dict()`` that caused the object to +become a garbage object is a good start to identify the problem code, and in +our example case it is easy to do. In more complex situations, it may be helpful +to split the complex test function into multiple simpler test functions. + +The ``garbage_tracked`` decorator can be combined with any other decorators. +Note that it always tracks the decorated function, so unless you want to track +what garbage other decorators create, you want to have it directly on the test +function, as the innermost decorator: .. code-block:: python - #!/usr/bin/env python + import pytest + from yagot import garbage_tracked + + @pytest.mark.parametrize('parm2', [ ... ]) + @pytest.mark.parametrize('parm1', [ ... ]) + @garbage_tracked + def test_something(parm1, parm2): + pass # some test code + + +Documentation +------------- + +* `Documentation `_ + + +Change History +-------------- - import yagot +* `Change history `_ - ... (tbd) ... Contributing ------------ -For information on how to contribute to the -Yagot project, see -`Contributing `_. +For information on how to contribute to the Yagot project, see +`Contributing `_. License diff --git a/docs/index.rst b/docs/index.rst index bae9a49..c185ff2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,9 +2,36 @@ Yagot - Yet Another Garbage Object Tracker for Python ***************************************************** -TBD +Yagot is Yet Another Garbage Object Tracker for Python. -The general project web site is: https://github.com/andy-maier/python-yagot +It provides a Python decorator named ``garbage_tracked`` which asserts that the +decorated function or method does not create any garbage objects. + +Garbage objects are Python objects that cannot be immediately released when +the object becomes unreachable and are therefore put into the generational +Python garbage collector where more elaborated algorithms are used at a later +point in time to release the objects. + +This may create problems for your Python application for two reasons: + +1. The time delay involved in this approach keeps memory allocated for longer + than necessary, causing increased memory consumption. + +2. There are cases where even the more elaborated algorithms cannot release a + garbage object. If that happens, it is a memory leak that remains until + the Python process ends. + +The ``garbage_tracked`` decorator can be used on any function or method, but +it makes most sense to use it on test functions. It is a signature-preserving +decorator that supports any number of positional and keyword arguments in the +decorated function or method, and any kind of return value(s). + +That decorator can be used with test functions or methods in all test +frameworks, for example `pytest`_, `nose`_, or `unittest`_. + +.. _pytest: https://docs.pytest.org/ +.. _nose: https://nose.readthedocs.io/ +.. _unittest: https://docs.python.org/3/library/unittest.html .. toctree:: :maxdepth: 2 diff --git a/docs/intro.rst b/docs/intro.rst index 073f4cb..92f43ed 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -8,12 +8,147 @@ Introduction :depth: 2 -.. _`Functionality`: +.. _`Quick start`: -Functionality -------------- +Quick start +----------- -TBD +Here is an example of using it with pytest: + +In ``examples/test_selfref_dict.py``: + +.. code-block:: python + + from yagot import garbage_tracked + + @garbage_tracked + def test_selfref_dict(): + + # Dictionary with self-referencing item: + d1 = dict() + d1['self'] = d1 + +Running pytest on this example reveals the garbage object with a test failure +raised by yagot: + +.. code-block:: text + + $ pytest examples -k test_selfref_dict.py + + ===================================== test session starts ====================================== + platform darwin -- Python 2.7.16, pytest-4.6.9, py-1.8.1, pluggy-0.13.1 + rootdir: /Users/maiera/PycharmProjects/python-yagot + plugins: cov-2.8.1 + collected 1 item + + examples/test_selfref_dict.py F [100%] + + =========================================== FAILURES =========================================== + ______________________________________ test_selfref_dict _______________________________________ + + args = (), kwargs = {}, tracker = + ret = None, location = 'test_selfref_dict::test_selfref_dict' + + def garbage_tracked_wrapper(*args, **kwargs): + """ + Wrapper function for the @garbage_tracked decorator. + """ + tracker = GarbageTracker.get_tracker('yagot.garbage_tracked') + tracker.enable() + tracker.start() + ret = func(*args, **kwargs) # The decorated function + tracker.stop() + location = "{module}::{function}".format( + module=func.__module__, function=func.__name__) + > assert not tracker.garbage, tracker.format_garbage(location) + E AssertionError: + E There was 1 garbage object(s) caused by function test_selfref_dict::test_selfref_dict: + E + E 1: object at 0x10e514d70: + E { 'self': } + + yagot/_decorators.py:43: AssertionError + =================================== 1 failed in 0.07 seconds =================================== + +The AssertionError shows that there was one garbage object detected, and +details about that object. In this case, the garbage object is a ``dict`` +object, and we can see that its 'self' item references back to the dict object. + +The failure location and source code shown by pytest is the wrapper function of +the ``garbage_tracked`` decorator, since this is where it is detected. +The decorated function that caused the garbage objects to be created is +reported by pytest as a failing test function, and is also mentioned in the +assertion message using a "module::function" notation. + +Knowing the test function ``test_selfref_dict()`` that caused the object to +become a garbage object is a good start to identify the problem code, and in +our example case it is easy to do. In more complex situations, it may be helpful +to split the complex test function into multiple simpler test functions. + +The ``garbage_tracked`` decorator can be combined with any other decorators. +Note that it always tracks the decorated function, so unless you want to track +what garbage other decorators create, you want to have it directly on the test +function, as the innermost decorator: + +.. code-block:: python + + import pytest + from yagot import garbage_tracked + + @pytest.mark.parametrize('parm2', [ ... ]) + @pytest.mark.parametrize('parm1', [ ... ]) + @garbage_tracked + def test_something(parm1, parm2): + pass # some test code + + +.. _`Reference cycles`: + +Reference cycles +---------------- + +In probably all cases, such garbage objects are caused by cyclic references +between objects. Here are some simple cases of objects with reference cycles: + +.. code-block:: python + + # Dictionary with self-referencing item: + d1 = dict() + d1['self'] = d1 + + # Object of a class with self-referencing attribute: + class SelfRef(object): + def __init__(self): + self.ref = self + obj = SelfRef() + +The garbage objects created as a result can be inspected by the standard Python +module ``gc`` that provides access to the garbage collector: + +.. code-block:: python + + $ python + >>> import gc + >>> gc.collect() + 0 # No garbage objects initially (in this simple case) + >>> d1 = dict(); d1['self'] = d1 + >>> d1 + {'self': {...}} + >>> gc.collect() + 0 # Still no garbage objects + >>> del d1 # The dict object becomes unreachable ... + >>> gc.collect() + 1 # ... and ends up as one garbage object + +The interesting part happens during the ``del d1`` statement, but let's first +level set on names vs. objects in Python: A variable (``d1``) is not an object +but a name that is bound to an object (of type ``dict``). The ``del d1`` +statement removes the name ``d1`` from its namespace. That causes the reference +count of the ``dict`` object to drop to 0 (in this case, where there is no other +variable name bound to it and no other object referencing it). The object is +then said to be "unreachable". That causes Python to try to immediately release +the ``dict`` object. This does not work because of the self-reference, so it is +put into the garbage collector for later treatment. .. _`Installation`: @@ -21,15 +156,12 @@ TBD Installation ------------ -TBD - - .. _`Supported environments`: Supported environments ^^^^^^^^^^^^^^^^^^^^^^ -Pywbem is supported in these environments: +Yagot is supported in these environments: * Operating Systems: Linux, Windows (native, and with UNIX-like environments), OS-X @@ -52,8 +184,8 @@ Installing - wheel - pip -* Install the yagot package and its prerequisite - Python packages into the active Python environment: +* Install the yagot package and its prerequisite Python packages into the + active Python environment: .. code-block:: bash @@ -194,16 +326,3 @@ the Yagot project, by version type: * New major release (M.N.P -> M+1.0.0): Deprecated functionality may get removed; functionality may be extended or changed; backwards compatibility may be broken. - - -.. _'Python namespaces`: - -Python namespaces ------------------ - -TBD - describe the python namespaces to clarify what is for external use -and what is internal. - -This documentation describes only the external APIs of the -Yagot project, and omits any internal symbols and -any sub-modules. diff --git a/examples/test_selfref_dict.py b/examples/test_selfref_dict.py new file mode 100644 index 0000000..a4c89e1 --- /dev/null +++ b/examples/test_selfref_dict.py @@ -0,0 +1,12 @@ +from yagot import garbage_tracked + +@garbage_tracked +def test_selfref_dict(): + + # Dictionary with self-referencing item: + d1 = dict() + d1['self'] = d1 + try: + x = y + except NameError: + pass