diff --git a/README.rst b/README.rst
index 88e09d6..a21b916 100644
--- a/README.rst
+++ b/README.rst
@@ -5,10 +5,6 @@ Yagot - Yet Another Garbage Object Tracker for Python
:target: https://pypi.python.org/pypi/yagot/
:alt: Version on Pypi
-.. # .. image:: https://img.shields.io/pypi/dm/yagot.svg
-.. # :target: https://pypi.python.org/pypi/yagot/
-.. # :alt: Pypi downloads
-
.. image:: https://travis-ci.org/andy-maier/python-yagot.svg?branch=master
:target: https://travis-ci.org/andy-maier/python-yagot/branches
:alt: Travis test status (master)
@@ -29,13 +25,43 @@ Yagot - Yet Another Garbage Object Tracker for Python
Overview
--------
-TBD
+Yagot is Yet Another Garbage Object Tracker for Python.
+
+It provides a Python decorator named ``garbage_tracked`` which asserts that the
+decorated function or method does not create any garbage objects.
+
+Garbage objects are Python objects that cannot be immediately released when
+the object becomes unreachable and are therefore put into the generational
+Python garbage collector where more elaborated algorithms are used at a later
+point in time to release the objects.
+
+This may create problems for your Python application for two reasons:
+
+1. The time delay involved in this approach keeps memory allocated for longer
+ than necessary, causing increased memory consumption.
+
+2. There are cases where even the more elaborated algorithms cannot release a
+ garbage object. If that happens, it is a memory leak that remains until
+ the Python process ends.
+
+The ``garbage_tracked`` decorator can be used on any function or method, but
+it makes most sense to use it on test functions. It is a signature-preserving
+decorator that supports any number of positional and keyword arguments in the
+decorated function or method, and any kind of return value(s).
+
+That decorator can be used with test functions or methods in all test
+frameworks, for example `pytest`_, `nose`_, or `unittest`_.
+
+.. _pytest: https://docs.pytest.org/
+.. _nose: https://nose.readthedocs.io/
+.. _unittest: https://docs.python.org/3/library/unittest.html
+
Installation
------------
-To install the latest released version of the yagot
-package into your active Python environment:
+To install the latest released version of the yagot package into your active
+Python environment:
.. code-block:: bash
@@ -43,41 +69,120 @@ package into your active Python environment:
This will also install any prerequisite Python packages.
-For more details and alternative ways to install, see
-`Installation`_.
+For more details and alternative ways to install, see `Installation`_.
.. _Installation: https://yagot.readthedocs.io/en/stable/intro.html#installation
-Documentation
--------------
-* `Documentation for latest released version `_
+Quick start
+-----------
-Change History
---------------
+Here is an example of using it with pytest:
-* `Change history for latest released version `_
+In ``examples/test_selfref_dict.py``:
-Quick Start
------------
+.. code-block:: python
-The following simple example script lists the namespaces and the Interop
-namespace in a particular WBEM server:
+ from yagot import garbage_tracked
+
+ @garbage_tracked
+ def test_selfref_dict():
+
+ # Dictionary with self-referencing item:
+ d1 = dict()
+ d1['self'] = d1
+
+Running pytest on this example reveals the garbage object with a test failure
+raised by yagot:
+
+.. code-block:: text
+
+ $ pytest examples -k test_selfref_dict.py
+
+ ===================================== test session starts ======================================
+ platform darwin -- Python 2.7.16, pytest-4.6.9, py-1.8.1, pluggy-0.13.1
+ rootdir: /Users/maiera/PycharmProjects/python-yagot
+ plugins: cov-2.8.1
+ collected 1 item
+
+ examples/test_selfref_dict.py F [100%]
+
+ =========================================== FAILURES ===========================================
+ ______________________________________ test_selfref_dict _______________________________________
+
+ args = (), kwargs = {}, tracker =
+ ret = None, location = 'test_selfref_dict::test_selfref_dict'
+
+ def garbage_tracked_wrapper(*args, **kwargs):
+ """
+ Wrapper function for the @garbage_tracked decorator.
+ """
+ tracker = GarbageTracker.get_tracker('yagot.garbage_tracked')
+ tracker.enable()
+ tracker.start()
+ ret = func(*args, **kwargs) # The decorated function
+ tracker.stop()
+ location = "{module}::{function}".format(
+ module=func.__module__, function=func.__name__)
+ > assert not tracker.garbage, tracker.format_garbage(location)
+ E AssertionError:
+ E There was 1 garbage object(s) caused by function test_selfref_dict::test_selfref_dict:
+ E
+ E 1: object at 0x10e514d70:
+ E { 'self': }
+
+ yagot/_decorators.py:43: AssertionError
+ =================================== 1 failed in 0.07 seconds ===================================
+
+The AssertionError shows that there was one garbage object detected, and
+details about that object. In this case, the garbage object is a ``dict``
+object, and we can see that its 'self' item references back to the dict object.
+
+The failure location and source code shown by pytest is the wrapper function of
+the ``garbage_tracked`` decorator, since this is where it is detected.
+The decorated function that caused the garbage objects to be created is
+reported by pytest as a failing test function, and is also mentioned in the
+assertion message using a "module::function" notation.
+
+Knowing the test function ``test_selfref_dict()`` that caused the object to
+become a garbage object is a good start to identify the problem code, and in
+our example case it is easy to do. In more complex situations, it may be helpful
+to split the complex test function into multiple simpler test functions.
+
+The ``garbage_tracked`` decorator can be combined with any other decorators.
+Note that it always tracks the decorated function, so unless you want to track
+what garbage other decorators create, you want to have it directly on the test
+function, as the innermost decorator:
.. code-block:: python
- #!/usr/bin/env python
+ import pytest
+ from yagot import garbage_tracked
+
+ @pytest.mark.parametrize('parm2', [ ... ])
+ @pytest.mark.parametrize('parm1', [ ... ])
+ @garbage_tracked
+ def test_something(parm1, parm2):
+ pass # some test code
+
+
+Documentation
+-------------
+
+* `Documentation `_
+
+
+Change History
+--------------
- import yagot
+* `Change history `_
- ... (tbd) ...
Contributing
------------
-For information on how to contribute to the
-Yagot project, see
-`Contributing `_.
+For information on how to contribute to the Yagot project, see
+`Contributing `_.
License
diff --git a/docs/index.rst b/docs/index.rst
index bae9a49..c185ff2 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -2,9 +2,36 @@
Yagot - Yet Another Garbage Object Tracker for Python
*****************************************************
-TBD
+Yagot is Yet Another Garbage Object Tracker for Python.
-The general project web site is: https://github.com/andy-maier/python-yagot
+It provides a Python decorator named ``garbage_tracked`` which asserts that the
+decorated function or method does not create any garbage objects.
+
+Garbage objects are Python objects that cannot be immediately released when
+the object becomes unreachable and are therefore put into the generational
+Python garbage collector where more elaborated algorithms are used at a later
+point in time to release the objects.
+
+This may create problems for your Python application for two reasons:
+
+1. The time delay involved in this approach keeps memory allocated for longer
+ than necessary, causing increased memory consumption.
+
+2. There are cases where even the more elaborated algorithms cannot release a
+ garbage object. If that happens, it is a memory leak that remains until
+ the Python process ends.
+
+The ``garbage_tracked`` decorator can be used on any function or method, but
+it makes most sense to use it on test functions. It is a signature-preserving
+decorator that supports any number of positional and keyword arguments in the
+decorated function or method, and any kind of return value(s).
+
+That decorator can be used with test functions or methods in all test
+frameworks, for example `pytest`_, `nose`_, or `unittest`_.
+
+.. _pytest: https://docs.pytest.org/
+.. _nose: https://nose.readthedocs.io/
+.. _unittest: https://docs.python.org/3/library/unittest.html
.. toctree::
:maxdepth: 2
diff --git a/docs/intro.rst b/docs/intro.rst
index 073f4cb..92f43ed 100644
--- a/docs/intro.rst
+++ b/docs/intro.rst
@@ -8,12 +8,147 @@ Introduction
:depth: 2
-.. _`Functionality`:
+.. _`Quick start`:
-Functionality
--------------
+Quick start
+-----------
-TBD
+Here is an example of using it with pytest:
+
+In ``examples/test_selfref_dict.py``:
+
+.. code-block:: python
+
+ from yagot import garbage_tracked
+
+ @garbage_tracked
+ def test_selfref_dict():
+
+ # Dictionary with self-referencing item:
+ d1 = dict()
+ d1['self'] = d1
+
+Running pytest on this example reveals the garbage object with a test failure
+raised by yagot:
+
+.. code-block:: text
+
+ $ pytest examples -k test_selfref_dict.py
+
+ ===================================== test session starts ======================================
+ platform darwin -- Python 2.7.16, pytest-4.6.9, py-1.8.1, pluggy-0.13.1
+ rootdir: /Users/maiera/PycharmProjects/python-yagot
+ plugins: cov-2.8.1
+ collected 1 item
+
+ examples/test_selfref_dict.py F [100%]
+
+ =========================================== FAILURES ===========================================
+ ______________________________________ test_selfref_dict _______________________________________
+
+ args = (), kwargs = {}, tracker =
+ ret = None, location = 'test_selfref_dict::test_selfref_dict'
+
+ def garbage_tracked_wrapper(*args, **kwargs):
+ """
+ Wrapper function for the @garbage_tracked decorator.
+ """
+ tracker = GarbageTracker.get_tracker('yagot.garbage_tracked')
+ tracker.enable()
+ tracker.start()
+ ret = func(*args, **kwargs) # The decorated function
+ tracker.stop()
+ location = "{module}::{function}".format(
+ module=func.__module__, function=func.__name__)
+ > assert not tracker.garbage, tracker.format_garbage(location)
+ E AssertionError:
+ E There was 1 garbage object(s) caused by function test_selfref_dict::test_selfref_dict:
+ E
+ E 1: object at 0x10e514d70:
+ E { 'self': }
+
+ yagot/_decorators.py:43: AssertionError
+ =================================== 1 failed in 0.07 seconds ===================================
+
+The AssertionError shows that there was one garbage object detected, and
+details about that object. In this case, the garbage object is a ``dict``
+object, and we can see that its 'self' item references back to the dict object.
+
+The failure location and source code shown by pytest is the wrapper function of
+the ``garbage_tracked`` decorator, since this is where it is detected.
+The decorated function that caused the garbage objects to be created is
+reported by pytest as a failing test function, and is also mentioned in the
+assertion message using a "module::function" notation.
+
+Knowing the test function ``test_selfref_dict()`` that caused the object to
+become a garbage object is a good start to identify the problem code, and in
+our example case it is easy to do. In more complex situations, it may be helpful
+to split the complex test function into multiple simpler test functions.
+
+The ``garbage_tracked`` decorator can be combined with any other decorators.
+Note that it always tracks the decorated function, so unless you want to track
+what garbage other decorators create, you want to have it directly on the test
+function, as the innermost decorator:
+
+.. code-block:: python
+
+ import pytest
+ from yagot import garbage_tracked
+
+ @pytest.mark.parametrize('parm2', [ ... ])
+ @pytest.mark.parametrize('parm1', [ ... ])
+ @garbage_tracked
+ def test_something(parm1, parm2):
+ pass # some test code
+
+
+.. _`Reference cycles`:
+
+Reference cycles
+----------------
+
+In probably all cases, such garbage objects are caused by cyclic references
+between objects. Here are some simple cases of objects with reference cycles:
+
+.. code-block:: python
+
+ # Dictionary with self-referencing item:
+ d1 = dict()
+ d1['self'] = d1
+
+ # Object of a class with self-referencing attribute:
+ class SelfRef(object):
+ def __init__(self):
+ self.ref = self
+ obj = SelfRef()
+
+The garbage objects created as a result can be inspected by the standard Python
+module ``gc`` that provides access to the garbage collector:
+
+.. code-block:: python
+
+ $ python
+ >>> import gc
+ >>> gc.collect()
+ 0 # No garbage objects initially (in this simple case)
+ >>> d1 = dict(); d1['self'] = d1
+ >>> d1
+ {'self': {...}}
+ >>> gc.collect()
+ 0 # Still no garbage objects
+ >>> del d1 # The dict object becomes unreachable ...
+ >>> gc.collect()
+ 1 # ... and ends up as one garbage object
+
+The interesting part happens during the ``del d1`` statement, but let's first
+level set on names vs. objects in Python: A variable (``d1``) is not an object
+but a name that is bound to an object (of type ``dict``). The ``del d1``
+statement removes the name ``d1`` from its namespace. That causes the reference
+count of the ``dict`` object to drop to 0 (in this case, where there is no other
+variable name bound to it and no other object referencing it). The object is
+then said to be "unreachable". That causes Python to try to immediately release
+the ``dict`` object. This does not work because of the self-reference, so it is
+put into the garbage collector for later treatment.
.. _`Installation`:
@@ -21,15 +156,12 @@ TBD
Installation
------------
-TBD
-
-
.. _`Supported environments`:
Supported environments
^^^^^^^^^^^^^^^^^^^^^^
-Pywbem is supported in these environments:
+Yagot is supported in these environments:
* Operating Systems: Linux, Windows (native, and with UNIX-like environments),
OS-X
@@ -52,8 +184,8 @@ Installing
- wheel
- pip
-* Install the yagot package and its prerequisite
- Python packages into the active Python environment:
+* Install the yagot package and its prerequisite Python packages into the
+ active Python environment:
.. code-block:: bash
@@ -194,16 +326,3 @@ the Yagot project, by version type:
* New major release (M.N.P -> M+1.0.0): Deprecated functionality may get
removed; functionality may be extended or changed; backwards compatibility
may be broken.
-
-
-.. _'Python namespaces`:
-
-Python namespaces
------------------
-
-TBD - describe the python namespaces to clarify what is for external use
-and what is internal.
-
-This documentation describes only the external APIs of the
-Yagot project, and omits any internal symbols and
-any sub-modules.
diff --git a/examples/test_selfref_dict.py b/examples/test_selfref_dict.py
new file mode 100644
index 0000000..a4c89e1
--- /dev/null
+++ b/examples/test_selfref_dict.py
@@ -0,0 +1,12 @@
+from yagot import garbage_tracked
+
+@garbage_tracked
+def test_selfref_dict():
+
+ # Dictionary with self-referencing item:
+ d1 = dict()
+ d1['self'] = d1
+ try:
+ x = y
+ except NameError:
+ pass
diff --git a/yagot/_decorators.py b/yagot/_decorators.py
index 622566d..02a6a39 100644
--- a/yagot/_decorators.py
+++ b/yagot/_decorators.py
@@ -29,25 +29,18 @@ def garbage_tracked(func):
any signature.
"""
- def wrapper_func(*args, **kwargs):
+ def garbage_tracked_wrapper(*args, **kwargs):
"""
- Wrapper function that is invoked instead of the decorated function.
-
- It puts garbage tracking around the invocation of the decorated function
- and reports any garbage objects by raising an AssertionError exception.
+ Wrapper function for the @garbage_tracked decorator.
"""
-
- tracker = GarbageTracker.get_tracker('_yagot.garbage_tracked')
+ tracker = GarbageTracker.get_tracker('yagot.garbage_tracked')
tracker.enable()
tracker.start()
-
- ret = func(*args, **kwargs)
-
+ ret = func(*args, **kwargs) # The decorated function
tracker.stop()
- location = "{file}::{func}". \
- format(file=func.__module__, func=func.__name__)
- tracker.assert_no_garbage(location)
-
+ location = "{module}::{function}".format(
+ module=func.__module__, function=func.__name__)
+ assert not tracker.garbage, tracker.format_garbage(location)
return ret
- return functools.update_wrapper(wrapper_func, func)
+ return functools.update_wrapper(garbage_tracked_wrapper, func)
diff --git a/yagot/_garbagetracker.py b/yagot/_garbagetracker.py
index 4fe686f..2178069 100644
--- a/yagot/_garbagetracker.py
+++ b/yagot/_garbagetracker.py
@@ -4,7 +4,6 @@
from __future__ import absolute_import, print_function
-import sys
import types
import re
import gc
@@ -153,62 +152,34 @@ def ignore(self):
if self.enabled:
self._ignored = True
- def print_if_garbage(self, location=None, max=10, stream=sys.stdout):
+ def format_garbage(self, location=None, max=10):
# pylint: disable=redefined-builtin
"""
- If there were garbage objects found during the last tracking period,
- print the garbage objects (up to a maximum number).
+ Return a formatted multi-line string for all garbage objects detected
+ during teh tracking period, up to a maximum number.
Parameters:
- location (string): Location of the tracked code, e.g. as
- "module::function".
-
- max (int): Maximum number of garbage objects to be printed.
-
- stream: Stream to be printed on.
- """
- if self.enabled and self.garbage:
- print("\n{num} garbage objects left by {loc}:".
- format(num=len(self.garbage), loc=location))
- for i, obj in enumerate(self.garbage):
- if i >= max:
- print("...", file=stream)
- break
- obj_str = self._format(obj)
- print(obj_str, file=stream)
- stream.flush()
-
- def assert_no_garbage(self, location=None, max=10):
- # pylint: disable=redefined-builtin
- """
- Assert that there were no garbage objects found during the last
- tracking period. Otherwise, raise AssertionError with a message that
- describes the garbage objects (up to a maximum number).
-
- Parameters:
-
- location (string): Location of the tracked code, e.g. as
- "module::function".
+ location (string): Location of the function that created the garbage
+ objects, e.g. in the notation "module::function".
max (int): Maximum number of garbage objects to be included in the
- exception message.
- """
- if self.enabled and self.garbage:
- ass_str = "{num} garbage objects left by {loc}:\n". \
- format(num=len(self.garbage), loc=location)
- for i, obj in enumerate(self.garbage):
- # self._generate_objgraph(obj)
- if i >= max:
- ass_str += "...\n"
- break
- ass_str += "{}: {}\n".format(i + 1, self._format(obj))
- raise AssertionError(ass_str)
+ returned string.
+ """
+ ret_str = "\nThere was {num} garbage object(s) caused by function " \
+ "{loc}:\n".format(num=len(self.garbage), loc=location)
+ for i, obj in enumerate(self.garbage):
+ # self._generate_objgraph(obj)
+ if i >= max:
+ ret_str += "\n...\n"
+ break
+ ret_str += "\n{}: {}\n".format(i + 1, self.format_obj(obj))
+ return ret_str
@staticmethod
- def _format(obj):
+ def format_obj(obj):
"""
- Return a formatted string for the garbage object.
+ Return a formatted string for a single garbage object.
Parameters: