Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix memory leak in cached_property backport #2011

Merged
merged 3 commits into from
Feb 8, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions doc/source/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -264,14 +264,14 @@ def __getattr__(cls, name):
'dask': ('https://docs.dask.org/en/latest', None),
'geoviews': ('http://geoviews.org', None),
'jobqueue': ('https://jobqueue.dask.org/en/latest', None),
'numpy': ('https://docs.scipy.org/doc/numpy', None),
'numpy': ('https://numpy.org/doc/stable', None),
'pydecorate': ('https://pydecorate.readthedocs.io/en/stable', None),
'pyorbital': ('https://pyorbital.readthedocs.io/en/stable', None),
'pyproj': ('https://pyproj4.github.io/pyproj/dev', None),
'pyresample': ('https://pyresample.readthedocs.io/en/stable', None),
'pytest': ('https://docs.pytest.org/en/stable/', None),
'python': ('https://docs.python.org/3', None),
'scipy': ('https://docs.scipy.org/doc/scipy/', None),
'scipy': ('http://scipy.github.io/devdocs', None),
'trollimage': ('https://trollimage.readthedocs.io/en/stable', None),
'trollsift': ('https://trollsift.readthedocs.io/en/stable', None),
'xarray': ('https://xarray.pydata.org/en/stable', None),
Expand Down
64 changes: 59 additions & 5 deletions satpy/_compat.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,69 @@
# satpy. If not, see <http://www.gnu.org/licenses/>.
"""Backports and compatibility fixes for satpy."""

from threading import RLock

_NOT_FOUND = object()


class CachedPropertyBackport:
"""Backport of cached_property from Python-3.8.

Source: https://github.com/python/cpython/blob/v3.8.0/Lib/functools.py#L930
"""

def __init__(self, func): # noqa
self.func = func
self.attrname = None
self.__doc__ = func.__doc__
self.lock = RLock()

def __set_name__(self, owner, name): # noqa
if self.attrname is None:
self.attrname = name
elif name != self.attrname:
raise TypeError(
"Cannot assign the same cached_property to two different names "
f"({self.attrname!r} and {name!r})."
)

def __get__(self, instance, owner=None): # noqa
if instance is None:
return self
if self.attrname is None:
raise TypeError(
"Cannot use cached_property instance without calling __set_name__ on it.")
try:
cache = instance.__dict__
except AttributeError: # not all objects have __dict__ (e.g. class defines slots)
msg = (
f"No '__dict__' attribute on {type(instance).__name__!r} "
f"instance to cache {self.attrname!r} property."
)
raise TypeError(msg) from None
val = cache.get(self.attrname, _NOT_FOUND)
if val is _NOT_FOUND:
with self.lock:
# check if another thread filled cache while we awaited lock
val = cache.get(self.attrname, _NOT_FOUND)
if val is _NOT_FOUND:
val = self.func(instance)
try:
cache[self.attrname] = val
except TypeError:
msg = (
f"The '__dict__' attribute on {type(instance).__name__!r} instance "
f"does not support item assignment for caching {self.attrname!r} property."
)
raise TypeError(msg) from None
return val


try:
from functools import cached_property # type: ignore
except ImportError:
# for python < 3.8
from functools import lru_cache

def cached_property(func): # type: ignore
"""Port back functools.cached_property."""
return property(lru_cache(maxsize=None)(func))
cached_property = CachedPropertyBackport # type: ignore


try:
Expand Down
48 changes: 48 additions & 0 deletions satpy/tests/test_compat.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (c) 2022 Satpy developers
#
# This file is part of satpy.
#
# satpy is free software: you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version.
#
# satpy is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# satpy. If not, see <http://www.gnu.org/licenses/>.
"""Test backports and compatibility fixes."""

import gc

from satpy._compat import CachedPropertyBackport


class ClassWithCachedProperty: # noqa
def __init__(self, x): # noqa
self.x = x

@CachedPropertyBackport
def property(self): # noqa
return 2 * self.x


def test_cached_property_backport():
"""Test cached property backport."""
c = ClassWithCachedProperty(1)
assert c.property == 2


def test_cached_property_backport_releases_memory():
"""Test that cached property backport releases memory."""
c1 = ClassWithCachedProperty(2)
del c1
instances = [
obj for obj in gc.get_objects()
if isinstance(obj, ClassWithCachedProperty)
]
assert len(instances) == 0