Merge pull request #2011 from sfinkens/fix-cached-property-backport

Fix memory leak in cached_property backport

doc/source/conf.py

@@ -264,14 +264,14 @@ def __getattr__(cls, name):
     'dask': ('https://docs.dask.org/en/latest', None),
     'geoviews': ('http://geoviews.org', None),
     'jobqueue': ('https://jobqueue.dask.org/en/latest', None),
-    'numpy': ('https://docs.scipy.org/doc/numpy', None),
+    'numpy': ('https://numpy.org/doc/stable', None),
     'pydecorate': ('https://pydecorate.readthedocs.io/en/stable', None),
     'pyorbital': ('https://pyorbital.readthedocs.io/en/stable', None),
     'pyproj': ('https://pyproj4.github.io/pyproj/dev', None),
     'pyresample': ('https://pyresample.readthedocs.io/en/stable', None),
     'pytest': ('https://docs.pytest.org/en/stable/', None),
     'python': ('https://docs.python.org/3', None),
-    'scipy': ('https://docs.scipy.org/doc/scipy/', None),
+    'scipy': ('http://scipy.github.io/devdocs', None),
     'trollimage': ('https://trollimage.readthedocs.io/en/stable', None),
     'trollsift': ('https://trollsift.readthedocs.io/en/stable', None),
     'xarray': ('https://xarray.pydata.org/en/stable', None),

satpy/_compat.py

@@ -17,15 +17,69 @@
 # satpy.  If not, see <http://www.gnu.org/licenses/>.
 """Backports and compatibility fixes for satpy."""
 
+from threading import RLock
+
+_NOT_FOUND = object()
+
+
+class CachedPropertyBackport:
+    """Backport of cached_property from Python-3.8.
+
+    Source: https://github.com/python/cpython/blob/v3.8.0/Lib/functools.py#L930
+    """
+
+    def __init__(self, func):  # noqa
+        self.func = func
+        self.attrname = None
+        self.__doc__ = func.__doc__
+        self.lock = RLock()
+
+    def __set_name__(self, owner, name):  # noqa
+        if self.attrname is None:
+            self.attrname = name
+        elif name != self.attrname:
+            raise TypeError(
+                "Cannot assign the same cached_property to two different names "
+                f"({self.attrname!r} and {name!r})."
+            )
+
+    def __get__(self, instance, owner=None):  # noqa
+        if instance is None:
+            return self
+        if self.attrname is None:
+            raise TypeError(
+                "Cannot use cached_property instance without calling __set_name__ on it.")
+        try:
+            cache = instance.__dict__
+        except AttributeError:  # not all objects have __dict__ (e.g. class defines slots)
+            msg = (
+                f"No '__dict__' attribute on {type(instance).__name__!r} "
+                f"instance to cache {self.attrname!r} property."
+            )
+            raise TypeError(msg) from None
+        val = cache.get(self.attrname, _NOT_FOUND)
+        if val is _NOT_FOUND:
+            with self.lock:
+                # check if another thread filled cache while we awaited lock
+                val = cache.get(self.attrname, _NOT_FOUND)
+                if val is _NOT_FOUND:
+                    val = self.func(instance)
+                    try:
+                        cache[self.attrname] = val
+                    except TypeError:
+                        msg = (
+                            f"The '__dict__' attribute on {type(instance).__name__!r} instance "
+                            f"does not support item assignment for caching {self.attrname!r} property."
+                        )
+                        raise TypeError(msg) from None
+        return val
+
+
 try:
     from functools import cached_property  # type: ignore
 except ImportError:
     # for python < 3.8
-    from functools import lru_cache
-
-    def cached_property(func):  # type: ignore
-        """Port back functools.cached_property."""
-        return property(lru_cache(maxsize=None)(func))
+    cached_property = CachedPropertyBackport  # type: ignore
 
 
 try:

satpy/tests/test_compat.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2022 Satpy developers
+#
+# This file is part of satpy.
+#
+# satpy is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# satpy is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# satpy.  If not, see <http://www.gnu.org/licenses/>.
+"""Test backports and compatibility fixes."""
+
+import gc
+
+from satpy._compat import CachedPropertyBackport
+
+
+class ClassWithCachedProperty:  # noqa
+    def __init__(self, x):  # noqa
+        self.x = x
+
+    @CachedPropertyBackport
+    def property(self):  # noqa
+        return 2 * self.x
+
+
+def test_cached_property_backport():
+    """Test cached property backport."""
+    c = ClassWithCachedProperty(1)
+    assert c.property == 2
+
+
+def test_cached_property_backport_releases_memory():
+    """Test that cached property backport releases memory."""
+    c1 = ClassWithCachedProperty(2)
+    del c1
+    instances = [
+        obj for obj in gc.get_objects()
+        if isinstance(obj, ClassWithCachedProperty)
+    ]
+    assert len(instances) == 0