From 4c5d091baece0734066d14a97a9d4e530771097d Mon Sep 17 00:00:00 2001 From: Brian McFee Date: Wed, 20 Feb 2019 15:30:03 -0500 Subject: [PATCH 1/3] step 1 of cache refactor #795; removed sys module hackery --- librosa/__init__.py | 2 +- librosa/{cache.py => _cache.py} | 6 +----- librosa/beat.py | 2 +- librosa/core/audio.py | 2 +- librosa/core/constantq.py | 2 +- librosa/core/pitch.py | 2 +- librosa/core/spectrum.py | 2 +- librosa/decompose.py | 2 +- librosa/feature/utils.py | 2 +- librosa/filters.py | 2 +- librosa/onset.py | 2 +- librosa/segment.py | 2 +- librosa/util/utils.py | 2 +- tests/test_cache.py | 23 +++++++++-------------- 14 files changed, 22 insertions(+), 31 deletions(-) rename librosa/{cache.py => _cache.py} (93%) diff --git a/librosa/__init__.py b/librosa/__init__.py index b1219001bb..8275a9dbe4 100644 --- a/librosa/__init__.py +++ b/librosa/__init__.py @@ -8,7 +8,7 @@ from .version import show_versions # And all the librosa sub-modules -from . import cache +from ._cache import cache from . import core from . import beat from . import decompose diff --git a/librosa/cache.py b/librosa/_cache.py similarity index 93% rename from librosa/cache.py rename to librosa/_cache.py index 07e93c13e5..d0cc03cdb8 100644 --- a/librosa/cache.py +++ b/librosa/_cache.py @@ -3,7 +3,6 @@ """Function caching""" import os -import sys from joblib import Memory @@ -55,11 +54,8 @@ def decorator_apply(dec, func): # Instantiate the cache from the environment -CACHE = CacheManager(os.environ.get('LIBROSA_CACHE_DIR', None), +cache = CacheManager(os.environ.get('LIBROSA_CACHE_DIR', None), mmap_mode=os.environ.get('LIBROSA_CACHE_MMAP', None), compress=os.environ.get('LIBROSA_CACHE_COMPRESS', False), verbose=int(os.environ.get('LIBROSA_CACHE_VERBOSE', 0)), level=int(os.environ.get('LIBROSA_CACHE_LEVEL', 10))) - -# Override the module's __call__ attribute -sys.modules[__name__] = CACHE diff --git a/librosa/beat.py b/librosa/beat.py index e2c591b47d..8127cb1045 100644 --- a/librosa/beat.py +++ b/librosa/beat.py @@ -13,7 +13,7 @@ import numpy as np import scipy -from . import cache +from ._cache import cache from . import core from . import onset from . import util diff --git a/librosa/core/audio.py b/librosa/core/audio.py index 004696b9c1..6f70053005 100644 --- a/librosa/core/audio.py +++ b/librosa/core/audio.py @@ -12,7 +12,7 @@ from .fft import get_fftlib from .time_frequency import frames_to_samples, time_to_samples -from .. import cache +from .._cache import cache from .. import util from ..util.exceptions import ParameterError diff --git a/librosa/core/constantq.py b/librosa/core/constantq.py index 49ff0fb01e..18b2bfe5ee 100644 --- a/librosa/core/constantq.py +++ b/librosa/core/constantq.py @@ -12,7 +12,7 @@ from .time_frequency import cqt_frequencies, note_to_hz from .spectrum import stft from .pitch import estimate_tuning -from .. import cache +from .._cache import cache from .. import filters from .. import util from ..util.exceptions import ParameterError diff --git a/librosa/core/pitch.py b/librosa/core/pitch.py index 3bf8001b6d..f98c8cfaf6 100644 --- a/librosa/core/pitch.py +++ b/librosa/core/pitch.py @@ -7,7 +7,7 @@ from .spectrum import _spectrogram from . import time_frequency -from .. import cache +from .._cache import cache from .. import util __all__ = ['estimate_tuning', 'pitch_tuning', 'piptrack'] diff --git a/librosa/core/spectrum.py b/librosa/core/spectrum.py index c83b1b5d9c..9ae2717ea8 100644 --- a/librosa/core/spectrum.py +++ b/librosa/core/spectrum.py @@ -15,7 +15,7 @@ from . import time_frequency from .fft import get_fftlib from .audio import resample -from .. import cache +from .._cache import cache from .. import util from ..util.exceptions import ParameterError from ..filters import get_window, semitone_filterbank diff --git a/librosa/decompose.py b/librosa/decompose.py index f49faf5a2f..b56ef4e12b 100644 --- a/librosa/decompose.py +++ b/librosa/decompose.py @@ -19,7 +19,7 @@ import sklearn.decomposition from . import core -from . import cache +from ._cache import cache from . import segment from . import util from .util.exceptions import ParameterError diff --git a/librosa/feature/utils.py b/librosa/feature/utils.py index 5ed0cd9ad0..cad94133fd 100644 --- a/librosa/feature/utils.py +++ b/librosa/feature/utils.py @@ -6,7 +6,7 @@ import numpy as np import scipy.signal -from .. import cache +from .._cache import cache from ..util.exceptions import ParameterError from ..util.deprecation import Deprecated __all__ = ['delta', 'stack_memory'] diff --git a/librosa/filters.py b/librosa/filters.py index b2a9c30130..4a84fb5660 100644 --- a/librosa/filters.py +++ b/librosa/filters.py @@ -50,7 +50,7 @@ from numba import jit -from . import cache +from ._cache import cache from . import util from .util.exceptions import ParameterError from .util.decorators import deprecated diff --git a/librosa/onset.py b/librosa/onset.py index 341842cc06..087a302fe0 100644 --- a/librosa/onset.py +++ b/librosa/onset.py @@ -15,7 +15,7 @@ import numpy as np import scipy -from . import cache +from ._cache import cache from . import core from . import util from .util.exceptions import ParameterError diff --git a/librosa/segment.py b/librosa/segment.py index b8e8e660a0..73db61cce8 100644 --- a/librosa/segment.py +++ b/librosa/segment.py @@ -34,7 +34,7 @@ import sklearn.feature_extraction import sklearn.neighbors -from . import cache +from ._cache import cache from . import util from .util.exceptions import ParameterError diff --git a/librosa/util/utils.py b/librosa/util/utils.py index bc2891bb20..f2f4f6130f 100644 --- a/librosa/util/utils.py +++ b/librosa/util/utils.py @@ -9,7 +9,7 @@ import numpy as np from numpy.lib.stride_tricks import as_strided -from .. import cache +from .._cache import cache from .exceptions import ParameterError # Constrain STFT block sizes to 256 KB diff --git a/tests/test_cache.py b/tests/test_cache.py index 37c997498a..055e2e7274 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -11,6 +11,7 @@ import numpy as np import pytest +import librosa._cache # Disable any initial cache settings @@ -24,9 +25,8 @@ @pytest.fixture def local_cache(): cache_dir = tempfile.mkdtemp() - os.environ['LIBROSA_CACHE_DIR'] = cache_dir - yield - os.environ.pop('LIBROSA_CACHE_DIR') + cache = librosa._cache.CacheManager(cache_dir, verbose=0, level=10) + yield cache shutil.rmtree(cache_dir) @@ -37,26 +37,21 @@ def func(x): def test_cache_disabled(): - os.environ.pop('LIBROSA_CACHE_DIR', None) - sys.modules.pop('librosa.cache', None) - import librosa.cache - - func_cache = librosa.cache(level=-10)(func) - # When there's no cache directory in the environment, # librosa.cache is a no-op. + cache = librosa._cache.CacheManager(None, verbose=0, level=10) + func_cache = cache(level=-10)(func) + assert func == func_cache def test_cache_enabled(local_cache): - sys.modules.pop('librosa.cache', None) - import librosa.cache - librosa.cache.clear() + local_cache.clear() - func_cache = librosa.cache(level=-10)(func) + func_cache = local_cache(level=-10)(func) - # The cache should be active now + # The cache should be active now, so func_cache should be a different object from func assert func_cache != func # issue three calls to func From 5a902bf62ca042dbd70ce2da805005a527b6b3cf Mon Sep 17 00:00:00 2001 From: Brian McFee Date: Wed, 20 Feb 2019 15:41:56 -0500 Subject: [PATCH 2/3] step 2 of cache refactor complete. fixes #795 --- librosa/_cache.py | 36 ++++++++++++++++++++++++++++-------- librosa/segment.py | 2 +- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/librosa/_cache.py b/librosa/_cache.py index d0cc03cdb8..5b823f0c19 100644 --- a/librosa/_cache.py +++ b/librosa/_cache.py @@ -6,16 +6,21 @@ from joblib import Memory -class CacheManager(Memory): - '''The librosa cache manager class extends joblib.Memory +class CacheManager(object): + '''The librosa cache manager class wraps joblib.Memory with a __call__ attribute, so that it may act as a function. - This allows us to override the librosa.cache module's __call__ - field, thereby allowing librosa.cache to act as a decorator function. + Additionally, it provides a caching level filter, so that + different functions can be cached or not depending on the user's + preference for speed vs. storage usage. ''' - def __init__(self, location, level=10, **kwargs): - super(CacheManager, self).__init__(location, **kwargs) + def __init__(self, *args, **kwargs): + + level = kwargs.pop('level', 10) + + # Initialize the memory object + self.memory = Memory(*args, **kwargs) # The level parameter controls which data we cache # smaller numbers mean less caching self.level = level @@ -45,13 +50,28 @@ def decorator_apply(dec, func): func, 'return decorated(%(signature)s)', dict(decorated=dec(func)), __wrapped__=func) - if self.location is not None and self.level >= level: - return decorator_apply(self.cache, function) + if self.memory.location is not None and self.level >= level: + return decorator_apply(self.memory.cache, function) else: return function return wrapper + def clear(self, *args, **kwargs): + return self.memory.clear(*args, **kwargs) + + def eval(self, *args, **kwargs): + return self.memory.eval(*args, **kwargs) + + def format(self, *args, **kwargs): + return self.memory.format(*args, **kwargs) + + def reduce_size(self, *args, **kwargs): + return self.memory.reduce_size(*args, **kwargs) + + def warn(self, *args, **kwargs): + return self.memory.warn(*args, **kwargs) + # Instantiate the cache from the environment cache = CacheManager(os.environ.get('LIBROSA_CACHE_DIR', None), diff --git a/librosa/segment.py b/librosa/segment.py index 73db61cce8..c9e58471cf 100644 --- a/librosa/segment.py +++ b/librosa/segment.py @@ -692,7 +692,7 @@ def agglomerative(data, k, clusterer=None, axis=-1): # Instantiate the clustering object clusterer = sklearn.cluster.AgglomerativeClustering(n_clusters=k, connectivity=grid, - memory=cache) + memory=cache.memory) # Fit the model clusterer.fit(data) From 375c1b2f10d75a5454365b1c4ba044143c6c4773 Mon Sep 17 00:00:00 2001 From: Brian McFee Date: Wed, 20 Feb 2019 15:43:20 -0500 Subject: [PATCH 3/3] updated documentation to reflect cache changes. --- docs/cache.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/cache.rst b/docs/cache.rst index 66bdb91492..63b92e6827 100644 --- a/docs/cache.rst +++ b/docs/cache.rst @@ -43,6 +43,9 @@ Please refer to the `joblib.Memory` `documentation `_ for a detailed explanation of these parameters. +As of 0.7, librosa's cache wraps (rather than extends) the `joblib.Memory` object. +The memory object can be directly accessed by `librosa.cache.memory`. + Cache levels ------------