From 66b0d346a36f994e24c86415bd7fa51207d7be98 Mon Sep 17 00:00:00 2001 From: Alberto Planas Date: Fri, 31 Jul 2015 15:07:58 +0200 Subject: [PATCH 1/3] Add external invalidation mechanism to memoize. --- osclib/memoize.py | 44 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/osclib/memoize.py b/osclib/memoize.py index 70fa3b18e..fb3b6019d 100644 --- a/osclib/memoize.py +++ b/osclib/memoize.py @@ -29,6 +29,7 @@ from xdg.BaseDirectory import save_cache_path except ImportError: from xdg.BaseDirectory import xdg_cache_home + def save_cache_path(*name): path = os.path.join(xdg_cache_home, *name) if not os.path.isdir(path): @@ -39,7 +40,7 @@ def save_cache_path(*name): CACHEDIR = save_cache_path('opensuse-repo-checker') -def memoize(ttl=None, session=False): +def memoize(ttl=None, session=False, is_method=False): """Decorator function to implement a persistent cache. >>> @memoize() @@ -100,9 +101,6 @@ def memoize(ttl=None, session=False): NCLEAN = 1024 # Number of slots to remove when limit reached TIMEOUT = 60*60*2 # Time to live for every cache slot (seconds) - # The session cache is only used when 'session' is True - session_cache = {} - def _memoize(fn): # Implement a POSIX lock / unlock extension for shelves. Inspired # on ActiveState Code recipe #576591 @@ -116,13 +114,16 @@ def _unlock(lckfile): lckfile.close() def _open_cache(cache_name): - cache = session_cache if not session: lckfile = _lock(cache_name) cache = shelve.open(cache_name, protocol=-1) # Store a reference to the lckfile to avoid to be # closed by gc cache.lckfile = lckfile + else: + if not hasattr(fn, '_memoize_session_cache'): + fn._memoize_session_cache = {} + cache = fn._memoize_session_cache return cache def _close_cache(cache): @@ -138,12 +139,43 @@ def _clean_cache(cache): for key in keys_to_delete: del cache[key] + def _key(obj): + # Pickle doesn't guarantee that there is a single + # representation for every serialization. We can try to + # picke / depickle twice to have a canonical + # representation. + key = pickle.dumps(obj, protocol=-1) + key = pickle.dumps(pickle.loads(key), protocol=-1) + return key + + def _invalidate(*args, **kwargs): + key = _key((args, kwargs)) + cache = _open_cache(cache_name) + if key in cache: + del cache[key] + + def _invalidate_all(): + cache = _open_cache(cache_name) + cache.clear() + + def _add_invalidate_method(_self): + name = '_invalidate_%s' % fn.__name__ + if not hasattr(_self, name): + setattr(_self, name, _invalidate) + + name = '_invalidate_all' + if not hasattr(_self, name): + setattr(_self, name, _invalidate_all) + @wraps(fn) def _fn(*args, **kwargs): def total_seconds(td): return (td.microseconds + (td.seconds + td.days * 24 * 3600.) * 10**6) / 10**6 now = datetime.now() - key = pickle.dumps((args, kwargs), protocol=-1) + if is_method: + _self = args[0] + _add_invalidate_method(_self) + key = _key((args[1:], kwargs)) updated = False cache = _open_cache(cache_name) if key in cache: From d7f445264fe518382a52ed68bbe0bb264a7bd56e Mon Sep 17 00:00:00 2001 From: Alberto Planas Date: Fri, 31 Jul 2015 15:12:46 +0200 Subject: [PATCH 2/3] Memoize `get_prj_pseudometa` and invalidate in `set_prj_pseudometa` --- osclib/stagingapi.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/osclib/stagingapi.py b/osclib/stagingapi.py index 9dc30f186..d0a4b331e 100644 --- a/osclib/stagingapi.py +++ b/osclib/stagingapi.py @@ -35,6 +35,7 @@ from osc.core import http_PUT from osclib.comments import CommentAPI +from osclib.memoize import memoize class StagingAPI(object): @@ -57,7 +58,7 @@ def __init__(self, apiurl, project): self.crebuild = conf.config[project]['rebuild'] self.cproduct = conf.config[project]['product'] self.copenqa = conf.config[project]['openqa'] - + # If the project support rings, inititialize some variables. self.ring_packages = {} if self.crings: @@ -259,7 +260,7 @@ def get_adi_projects(self): :return list of known ADI projects """ - projects = [p for p in self.get_staging_projects() if self.is_adi_project(p) ] + projects = [p for p in self.get_staging_projects() if self.is_adi_project(p)] return sorted(projects, key=lambda project: self.extract_adi_number(project)) def do_change_review_state(self, request_id, newstate, message=None, @@ -415,10 +416,11 @@ def dispatch_open_requests(self): requests = self.get_open_requests() # check if we can reduce it down by accepting some for rq in requests: - #if self.crings: - # self.accept_non_ring_request(rq) + # if self.crings: + # self.accept_non_ring_request(rq) self.update_superseded_request(rq) + @memoize(ttl=60, session=True, is_method=True) def get_prj_pseudometa(self, project): """ Gets project data from YAML in project description @@ -470,6 +472,9 @@ def set_prj_pseudometa(self, project, meta): url = make_meta_url('prj', project, self.apiurl, force=True) http_PUT(url, data=ET.tostring(root)) + # Invalidate here the cache for this stating project + self._invalidate_get_prj_pseudometa(project) + def _add_rq_to_prj_pseudometa(self, project, request_id, package): """ Records request as part of the project within metadata From 90412c30a443341c562d000c23fdbc64c14ade62 Mon Sep 17 00:00:00 2001 From: Alberto Planas Date: Mon, 3 Aug 2015 13:21:12 +0200 Subject: [PATCH 3/3] Clean the cache after every test. --- tests/api_tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/api_tests.py b/tests/api_tests.py index 8edf784c1..149490963 100644 --- a/tests/api_tests.py +++ b/tests/api_tests.py @@ -46,6 +46,10 @@ def setUp(self): Config('openSUSE:Factory') self.api = StagingAPI(APIURL, 'openSUSE:Factory') + def tearDown(self): + """Clean internal cache""" + self.api._invalidate_all() + def test_ring_packages(self): """ Validate the creation of the rings.