From 767e45177274db6a854ad7e30d5af25f64ed8cbe Mon Sep 17 00:00:00 2001 From: Jim Laney Date: Tue, 24 Nov 2020 09:38:10 -0800 Subject: [PATCH 1/2] drops memcached cache implementation --- rc_django/cache_implementation/memcache.py | 155 ------------------ .../cache_implementation/test_memcached.py | 146 ----------------- setup.py | 1 - 3 files changed, 302 deletions(-) delete mode 100644 rc_django/cache_implementation/memcache.py delete mode 100644 rc_django/tests/cache_implementation/test_memcached.py diff --git a/rc_django/cache_implementation/memcache.py b/rc_django/cache_implementation/memcache.py deleted file mode 100644 index c4c91ae..0000000 --- a/rc_django/cache_implementation/memcache.py +++ /dev/null @@ -1,155 +0,0 @@ -""" -Contains memcached cache implementations -""" - -from bmemcached import Client -from bmemcached.exceptions import MemcachedException -import logging -import pickle -import threading -from dateutil.parser import parse -from django.conf import settings -from django.utils import timezone -from restclients_core.models import MockHTTP -from rc_django.cache_implementation.logger import log_err, log_info - - -logger = logging.getLogger(__name__) - - -class MemcachedCache(object): - - def __init__(self): - self._set_client() - - def deleteCache(self, service, url): - key = self._get_key(service, url) - try: - return self.client.delete(key) - except MemcachedException as ex: - log_err(logger, "MemCache Delete(key: {}) => {}".format(key, ex)) - - def getCache(self, service, url, headers): - expire_seconds = self.get_cache_expiration_time(service, url) - if expire_seconds is None: - return - - key = self._get_key(service, url) - try: - data = self.client.get(key) - except MemcachedException as ex: - log_err(logger, "MemCache Get(key: {}) => {}".format(key, ex)) - return - - if not data: - return - - values = self._get_cache_data(data) - response = MockHTTP() - response.headers = values["headers"] - response.status = values["status"] - response.data = values["data"] - - return {"response": response} - - def processResponse(self, service, url, response): - expire_seconds = self.get_cache_expiration_time(service, url) - if expire_seconds is None: - return - - header_data = {} - for header in response.headers: - header_data[header] = response.getheader(header) - - key = self._get_key(service, url) - cdata = self._make_cache_data( - service, url, response.data, header_data, response.status, - timezone.now()) - try: - self.client.set(key, cdata, time=expire_seconds) - log_info(logger, "MemCache Set(key: {})".format(key)) - except MemcachedException as ex: - log_err(logger, "MemCache Set(key: {}) => {}".format(key, ex)) - - def updateCache(self, service, url, new_data, new_data_dt): - """ - :param new_data: a string representation of the data - :param new_data_dt: a timezone aware datetime object giving - the timestamp of the new_data - :raise MemcachedException: if update failed - """ - expire_seconds = self.get_cache_expiration_time(service, url) - if expire_seconds is None: - return - - key = self._get_key(service, url) - cdata = self._make_cache_data( - service, url, new_data, {}, 200, new_data_dt) - try: - value = self.client.get(key) - if value: - data = self._get_cache_data(value) - if "time_stamp" in data: - cached_data_dt = parse(data["time_stamp"]) - if new_data_dt <= cached_data_dt: - log_info(logger, "IN cache (key: {})".format(key)) - return - # replace existing value in cache - self.client.replace(key, cdata, time=expire_seconds) - log_info(logger, "MemCache replace(key: {})".format(key)) - return - except MemcachedException as ex: - log_err(logger, "MemCache replace(key: {}) => {}".format(key, ex)) - raise - - # not in cache - try: - self.client.set(key, cdata, time=expire_seconds) - log_info(logger, "MemCache Set(key {})".format(key)) - except MemcachedException as ex: - log_err(logger, "MemCache Set(key: {}) => {}".format(key, ex)) - raise - - def _get_cache_data(self, data_from_cache): - return pickle.loads(data_from_cache, encoding="utf8") - - def _make_cache_data(self, service, url, data_to_cache, - header_data, status, time_stamp): - return pickle.dumps({ - "status": status, - "headers": header_data, - "data": data_to_cache, - "time_stamp": time_stamp.isoformat(), - }) - - """ - Returns an integer representing seconds until a document expires, - overridden to set per-URL expiration times. The default is 4 hours. - - Following memcached documentation: - # Can be set from 0, meaning "never expire", to 30 days (60*60*24*30). - # Any time higher than 30 days is interpreted as a unix timestamp date. - - A value of None indicates "Do not cache", and will not use the cache. - """ - def get_cache_expiration_time(self, service, url): - return 60 * 60 * 4 - - def _get_key(self, service, url): - return "{}-{}".format(service, url) - - def _set_client(self): - thread_id = threading.current_thread().ident - if not hasattr(MemcachedCache, "_memcached_cache"): - MemcachedCache._memcached_cache = {} - - if thread_id in MemcachedCache._memcached_cache: - self.client = MemcachedCache._memcached_cache[thread_id] - return - - servers = settings.RESTCLIENTS_MEMCACHED_SERVERS - username = getattr(settings, "RESTCLIENTS_MEMCACHED_USER", None) - password = getattr(settings, "RESTCLIENTS_MEMCACHED_PASS", None) - - self.client = Client(servers, username, password) - MemcachedCache._memcached_cache[thread_id] = self.client diff --git a/rc_django/tests/cache_implementation/test_memcached.py b/rc_django/tests/cache_implementation/test_memcached.py deleted file mode 100644 index bd099cd..0000000 --- a/rc_django/tests/cache_implementation/test_memcached.py +++ /dev/null @@ -1,146 +0,0 @@ -import json -from unittest import skipIf -from django.conf import settings -from django.test import TestCase -from django.utils import timezone -from restclients_core.models import MockHTTP -from rc_django.cache_implementation.memcache import ( - MemcachedCache, Client, MemcachedException) - - -@skipIf(not getattr(settings, 'RESTCLIENTS_MEMCACHED_SERVERS', None), - "Memcached cache not configured") -class MemcachedCacheTest(TestCase): - def setUp(self): - cache = MemcachedCache() - cache.client.flush_all() - - def test_cacheGet(self): - cache = MemcachedCache() - key = cache._get_key('mem', '/same') - self.assertEquals(key, "mem-/same") - - data = cache.getCache('mem', '/same', {}) - self.assertIsNone(data) - - expires = cache.get_cache_expiration_time('mem', '/same') - cdata = cache._make_cache_data( - 'mem', '/same', json.dumps({"data": "Body Content"}), - {}, 200, timezone.now()) - cache.client.set(key, cdata, time=expires) - - hit = cache.getCache('mem', '/same', {}) - response = hit["response"] - self.assertEquals(response.status, 200) - self.assertEquals(response.data, '{"data": "Body Content"}') - - # delete existing entry - self.assertTrue(cache.deleteCache('mem', '/same')) - - cache.client = MockClient1() - # test get err - cache.getCache('mem', '/same', {}) - # test delete err - self.assertFalse(cache.deleteCache('mem', '/same')) - - def test_updateCache(self): - cache = MemcachedCache() - # cache no data - cache.updateCache('mem', '/same', '{"data": "Content1"}', - timezone.now()) - - time1 = timezone.now() - # cache has older data - cache.updateCache('mem', '/same', '{"data": "Content2"}', time1) - - hit = cache.getCache('mem', '/same', {}) - response = hit["response"] - self.assertEquals(response.headers, {}) - self.assertEquals(response.status, 200) - self.assertEquals(response.data, '{"data": "Content2"}') - - # update with no newer data - cache.updateCache('mem', '/same', '{"data": "Content3"}', time1) - hit = cache.getCache('mem', '/same', {}) - response = hit["response"] - self.assertEquals(response.data, '{"data": "Content2"}') - - # test replace err - cache.client = MockClient2() - self.assertRaises(MemcachedException, - cache.updateCache, - 'mem', '/same', '{}', timezone.now()) - - # test set err - cache.deleteCache('mem', '/same') - self.assertRaises(MemcachedException, - cache.updateCache, - 'mem', '/same', '{}', timezone.now()) - - def test_processResponse(self): - mock_resp = MockHTTP() - mock_resp.status = 200 - mock_resp.data = "Content4" - mock_resp.headers = {"Content-type": "text/html"} - - cache = MemcachedCache() - cache.processResponse('mem', '/same1', mock_resp) - - hit = cache.getCache('mem', '/same1', {}) - response = hit["response"] - self.assertEquals(response.headers, {"Content-type": "text/html"}) - self.assertEquals(response.status, 200) - self.assertEquals(response.data, "Content4") - - def test_processResponse_err(self): - mock_resp = MockHTTP() - mock_resp.status = 200 - mock_resp.data = "Content4" - mock_resp.headers = {"Content-type": "text/html"} - cache = MemcachedCache() - cache.client = MockClient2() - cache.processResponse('mem', '/same1', mock_resp) - - def test_binary_processResponse(self): - mock_resp = MockHTTP() - mock_resp.status = 200 - mock_resp.data = b'content to be encoded' - mock_resp.headers = {"Content-type": "image/png"} - - cache = MemcachedCache() - cache.processResponse('mem', '/same2', mock_resp) - - hit = cache.getCache('mem', '/same2', {}) - response = hit["response"] - self.assertEquals(response.headers, {"Content-type": "image/png"}) - self.assertEquals(response.status, 200) - self.assertEquals(response.data, b'content to be encoded') - - def test_memcached_client(self): - cache = MemcachedCache() - key = cache._get_key('mem', '/same') - self.assertEquals(key, "mem-/same") - cache.updateCache('mem', '/same', '{"data": "Content"}', - timezone.now()) - hit = cache.getCache('mem', '/same', {}) - response = hit["response"] - self.assertEquals(response.headers, {}) - self.assertEquals(response.status, 200) - - -class MockClient1(Client): - - def delete(self, key): - raise MemcachedException("err", 400) - - def get(self, key, get_cas=False): - raise MemcachedException("err", 400) - - -class MockClient2(Client): - - def replace(self, key, value, time=0, compress_level=-1): - raise MemcachedException("err", 400) - - def set(self, key, value, time=0, compress_level=-1): - raise MemcachedException("err", 400) diff --git a/setup.py b/setup.py index 463eda2..b98da01 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,6 @@ 'Django>=2.1,<3.0', 'UW-RestClients-Core<2.0', 'django-userservice<4.0', - 'python-binary-memcached', ], license='Apache License, Version 2.0', description=('UW-RestClients-Django-Utils'), From 53308c800282449bd8ea4feba5221803045c8f6e Mon Sep 17 00:00:00 2001 From: Jim Laney Date: Tue, 24 Nov 2020 09:44:55 -0800 Subject: [PATCH 2/2] drops cache_implementation logger --- rc_django/cache_implementation/logger.py | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 rc_django/cache_implementation/logger.py diff --git a/rc_django/cache_implementation/logger.py b/rc_django/cache_implementation/logger.py deleted file mode 100644 index 108736a..0000000 --- a/rc_django/cache_implementation/logger.py +++ /dev/null @@ -1,13 +0,0 @@ -from django.conf import settings - -enable_logging = getattr(settings, 'ENABLE_MEMCACHE_LOGGING', None) - - -def log_err(logger, msg): - if enable_logging: - logger.error(msg) - - -def log_info(logger, msg): - if enable_logging: - logger.info(msg)