diff --git a/.gitignore b/.gitignore index b288303..9adce58 100644 --- a/.gitignore +++ b/.gitignore @@ -1,37 +1,101 @@ +### https://raw.github.com/github/gitignore/f57304e9762876ae4c9b02867ed0cb887316387e/python.gitignore + +# Byte-compiled / optimized / DLL files +__pycache__/ *.py[cod] +*$py.class # C extensions *.so -# Packages -*.egg -*.egg-info -dist -build -eggs -parts -bin -var -sdist -develop-eggs +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ .installed.cfg -lib -lib64 -__pycache__ +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec # Installer logs pip-log.txt +pip-delete-this-directory.txt # Unit test / coverage reports +htmlcov/ +.tox/ .coverage -.tox +.coverage.* +.cache nosetests.xml +coverage.xml +*,cover +.hypothesis/ # Translations *.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + -# Mr Developer -.mr.developer.cfg -.project -.pydevproject -.idea diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..4edb0df --- /dev/null +++ b/.travis.yml @@ -0,0 +1,69 @@ +sudo: false +language: python +cache: pip +matrix: + fast_finish: true + include: + - python: 2.7 + env: TOX_ENV=py27-dj18 + - python: 3.3 + env: TOX_ENV=py33-dj18 + - python: 3.4 + env: TOX_ENV=py34-dj18 + - python: 3.5 + env: TOX_ENV=py34-dj18 + - python: 2.7 + env: TOX_ENV=py27-dj19 + - python: 3.4 + env: TOX_ENV=py34-dj19 + - python: 3.5 + env: TOX_ENV=py35-dj19 + - python: 2.7 + env: TOX_ENV=py27-dj110 + - python: 3.4 + env: TOX_ENV=py34-dj110 + - python: 3.5 + env: TOX_ENV=py35-dj110 + - python: 2.7 + env: TOX_ENV=py27-dj111 + - python: pypy + env: TOX_ENV=pypy-dj111 + - python: 3.4 + env: TOX_ENV=py34-dj111 + - python: 3.5 + env: TOX_ENV=py35-dj111 + - python: 3.6 + env: TOX_ENV=py36-dj111 + - python: pypy3 + env: TOX_ENV=pypy3-dj111 + - python: 3.5 + env: TOX_ENV=py35-djdev + - python: 3.6 + env: TOX_ENV=py36-djdev + - python: 3.6 + env: TOX_ENV=flake8 + - python: 3.6 + env: TOX_ENV=isort + - python: 3.6 + env: TOX_ENV=readme + - python: 3.6 + env: TOX_ENV=check-manifest + allow_failures: + - env: TOX_ENV=py35-djdev + - env: TOX_ENV=py36-djdev + - env: TOX_ENV=pypy3-dj111 +install: +- pip install tox +script: +- tox -e "$TOX_ENV" +deploy: + provider: pypi + user: uncovertruth + password: + secure: FtNY+8/F79RYpJzfusYBC9HFfDWlevUbS4J5TECm8tyMxAxeHMpLOaL4dd/wPkRkuctUM5LyXux5rpVQTfMA4Dvjdufc7BqexCt056GznJPFRjHW19ExeguTHXW810RIVrFmo5Dk1/AlMpGEeiL3OXoI041h7/vRZe6QWP6P0NCz9UrkZ/Qc9JDZqkXBmhiwvt4P3gu5sbxaiM4IqxhSQ/4DoI87SQlD3nUfgRT2TkqN/Jx0ME0H7l+40HeqZvVq6sh9Muc5/XhDEx5HWt9BSgymeXXG6mT3sewBfinJlQ5/1Rrm+IHa48m/3mLJkfYEs15BQCizvDc6/dRfdtCc/9jYl3e67yEM6Akn+52pTOU4Oa727ZUpAYNCzfY+pHc3E9oI0YFuOl+WYNo5bWUh3IlWAl/eXDGWOvRMF4waUbyjUMiZ+C4pRmRKznYZB9xgbPxHDlR6CMn8P5F5j3ZVbW0BxUEjlDFcDT65gflBhKS1eWTOQRMobR+yjQKlYMlIeCgITbuEYbn2GbL84aN42U910rSQ4SryzqjaOIyxtZG53cGDw2NMm4q7bIwRkj6gbvliA36HIFaRQrI/wtrmr3guy4raPRpEGi7OhFoaPxaqkEs+yF55MEafNM4eJc9swM/K/mbN6FFKIWoWVyXUCUDIWvcPYLWPR/kLidWCNzw= + distributions: sdist bdist_wheel + skip_upload_docs: true + on: + repo: uncovertruth/django-elastipymemcache + tags: true + condition: $TOX_ENV = py36-dj111 diff --git a/LICENSE b/LICENSE index 9b5f8a5..e4835f6 100644 --- a/LICENSE +++ b/LICENSE @@ -18,3 +18,4 @@ FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/MANIFEST.in b/MANIFEST.in index 4c4c31e..2be18b6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ -include README.rst -include MANIFEST.in -include setup.py \ No newline at end of file +graft django_elastipymemcache +graft tests +include README.rst AUTHORS LICENSE CHANGELOG.rst setup.py setup.cfg requirements.txt tox.ini +global-exclude __pycache__ *.pyc diff --git a/README.rst b/README.rst index 644a037..09889a1 100644 --- a/README.rst +++ b/README.rst @@ -1,31 +1,44 @@ -Amazon ElastiCache backend for Django -===================================== +======================= +django-elastipymemcache +======================= + +:Info: Simple Django cache backend for Amazon ElastiCache (memcached based). +:Author: UNCOVER TRUTH Inc. +:Copyright: © UNCOVER TRUTH Inc. +:Date: 2017-04-11 +:Version: 0.0.6 + +.. index: README +.. image:: https://travis-ci.org/uncovertruth/django-elastipymemcache.svg?branch=master + :target: https://travis-ci.org/uncovertruth/django-elastipymemcache +.. image:: https://codecov.io/gh/uncovertruth/django-elastipymemcache/branch/master/graph/badge.svg + :target: https://codecov.io/gh/uncovertruth/django-elastipymemcache +.. image:: https://requires.io/github/uncovertruth/django-elastipymemcache/requirements.svg?branch=master + :target: https://requires.io/github/uncovertruth/django-elastipymemcache/requirements/?branch=master +.. image:: https://badge.fury.io/py/django-elastipymemcache.svg + :target: https://badge.fury.io/py/django-elastipymemcache + +Purpose +------- Simple Django cache backend for Amazon ElastiCache (memcached based). It uses -`pylibmc `_ and sets up a connection to each +`pymemcache `_ and sets up a connection to each node in the cluster using `auto discovery `_. - +Originally forked from `django-elasticache `_. Requirements ------------ -* pylibmc -* Django 1.5+. - -It was written and tested on Python 2.7 and 3.4. +* pymemcache +* Django>=1.8 Installation ------------ -Get it from `pypi `_:: - - pip install django-elasticache - -or `github `_:: - - pip install -e git://github.com/gusdan/django-elasticache.git#egg=django-elasticache +Get it from `pypi `_:: + pip install django-elastipymemcache Usage ----- @@ -34,95 +47,16 @@ Your cache backend should look something like this:: CACHES = { 'default': { - 'BACKEND': 'django_elasticache.memcached.ElastiCache', - 'LOCATION': 'cache-c.draaaf.cfg.use1.cache.amazonaws.com:11211', - 'OPTIONS' { - 'IGNORE_CLUSTER_ERRORS': [True,False], - }, - } - } - -By the first call to cache it connects to cluster (using ``LOCATION`` param), -gets list of all nodes and setup pylibmc client using full -list of nodes. As result your cache will work with all nodes in cluster and -automatically detect new nodes in cluster. List of nodes are stored in class-level -cached, so any changes in cluster take affect only after restart of working process. -But if you're using gunicorn or mod_wsgi you usually have max_request settings which -restart process after some count of processed requests, so auto discovery will work -fine. - -The ``IGNORE_CLUSTER_ERRORS`` option is useful when ``LOCATION`` doesn't have support -for ``config get cluster``. When set to ``True``, and ``config get cluster`` fails, -it returns a list of a single node with the same endpoint supplied to ``LOCATION``. - -Django-elasticache changes default pylibmc params to increase performance. - -Another solutions ------------------ - -ElastiCache provides memcached interface so there are three solution of using it: - -1. Memcached configured with location = Configuration Endpoint -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In this case your application -will randomly connect to nodes in cluster and cache will be used with not optimal -way. At some moment you will be connected to first node and set item. Minute later -you will be connected to another node and will not able to get this item. - - :: - - CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache', - 'LOCATION': 'cache.gasdbp.cfg.use1.cache.amazonaws.com:11211', - } - } - - -2. Memcached configured with all nodes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -It will work fine, memcache client will -separate items between all nodes and will balance loading on client side. You will -have problems only after adding new nodes or delete old nodes. In this case you should -add new nodes manually and don't forget update your app after all changes on AWS. - - :: - - CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache', - 'LOCATION': [ - 'cache.gqasdbp.0001.use1.cache.amazonaws.com:11211', - 'cache.gqasdbp.0002.use1.cache.amazonaws.com:11211', - ] - } - } - - -3. Use django-elasticache -~~~~~~~~~~~~~~~~~~~~~~~~~ - -It will connect to cluster and retrieve ip addresses -of all nodes and configure memcached to use all nodes. - - :: - - CACHES = { - 'default': { - 'BACKEND': 'django_elasticache.memcached.ElastiCache', - 'LOCATION': 'cache-c.draaaf.cfg.use1.cache.amazonaws.com:11211', + 'BACKEND': 'django_elastipymemcache.memcached.ElastiPyMemCache', + 'LOCATION': '[configuration endpoint]:11211', + 'OPTIONS': { + 'cluster_timeout': 1, # its used when get cluster info + 'ignore_exc': True, # pymemcache Client params + 'ignore_cluster_errors': True, # ignore get cluster info error + } } } - -Difference between setup with nodes list (django-elasticache) and -connection to only one configuration Endpoint (using dns routing) you can see on -this graph: - -.. image:: https://raw.github.com/gusdan/django-elasticache/master/docs/images/get%20operation%20in%20cluster.png - Testing ------- diff --git a/django_elasticache/memcached.py b/django_elasticache/memcached.py deleted file mode 100644 index f7c19fc..0000000 --- a/django_elasticache/memcached.py +++ /dev/null @@ -1,121 +0,0 @@ -""" -Backend for django cache -""" -import socket -from functools import wraps -from django.core.cache import InvalidCacheBackendError -from django.core.cache.backends.memcached import PyLibMCCache -from .cluster_utils import get_cluster_info - - -def invalidate_cache_after_error(f): - """ - catch any exception and invalidate internal cache with list of nodes - """ - @wraps(f) - def wrapper(self, *args, **kwds): - try: - return f(self, *args, **kwds) - except Exception: - self.clear_cluster_nodes_cache() - raise - return wrapper - - -class ElastiCache(PyLibMCCache): - """ - backend for Amazon ElastiCache (memcached) with auto discovery mode - it used pylibmc in binary mode - """ - def __init__(self, server, params): - self.update_params(params) - super(ElastiCache, self).__init__(server, params) - if len(self._servers) > 1: - raise InvalidCacheBackendError( - 'ElastiCache should be configured with only one server ' - '(Configuration Endpoint)') - if len(self._servers[0].split(':')) != 2: - raise InvalidCacheBackendError( - 'Server configuration should be in format IP:port') - - self._ignore_cluster_errors = self._options.get( - 'IGNORE_CLUSTER_ERRORS', False) - - def update_params(self, params): - """ - update connection params to maximize performance - """ - if not params.get('BINARY', True): - raise Warning('To increase performance please use ElastiCache' - ' in binary mode') - else: - params['BINARY'] = True # patch params, set binary mode - if 'OPTIONS' not in params: - # set special 'behaviors' pylibmc attributes - params['OPTIONS'] = { - 'tcp_nodelay': True, - 'ketama': True - } - - def clear_cluster_nodes_cache(self): - """clear internal cache with list of nodes in cluster""" - if hasattr(self, '_cluster_nodes_cache'): - del self._cluster_nodes_cache - - def get_cluster_nodes(self): - """ - return list with all nodes in cluster - """ - if not hasattr(self, '_cluster_nodes_cache'): - server, port = self._servers[0].split(':') - try: - self._cluster_nodes_cache = ( - get_cluster_info(server, port, - self._ignore_cluster_errors)['nodes']) - except (socket.gaierror, socket.timeout) as err: - raise Exception('Cannot connect to cluster {} ({})'.format( - self._servers[0], err - )) - return self._cluster_nodes_cache - - @property - def _cache(self): - # PylibMC uses cache options as the 'behaviors' attribute. - # It also needs to use threadlocals, because some versions of - # PylibMC don't play well with the GIL. - - # instance to store cached version of client - # in Django 1.7 use self - # in Django < 1.7 use thread local - container = getattr(self, '_local', self) - client = getattr(container, '_client', None) - if client: - return client - - client = self._lib.Client(self.get_cluster_nodes()) - if self._options: - client.behaviors = self._options - - container._client = client - - return client - - @invalidate_cache_after_error - def get(self, *args, **kwargs): - return super(ElastiCache, self).get(*args, **kwargs) - - @invalidate_cache_after_error - def get_many(self, *args, **kwargs): - return super(ElastiCache, self).get_many(*args, **kwargs) - - @invalidate_cache_after_error - def set(self, *args, **kwargs): - return super(ElastiCache, self).set(*args, **kwargs) - - @invalidate_cache_after_error - def set_many(self, *args, **kwargs): - return super(ElastiCache, self).set_many(*args, **kwargs) - - @invalidate_cache_after_error - def delete(self, *args, **kwargs): - return super(ElastiCache, self).delete(*args, **kwargs) diff --git a/django_elasticache/__init__.py b/django_elastipymemcache/__init__.py similarity index 67% rename from django_elasticache/__init__.py rename to django_elastipymemcache/__init__.py index e3114ec..d7e04e4 100644 --- a/django_elasticache/__init__.py +++ b/django_elastipymemcache/__init__.py @@ -1,2 +1,2 @@ -VERSION = (1, 0, 1) +VERSION = (0, 0, 6) __version__ = '.'.join(map(str, VERSION)) diff --git a/django_elastipymemcache/client.py b/django_elastipymemcache/client.py new file mode 100644 index 0000000..9068eb9 --- /dev/null +++ b/django_elastipymemcache/client.py @@ -0,0 +1,38 @@ +from pymemcache.client.hash import HashClient + + +class Client(HashClient): + def get_many(self, keys, gets=False, *args, **kwargs): + client_batches = {} + end = {} + + for key in keys: + client = self._get_client(key) + + if client is None: + continue + + if client.server not in client_batches: + client_batches[client.server] = [] + + client_batches[client.server].append(key) + + for server, keys in client_batches.items(): + client = self.clients['%s:%s' % server] + new_args = list(args) + new_args.insert(0, keys) + + if gets: + get_func = client.gets_many + else: + get_func = client.get_many + + result = self._safely_run_func( + client, + get_func, {}, *new_args, **kwargs + ) + end.update(result) + + return end + + get_multi = get_many diff --git a/django_elasticache/cluster_utils.py b/django_elastipymemcache/cluster_utils.py similarity index 85% rename from django_elasticache/cluster_utils.py rename to django_elastipymemcache/cluster_utils.py index 740e4cf..15f2d9f 100644 --- a/django_elasticache/cluster_utils.py +++ b/django_elastipymemcache/cluster_utils.py @@ -1,11 +1,13 @@ """ utils for discovery cluster """ -from distutils.version import StrictVersion -from django.utils.encoding import smart_text import re +from distutils.version import StrictVersion +import socket from telnetlib import Telnet +from django.utils.encoding import smart_text + class WrongProtocolData(ValueError): """ @@ -17,7 +19,11 @@ def __init__(self, cmd, response): 'Unexpected response {} for command {}'.format(response, cmd)) -def get_cluster_info(host, port, ignore_cluster_errors=False): +def get_cluster_info( + host, + port, + ignore_cluster_errors=False, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT): """ return dict with info about nodes in cluster and current version { @@ -28,7 +34,7 @@ def get_cluster_info(host, port, ignore_cluster_errors=False): 'version': '1.4.4' } """ - client = Telnet(host, int(port)) + client = Telnet(host, int(port), timeout=timeout) client.write(b'version\n') res = client.read_until(b'\r\n').strip() version_list = res.split(b' ') @@ -50,8 +56,7 @@ def get_cluster_info(host, port, ignore_cluster_errors=False): return { 'version': version, 'nodes': [ - '{}:{}'.format(smart_text(host), - smart_text(port)) + (smart_text(host), int(port)) ] } @@ -67,8 +72,7 @@ def get_cluster_info(host, port, ignore_cluster_errors=False): try: for node in ls[2].split(b' '): host, ip, port = node.split(b'|') - nodes.append('{}:{}'.format(smart_text(ip or host), - smart_text(port))) + nodes.append((smart_text(ip or host), int(port))) except ValueError: raise WrongProtocolData(cmd, res) return { diff --git a/django_elastipymemcache/memcached.py b/django_elastipymemcache/memcached.py new file mode 100644 index 0000000..cf59f77 --- /dev/null +++ b/django_elastipymemcache/memcached.py @@ -0,0 +1,141 @@ +""" +Backend for django cache +""" +import logging +import socket +from functools import wraps + +try: + import cPickle as pickle +except ImportError: + import pickle + +from django.core.cache import InvalidCacheBackendError +from django.core.cache.backends.memcached import BaseMemcachedCache + +from . import client as pyMemcache_client +from .cluster_utils import get_cluster_info + + +logger = logging.getLogger(__name__) + + +def serialize_pickle(key, value): + if isinstance(value, str): + return value, 1 + return pickle.dumps(value), 2 + + +def deserialize_pickle(key, value, flags): + if flags == 1: + return value + if flags == 2: + return pickle.loads(value) + + +def invalidate_cache_after_error(f): + """ + catch any exception and invalidate internal cache with list of nodes + """ + @wraps(f) + def wrapper(self, *args, **kwds): + try: + return f(self, *args, **kwds) + except Exception: + self.clear_cluster_nodes_cache() + raise + return wrapper + + +class ElastiPyMemCache(BaseMemcachedCache): + """ + backend for Amazon ElastiCache (memcached) with auto discovery mode + it used pyMemcache + """ + def __init__(self, server, params): + super(ElastiPyMemCache, self).__init__( + server, + params, + library=pyMemcache_client, + value_not_found_exception=ValueError) + if len(self._servers) > 1: + raise InvalidCacheBackendError( + 'ElastiCache should be configured with only one server ' + '(Configuration Endpoint)') + if len(self._servers[0].split(':')) != 2: + raise InvalidCacheBackendError( + 'Server configuration should be in format IP:port') + + # Patch for django<1.11 + self._options = self._options or dict() + self._cluster_timeout = self._options.get( + 'cluster_timeout', socket._GLOBAL_DEFAULT_TIMEOUT) + self._ignore_cluster_errors = self._options.get( + 'ignore_cluster_errors', False) + + def clear_cluster_nodes_cache(self): + """clear internal cache with list of nodes in cluster""" + if hasattr(self, '_client'): + del self._client + + def get_cluster_nodes(self): + """ + return list with all nodes in cluster + """ + server, port = self._servers[0].split(':') + try: + return get_cluster_info( + server, + port, + self._ignore_cluster_errors, + self._cluster_timeout + )['nodes'] + except (OSError, socket.gaierror, socket.timeout) as err: + logger.debug( + 'Cannot connect to cluster %s, err: %s', + self._servers[0], + err + ) + return [] + + @property + def _cache(self): + + if getattr(self, '_client', None) is None: + + options = self._options + options['serializer'] = serialize_pickle + options['deserializer'] = deserialize_pickle + options.setdefault('ignore_exc', True) + options.pop('cluster_timeout', None) + options.pop('ignore_cluster_errors', None) + + self._client = self._lib.Client( + self.get_cluster_nodes(), **options) + + return self._client + + def close(self, **kwargs): + # libmemcached manages its own connections. Don't call disconnect_all() + # as it resets the failover state and creates unnecessary reconnects. + pass + + @invalidate_cache_after_error + def get(self, *args, **kwargs): + return super(ElastiPyMemCache, self).get(*args, **kwargs) + + @invalidate_cache_after_error + def get_many(self, *args, **kwargs): + return super(ElastiPyMemCache, self).get_many(*args, **kwargs) + + @invalidate_cache_after_error + def set(self, *args, **kwargs): + return super(ElastiPyMemCache, self).set(*args, **kwargs) + + @invalidate_cache_after_error + def set_many(self, *args, **kwargs): + return super(ElastiPyMemCache, self).set_many(*args, **kwargs) + + @invalidate_cache_after_error + def delete(self, *args, **kwargs): + return super(ElastiPyMemCache, self).delete(*args, **kwargs) diff --git a/docs/images/get operation in cluster.png b/docs/images/get operation in cluster.png deleted file mode 100644 index a85f01d..0000000 Binary files a/docs/images/get operation in cluster.png and /dev/null differ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e029246 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +check-manifest==0.35 +coverage==4.4.1 +flake8==3.5.0 +isort==4.2.15 +mock==2.0.0 +nose==1.3.7 +pymemcache==1.4.3 +readme-renderer==17.2 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..ca864d7 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,17 @@ +[wheel] +universal = 1 + +[isort] +include_trailing_comma=True +line_length=80 +multi_line_output=3 +not_skip=__init__.py +known_first_party=django_elastipymemcache + +[check-manifest] +ignore = + *.swp + +[coverage:run] +branch = True +omit = tests/* diff --git a/setup.py b/setup.py index f65143c..71b8992 100644 --- a/setup.py +++ b/setup.py @@ -1,30 +1,47 @@ -from setuptools import setup +#!/usr/bin/env python +# -*- encoding: utf-8 -*- -import django_elasticache +import io +from setuptools import find_packages, setup + +import django_elastipymemcache setup( - name='django-elasticache', - version=django_elasticache.__version__, + name='django-elastipymemcache', + version=django_elastipymemcache.__version__, description='Django cache backend for Amazon ElastiCache (memcached)', - long_description=open('README.rst').read(), - author='Danil Gusev', - platforms='any', - author_email='danil.gusev@gmail.com', - url='http://github.com/gusdan/django-elasticache', + keywords='elasticache amazon cache pymemcache memcached aws', + author='UNCOVER TRUTH Inc.', + author_email='develop@uncovertruth.co.jp', + url='http://github.com/uncovertruth/django-elastipymemcache', license='MIT', - keywords='elasticache amazon cache pylibmc memcached aws', - packages=['django_elasticache'], - install_requires=['pylibmc', 'Django>=1.3'], + long_description=io.open('README.rst').read(), + platforms='any', + zip_safe=False, classifiers=[ 'Development Status :: 4 - Beta', 'Environment :: Web Environment', - 'Environment :: Web Environment :: Mozilla', - 'Framework :: Django', + 'Framework :: Django :: 1.8', + 'Framework :: Django :: 1.9', + 'Framework :: Django :: 1.10', + 'Framework :: Django :: 1.11', 'Intended Audience :: Developers', - 'License :: OSI Approved :: BSD License', + 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', - 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Software Development :: Libraries :: Python Modules', ], + packages=find_packages(exclude=('tests',)), + include_package_data=True, + install_requires=[ + 'pymemcache', + 'Django>=1.8', + ], ) diff --git a/tests/test_backend.py b/tests/test_backend.py index 9b6f425..e46feb4 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -1,6 +1,9 @@ -from django.conf import global_settings, settings -from nose.tools import eq_, raises +import socket import sys + +from django.conf import global_settings, settings +from nose.tools import eq_ + if sys.version < '3': from mock import patch, Mock else: @@ -13,76 +16,72 @@ @patch('django.conf.settings', global_settings) -def test_patch_params(): - from django_elasticache.memcached import ElastiCache - params = {} - ElastiCache('qew:12', params) - eq_(params['BINARY'], True) - eq_(params['OPTIONS']['tcp_nodelay'], True) - eq_(params['OPTIONS']['ketama'], True) - - -@raises(Exception) -@patch('django.conf.settings', global_settings) -def test_wrong_params(): - from django_elasticache.memcached import ElastiCache - ElastiCache('qew', {}) - - -@raises(Warning) -@patch('django.conf.settings', global_settings) -def test_wrong_params_warning(): - from django_elasticache.memcached import ElastiCache - ElastiCache('qew', {'BINARY': False}) - - -@patch('django.conf.settings', global_settings) -@patch('django_elasticache.memcached.get_cluster_info') +@patch('django_elastipymemcache.memcached.get_cluster_info') def test_split_servers(get_cluster_info): - from django_elasticache.memcached import ElastiCache - backend = ElastiCache('h:0', {}) - servers = ['h1:p', 'h2:p'] + from django_elastipymemcache.memcached import ( + ElastiPyMemCache, + deserialize_pickle, + serialize_pickle, + ) + backend = ElastiPyMemCache('h:0', {}) + servers = [('h1', 0), ('h2', 0)] get_cluster_info.return_value = { 'nodes': servers } backend._lib.Client = Mock() assert backend._cache - get_cluster_info.assert_called_once_with('h', '0', False) - backend._lib.Client.assert_called_once_with(servers) + get_cluster_info.assert_called_once_with( + 'h', '0', False, socket._GLOBAL_DEFAULT_TIMEOUT) + backend._lib.Client.assert_called_once_with( + servers, + deserializer=deserialize_pickle, + ignore_exc=True, + serializer=serialize_pickle + ) @patch('django.conf.settings', global_settings) -@patch('django_elasticache.memcached.get_cluster_info') +@patch('django_elastipymemcache.memcached.get_cluster_info') def test_node_info_cache(get_cluster_info): - from django_elasticache.memcached import ElastiCache - servers = ['h1:p', 'h2:p'] + from django_elastipymemcache.memcached import ( + ElastiPyMemCache, + deserialize_pickle, + serialize_pickle, + ) + servers = [('h1', 0), ('h2', 0)] get_cluster_info.return_value = { 'nodes': servers } - backend = ElastiCache('h:0', {}) + backend = ElastiPyMemCache('h:0', {}) backend._lib.Client = Mock() backend.set('key1', 'val') backend.get('key1') backend.set('key2', 'val') backend.get('key2') - backend._lib.Client.assert_called_once_with(servers) + backend._lib.Client.assert_called_once_with( + servers, + deserializer=deserialize_pickle, + ignore_exc=True, + serializer=serialize_pickle + ) eq_(backend._cache.get.call_count, 2) eq_(backend._cache.set.call_count, 2) - get_cluster_info.assert_called_once_with('h', '0', False) + get_cluster_info.assert_called_once_with( + 'h', '0', False, socket._GLOBAL_DEFAULT_TIMEOUT) @patch('django.conf.settings', global_settings) -@patch('django_elasticache.memcached.get_cluster_info') +@patch('django_elastipymemcache.memcached.get_cluster_info') def test_invalidate_cache(get_cluster_info): - from django_elasticache.memcached import ElastiCache - servers = ['h1:p', 'h2:p'] + from django_elastipymemcache.memcached import ElastiPyMemCache + servers = [('h1', 0), ('h2', 0)] get_cluster_info.return_value = { 'nodes': servers } - backend = ElastiCache('h:0', {}) + backend = ElastiPyMemCache('h:0', {}) backend._lib.Client = Mock() assert backend._cache backend._cache.get = Mock() @@ -99,4 +98,34 @@ def test_invalidate_cache(get_cluster_info): except Exception: pass eq_(backend._cache.get.call_count, 2) - eq_(get_cluster_info.call_count, 2) + eq_(get_cluster_info.call_count, 3) + + +@patch('django.conf.settings', global_settings) +@patch('django_elastipymemcache.memcached.get_cluster_info') +def test_client_get_many(get_cluster_info): + from django_elastipymemcache.memcached import ElastiPyMemCache + + servers = [('h1', 0), ('h2', 0)] + get_cluster_info.return_value = { + 'nodes': servers + } + + backend = ElastiPyMemCache('h:0', {}) + ret = backend.get_many(['key1']) + eq_(ret, {}) + + # When server does not found... + with patch('pymemcache.client.hash.HashClient._get_client') as p: + p.return_value = None + ret = backend.get_many(['key2']) + eq_(ret, {}) + + with patch('django_elastipymemcache.client.Client.get_many') as p: + with patch('pymemcache.client.hash.HashClient._safely_run_func') as p2: + p2.return_value = { + ':1:key3': 1509111630.048594 + } + + ret = backend.get_many(['key3']) + eq_(ret, {'key3': 1509111630.048594}) diff --git a/tests/test_protocol.py b/tests/test_protocol.py index 7232a1a..a536238 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -1,7 +1,11 @@ -from django_elasticache.cluster_utils import ( - get_cluster_info, WrongProtocolData) -from nose.tools import eq_, raises import sys + +from django_elastipymemcache.cluster_utils import ( + WrongProtocolData, + get_cluster_info, +) +from nose.tools import eq_, raises + if sys.version < '3': from mock import patch, call, MagicMock else: @@ -13,7 +17,7 @@ ] TEST_PROTOCOL_1_EXPECT = [ - (0, None, b'CONFIG cluster 0 138\r\n1\nhost|ip|port host||port\n\r\nEND\r\n'), # NOQA + (0, None, b'CONFIG cluster 0 138\r\n1\nhost|ip|11211 host||11211\n\r\nEND\r\n'), # NOQA ] TEST_PROTOCOL_2_READ_UNTIL = [ @@ -21,7 +25,7 @@ ] TEST_PROTOCOL_2_EXPECT = [ - (0, None, b'CONFIG cluster 0 138\r\n1\nhost|ip|port host||port\n\r\nEND\r\n'), # NOQA + (0, None, b'CONFIG cluster 0 138\r\n1\nhost|ip|11211 host||11211\n\r\nEND\r\n'), # NOQA ] TEST_PROTOCOL_3_READ_UNTIL = [ @@ -29,7 +33,7 @@ ] TEST_PROTOCOL_3_EXPECT = [ - (0, None, b'CONFIG cluster 0 138\r\n1\nhost|ip|port host||port\n\r\nEND\r\n'), # NOQA + (0, None, b'CONFIG cluster 0 138\r\n1\nhost|ip|11211 host||11211\n\r\nEND\r\n'), # NOQA ] TEST_PROTOCOL_4_READ_UNTIL = [ @@ -41,23 +45,23 @@ ] -@patch('django_elasticache.cluster_utils.Telnet') +@patch('django_elastipymemcache.cluster_utils.Telnet') def test_happy_path(Telnet): client = Telnet.return_value client.read_until.side_effect = TEST_PROTOCOL_1_READ_UNTIL client.expect.side_effect = TEST_PROTOCOL_1_EXPECT info = get_cluster_info('', 0) eq_(info['version'], 1) - eq_(info['nodes'], ['ip:port', 'host:port']) + eq_(info['nodes'], [('ip', 11211), ('host', 11211)]) @raises(WrongProtocolData) -@patch('django_elasticache.cluster_utils.Telnet', MagicMock()) +@patch('django_elastipymemcache.cluster_utils.Telnet', MagicMock()) def test_bad_protocol(): get_cluster_info('', 0) -@patch('django_elasticache.cluster_utils.Telnet') +@patch('django_elastipymemcache.cluster_utils.Telnet') def test_last_versions(Telnet): client = Telnet.return_value client.read_until.side_effect = TEST_PROTOCOL_1_READ_UNTIL @@ -69,7 +73,7 @@ def test_last_versions(Telnet): ]) -@patch('django_elasticache.cluster_utils.Telnet') +@patch('django_elastipymemcache.cluster_utils.Telnet') def test_prev_versions(Telnet): client = Telnet.return_value client.read_until.side_effect = TEST_PROTOCOL_2_READ_UNTIL @@ -81,44 +85,31 @@ def test_prev_versions(Telnet): ]) -@patch('django_elasticache.cluster_utils.Telnet') +@patch('django_elastipymemcache.cluster_utils.Telnet') def test_ubuntu_protocol(Telnet): client = Telnet.return_value client.read_until.side_effect = TEST_PROTOCOL_3_READ_UNTIL client.expect.side_effect = TEST_PROTOCOL_3_EXPECT - try: - get_cluster_info('', 0) - except WrongProtocolData: - raise AssertionError('Raised WrongProtocolData with Ubuntu version.') - - client.write.assert_has_calls([ - call(b'version\n'), - call(b'config get cluster\n'), - ]) - + # try: + # get_cluster_info('', 0) + # except WrongProtocolData: + # raise AssertionError('Raised WrongProtocolData with Ubuntu version.') + get_cluster_info('', 0) -@patch('django_elasticache.cluster_utils.Telnet') -def test_no_configuration_protocol_support_with_errors_ignored(Telnet): - client = Telnet.return_value - client.read_until.side_effect = TEST_PROTOCOL_4_READ_UNTIL - client.expect.side_effect = TEST_PROTOCOL_4_EXPECT - info = get_cluster_info('test', 0, ignore_cluster_errors=True) client.write.assert_has_calls([ call(b'version\n'), call(b'config get cluster\n'), ]) - eq_(info['version'], '1.4.34') - eq_(info['nodes'], ['test:0']) @raises(WrongProtocolData) -@patch('django_elasticache.cluster_utils.Telnet') +@patch('django_elastipymemcache.cluster_utils.Telnet') def test_no_configuration_protocol_support_with_errors(Telnet): client = Telnet.return_value client.read_until.side_effect = TEST_PROTOCOL_4_READ_UNTIL client.expect.side_effect = TEST_PROTOCOL_4_EXPECT - get_cluster_info('test', 0, ignore_cluster_errors=False) + get_cluster_info('test', 0) client.write.assert_has_calls([ call(b'version\n'), call(b'config get cluster\n'), diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..72aeba4 --- /dev/null +++ b/tox.ini @@ -0,0 +1,53 @@ +[tox] +envlist = + py{py,py3,27,33,34,35,36}-dj{18,19,110,111,dev} + flake8, + isort, + readme + check-manifest + +[testenv] +basepython = + py27: python2.7 + py33: python3.3 + py34: python3.4 + py35: python3.5 + py36: python3.6 + pypy: pypy + pypy3: pypy3 +deps = + dj18: Django>=1.8,<1.9 + dj19: Django>=1.9,<1.10 + dj110: Django>=1.10,<1.11 + dj111: Django>=1.11,<2.0 + djdev: https://github.com/django/django/archive/master.tar.gz + -r{toxinidir}/requirements.txt + py36-dj111: codecov +setenv = + PYTHONPATH = {toxinidir} + CODECOV_TOKEN="8ea69bfe-d9b5-45fc-97f7-ef843bd3d9d2" +commands = + coverage run --source=django_elastipymemcache -m nose + py36-dj111: coverage report + py36-dj111: coverage xml + py36-dj111: codecov + +[testenv:flake8] +basepython = python3.6 +commands = flake8 +deps = flake8 + +[testenv:isort] +basepython = python3.6 +commands = isort --verbose --check-only --diff django_elastipymemcache tests setup.py +deps = isort + +[testenv:readme] +basepython = python3.6 +commands = python setup.py check -r -s +deps = readme_renderer + +[testenv:check-manifest] +basepython = python3.6 +commands = check-manifest {toxinidir} +deps = check-manifest