Permalink
Browse files

Bug 934594: add memcached support to buildapi, fix redis support; r=c…

…atlee
  • Loading branch information...
1 parent fb197d3 commit 3a1be95ea89d0c323dbfd35587ad71dcfbe8c199 @djmitche djmitche committed Dec 4, 2013
Showing with 165 additions and 23 deletions.
  1. +14 −2 README.txt
  2. +4 −1 buildapi/config/deployment.ini_tmpl
  3. +9 −5 buildapi/lib/app_globals.py
  4. +52 −15 buildapi/lib/cacher.py
  5. +86 −0 buildapi/lib/test_cacher.py
View
@@ -1,10 +1,14 @@
Installation and Setup
======================
-Install ``buildapi`` and ``redis`` using easy_install::
+Install ``buildapi`` using easy_install::
easy_install buildapi
+
+Then install either redis or memcached:
+
easy_install redis
+ easy_install python-memcached
Make a config file as follows::
@@ -20,6 +24,14 @@ Tweak the config file as appropriate::
use = egg:PasteDeploy#prefix
prefix = /~(username)/wsgi
+Also set up your cache configuration:
+
+ buildapi.cache = redis:HOSTNAME:PORT
+
+or
+
+ buildapi.cache = memcached:HOSTNAME:PORT,HOSTNAME:PORT,..
+
Now setup the application::
paster setup-app config.ini
@@ -44,4 +56,4 @@ Installing google viz::
python setup.py test
Now you should be able to see reports like http://cruncher.build.mozilla.org/~(username)/wsgi/reports/pushes
-which use the google visualization library (make sure you have the statusdb set in your config.ini
+which use the google visualization library (make sure you have the statusdb set in your config.ini
@@ -58,7 +58,10 @@ masters_url = http://hg.mozilla.org/build/tools/raw-file/default/buildfarm/maint
# Similarly a JSON file defining the branches
branches_url = http://hg.mozilla.org/build/tools/raw-file/default/buildfarm/maintenance/production-branches.json
-# Path to the redis backend cache
+# Path to the backend cache:
+# redis:HOSTNAME:PORT
+# or
+# memcached:HOSTNAME:PORT,HOSTNAME:PORT,..
buildapi.cache = redis:HOSTNAME:PORT
# What timezone we're in
@@ -29,16 +29,20 @@ def __init__(self, config):
self.masters_url = config['masters_url']
self.branches_url = config['branches_url']
- # TODO: handle other hosts/ports
- if cache_spec.startswith('redis:'):
+ if hasattr(cacher, 'RedisCache') and cache_spec.startswith('redis:'):
+ # TODO: handle other hosts/ports
bits = cache_spec.split(':')
kwargs = {}
if len(bits) >= 2:
kwargs['host'] = bits[1]
if len(bits) == 3:
kwargs['port'] = int(bits[2])
-
- self.buildapi_cache = cache.BuildapiCache(cacher.RedisCache(**kwargs), tz)
+ buildapi_cacher = cacher.RedisCache(**kwargs)
+ elif hasattr(cacher, 'MemcacheCache') and cache_spec.startswith('memcached:'):
+ hosts = cache_spec[10:].split(',')
+ buildapi_cacher = cacher.MemcacheCache(hosts)
else:
- self.buildapi_cache = None
+ raise RuntimeError("invalid cache spec %r" % (cache_spec,))
+
+ self.buildapi_cache = cache.BuildapiCache(buildapi_cacher, tz)
View
@@ -1,3 +1,4 @@
+import threading
import time
try:
import simplejson as json
@@ -57,20 +58,18 @@ def put(self, key, val, expire=0):
class RedisCache(BaseCache):
def __init__(self, host='localhost', port=6379):
self.r = redis.client.Redis(host, port)
- self.locks = {}
+ # use a thread-local object for holding locks, so that different
+ # threads can use locks without stepping on feet
+ self.local = threading.local()
def _get(self, key):
if not self.has_key(key):
raise KeyError
- t = self.r.type(key)
- if t == 'list':
- return [json.loads(x) for x in self.r.lrange(key, 0, -1)]
- else:
- retval = self.r.get(key)
- if retval is not None:
- return json.loads(retval)
- return None
+ retval = self.r.get(key)
+ if retval is not None:
+ return json.loads(retval)
+ return None
def _put(self, key, val, expire=0):
val = json.dumps(val)
@@ -83,17 +82,55 @@ def _put(self, key, val, expire=0):
def has_key(self, key):
return self.r.exists(key)
- #def push(self, key, value):
- #return self.r.rpush(key, json.dumps(value))
-
def _getlock(self, key, expire):
+ if not hasattr(self.local, 'locks'):
+ self.local.locks = {}
+ assert key not in self.local.locks
l = redis.client.Lock(self.r, key, timeout=int(expire-time.time()))
l.acquire()
- self.locks[key] = l
+ self.local.locks[key] = l
+
+ def _releaselock(self, key):
+ self.local.locks[key].release()
+ del self.local.locks[key]
+
+except ImportError:
+ pass
+
+try:
+ import memcache
+ class MemcacheCache(BaseCache):
+ def __init__(self, hosts=['localhost:11211']):
+ self.m = memcache.Client(hosts)
+
+ def _get(self, key):
+ retval = self.m.get(key)
+ if retval is None:
+ raise KeyError
+ else:
+ return json.loads(retval)
+
+ def _put(self, key, val, expire=0):
+ val = json.dumps(val)
+ if expire == 0:
+ self.m.set(key, val)
+ else:
+ expire = int(expire - time.time())
+ self.m.set(key, val, expire)
+
+ def has_key(self, key):
+ return self.m.get(key) is not None
+
+ def _getlock(self, key, expire):
+ # try repeatedly to add the key, which will fail if the key
+ # already exists, until we are the one to add it
+ delay = 0.001
+ while not self.m.add(key, '', expire):
+ time.sleep(delay)
+ delay = min(delay * 1.1, 1)
def _releaselock(self, key):
- self.locks[key].release()
- del self.locks[key]
+ self.m.delete(key)
except ImportError:
pass
@@ -0,0 +1,86 @@
+import threading
+import time
+import mock
+from buildapi.lib import cacher
+from unittest import TestCase
+
+class Cases(object):
+
+ def test_get(self):
+ m = mock.Mock()
+ m.return_value = 7
+ self.assertEqual(self.c.get('not-there', m,
+ args=(1, 2), kwargs=dict(a='a', b='b')),
+ 7)
+ m.assert_called_with(1, 2, a='a', b='b')
+ m.clear()
+
+ # and the second time, it's in the cache
+ self.assertEqual(self.c.get('not-there', m), 7)
+ m.assert_not_called()
+
+ def test_put(self):
+ m = mock.Mock()
+ self.c.put('not-there', 7)
+ self.assertEqual(self.c.get('not-there', m), 7)
+ m.assert_not_called()
+ self.c.put('not-there', 8)
+ self.assertEqual(self.c.get('not-there', m), 8)
+ m.assert_not_called()
+
+ def test_has_key(self):
+ self.c.put('there', 'there')
+ self.assertFalse(self.c.has_key('not-there'))
+ self.assertTrue(self.c.has_key('there'))
+
+ def test_race(self):
+ # this is to test a thundering-herd problem when a key is missing and
+ # multiple gets occur at the same time. It uses a "slow" function to
+ # generate the result, and then just throws a lot of threads at it.
+ # This will either fail "sometimes" or not at all, unfortunately.
+ # Testing for race conditions is hard, since they are inherently
+ # sometimes harmless.
+ calls = []
+ def generate(thd):
+ calls.append(thd)
+ time.sleep(0.1)
+ def get(thd):
+ c = self.newCache() if thd else self.c
+ c.get('not-there', generate, args=(thd,), lock_time=2)
+ thds = [ threading.Thread(target=get, args=(i,))
+ for i in range(10) ]
+ for thd in thds:
+ thd.start()
+ for thd in thds:
+ thd.join()
+ self.assertEqual(len(calls), 1, calls)
+
+ # TODO: lists?
+
+class TestRedisCacher(TestCase, Cases):
+
+ def newCache(self):
+ return cacher.RedisCache()
+
+ def setUp(self):
+ self.c = self.newCache()
+ self.c.r.delete('not-there')
+
+ def tearDown(self):
+ self.c.r.delete('there')
+ self.c.r.delete('not-there')
+
+
+class TestMemcacheCacher(TestCase, Cases):
+
+ def newCache(self):
+ return cacher.MemcacheCache()
+
+ def setUp(self):
+ self.c = self.newCache()
+ self.c.m.delete('not-there')
+
+ def tearDown(self):
+ self.c.m.delete('there')
+ self.c.m.delete('not-there')
+

0 comments on commit 3a1be95

Please sign in to comment.