From 2db0d2b9d826be47b4c9c2fd53eea2d4e414f549 Mon Sep 17 00:00:00 2001 From: Evan Borgstrom Date: Sat, 23 May 2015 15:49:53 -0700 Subject: [PATCH] Refactor Storage to allow different backends I am unable to run Redis in my environment, but am able to run Couchbase and also use local disk storage. I refactored the StorageMixin so that it's abstract and a new setting named STORAGE_BACKEND defines which backend actually handles the persisting of the values. This doesn't introduce anything that's not backwards compatible and allows existing deployments of Will to run unchanged. As part of this I refactored the requirements so that there could be a requirements file for couchbase as well as redis. --- .gitignore | 3 + AUTHORS | 1 + README.md | 2 +- docs/index.md | 38 ++++++++++- requirements.base.txt | 31 +++++++++ requirements.couchbase.txt | 2 + requirements.dev.txt | 19 +----- requirements.txt | 32 +-------- setup.py | 9 ++- will/main.py | 14 ++-- will/mixins/storage.py | 89 ++++++++++++------------- will/plugins/admin/storage.py | 10 ++- will/settings.py | 62 ++++++++++------- will/storage/__init__.py | 0 will/storage/couchbase_storage.py | 62 +++++++++++++++++ will/storage/file_storage.py | 107 ++++++++++++++++++++++++++++++ will/storage/redis_storage.py | 47 +++++++++++++ will/utils.py | 9 +++ 18 files changed, 407 insertions(+), 130 deletions(-) create mode 100644 requirements.base.txt create mode 100644 requirements.couchbase.txt create mode 100644 will/storage/__init__.py create mode 100644 will/storage/couchbase_storage.py create mode 100644 will/storage/file_storage.py create mode 100644 will/storage/redis_storage.py diff --git a/.gitignore b/.gitignore index cb721d54..48fe4a57 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,6 @@ shelf.db .idea/* site docs/.DS_Store + +# VIM swap files +.*.sw[a-z] diff --git a/AUTHORS b/AUTHORS index 877e9f47..2af2be14 100644 --- a/AUTHORS +++ b/AUTHORS @@ -24,3 +24,4 @@ Ryan Murfitt, https://github.com/puug Piotr 'keNzi' Czajkowski, http://www.videotesty.pl/ Dmitri Muntean, https://github.com/dmuntean Ben lau, Mashery, https://github.com/netjunki +Evan Borgstrom, https://github.com/borgstrom diff --git a/README.md b/README.md index 9d6ba416..19e5b62a 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ He makes teaching your chat bot this simple: ```python @respond_to("hi") def say_hello(self, message): - self.say("oh, hello!") + self.say("oh, hello!", message=message) ``` Will was first built by [Steven Skoczen](http://stevenskoczen.com) while in the [Greenkahuna Skunkworks](http://skunkworks.greenkahuna.com), and has been [contributed to by lots of folks](http://skoczen.github.io/will/improve/#the-shoulders-of-giants). diff --git a/docs/index.md b/docs/index.md index 03158314..9042ca8b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -10,7 +10,7 @@ He makes teaching your chat bot this simple: ``` @respond_to("hi") def say_hello(self, message): - self.say("oh, hello!") + self.say("oh, hello!", message=message) ``` Lots of batteries are included, and you can get your own will up and running in a couple of minutes. @@ -123,4 +123,38 @@ Once your will is up and running, hop into any of your hipchat rooms, and say he ![Help, will](img/help.gif) -You're up and running - now it's time to [teach your will a few things](plugins/basics.md)! \ No newline at end of file +You're up and running - now it's time to [teach your will a few things](plugins/basics.md)! + + +## Storage Backends + +Will's default storage backend is Redis, but he supports some others if you can't run Redis. + +To change the backend you just need to set `STORAGE_BACKEND` in your config and then supply any other needed settings for the new storage backend. The currently supported backends are: + + * `redis` - The default Redis backend + * `couchbase` - A Couchbase backend + * `file` - Keeps the settings as files on a local filesystem + + +#### Couchbase + +Couchbase requries you set `COUCHBASE_URL` in your config. + +You are also required to have the python Couchbase client (and thus, libcouchbase) installed. If you are installing for development you can use `pip install -r requirements.couchbase.txt` to pull in the Couchbase client. See [the Python Couchbase client repo](https://github.com/couchbase/couchbase-python-client) for more info. + +Examples: + + * `COUCHBASE_URL='couchbase:///bucket'` + * `COUCHBASE_URL='couchbase://hostname/bucket'` + * `COUCHBASE_URL='couchbase://host1,host2/bucket'` + * `COUCHBASE_URL='couchbase://hostname/bucket?password=123abc&timeout=5'` + +#### File + +File requires you set `FILE_DIR` in your config to point to an empty directory. + +Examples: + + * `FILE_DIR='/var/run/will/settings/'` + * `FILE_DIR='~will/settings/'` diff --git a/requirements.base.txt b/requirements.base.txt new file mode 100644 index 00000000..a2f464fb --- /dev/null +++ b/requirements.base.txt @@ -0,0 +1,31 @@ +bottle==0.12.7 +clint==0.3.7 +dill==0.2.1 +dnspython==1.12.0 +natural==0.1.5 +requests==2.4.1 +parsedatetime==1.1.2 +pyasn1==0.1.7 +pyasn1-modules==0.0.5 +sleekxmpp==1.3.1 +APScheduler==2.1.2 +CherryPy==3.6.0 +Fabric==1.10.0 +Jinja2==2.7.3 +Markdown==2.3.1 +MarkupSafe==0.23 +PyYAML==3.10 +argh==0.25.0 +args==0.1.0 +ecdsa==0.11 +futures==2.1.6 +ghp-import==0.4.1 +greenlet==0.4.4 +paramiko==1.14.1 +pathtools==0.1.2 +pycrypto==2.6.1 +pytz==2014.7 +six==1.8.0 +tzlocal==1.1.1 +watchdog==0.7.0 +wsgiref==0.1.2 diff --git a/requirements.couchbase.txt b/requirements.couchbase.txt new file mode 100644 index 00000000..dfb385d9 --- /dev/null +++ b/requirements.couchbase.txt @@ -0,0 +1,2 @@ +-e git+git://github.com/couchbase/couchbase-python-client#egg=couchbase-python-client +-r requirements.base.txt diff --git a/requirements.dev.txt b/requirements.dev.txt index f8bc8c0f..ce2bf7b4 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -1,19 +1,4 @@ -apscheduler<3.0 -bottle>=0.12.6 -cherrypy -coverage -clint -dill>=0.2b1 -dnspython -jinja2 -hiredis -natural -redis -requests -parsedatetime==1.1.2 -pyasn1 -pyasn1_modules -sleekxmpp>=1.2 +-r requirements.txt # Dev only pyandoc @@ -21,4 +6,4 @@ mkdocs fabric flake8 mock -nose \ No newline at end of file +nose diff --git a/requirements.txt b/requirements.txt index d976ef75..087636ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,33 +1,3 @@ -bottle==0.12.7 -clint==0.3.7 -dill==0.2.1 -dnspython==1.12.0 hiredis==0.1.4 -natural==0.1.5 redis==2.10.3 -requests==2.4.1 -parsedatetime==1.1.2 -pyasn1==0.1.7 -pyasn1-modules==0.0.5 -sleekxmpp==1.3.1 -APScheduler==2.1.2 -CherryPy==3.6.0 -Fabric==1.10.0 -Jinja2==2.7.3 -Markdown==2.3.1 -MarkupSafe==0.23 -PyYAML==3.10 -argh==0.25.0 -args==0.1.0 -ecdsa==0.11 -futures==2.1.6 -ghp-import==0.4.1 -greenlet==0.4.4 -paramiko==1.14.1 -pathtools==0.1.2 -pycrypto==2.6.1 -pytz==2014.7 -six==1.8.0 -tzlocal==1.1.1 -watchdog==0.7.0 -wsgiref==0.1.2 +-r requirements.base.txt diff --git a/setup.py b/setup.py index ef88f9ba..abb63dfc 100644 --- a/setup.py +++ b/setup.py @@ -9,9 +9,12 @@ SOURCE_DIR = os.path.join(ROOT_DIR) reqs = [] -with open("requirements.txt", "r+") as f: - for line in f.readlines(): - reqs.append(line.strip()) +for req_file in ("requirements.base.txt", "requirements.txt"): + with open(req_file, "r+") as f: + for line in f.readlines(): + if line[0] == "-": + continue + reqs.append(line.strip()) try: import pypandoc diff --git a/will/main.py b/will/main.py index 29b2b1a7..2fea4e45 100644 --- a/will/main.py +++ b/will/main.py @@ -146,6 +146,9 @@ def bootstrap(self): time.sleep(0.5) def verify_individual_setting(self, test_setting, quiet=False): + if not test_setting.get("only_if", True): + return True + if hasattr(settings, test_setting["name"][5:]): with indent(2): show_valid(test_setting["name"]) @@ -189,9 +192,10 @@ def verify_environment(self): }, { "name": "WILL_REDIS_URL", + "only_if": getattr(settings, "STORAGE_BACKEND", "redis") == "redis", "obtain_at": """1. Set up an accessible redis host locally or in production 2. Set WILL_REDIS_URL to its full value, i.e. redis://localhost:6379/7""", - } + }, ] puts("") @@ -211,8 +215,8 @@ def verify_environment(self): puts("") puts("Verifying credentials...") - # Parse 11111_222222@chat.hipchat.com into id, where 222222 is the id. Yup. - user_id = settings.USERNAME[0:settings.USERNAME.find("@")][settings.USERNAME.find("_") + 1:] + # Parse 11111_222222@chat.hipchat.com into id, where 222222 is the id. + user_id = settings.USERNAME.split('@')[0].split('_')[1] # Splitting into a thread. Necessary because *BSDs (including OSX) don't have threadsafe DNS. # http://stackoverflow.com/questions/1212716/python-interpreter-blocks-multithreaded-dns-requests @@ -295,10 +299,10 @@ def bootstrap_storage_mixin(self): try: self.bootstrap_storage() with indent(2): - show_valid("Connection to %s successful." % settings.REDIS_URL) + show_valid("Bootstrapped!") puts("") except: - error("Unable to connect to %s" % settings.REDIS_URL) + error("Unable to bootstrap!") sys.exit(1) def bootstrap_scheduler(self): diff --git a/will/mixins/storage.py b/will/mixins/storage.py index 6118e184..1f523dda 100644 --- a/will/mixins/storage.py +++ b/will/mixins/storage.py @@ -1,10 +1,8 @@ +import importlib import logging -import redis -import traceback -import urlparse import dill as pickle +import functools from will import settings -from will.utils import show_valid, error, warn, note class StorageMixin(object): @@ -13,58 +11,59 @@ def bootstrap_storage(self): if hasattr(self, "bot") and hasattr(self.bot, "storage"): self.storage = self.bot.storage else: - # redis://localhost:6379/7 - # or - # redis://rediscloud:asdfkjaslkdjflasdf@pub-redis-12345.us-east-1-1.2.ec2.garantiadata.com:12345 - url = urlparse.urlparse(settings.REDIS_URL) + # The STORAGE_BACKEND setting points to a specific module namespace + # redis => will.storage.redis_backend + # couchbase => will.storage.couchbase_backend + # etc... + module_name = ''.join([ + 'will.storage.', + getattr(settings, 'STORAGE_BACKEND', 'redis'), + '_storage' + ]) - if hasattr(url, "path"): - db = url.path[1:] - else: - db = 0 - max_connections = getattr(settings, 'REDIS_MAX_CONNECTIONS', - None) - connection_pool = redis.ConnectionPool( - max_connections=max_connections, host=url.hostname, - port=url.port, db=db, password=url.password - ) - self.storage = redis.Redis(connection_pool=connection_pool) + # XXX TODO How to handle import errors here? + # Since this is required to work (I think), I've just let + # the exceptions bubble up for now. + storage_module = importlib.import_module(module_name) - def save(self, key, value, expire=0): - if not hasattr(self, "storage"): - self.bootstrap_storage() + # Now create our storage object using the bootstrap function + # from within the import + self.storage = storage_module.bootstrap(settings) + def save(self, key, value, expire=None): + self.bootstrap_storage() try: - if expire: - ret = self.storage.setex(key, pickle.dumps(value), expire) - else: - ret = self.storage.set(key, pickle.dumps(value)) - - return ret - except: - logging.critical("Unable to save %s: \n%s" % - (key, traceback.format_exc())) + return self.storage.save(key, pickle.dumps(value), expire=expire) + except Exception: + logging.exception("Unable to save %s", key) def clear(self, key): - if not hasattr(self, "storage"): - self.bootstrap_storage() - return self.storage.delete(key) + self.bootstrap_storage() + try: + return self.storage.clear(key) + except Exception: + logging.exception("Unable to clear %s", key) def clear_all_keys(self): - if not hasattr(self, "storage"): - self.bootstrap_storage() - return self.storage.flushdb() + self.bootstrap_storage() + try: + return self.storage.clear_all_keys() + except Exception: + logging.exception("Unable to clear all keys") def load(self, key, default=None): - if not hasattr(self, "storage"): - self.bootstrap_storage() - + self.bootstrap_storage() try: - val = self.storage.get(key) + val = self.storage.load(key) if val is not None: return pickle.loads(val) + return default + except Exception: + logging.exception("Failed to load %s", key) - except: - logging.warn("Unable to load %s" % key) - - return default + def size(self): + self.bootstrap_storage() + try: + return self.storage.size() + except Exception: + logging.exception("Failed to get the size of our storage") diff --git a/will/plugins/admin/storage.py b/will/plugins/admin/storage.py index 0ecae5df..d0107a9b 100644 --- a/will/plugins/admin/storage.py +++ b/will/plugins/admin/storage.py @@ -7,15 +7,17 @@ class StoragePlugin(WillPlugin): @respond_to("^How big is the db?", admin_only=True) def db_size(self, message): self.bootstrap_storage() - self.say("It's %s." % self.storage.info()["used_memory_human"], message=message) + self.say("It's %s." % self.storage.size(), message=message) @respond_to("^SERIOUSLY. Clear (?P.*)", case_sensitive=True, admin_only=True) def clear_storage(self, message, key=None): if not key: self.say("Sorry, you didn't say what to clear.", message=message) else: - self.clear(key) self.say("Ok. Clearing the storage for %s" % key, message=message) + res = self.clear(key) + if res not in (None, True, False): + self.say("Something happened while clearing: %s" % res, message=message) @respond_to("^SERIOUSLY. REALLY. Clear all keys.", case_sensitive=True, admin_only=True) def clear_all_keys_listener(self, message): @@ -23,7 +25,9 @@ def clear_all_keys_listener(self, message): "Ok, I'm clearing them. You're probably going to want to restart me." "I just forgot everything, including who I am and where the chat room is.", message=message ) - self.clear_all_keys() + res = self.clear_all_keys() + if res not in (None, True, False): + self.say("Something happened while clearing all keys: %s" % res, message=message) @respond_to("^Show (?:me )?(?:the )?storage for (?P.*)", admin_only=True) def show_storage(self, message, key=None): diff --git a/will/settings.py b/will/settings.py index b57ac618..80a4b0f4 100644 --- a/will/settings.py +++ b/will/settings.py @@ -93,27 +93,48 @@ def import_settings(quiet=True): warn("no HTTPSERVER_PORT found in the environment or config. Defaulting to ':80'.") settings["HTTPSERVER_PORT"] = "80" - if "REDIS_URL" not in settings: - # For heroku - if "REDISCLOUD_URL" in os.environ: - settings["REDIS_URL"] = os.environ["REDISCLOUD_URL"] - if not quiet: - note("WILL_REDIS_URL not set, but it appears you're using RedisCloud. If so, all good.") - elif "REDISTOGO_URL" in os.environ: - settings["REDIS_URL"] = os.environ["REDISTOGO_URL"] + if "STORAGE_BACKEND" not in settings: + settings["STORAGE_BACKEND"] = "redis" + + if settings["STORAGE_BACKEND"] == "redis": + if "REDIS_URL" not in settings: + # For heroku + if "REDISCLOUD_URL" in os.environ: + settings["REDIS_URL"] = os.environ["REDISCLOUD_URL"] + if not quiet: + note("WILL_REDIS_URL not set, but it appears you're using RedisCloud. If so, all good.") + elif "REDISTOGO_URL" in os.environ: + settings["REDIS_URL"] = os.environ["REDISTOGO_URL"] + if not quiet: + note("WILL_REDIS_URL not set, but it appears you're using RedisToGo. If so, all good.") + elif "OPENREDIS_URL" in os.environ: + settings["REDIS_URL"] = os.environ["OPENREDIS_URL"] + if not quiet: + note("WILL_REDIS_URL not set, but it appears you're using OpenRedis. If so, all good.") + else: + settings["REDIS_URL"] = "redis://localhost:6379/7" + if not quiet: + note("WILL_REDIS_URL not set. Defaulting to redis://localhost:6379/7.") + + if not settings["REDIS_URL"].startswith("redis://"): + settings["REDIS_URL"] = "redis://%s" % settings["REDIS_URL"] + + if "REDIS_MAX_CONNECTIONS" not in settings: + settings["REDIS_MAX_CONNECTIONS"] = 4 if not quiet: - note("WILL_REDIS_URL not set, but it appears you're using RedisToGo. If so, all good.") - elif "OPENREDIS_URL" in os.environ: - settings["REDIS_URL"] = os.environ["OPENREDIS_URL"] - if not quiet: - note("WILL_REDIS_URL not set, but it appears you're using OpenRedis. If so, all good.") - else: - settings["REDIS_URL"] = "redis://localhost:6379/7" + note("REDIS_MAX_CONNECTIONS not set. Defaulting to 4.") + + if settings["STORAGE_BACKEND"] == "file": + if "FILE_DIR" not in settings: + settings["FILE_DIR"] = "~/.will/" if not quiet: - note("WILL_REDIS_URL not set. Defaulting to redis://localhost:6379/7.") + note("FILE_DIR not set. Defaulting to ~/.will/") - if not settings["REDIS_URL"].startswith("redis://"): - settings["REDIS_URL"] = "redis://%s" % settings["REDIS_URL"] + if settings["STORAGE_BACKEND"] == "couchbase": + if "COUCHBASE_URL" not in settings: + settings["COUCHBASE_URL"] = "couchbase:///will" + if not quiet: + note("COUCHBASE_URL not set. Defaulting to couchbase:///will") if "PUBLIC_URL" not in settings: default_public = "http://localhost:%s" % settings["HTTPSERVER_PORT"] @@ -129,11 +150,6 @@ def import_settings(quiet=True): "you may recieve rate-limit errors without one." ) - if "REDIS_MAX_CONNECTIONS" not in settings: - settings["REDIS_MAX_CONNECTIONS"] = 4 - if not quiet: - note("REDIS_MAX_CONNECTIONS not set. Defaulting to 4.") - if "TEMPLATE_DIRS" not in settings: if "WILL_TEMPLATE_DIRS_PICKLED" in os.environ: # All good diff --git a/will/storage/__init__.py b/will/storage/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/will/storage/couchbase_storage.py b/will/storage/couchbase_storage.py new file mode 100644 index 00000000..60d9dd56 --- /dev/null +++ b/will/storage/couchbase_storage.py @@ -0,0 +1,62 @@ +import urlparse + +from couchbase import Couchbase, exceptions as cb_exc + + +class CouchbaseStorage(object): + """ + A storage backend using Couchbase + + You must supply a COUCHBASE_URL setting that is passed through urlparse. + All parameters supplied get passed through to Couchbase + + Examples: + + * couchbase:///bucket + * couchbase://hostname/bucket + * couchbase://host1,host2/bucket + * couchbase://hostname/bucket?password=123abc&timeout=5 + """ + def __init__(self, settings): + url = urlparse.urlparse(settings.COUCHBASE_URL) + params = dict([ + param.split('=') + for param in url.query.split('&') + ]) + self.couchbase = Couchbase(host=url.hostname.split(','), + bucket=url.path.strip('/'), + port=url.port or 8091, + **params) + + def save(self, key, value, expire=None): + res = self.couchbase.set(key, value, ttl=expire) + return res.success + + def clear(self, key): + res = self.couchbase.delete(key) + return res.success + + def clear_all_keys(self): + """ + Couchbase doesn't support clearing all keys (flushing) without the + Admin username and password. It's not appropriate for Will to have + this information so we don't support clear_all_keys for CB. + """ + return "Sorry, you must flush the Couchbase bucket from the Admin UI" + + def load(self, key): + try: + res = self.couchbase.get(key) + return res.value + except cb_exc.NotFoundError: + pass + + def size(self): + """ + Couchbase doesn't support getting the size of the DB + """ + return "Unknown (See Couchbase Admin UI)" + + +def bootstrap(settings): + return CouchbaseStorage(settings) diff --git a/will/storage/file_storage.py b/will/storage/file_storage.py new file mode 100644 index 00000000..b74fabee --- /dev/null +++ b/will/storage/file_storage.py @@ -0,0 +1,107 @@ +import logging +import os +import time + +from will.utils import sizeof_fmt + + +class FileStorageException(Exception): + """ + A condition that should not occur happened in the FileStorage module + """ + pass + + +class FileStorage(object): + """ + A storage backend using a local filesystem directory. + + Each setting is its own file. + + You must supply a FILE_DIR setting that is a path to a directory. + + Examples: + + * /var/run/will/settings/ + * ~will/settings/ + """ + def __init__(self, settings): + self.dirname = os.path.abspath(os.path.expanduser(settings.FILE_DIR)) + self.dotfile = os.path.join(self.dirname, ".will_settings") + logging.debug("Using %s for local setting storage", self.dirname) + + if not os.path.exists(self.dirname): + # the directory doesn't exist, try to create it + os.makedirs(self.dirname, mode=0700) + elif not os.path.exists(self.dotfile): + # the directory exists, but doesn't have our dot file in it + # if it has any other files in it then we bail out since we want to + # have full control over wiping out the contents of the directory + if len(self._all_setting_files()) > 0: + raise FileStorageException("%s is not empty, " + "will needs an empty directory for " + "settings" % (self.dirname,)) + + # update our dir & dotfile + os.chmod(self.dirname, 0700) + with open(self.dotfile, 'a'): + os.utime(self.dotfile, None) + + def _all_setting_files(self): + return [ + os.path.join(self.dirname, f) + for f in os.listdir(self.dirname) + if os.path.isfile(os.path.join(self.dirname, f)) + ] + + def _key_paths(self, key): + key_path = os.path.join(self.dirname, key) + expire_path = os.path.join(self.dirname, '.' + key + '.expires') + return key_path, expire_path + + def save(self, key, value, expire=None): + key_path, expire_path = self._key_paths(key) + with open(key_path, 'w') as f: + f.write(value) + + if expire is not None: + with open(expire_path, 'w') as f: + f.write(expire) + elif os.path.exists(expire_path): + os.unlink(expire_path) + + def clear(self, key): + key_path, expire_path = self._key_paths(key) + if os.path.exists(key_path): + os.unlink(key_path) + if os.path.exists(expire_path): + os.unlink(expire_path) + + def clear_all_keys(self): + for filename in self._all_setting_files(): + os.unlink(filename) + + def load(self, key): + key_path, expire_path = self._key_paths(key) + + if os.path.exists(expire_path): + with open(expire_path, 'r') as f: + expire_at = f.read() + if time.time() > int(expire_at): + # the current value has expired + self.clear(key) + return + + if os.path.exists(key_path): + with open(key_path, 'r') as f: + return f.read() + + def size(self): + return sizeof_fmt(sum([ + os.path.getsize(filename) + for filename in self._all_setting_files() + ])) + + +def bootstrap(settings): + return FileStorage(settings) diff --git a/will/storage/redis_storage.py b/will/storage/redis_storage.py new file mode 100644 index 00000000..ea74bf6d --- /dev/null +++ b/will/storage/redis_storage.py @@ -0,0 +1,47 @@ +import redis +import urlparse + + +class RedisStorage(object): + """ + A storage backend using Redis. + + You must supply a REDIS_URL setting that is passed through urlparse. + + Examples: + + * redis://localhost:6379/7 + * redis://rediscloud:asdfkjaslkdjflasdf@pub-redis-12345.us-east-1-1.2.ec2.garantiadata.com:12345 + """ + def __init__(self, settings): + url = urlparse.urlparse(settings.REDIS_URL) + + if hasattr(url, "path"): + db = url.path[1:] + else: + db = 0 + max_connections = getattr(settings, 'REDIS_MAX_CONNECTIONS', None) + connection_pool = redis.ConnectionPool( + max_connections=max_connections, host=url.hostname, + port=url.port, db=db, password=url.password + ) + self.redis = redis.Redis(connection_pool=connection_pool) + + def save(self, key, value, expire=None): + return self.redis.set(key, value, ex=expire) + + def clear(self, key): + return self.redis.delete(key) + + def clear_all_keys(self): + return self.redis.flushdb() + + def load(self, key): + return self.redis.get(key) + + def size(self): + return self.redis.info()["used_memory_human"] + + +def bootstrap(settings): + return RedisStorage(settings) diff --git a/will/utils.py b/will/utils.py index d18cc65f..f766b54a 100644 --- a/will/utils.py +++ b/will/utils.py @@ -73,3 +73,12 @@ def print_head(): Will: Hi! """) + + +def sizeof_fmt(num, suffix='B'): + # http://stackoverflow.com/a/1094933 + for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']: + if abs(num) < 1024.0: + return "%3.1f%s%s" % (num, unit, suffix) + num /= 1024.0 + return "%.1f%s%s" % (num, 'Yi', suffix)