Permalink
Browse files

First commit

  • Loading branch information...
alexksikes committed Apr 16, 2011
0 parents commit 06fa28ba1c68bb2cb7f3b6e579b15c3272a0329b
Showing with 928 additions and 0 deletions.
  1. +5 −0 INSTALL.md
  2. +674 −0 LICENSE
  3. +34 −0 README.md
  4. +11 −0 TODO
  5. +50 −0 make_sql_cache.py
  6. +124 −0 sql_cache.py
  7. +30 −0 test.py
@@ -0,0 +1,5 @@
+Download and extract the latest tar ball:
+
+ wget https://github.com/alexksikes/SQL-Cache/tarball/master; tar xvzf "the tar ball";
+
+and add sql_cache.py to your PYTHONPATH.
674 LICENSE

Large diffs are not rendered by default.

Oops, something went wrong.
@@ -0,0 +1,34 @@
+A simple key-value MySQL store for small chunks of arbitrary data (strings, objects).
+
+Here is an example of usage:
+
+ import sql_cache
+
+ # setup the database
+ sql_cache.set_DB(db='db', user='user', passwd='passwd')
+
+ # setup cache size
+ sql_cache.MAX_SIZE = 10000
+
+ # make the cache table, drop it if it already exists
+ sql_cache.make_sql_table(drop=True)
+
+ # set some key value pairs
+ sql_cache.set('query_1', [1,2,3])
+ sql_cache.set('query_2', 'a string', sticky=True)
+ sql_cache.set('query_3', {'a':1, 'b':2, 'c':3})
+
+ # returns {'a':1, 'b':2, 'c':3}
+ print sql_cache.get('query_3')
+
+ # returns 'a string'
+ print sql_cache.get('query_2')
+
+ # clears the entire but not the sticky values
+ sql_cache.clear(also_sticky=False)
+
+ # returns None
+ print sql_cache.get('query_1')
+
+ # returns 'a string'
+ print sql_cache.get('query_2')
11 TODO
@@ -0,0 +1,11 @@
+# TODO
+# [*] make_sql_cache_table.py
+# [ ] sql_cache.cache decorator to cache the ouput a function
+# [ ] what about compression?
+# [ ] get_multi and set_multi in one SQL query.
+# [ ] per facet cache with sql_cache.get_many(*keys) for still one sql query
+# [ ] value should probably be a blob
+
+# NOTE:
+# We had to use pickle + base64 see http://bugs.python.org/issue2980
+# This table could get very large, very quickly, we should instead have a max KB.
@@ -0,0 +1,50 @@
+#! /usr/bin/env python
+# Author: Alex Ksikes (alex.ksikes@gmail.com)
+
+import getpass
+import web
+
+import sql_cache
+
+def run(db_name, drop=False):
+ db = web.database(dbn='mysql',
+ db=db_name,
+ user=raw_input('User: '),
+ passwd=getpass.unix_getpass('Password: ')
+ )
+ sql_cache.Cache.make_sql_table(db, drop)
+
+def usage():
+ print 'Usage:'
+ print ' python make_sql_cache.py [options] db_name'
+ print
+ print 'Description:'
+ print ' Makes the required sql table.'
+ print
+ print 'Options:'
+ print ' -d, --drop : drop cache sql table if it exists'
+ print ' -h, --help : this help message'
+ print
+ print 'Email bugs/suggestions to Alex Ksikes (alex.ksikes@gmail.com)'
+
+import sys, getopt
+def main():
+ try:
+ opts, args = getopt.getopt(sys.argv[1:], 'dh', ['drop', 'help'])
+ except getopt.GetoptError:
+ usage(); sys.exit(2)
+
+ drop = False
+ for o, a in opts:
+ if o in ('-d', '--drop'):
+ drop = True
+ elif o in ('-h', '--help'):
+ usage(); sys.exit()
+
+ if len(args) < 1:
+ usage()
+ else:
+ run(args[0], drop)
+
+if __name__ == '__main__':
+ main()
@@ -0,0 +1,124 @@
+'''A simple key-value MySQL store for small chunks of arbitrary data.
+
+You need a table of the following schema:
+
+create table cache (
+ _key varchar(32) primary key,
+ query varchar(250),
+ value text,
+ sticky boolean,
+ datetime timestamp default current_timestamp,
+ index (sticky),
+ index (datetime)
+) charset utf8 engine MyISAM;
+'''
+
+__author__ = 'Alex Ksikes (alex.ksikes@gmail.com)'
+
+import base64
+import md5
+import cPickle as pickle
+import web
+
+DB = None
+MAX_SIZE = 10000 # maximum allowed size including the sticky values
+
+class Cache(object):
+ def __init__(self, **db_params):
+ if not db_params:
+ db = DB
+ elif len(db_params) == 1 and db_params.get('db'):
+ db = db_params.pop('db')
+ else:
+ db = web.database(**db_params)
+ self.db = db
+
+ def get(self, query):
+ key = self._make_key(query)
+ r = self.db.select('cache', vars=dict(key=key), where='_key = $key', limit=1)
+ r = web.listget(r, 0)
+
+ value = None
+ if r:
+ if not r.sticky:
+ self.db.query('update cache set datetime=now() where _key = $key', vars=dict(key=key))
+ value = pickle.loads(base64.b64decode(r.value))
+ return value
+
+ def set(self, query, value, replace=False, sticky=False):
+ key = self._make_key(query)
+ value = base64.b64encode(pickle.dumps(value))
+
+ sql = ('insert cache (_key, query, value, sticky) values ($key, $query, $value, $sticky) '
+ 'on duplicate key update datetime=now()')
+
+ if replace:
+ sql += ', query=$query, value=$value, sticky=$sticky'
+
+ self.db.query(sql, vars=dict(key=key, query=query, value=value, sticky=sticky))
+ self._delete_lru()
+
+ def get_ifnot_set(self, query, value, replace=False, sticky=False):
+ value = self.get(query)
+ if not value:
+ self.set(query, value, replace, sticky)
+ return value
+
+ def _delete_lru(self):
+ r = self.db.query('select count(*) as size from cache')
+ r = web.listget(r, 0)
+ n = r.size - MAX_SIZE
+ if n > 0:
+ self.db.query('delete from cache where sticky !=1 order by datetime limit %s' % n)
+
+ def _make_key(self, query):
+ return md5.new(_utf8(query)).hexdigest()
+
+ @classmethod
+ def clear(cls, db, also_sticky=False):
+ if not also_sticky:
+ where = 'sticky != 1'
+ else:
+ where = ''
+ db.delete('cache', where=where)
+
+ @classmethod
+ def make_sql_table(cls, db, drop=False):
+ if drop:
+ db.query('drop table if exists cache')
+ db.query(
+ 'create table cache ('
+ ' _key varchar(32) primary key,'
+ ' query varchar(250),'
+ ' value text,'
+ ' sticky boolean,'
+ ' datetime timestamp default current_timestamp,'
+ ' index (sticky),'
+ ' index (datetime)'
+ ') charset utf8 engine MyISAM;'
+ )
+
+def set_DB(**db_params):
+ global DB
+ DB = web.database(**db_params)
+ DB.printing = False
+
+def get(query):
+ return Cache(db=DB).get(query)
+
+def set(query, value, replace=False, sticky=False):
+ return Cache(db=DB).set(query, value, replace, sticky)
+
+def clear(db=None, also_sticky=False):
+ db = db or DB
+ Cache.clear(db, also_sticky)
+
+def make_sql_table(db=None, drop=False):
+ db = db or DB
+ Cache.make_sql_table(db, drop)
+
+def _utf8(s):
+ if isinstance(s, unicode):
+ return s.encode("utf-8")
+ assert isinstance(s, str)
+ return s
30 test.py
@@ -0,0 +1,30 @@
+import sql_cache
+
+# setup the database
+sql_cache.set_DB(db='db', user='user', passwd='passwd')
+
+# setup cache size
+sql_cache.MAX_SIZE = 10000
+
+# make the cache table, drop it if it already exists
+sql_cache.make_sql_table(drop=True)
+
+# set some key value pairs
+sql_cache.set('query_1', [1,2,3])
+sql_cache.set('query_2', 'a string', sticky=True)
+sql_cache.set('query_3', {'a':1, 'b':2, 'c':3})
+
+# returns {'a':1, 'b':2, 'c':3}
+print sql_cache.get('query_3')
+
+# returns 'a string'
+print sql_cache.get('query_2')
+
+# clears the entire but not the sticky values
+sql_cache.clear(also_sticky=False)
+
+# returns None
+print sql_cache.get('query_1')
+
+# returns 'a string'
+print sql_cache.get('query_2')

0 comments on commit 06fa28b

Please sign in to comment.