Browse files

move to sqlite

  • Loading branch information...
1 parent 8e3906d commit 6d50b396bce47171ebd9e2c06ab1db7c62a01f37 @jvasile committed Nov 22, 2011
Showing with 164 additions and 16 deletions.
  1. +22 −0 dbconvert.py
  2. +12 −15 planet.py
  3. +9 −1 util.py
  4. +121 −0 withsqlite.py
View
22 dbconvert.py
@@ -0,0 +1,22 @@
+#!/usr/bin/python
+
+# convert from shelf to sqlite
+
+import sys
+from util import our_db, sqlite_db
+import simplejson as json
+
+with sqlite_db('planets') as sdb:
+ sdb.clear()
+ with our_db('planets') as odb:
+ for key, val in odb.items():
+ val = json.loads(val)
+ sdb[key]=val
+sys.exit()
+with sqlite_db('cache') as sdb:
+ sdb.clear()
+ with our_db('cache') as odb:
+ for key, val in odb.items():
+ val = json.loads(val)
+ sdb[key]=val
+
View
27 planet.py
@@ -32,14 +32,11 @@ def to_json(python_object):
#raise TypeError(repr(python_object) + ' is not JSON serializable')
-def serialize_feedparse(parse_object):
- return json.dumps(parse_object, default=to_json)
-
class Planet():
def __init__(self, *args, **kwargs):
if 'direc' in kwargs:
with our_db('planets') as db:
- self.load_json(db[kwargs['direc']])
+ self.load_dict(db[kwargs['direc']])
elif isinstance(args[0], basestring):
self.load_json(args[0])
elif isinstance(args[0], dict):
@@ -82,8 +79,8 @@ def load_json(self, j):
def save_cache(self, cache, url):
with our_db('cache') as db:
- j = json.dumps(cache, sort_keys=True, indent=3)
- db[url.encode("utf-8")] = j
+ #db[url.encode("utf-8")] = cache
+ db[url] = cache
def save(self, update_config_timestamp=False, ignore_missing_dir=False):
output_dir = os.path.join(cfg.OUTPUT_DIR, self.direc)
@@ -95,7 +92,8 @@ def save(self, update_config_timestamp=False, ignore_missing_dir=False):
if update_config_timestamp:
self.last_config_change = time.time()
with our_db('planets') as db:
- db[self.direc.encode("utf-8")] = self.json()
+ #db[self.direc.encode("utf-8")] = self.serializable()
+ db[self.direc] = self.serializable()
def serializable(self):
return {'direc':self.direc,
@@ -117,7 +115,8 @@ def update_feed(self, url):
force_check = opt['force_check']
with our_db('cache') as db:
try:
- cache = json.loads(db[url.encode("utf-8")])
+ #cache = db[url.encode("utf-8")]
+ cache = db[url]
except KeyError:
log.info("Can't find %s in cache. Making default." % url)
cache = {'data':'', 'last_downloaded':0, 'dload_fail':False}
@@ -143,7 +142,8 @@ def update_feed(self, url):
cache['last_downloaded'] = time.time()
with our_db('cache') as db:
try:
- db[url.encode("utf8")] = json.dumps(cache, default=to_json, sort_keys=True, indent=3)
+ #db[url.encode("utf8")] = cache
+ db[url] = cache
except TypeError, e:
log.debug("Can't save feed (%s): %s" % (url, e))
cache['error'] = str(e)
@@ -184,10 +184,11 @@ def generate(self):
lopt['Feeds']=[]
for url, f in self.feeds.items():
with our_db('cache') as db:
- if not url.encode("utf-8") in db:
+ if not url in db:
continue
try:
- cache = json.loads(db[url.encode("utf-8")])
+ #cache = db[url.encode("utf-8")]
+ cache = db[url]
except json.decoder.JSONDecodeError, e:
log.debug("Json error on generating url %s: %s" % (url, e))
continue
@@ -304,10 +305,6 @@ def dump_cache(self):
print url
with our_db('cache') as db:
cache = db[url]
- try:
- cache = json.loads(cache)
- except json.decoder.JSONDecodeError:
- return
print json.dumps(cache)
View
10 util.py
@@ -14,6 +14,7 @@
import logging
log = logging.getLogger('planeteria')
import dateutil.parser
+import withsqlite
try:
import tidy
@@ -40,7 +41,11 @@ def just_body(xhtml):
except IndexError:
return ''
-class our_db:
+class sqlite_db(withsqlite.sqlite_db):
+ def __init__(self, fname):
+ withsqlite.sqlite_db.__init__(self, os.path.join(cfg.data_dir, fname))
+
+class shelve_db:
def __init__(self, fname):
self.fname = fname + ".shelf"
def __enter__(self):
@@ -49,6 +54,9 @@ def __enter__(self):
def __exit__(self, type, value, traceback):
self.db.close()
+class our_db(sqlite_db):
+ pass
+
class berkeley_db:
def __init__(self, fname):
self.fname = fname
View
121 withsqlite.py
@@ -0,0 +1,121 @@
+#!/usr/bin/env python
+
+import os, sys, sqlite3
+import simplejson as json
+
+class sqlite_db():
+ """
+Backends a dict on an sqlite db. This class aims to present like a
+dict wherever it can.
+
+USE:
+import sqlite_db from withsqlite
+with sqlite_db("filename") as db:
+ db['aaa'] = {'test':'ok'}
+ print db.items()
+
+BUGS:
+
+vals are json serialized before being written, so if you can't
+serialize it, you can't put it in the dict.
+
+Unimplemented mapping API:
+a.copy() a (shallow) copy of a
+k in a True if a has a key k, else False
+k not in a Equivalent to not k in a
+a.has_key(k) Equivalent to k in a, use that form in new code
+a.update([b]) updates a with key/value pairs from b, overwriting existing keys, returns None
+a.fromkeys(seq[, value]) Creates a new dictionary with keys from seq and values set to value
+a.setdefault(k[, x]) a[k] if k in a, else x (also setting it)
+a.pop(k[, x]) a[k] if k in a, else x (and remove k)
+a.popitem() remove and return an arbitrary (key, value) pair
+a.iteritems() return an iterator over (key, value) pairs
+a.iterkeys() return an iterator over the mapping's keys
+a.itervalues() return an iterator over the mapping's values
+"""
+
+ def __init__(self, fname):
+ self.fname = fname + ".sqlite3"
+ def __enter__(self):
+ if not os.path.exists(self.fname):
+ self.make_db()
+ self.conn = sqlite3.connect(self.fname)
+ self.crsr = self.conn.cursor()
+ return self
+ def __exit__(self, type, value, traceback):
+ self.conn.commit()
+ self.crsr.close()
+ def make_db(self):
+ conn = sqlite3.connect(self.fname)
+ c = conn.cursor()
+ c.execute('''create table store (key text unique, val text)''')
+ conn.commit()
+ c.close()
+ def __delitem__(self, key):
+ """del a[k] remove a[k] from a"""
+ self.crsr.execute("delete from store where key=?", [key])
+ def jsonize(self,val):
+ "If it's just a string, serialize it ourselves"
+ if isinstance(val, basestring):
+ return '"%s"' % val
+ return json.dumps(val, sort_keys=True, indent=3)
+ def __setitem__(self, key, val):
+ """a[k] = v set a[k] to v """
+
+ try:
+ if val == self.__getitem__(key):
+ return
+ self.crsr.execute("update or fail store set val=? where key==?", [self.jsonize(val), key])
+ except KeyError:
+ self.crsr.execute("insert into store values (?, ?)", [key, self.jsonize(val)])
+ def __getitem__(self, key):
+ """a[k] the item of a with key k (1), (10)"""
+ self.crsr.execute('select val from store where key=?', [key])
+ try:
+ f = self.crsr.fetchone()[0]
+ except TypeError:
+ raise KeyError, key
+ return json.loads(f)
+ def __len__(self):
+ """len(a) the number of items in a"""
+ self.crsr.execute("select COUNT(*) from store")
+ return self.crsr.fetchone()[0]
+ def keys(self):
+ """a.keys() a copy of a's list of keys"""
+ self.crsr.execute("select key from store")
+ return [f[0] for f in self.crsr.fetchall()]
+ def values(self):
+ """a.values() a copy of a's list of values"""
+ self.crsr.execute("select val from store")
+ return [f[0] for f in self.crsr.fetchall()]
+ def items(self):
+ """a.items() a copy of a's list of (key, value) pairs"""
+ self.crsr.execute("select * from store")
+ return self.crsr.fetchall()
+ def get(self, k, x=None):
+ """a.get(k[, x]) a[k] if k in a, else x """
+ try:
+ return self.__getitem__(k)
+ except KeyError:
+ return x
+
+ def clear(self):
+ """a.clear() remove all items from a"""
+ self.crsr.execute("delete from store")
+
+if __name__=="__main__":
+ with sqlite_db("data/test") as db:
+ db.clear()
+ db['a']="test"
+ db['as']="test"
+ db['b']=[1,2,3,4,5]
+ db['c']=[1,2,3,4,5]
+ db['d']="who?"
+ del db['b']
+ print len(db)
+ print db.keys()
+ print db.values()
+ print db.items()
+ print db.get('b',5)
+ print db.get('b')
+ print db.get('c',5)

0 comments on commit 6d50b39

Please sign in to comment.