Skip to content
This repository
Browse code

move to sqlite

  • Loading branch information...
commit 6d50b396bce47171ebd9e2c06ab1db7c62a01f37 1 parent 8e3906d
James Vasile authored

Showing 4 changed files with 164 additions and 16 deletions. Show diff stats Hide diff stats

  1. +22 0 dbconvert.py
  2. +12 15 planet.py
  3. +9 1 util.py
  4. +121 0 withsqlite.py
22 dbconvert.py
... ... @@ -0,0 +1,22 @@
  1 +#!/usr/bin/python
  2 +
  3 +# convert from shelf to sqlite
  4 +
  5 +import sys
  6 +from util import our_db, sqlite_db
  7 +import simplejson as json
  8 +
  9 +with sqlite_db('planets') as sdb:
  10 + sdb.clear()
  11 + with our_db('planets') as odb:
  12 + for key, val in odb.items():
  13 + val = json.loads(val)
  14 + sdb[key]=val
  15 +sys.exit()
  16 +with sqlite_db('cache') as sdb:
  17 + sdb.clear()
  18 + with our_db('cache') as odb:
  19 + for key, val in odb.items():
  20 + val = json.loads(val)
  21 + sdb[key]=val
  22 +
27 planet.py
@@ -32,14 +32,11 @@ def to_json(python_object):
32 32
33 33 #raise TypeError(repr(python_object) + ' is not JSON serializable')
34 34
35   -def serialize_feedparse(parse_object):
36   - return json.dumps(parse_object, default=to_json)
37   -
38 35 class Planet():
39 36 def __init__(self, *args, **kwargs):
40 37 if 'direc' in kwargs:
41 38 with our_db('planets') as db:
42   - self.load_json(db[kwargs['direc']])
  39 + self.load_dict(db[kwargs['direc']])
43 40 elif isinstance(args[0], basestring):
44 41 self.load_json(args[0])
45 42 elif isinstance(args[0], dict):
@@ -82,8 +79,8 @@ def load_json(self, j):
82 79
83 80 def save_cache(self, cache, url):
84 81 with our_db('cache') as db:
85   - j = json.dumps(cache, sort_keys=True, indent=3)
86   - db[url.encode("utf-8")] = j
  82 + #db[url.encode("utf-8")] = cache
  83 + db[url] = cache
87 84
88 85 def save(self, update_config_timestamp=False, ignore_missing_dir=False):
89 86 output_dir = os.path.join(cfg.OUTPUT_DIR, self.direc)
@@ -95,7 +92,8 @@ def save(self, update_config_timestamp=False, ignore_missing_dir=False):
95 92 if update_config_timestamp:
96 93 self.last_config_change = time.time()
97 94 with our_db('planets') as db:
98   - db[self.direc.encode("utf-8")] = self.json()
  95 + #db[self.direc.encode("utf-8")] = self.serializable()
  96 + db[self.direc] = self.serializable()
99 97
100 98 def serializable(self):
101 99 return {'direc':self.direc,
@@ -117,7 +115,8 @@ def update_feed(self, url):
117 115 force_check = opt['force_check']
118 116 with our_db('cache') as db:
119 117 try:
120   - cache = json.loads(db[url.encode("utf-8")])
  118 + #cache = db[url.encode("utf-8")]
  119 + cache = db[url]
121 120 except KeyError:
122 121 log.info("Can't find %s in cache. Making default." % url)
123 122 cache = {'data':'', 'last_downloaded':0, 'dload_fail':False}
@@ -143,7 +142,8 @@ def update_feed(self, url):
143 142 cache['last_downloaded'] = time.time()
144 143 with our_db('cache') as db:
145 144 try:
146   - db[url.encode("utf8")] = json.dumps(cache, default=to_json, sort_keys=True, indent=3)
  145 + #db[url.encode("utf8")] = cache
  146 + db[url] = cache
147 147 except TypeError, e:
148 148 log.debug("Can't save feed (%s): %s" % (url, e))
149 149 cache['error'] = str(e)
@@ -184,10 +184,11 @@ def generate(self):
184 184 lopt['Feeds']=[]
185 185 for url, f in self.feeds.items():
186 186 with our_db('cache') as db:
187   - if not url.encode("utf-8") in db:
  187 + if not url in db:
188 188 continue
189 189 try:
190   - cache = json.loads(db[url.encode("utf-8")])
  190 + #cache = db[url.encode("utf-8")]
  191 + cache = db[url]
191 192 except json.decoder.JSONDecodeError, e:
192 193 log.debug("Json error on generating url %s: %s" % (url, e))
193 194 continue
@@ -304,10 +305,6 @@ def dump_cache(self):
304 305 print url
305 306 with our_db('cache') as db:
306 307 cache = db[url]
307   - try:
308   - cache = json.loads(cache)
309   - except json.decoder.JSONDecodeError:
310   - return
311 308
312 309 print json.dumps(cache)
313 310
10 util.py
@@ -14,6 +14,7 @@
14 14 import logging
15 15 log = logging.getLogger('planeteria')
16 16 import dateutil.parser
  17 +import withsqlite
17 18
18 19 try:
19 20 import tidy
@@ -40,7 +41,11 @@ def just_body(xhtml):
40 41 except IndexError:
41 42 return ''
42 43
43   -class our_db:
  44 +class sqlite_db(withsqlite.sqlite_db):
  45 + def __init__(self, fname):
  46 + withsqlite.sqlite_db.__init__(self, os.path.join(cfg.data_dir, fname))
  47 +
  48 +class shelve_db:
44 49 def __init__(self, fname):
45 50 self.fname = fname + ".shelf"
46 51 def __enter__(self):
@@ -49,6 +54,9 @@ def __enter__(self):
49 54 def __exit__(self, type, value, traceback):
50 55 self.db.close()
51 56
  57 +class our_db(sqlite_db):
  58 + pass
  59 +
52 60 class berkeley_db:
53 61 def __init__(self, fname):
54 62 self.fname = fname
121 withsqlite.py
... ... @@ -0,0 +1,121 @@
  1 +#!/usr/bin/env python
  2 +
  3 +import os, sys, sqlite3
  4 +import simplejson as json
  5 +
  6 +class sqlite_db():
  7 + """
  8 +Backends a dict on an sqlite db. This class aims to present like a
  9 +dict wherever it can.
  10 +
  11 +USE:
  12 +import sqlite_db from withsqlite
  13 +with sqlite_db("filename") as db:
  14 + db['aaa'] = {'test':'ok'}
  15 + print db.items()
  16 +
  17 +BUGS:
  18 +
  19 +vals are json serialized before being written, so if you can't
  20 +serialize it, you can't put it in the dict.
  21 +
  22 +Unimplemented mapping API:
  23 +a.copy() a (shallow) copy of a
  24 +k in a True if a has a key k, else False
  25 +k not in a Equivalent to not k in a
  26 +a.has_key(k) Equivalent to k in a, use that form in new code
  27 +a.update([b]) updates a with key/value pairs from b, overwriting existing keys, returns None
  28 +a.fromkeys(seq[, value]) Creates a new dictionary with keys from seq and values set to value
  29 +a.setdefault(k[, x]) a[k] if k in a, else x (also setting it)
  30 +a.pop(k[, x]) a[k] if k in a, else x (and remove k)
  31 +a.popitem() remove and return an arbitrary (key, value) pair
  32 +a.iteritems() return an iterator over (key, value) pairs
  33 +a.iterkeys() return an iterator over the mapping's keys
  34 +a.itervalues() return an iterator over the mapping's values
  35 +"""
  36 +
  37 + def __init__(self, fname):
  38 + self.fname = fname + ".sqlite3"
  39 + def __enter__(self):
  40 + if not os.path.exists(self.fname):
  41 + self.make_db()
  42 + self.conn = sqlite3.connect(self.fname)
  43 + self.crsr = self.conn.cursor()
  44 + return self
  45 + def __exit__(self, type, value, traceback):
  46 + self.conn.commit()
  47 + self.crsr.close()
  48 + def make_db(self):
  49 + conn = sqlite3.connect(self.fname)
  50 + c = conn.cursor()
  51 + c.execute('''create table store (key text unique, val text)''')
  52 + conn.commit()
  53 + c.close()
  54 + def __delitem__(self, key):
  55 + """del a[k] remove a[k] from a"""
  56 + self.crsr.execute("delete from store where key=?", [key])
  57 + def jsonize(self,val):
  58 + "If it's just a string, serialize it ourselves"
  59 + if isinstance(val, basestring):
  60 + return '"%s"' % val
  61 + return json.dumps(val, sort_keys=True, indent=3)
  62 + def __setitem__(self, key, val):
  63 + """a[k] = v set a[k] to v """
  64 +
  65 + try:
  66 + if val == self.__getitem__(key):
  67 + return
  68 + self.crsr.execute("update or fail store set val=? where key==?", [self.jsonize(val), key])
  69 + except KeyError:
  70 + self.crsr.execute("insert into store values (?, ?)", [key, self.jsonize(val)])
  71 + def __getitem__(self, key):
  72 + """a[k] the item of a with key k (1), (10)"""
  73 + self.crsr.execute('select val from store where key=?', [key])
  74 + try:
  75 + f = self.crsr.fetchone()[0]
  76 + except TypeError:
  77 + raise KeyError, key
  78 + return json.loads(f)
  79 + def __len__(self):
  80 + """len(a) the number of items in a"""
  81 + self.crsr.execute("select COUNT(*) from store")
  82 + return self.crsr.fetchone()[0]
  83 + def keys(self):
  84 + """a.keys() a copy of a's list of keys"""
  85 + self.crsr.execute("select key from store")
  86 + return [f[0] for f in self.crsr.fetchall()]
  87 + def values(self):
  88 + """a.values() a copy of a's list of values"""
  89 + self.crsr.execute("select val from store")
  90 + return [f[0] for f in self.crsr.fetchall()]
  91 + def items(self):
  92 + """a.items() a copy of a's list of (key, value) pairs"""
  93 + self.crsr.execute("select * from store")
  94 + return self.crsr.fetchall()
  95 + def get(self, k, x=None):
  96 + """a.get(k[, x]) a[k] if k in a, else x """
  97 + try:
  98 + return self.__getitem__(k)
  99 + except KeyError:
  100 + return x
  101 +
  102 + def clear(self):
  103 + """a.clear() remove all items from a"""
  104 + self.crsr.execute("delete from store")
  105 +
  106 +if __name__=="__main__":
  107 + with sqlite_db("data/test") as db:
  108 + db.clear()
  109 + db['a']="test"
  110 + db['as']="test"
  111 + db['b']=[1,2,3,4,5]
  112 + db['c']=[1,2,3,4,5]
  113 + db['d']="who?"
  114 + del db['b']
  115 + print len(db)
  116 + print db.keys()
  117 + print db.values()
  118 + print db.items()
  119 + print db.get('b',5)
  120 + print db.get('b')
  121 + print db.get('c',5)

0 comments on commit 6d50b39

Please sign in to comment.
Something went wrong with that request. Please try again.