Added first implementation of 3store and relational queries for data …

…retrieval.

Should collate data into identical hierarchical format.

dbperf.py

@@ -4,7 +4,39 @@
 
 from datageneration import generate
 from database import insert
+from dbquery import *
+from timeit import Timer
+
+import datetime
+from decimal import Decimal
 
 if __name__ == "__main__":
-    for x in generate(1000000):
-        insert(x)
+    owners = choose_owners()
+    d1 = get_data_3store(owners)
+    d2 = get_data_rel(owners)
+    #print '>> 3store:\t\t%s' % Timer('from __main__ import *; get_data_3store(%s)' % owners).repeat(5, 100)
+    #print '>> relational:\t%s' % Timer('from __main__ import *; get_data_rel(%s)' % owners).repeat(5, 100)
+    #for x in generate(1000000):
+    #    insert(x)
+    print 'D1.keys - D2.keys: %s' % (set(d1.keys()) - set(d2.keys()))
+    print 'D2.keys - D1.keys: %s' % (set(d2.keys()) - set(d1.keys()))
+    for owner in d1:
+        a = d1[owner]
+        b = d2[owner]
+        if len(set(a.keys()) - set(b.keys())) > 0:
+            print 'a.keys - b.keys: %s' % (set(a.keys()) - set(b.keys()))
+        if len(set(b.keys()) - set(a.keys())) > 0:
+            print 'b.keys - a.keys: %s' % (set(b.keys()) - set(a.keys()))
+        for item in a:
+            ia = a[item]
+            ib = b[item]
+            print 'Check %s -> %s of type %s' % (owner, item, ia['TYPE'])
+            if len(set(ia.keys()) - set(ib.keys())) > 0:
+                print 'ia.keys - ib.keys: %s' % (set(ia.keys()) - set(ib.keys()))
+            if len(set(ib.keys()) - set(ia.keys())) > 0:
+                print 'ib.keys - ia.keys: %s' % (set(ib.keys()) - set(ia.keys()))
+            for pred in ia.keys():
+                if not pred in ib:
+                    continue
+                if not str(ia[pred]) == str(ib[pred]):
+                    print '\t%s: %s != %s' % (pred, str(ia[pred]), str(ib[pred]))

dbquery.py

@@ -0,0 +1,58 @@
+
+__author__="majorerror"
+__date__ ="$31-Oct-2010 09:42:52$"
+
+from MySQLdb import *
+from itemtypes import TYPES
+
+DB = connect(user="root", db="3store")
+
+def choose_owners(num=10):
+    c = DB.cursor()
+    c.execute("""SELECT DISTINCT Owner.ownerid FROM Owner
+LEFT JOIN Item ON Owner.ownerid = Item.ownerid
+GROUP BY Owner.ownerid
+ORDER BY COUNT(itemid) DESC LIMIT %d""" % num)
+    return [str(o[0]) for o in c.fetchall()]
+
+def get_data_3store(owners):
+    c = DB.cursor()
+    c.execute("""SELECT Item.itemid, Item.type, ownerid, predicate, value
+    FROM Item LEFT JOIN ItemDetails ON Item.itemid = ItemDetails.itemid
+    WHERE Item.Ownerid IN (%s)""" % ', '.join(owners))
+    data = {}
+    for row in c.fetchall():
+        itemid, type, ownerid, key, val = row
+        if not key in TYPES[type]:
+            continue
+        if not str(ownerid) in data:
+            data[str(ownerid)] = {}
+        if not str(itemid) in data[str(ownerid)]:
+            data[str(ownerid)][str(itemid)] = {}
+            data[str(ownerid)][str(itemid)]['TYPE'] = type
+        data[str(ownerid)][str(itemid)][key] = val
+    return data
+
+def get_data_rel(owners):
+    c = DB.cursor()
+    c.execute("""SELECT Item.itemid, type FROM Item WHERE Item.Ownerid IN (%s)""" % ', '.join(owners))
+    items = {}
+    for row in c.fetchall():
+        itemid, type = row
+        if not type in items:
+            items[type] = []
+        items[type].append(itemid)
+    data = {}
+    for type in items:
+        keys = TYPES[type].keys()
+        c = DB.cursor()
+        c.execute('SELECT Item.itemid, ownerid, %s FROM Item LEFT JOIN %s ON Item.itemid = %s.itemid WHERE Item.itemid IN (%s)' %
+            (', '.join(keys), type, type, ', '.join([str(x) for x in items[type]])))
+        for row in c.fetchall():
+            itemid = row[0]
+            ownerid = row[1]
+            if not str(ownerid) in data:
+                data[str(ownerid)] = {}
+            data[str(ownerid)][str(itemid)] = dict([(keys[i], row[i + 2]) for i in xrange(0, len(keys) - 1)])
+            data[str(ownerid)][str(itemid)]['TYPE'] = type
+    return data