Permalink
Switch branches/tags
Nothing to show
Find file
Fetching contributors…
Cannot retrieve contributors at this time
executable file 157 lines (120 sloc) 3.61 KB
#!/usr/bin/python2.5
import os
import sys
import md5
import bsddb
import time
import datetime
import string
import array
import dbm
import gdbm
def random_string():
ttab = string.letters*4 + '\0'*48
a = array.array('B', open("/dev/urandom").read(150).translate(ttab))
a = array.array('B', filter(abs,a)).tostring()[:10]
return a
class DBFile(object):
def __init__(self, directory = '/tmp'):
self.file = bsddb.btopen("%s/insert-key-test.db"%(directory),'c')
def add_key(self, key, value='1'):
self.file[key] = value
def check_key(self, key):
return self.file.has_key(key)
def close(self):
self.file.close()
def name(self):
return "bsddb.btopen"
class DBFileHash(DBFile):
def __init__(self, directory = '/tmp'):
self.file = bsddb.hashopen("%s/insert-key-test.hash"%(directory),'c')
def add_key(self, key, value='1'):
self.file[key] = value
def check_key(self, key):
return self.file.has_key(key)
def close(self):
self.file.close()
def name(self):
return "bsddb.hashopen"
class DBFileDBM(DBFile):
def __init__(self, directory = '/tmp'):
self.file = dbm.open("%s/insert-key-test.dbm"%(directory),'c')
def add_key(self, key, value='1'):
self.file[key] = value
def check_key(self, key):
return self.file.has_key(key)
def close(self):
self.file.close()
def name(self):
return "dbm"
class DBFileGDBM(DBFile):
def __init__(self, directory = '/tmp'):
self.file = gdbm.open("%s/insert-key-test.gdbm"%(directory),'cf')
def add_key(self, key, value='1'):
self.file[key] = value
def check_key(self, key):
return self.file.has_key(key)
def close(self):
self.file.close()
def name(self):
return "gdbm"
class MySQLdbfile(object):
def __init__(self, user, passwd, db):
self.db=MySQLdb.connect(user=user, passwd=passwd, db=db)
self.c=self.db.cursor()
self.d=[]
self.count=0
self.errors=0
def add_key(self, key, value='1'):
self.d.append((key,value))
self.count+=1
if self.count > 50000:
try:
self.c.executemany('replace into msgids values(%s,%s)', self.d)
except:
print "erreur: %s" % traceback.format_exc()
self.errors+=1
# would do it the normal way (with simple execute)
self.d=[]
self.count=0
def check_key(self,key):
self.c.execute('select id from msgids where id = %s',key)
if self.c.rowcount > 0:
return True
else:
return False
def close(self):
self.c.close()
self.db.close()
def test_keys(dbfile):
filein = open('test.data', 'r')
counter = 0
t=time.time()
for stri in filein:
stri = stri.strip()
# stri = random_string()
# stri = str(counter)
if not dbfile.check_key(stri):
dbfile.add_key(stri)
counter += 1
if (counter%10000) == 0:
sys.stdout.write("\r%i"%counter)
sys.stdout.flush()
print "\n%i/sec for %s before saving"%(counter/(time.time()-t), dbfile.name())
filein.close()
dbfile.close()
print "%i/sec for %s after saving"%(counter/(time.time()-t), dbfile.name())
return counter
def main():
dbfile = DBFile('tmp/')
counter = test_keys(dbfile)
dbfile = DBFileHash('tmp/')
counter = test_keys(dbfile)
dbfile = DBFileDBM('tmp/')
t=time.time()
counter = test_keys(dbfile)
dbfile = DBFileGDBM('tmp/')
t=time.time()
counter = test_keys(dbfile)
if __name__ == '__main__':
main()