Permalink
Browse files

first pass at persistance api

  • Loading branch information...
0 parents commit 60c951a8a20da9818aef96510449431b1855c15d BuzzTroll committed Oct 1, 2010
0 README
No changes.
@@ -0,0 +1 @@
+from cloudminer import *
@@ -0,0 +1,83 @@
+import datetime
+import uuid
+import simplejson as json
+import sqlalchemy
+from sqlalchemy import ForeignKey
+from sqlalchemy.orm import relation, backref
+from sqlalchemy.orm import mapper
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy import Table
+from sqlalchemy import Integer
+from sqlalchemy import String, MetaData, Sequence
+from sqlalchemy import Column
+import cloudyvents.cyvents as cyvents
+
+class _CYventExtra(object):
+ def __init__(self, key, value):
+ self.key = key
+ self.value = value
+
+class _CYvent(object):
+ """Convenience class for a parsed event.
+ """
+
+ def __init__(self, runname, iaasid, source, name, key, timestamp, extra):
+
+ self.runname = runname
+ self.iaasid = iaasid
+ self.source = source
+ self.name = name
+ self.key = key
+ self.timestamp = timestamp
+ self.extra = extra
+
+metadata = MetaData()
+event_table = Table('events', metadata,
+ Column('id', Integer, Sequence('event_id_seq'), primary_key=True),
+ Column('source', String(50)),
+ Column('name', String(50)),
+ Column('key', String(50)),
+ Column('timestamp', sqlalchemy.types.Time),
+ Column('runname', String(50)),
+ Column('iaasid', String(50)),
+ )
+
+xtra_table = Table('extras', metadata,
+ Column('id', Integer, Sequence('extra_id_seq'), primary_key=True),
+ Column('key', String(50)),
+ Column('value', String(50)),
+ Column('event_id', Integer, ForeignKey('events.id'))
+ )
+
+mapper(_CYventExtra, xtra_table)
+mapper(_CYvent, event_table, properties={
+ 'extra': relation(_CYventExtra)})
+
+
+class CloudMiner(object):
+
+ def __init__(self, dburl, module=None):
+ if module == None:
+ self.engine = sqlalchemy.create_engine(dburl)
+ else:
+ self.engine = sqlalchemy.create_engine(dburl, module=module)
+ metadata.create_all(self.engine)
+ self.Session = sessionmaker(bind=self.engine)
+ self.session = self.Session()
+
+ def add_cloudyvent(self, runname, iaasid, cyv):
+ xtras_list = []
+ if cyv.extra != None:
+ for k in cyv.extra.keys():
+ e = _CYventExtra(k, cyv.extra[k])
+ xtras_list.append(e)
+
+ _cyv = _CYvent(runname, iaasid, cyv.source, cyv.name, cyv.key, cyv.timestamp, xtras_list)
+ self.session.add(_cyv)
+
+ def get_events_by_runname(self, runname):
+ return self.session.query(_CYvent).filter(_CYvent.runname == runname).all()
+
+ def commit(self):
+ self.session.commit()
+
@@ -0,0 +1,78 @@
+import unittest
+import datetime
+import logging
+import os
+import shutil
+import uuid
+import tempfile
+import time
+import cloudyvents.cyvents as cyvents
+from cloudminer import CloudMiner
+from cloudyvents.cyvents import CYvent
+import tempfile
+
+# Set this to False to look at generated log files afterwards. There will be
+# many directories like /tmp/cytestlog*
+DESTROY_LOGDIR = True
+
+logger = logging.getLogger(__name__)
+
+class CloudMinerRealFileTestCase(unittest.TestCase):
+
+ def setUp(self):
+ (osf, self.filename) = tempfile.mkstemp()
+ os.close(osf)
+ print self.filename
+ self.cdb = CloudMiner('sqlite:///' + self.filename)
+ self.runname = "runitk"
+ self.iaasid = "iceicebaby"
+
+ def tearDown(self):
+ os.remove(self.filename)
+ pass
+
+ def test_file_db(self):
+ extras = {}
+ key = "key1"
+ source = "src1"
+ name = "name1"
+ extras['hi'] = 'there'
+ cye = CYvent(source, name, key, datetime.datetime.now(), extras)
+ self.cdb.add_cloudyvent(self.runname, self.iaasid, cye)
+ self.cdb.commit()
+
+ def test_multiply_cms_simple(self):
+ extras = {}
+ extras['hi'] = 'there'
+ cye = CYvent('src1', 'name1', 'key', datetime.datetime.now(), extras)
+ self.cdb.add_cloudyvent(self.runname, self.iaasid, cye)
+ self.cdb.commit()
+
+ cdb2 = CloudMiner('sqlite:///' + self.filename)
+ cdb2.add_cloudyvent(self.runname, self.iaasid, cye)
+ cdb2.commit()
+
+ rc = self.cdb.get_events_by_runname(self.runname)
+ self.assertEqual(len(rc), 2)
+ self.assertEqual(rc[0].source, "src1")
+ self.assertEqual(rc[0].key, 'key')
+ self.assertEqual(rc[0].name, 'name1')
+
+ def test_multiple_cms_commit(self):
+ extras = {}
+ extras['hi'] = 'there'
+ cye = CYvent('src1', 'name1', 'key', datetime.datetime.now(), extras)
+ self.cdb.add_cloudyvent(self.runname, self.iaasid, cye)
+
+ cdb2 = CloudMiner('sqlite:///' + self.filename)
+
+ rc = cdb2.get_events_by_runname(self.runname)
+ self.assertEqual(len(rc), 0)
+
+ self.cdb.commit()
+ cdb2.commit()
+ rc = cdb2.get_events_by_runname(self.runname)
+ self.assertEqual(len(rc), 1)
+ self.assertEqual(rc[0].source, "src1")
+ self.assertEqual(rc[0].key, 'key')
+ self.assertEqual(rc[0].name, 'name1')
@@ -0,0 +1,120 @@
+import unittest
+import datetime
+import logging
+import os
+import shutil
+import uuid
+import tempfile
+import time
+import cloudyvents.cyvents as cyvents
+from cloudminer import CloudMiner
+from cloudyvents.cyvents import CYvent
+import tempfile
+
+# Set this to False to look at generated log files afterwards. There will be
+# many directories like /tmp/cytestlog*
+DESTROY_LOGDIR = True
+
+logger = logging.getLogger(__name__)
+
+class CloudMinerTestCase(unittest.TestCase):
+
+ def setUp(self):
+ self.cdb = CloudMiner('sqlite:///:memory:')
+ self.runname = "run1"
+ self.iaasid = "iaasid1"
+
+ def tearDown(self):
+ pass
+
+ def test_simple_insert_with_extra_none(self):
+ cye = CYvent('src1', 'name1', 'key', datetime.datetime.now(), None)
+ self.cdb.add_cloudyvent(self.runname, self.iaasid, cye)
+ self.cdb.commit()
+
+ def test_simple_insert_with_extra(self):
+ extras = {}
+ extras['hi'] = 'there'
+ cye = CYvent('src1', 'name1', 'key', datetime.datetime.now(), extras)
+ self.cdb.add_cloudyvent(self.runname, self.iaasid, cye)
+ self.cdb.commit()
+
+ def test_simple_insert_with_extra_empty(self):
+ extras = {}
+ cye = CYvent('src1', 'name1', 'key', datetime.datetime.now(), extras)
+ self.cdb.add_cloudyvent(self.runname, self.iaasid, cye)
+ self.cdb.commit()
+
+ def test_simple_query(self):
+ extras = {}
+ key = "key1"
+ source = "src1"
+ name = "name1"
+ extras['hi'] = 'there'
+ cye = CYvent(source, name, key, datetime.datetime.now(), extras)
+ self.cdb.add_cloudyvent(self.runname, self.iaasid, cye)
+ self.cdb.commit()
+
+ rc = self.cdb.get_events_by_runname(self.runname)
+ self.assertEqual(len(rc), 1)
+ self.assertEqual(rc[0].source, source)
+ self.assertEqual(rc[0].key, key)
+ self.assertEqual(rc[0].name, name)
+
+ def test_man_run_query(self):
+ run2 = str(uuid.uuid1())
+
+ extras = {}
+ key = "key1"
+ source = "src1"
+ name = "name1"
+ key2 = "key2"
+ source2 = "src2"
+ name2 = "name2"
+ extras['hi'] = 'there'
+ cye = CYvent(source, name, key, datetime.datetime.now(), extras)
+ self.cdb.add_cloudyvent(self.runname, self.iaasid, cye)
+ cye = CYvent(source2, name2, key2, datetime.datetime.now(), extras)
+ self.cdb.add_cloudyvent(run2, self.iaasid, cye)
+ self.cdb.commit()
+
+ rc = self.cdb.get_events_by_runname(self.runname)
+ self.assertEqual(len(rc), 1, "len is %d should be 0" % (len(rc)))
+ self.assertEqual(rc[0].source, source)
+ self.assertEqual(rc[0].key, key)
+ self.assertEqual(rc[0].name, name)
+
+ rc = self.cdb.get_events_by_runname(run2)
+ self.assertEqual(len(rc), 1)
+ self.assertEqual(rc[0].source, source2)
+ self.assertEqual(rc[0].key, key2)
+ self.assertEqual(rc[0].name, name2)
+
+ def test_multiply_cms_simple(self):
+ extras = {}
+ extras['hi'] = 'there'
+ cye = CYvent('src1', 'name1', 'key', datetime.datetime.now(), extras)
+ self.cdb.add_cloudyvent(self.runname, self.iaasid, cye)
+ self.cdb.commit()
+
+ cdb2 = CloudMiner('sqlite:///:memory:')
+ cdb2.add_cloudyvent(self.runname, self.iaasid, cye)
+ cdb2.commit()
+
+ def test_multiply_cms_simple(self):
+ extras = {}
+ extras['hi'] = 'there'
+ cye = CYvent('src1', 'name1', 'key', datetime.datetime.now(), extras)
+ self.cdb.add_cloudyvent(self.runname, self.iaasid, cye)
+ self.cdb.commit()
+
+ cdb2 = CloudMiner('sqlite:///:memory:')
+ cdb2.add_cloudyvent(self.runname, self.iaasid, cye)
+ cdb2.commit()
+
+ rc = self.cdb.get_events_by_runname(self.runname)
+ self.assertEqual(len(rc), 1)
+ self.assertEqual(rc[0].source, "src1")
+ self.assertEqual(rc[0].key, 'key')
+ self.assertEqual(rc[0].name, 'name1')
+

0 comments on commit 60c951a

Please sign in to comment.