Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

first pass at persistance api

  • Loading branch information...
commit 60c951a8a20da9818aef96510449431b1855c15d 0 parents
BuzzTroll authored
0  README
No changes.
1  cloudminer/__init__.py
... ... @@ -0,0 +1 @@
  1 +from cloudminer import *
83 cloudminer/cloudminer.py
... ... @@ -0,0 +1,83 @@
  1 +import datetime
  2 +import uuid
  3 +import simplejson as json
  4 +import sqlalchemy
  5 +from sqlalchemy import ForeignKey
  6 +from sqlalchemy.orm import relation, backref
  7 +from sqlalchemy.orm import mapper
  8 +from sqlalchemy.orm import sessionmaker
  9 +from sqlalchemy import Table
  10 +from sqlalchemy import Integer
  11 +from sqlalchemy import String, MetaData, Sequence
  12 +from sqlalchemy import Column
  13 +import cloudyvents.cyvents as cyvents
  14 +
  15 +class _CYventExtra(object):
  16 + def __init__(self, key, value):
  17 + self.key = key
  18 + self.value = value
  19 +
  20 +class _CYvent(object):
  21 + """Convenience class for a parsed event.
  22 + """
  23 +
  24 + def __init__(self, runname, iaasid, source, name, key, timestamp, extra):
  25 +
  26 + self.runname = runname
  27 + self.iaasid = iaasid
  28 + self.source = source
  29 + self.name = name
  30 + self.key = key
  31 + self.timestamp = timestamp
  32 + self.extra = extra
  33 +
  34 +metadata = MetaData()
  35 +event_table = Table('events', metadata,
  36 + Column('id', Integer, Sequence('event_id_seq'), primary_key=True),
  37 + Column('source', String(50)),
  38 + Column('name', String(50)),
  39 + Column('key', String(50)),
  40 + Column('timestamp', sqlalchemy.types.Time),
  41 + Column('runname', String(50)),
  42 + Column('iaasid', String(50)),
  43 + )
  44 +
  45 +xtra_table = Table('extras', metadata,
  46 + Column('id', Integer, Sequence('extra_id_seq'), primary_key=True),
  47 + Column('key', String(50)),
  48 + Column('value', String(50)),
  49 + Column('event_id', Integer, ForeignKey('events.id'))
  50 + )
  51 +
  52 +mapper(_CYventExtra, xtra_table)
  53 +mapper(_CYvent, event_table, properties={
  54 + 'extra': relation(_CYventExtra)})
  55 +
  56 +
  57 +class CloudMiner(object):
  58 +
  59 + def __init__(self, dburl, module=None):
  60 + if module == None:
  61 + self.engine = sqlalchemy.create_engine(dburl)
  62 + else:
  63 + self.engine = sqlalchemy.create_engine(dburl, module=module)
  64 + metadata.create_all(self.engine)
  65 + self.Session = sessionmaker(bind=self.engine)
  66 + self.session = self.Session()
  67 +
  68 + def add_cloudyvent(self, runname, iaasid, cyv):
  69 + xtras_list = []
  70 + if cyv.extra != None:
  71 + for k in cyv.extra.keys():
  72 + e = _CYventExtra(k, cyv.extra[k])
  73 + xtras_list.append(e)
  74 +
  75 + _cyv = _CYvent(runname, iaasid, cyv.source, cyv.name, cyv.key, cyv.timestamp, xtras_list)
  76 + self.session.add(_cyv)
  77 +
  78 + def get_events_by_runname(self, runname):
  79 + return self.session.query(_CYvent).filter(_CYvent.runname == runname).all()
  80 +
  81 + def commit(self):
  82 + self.session.commit()
  83 +
78 cloudminer/tests/test_filedb_cm.py
... ... @@ -0,0 +1,78 @@
  1 +import unittest
  2 +import datetime
  3 +import logging
  4 +import os
  5 +import shutil
  6 +import uuid
  7 +import tempfile
  8 +import time
  9 +import cloudyvents.cyvents as cyvents
  10 +from cloudminer import CloudMiner
  11 +from cloudyvents.cyvents import CYvent
  12 +import tempfile
  13 +
  14 +# Set this to False to look at generated log files afterwards. There will be
  15 +# many directories like /tmp/cytestlog*
  16 +DESTROY_LOGDIR = True
  17 +
  18 +logger = logging.getLogger(__name__)
  19 +
  20 +class CloudMinerRealFileTestCase(unittest.TestCase):
  21 +
  22 + def setUp(self):
  23 + (osf, self.filename) = tempfile.mkstemp()
  24 + os.close(osf)
  25 + print self.filename
  26 + self.cdb = CloudMiner('sqlite:///' + self.filename)
  27 + self.runname = "runitk"
  28 + self.iaasid = "iceicebaby"
  29 +
  30 + def tearDown(self):
  31 + os.remove(self.filename)
  32 + pass
  33 +
  34 + def test_file_db(self):
  35 + extras = {}
  36 + key = "key1"
  37 + source = "src1"
  38 + name = "name1"
  39 + extras['hi'] = 'there'
  40 + cye = CYvent(source, name, key, datetime.datetime.now(), extras)
  41 + self.cdb.add_cloudyvent(self.runname, self.iaasid, cye)
  42 + self.cdb.commit()
  43 +
  44 + def test_multiply_cms_simple(self):
  45 + extras = {}
  46 + extras['hi'] = 'there'
  47 + cye = CYvent('src1', 'name1', 'key', datetime.datetime.now(), extras)
  48 + self.cdb.add_cloudyvent(self.runname, self.iaasid, cye)
  49 + self.cdb.commit()
  50 +
  51 + cdb2 = CloudMiner('sqlite:///' + self.filename)
  52 + cdb2.add_cloudyvent(self.runname, self.iaasid, cye)
  53 + cdb2.commit()
  54 +
  55 + rc = self.cdb.get_events_by_runname(self.runname)
  56 + self.assertEqual(len(rc), 2)
  57 + self.assertEqual(rc[0].source, "src1")
  58 + self.assertEqual(rc[0].key, 'key')
  59 + self.assertEqual(rc[0].name, 'name1')
  60 +
  61 + def test_multiple_cms_commit(self):
  62 + extras = {}
  63 + extras['hi'] = 'there'
  64 + cye = CYvent('src1', 'name1', 'key', datetime.datetime.now(), extras)
  65 + self.cdb.add_cloudyvent(self.runname, self.iaasid, cye)
  66 +
  67 + cdb2 = CloudMiner('sqlite:///' + self.filename)
  68 +
  69 + rc = cdb2.get_events_by_runname(self.runname)
  70 + self.assertEqual(len(rc), 0)
  71 +
  72 + self.cdb.commit()
  73 + cdb2.commit()
  74 + rc = cdb2.get_events_by_runname(self.runname)
  75 + self.assertEqual(len(rc), 1)
  76 + self.assertEqual(rc[0].source, "src1")
  77 + self.assertEqual(rc[0].key, 'key')
  78 + self.assertEqual(rc[0].name, 'name1')
120 cloudminer/tests/test_simple_cm.py
... ... @@ -0,0 +1,120 @@
  1 +import unittest
  2 +import datetime
  3 +import logging
  4 +import os
  5 +import shutil
  6 +import uuid
  7 +import tempfile
  8 +import time
  9 +import cloudyvents.cyvents as cyvents
  10 +from cloudminer import CloudMiner
  11 +from cloudyvents.cyvents import CYvent
  12 +import tempfile
  13 +
  14 +# Set this to False to look at generated log files afterwards. There will be
  15 +# many directories like /tmp/cytestlog*
  16 +DESTROY_LOGDIR = True
  17 +
  18 +logger = logging.getLogger(__name__)
  19 +
  20 +class CloudMinerTestCase(unittest.TestCase):
  21 +
  22 + def setUp(self):
  23 + self.cdb = CloudMiner('sqlite:///:memory:')
  24 + self.runname = "run1"
  25 + self.iaasid = "iaasid1"
  26 +
  27 + def tearDown(self):
  28 + pass
  29 +
  30 + def test_simple_insert_with_extra_none(self):
  31 + cye = CYvent('src1', 'name1', 'key', datetime.datetime.now(), None)
  32 + self.cdb.add_cloudyvent(self.runname, self.iaasid, cye)
  33 + self.cdb.commit()
  34 +
  35 + def test_simple_insert_with_extra(self):
  36 + extras = {}
  37 + extras['hi'] = 'there'
  38 + cye = CYvent('src1', 'name1', 'key', datetime.datetime.now(), extras)
  39 + self.cdb.add_cloudyvent(self.runname, self.iaasid, cye)
  40 + self.cdb.commit()
  41 +
  42 + def test_simple_insert_with_extra_empty(self):
  43 + extras = {}
  44 + cye = CYvent('src1', 'name1', 'key', datetime.datetime.now(), extras)
  45 + self.cdb.add_cloudyvent(self.runname, self.iaasid, cye)
  46 + self.cdb.commit()
  47 +
  48 + def test_simple_query(self):
  49 + extras = {}
  50 + key = "key1"
  51 + source = "src1"
  52 + name = "name1"
  53 + extras['hi'] = 'there'
  54 + cye = CYvent(source, name, key, datetime.datetime.now(), extras)
  55 + self.cdb.add_cloudyvent(self.runname, self.iaasid, cye)
  56 + self.cdb.commit()
  57 +
  58 + rc = self.cdb.get_events_by_runname(self.runname)
  59 + self.assertEqual(len(rc), 1)
  60 + self.assertEqual(rc[0].source, source)
  61 + self.assertEqual(rc[0].key, key)
  62 + self.assertEqual(rc[0].name, name)
  63 +
  64 + def test_man_run_query(self):
  65 + run2 = str(uuid.uuid1())
  66 +
  67 + extras = {}
  68 + key = "key1"
  69 + source = "src1"
  70 + name = "name1"
  71 + key2 = "key2"
  72 + source2 = "src2"
  73 + name2 = "name2"
  74 + extras['hi'] = 'there'
  75 + cye = CYvent(source, name, key, datetime.datetime.now(), extras)
  76 + self.cdb.add_cloudyvent(self.runname, self.iaasid, cye)
  77 + cye = CYvent(source2, name2, key2, datetime.datetime.now(), extras)
  78 + self.cdb.add_cloudyvent(run2, self.iaasid, cye)
  79 + self.cdb.commit()
  80 +
  81 + rc = self.cdb.get_events_by_runname(self.runname)
  82 + self.assertEqual(len(rc), 1, "len is %d should be 0" % (len(rc)))
  83 + self.assertEqual(rc[0].source, source)
  84 + self.assertEqual(rc[0].key, key)
  85 + self.assertEqual(rc[0].name, name)
  86 +
  87 + rc = self.cdb.get_events_by_runname(run2)
  88 + self.assertEqual(len(rc), 1)
  89 + self.assertEqual(rc[0].source, source2)
  90 + self.assertEqual(rc[0].key, key2)
  91 + self.assertEqual(rc[0].name, name2)
  92 +
  93 + def test_multiply_cms_simple(self):
  94 + extras = {}
  95 + extras['hi'] = 'there'
  96 + cye = CYvent('src1', 'name1', 'key', datetime.datetime.now(), extras)
  97 + self.cdb.add_cloudyvent(self.runname, self.iaasid, cye)
  98 + self.cdb.commit()
  99 +
  100 + cdb2 = CloudMiner('sqlite:///:memory:')
  101 + cdb2.add_cloudyvent(self.runname, self.iaasid, cye)
  102 + cdb2.commit()
  103 +
  104 + def test_multiply_cms_simple(self):
  105 + extras = {}
  106 + extras['hi'] = 'there'
  107 + cye = CYvent('src1', 'name1', 'key', datetime.datetime.now(), extras)
  108 + self.cdb.add_cloudyvent(self.runname, self.iaasid, cye)
  109 + self.cdb.commit()
  110 +
  111 + cdb2 = CloudMiner('sqlite:///:memory:')
  112 + cdb2.add_cloudyvent(self.runname, self.iaasid, cye)
  113 + cdb2.commit()
  114 +
  115 + rc = self.cdb.get_events_by_runname(self.runname)
  116 + self.assertEqual(len(rc), 1)
  117 + self.assertEqual(rc[0].source, "src1")
  118 + self.assertEqual(rc[0].key, 'key')
  119 + self.assertEqual(rc[0].name, 'name1')
  120 +

0 comments on commit 60c951a

Please sign in to comment.
Something went wrong with that request. Please try again.