Permalink
Browse files

Added data entry

  • Loading branch information...
1 parent b072419 commit 9badc17f40438b061c4128d42b3a3bd2ac6c1734 @jrconlin committed Oct 10, 2012
Showing with 211 additions and 109 deletions.
  1. +27 −10 campaign/storage/__init__.py
  2. +96 −64 campaign/storage/sql.py
  3. +48 −15 campaign/templates/main.mako
  4. +35 −18 campaign/views.py
  5. +2 −0 prod-reqs.txt
  6. +3 −2 work/loader.py
@@ -1,21 +1,38 @@
#import memcache
from time import time
+import math
+import string
+class StorageException(Exception):
+ pass
class StorageBase(object):
def __init__(self, config, **kw):
self.config = config
+ self.alphabet = string.digits + string.letters
# self._mcache = memcache.Client(kw.get(servers,['localhost']))
- def _gen_key(self, **kw):
- key = ''
- for k in ['idle_time', 'lang', 'locale', 'client', 'platform']:
- key +='%s=%s|' % (k, kw.get(k, ''))
- return key
-
- def put(self, **data):
- key = self._gen_key(**data)
+ def _encode_num(self, num=0):
+ if num == 0:
+ return self.alphabet[0]
+ barray = []
+ base = len(self.alphabet)
+ while num:
+ remain = num % base
+ num = num // base
+ barray.append(self.alphabet[remain])
+ barray.reverse
+ return ''.join(barray)
+
+ def _gen_key(self, data):
+ """ Create a semi-arbitrary unique key for this record """
+ nowbits = math.modf(time())
+ return '%s%s' % (self._encode_num(int(nowbits[1])),
+ self._encode_num(int(nowbits[0]*100000)))
+
+ def put_announce(self, data):
+ data['id'] = self._gen_key(data)
now = time()
expry = None
# auto expire if need be
@@ -24,8 +41,8 @@ def put(self, **data):
#self._mcache.add(key, value=data.get('note'),
# time=expry)
- def get(self, **data):
- key = self._gen_key(**data)
+ def get_announce(self, data):
+ key = self._gen_key(data)
# if there are less entries in memcache than the total number of
# possible iterations, just walk the list of items in memcache.
View
@@ -1,14 +1,32 @@
import logging
import json
+from dateutil import parser
from time import time
-from sqlalchemy import (Table, Column, Integer, String, Text, Float,
+from . import StorageBase
+from sqlalchemy import (Column, Integer, String, Text, Float,
create_engine, MetaData, text)
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import scoped_session, sessionmaker
-from campaign.storage import StorageBase
+
Base = declarative_base()
-Session = scoped_session(sessionmaker())
+
+class Campaign(Base):
+ __tablename__ = 'campaigns'
+
+ id = Column('id', String(25), primary_key=True)
+ channel = Column('channel', String(24), index=True, nullable=True)
+ version = Column('version', Float, index=True, nullable=True)
+ platform = Column('platform', String(24), index=True, nullable=True)
+ lang = Column('lang', String(24), index=True)
+ locale = Column('locale', String(24), index=True, nullable=True)
+ start_time = Column('start_time', Integer, index=True)
+ end_time = Column('end_time', Integer, index=True, nullable=True)
+ idle_time = Column('idle_time', Integer, index=True, nullable=True)
+ note = Column('note', Text)
+ dest_url = Column('dest_url', Text)
+ author = Column('author', String(255), index=True)
+ created = Column('created', Integer, index=True)
class Storage(StorageBase):
@@ -19,23 +37,6 @@ def __init__(self, config, **kw):
try:
super(Storage, self).__init__(config, **kw)
self.metadata = MetaData()
- self.campaigns = Table(self.__tablename__, self.metadata,
- Column('id', String(25), primary_key=True),
- Column('channel', String(24), index=True, nullable=True),
- Column('version', Float, index=True, nullable=True),
- Column('platform', String(24), index=True, nullable=True),
- Column('lang', String(24), index=True),
- Column('locale', String(24), index=True, nullable=True),
- Column('start_time', Integer, index=True),
- Column('end_time', Integer, index=True, nullable=True),
- Column('idle_time', Integer, index=True, nullable=True),
- Column('note', Text),
- Column('author', String(255), index=True),
- Column('created', Integer, index=True))
- self.authors = Table('authors', self.metadata,
- Column('id', Integer, autoincrement=True, primary_key=True),
- Column('user', String(255), index=True),
- Column('name', String(255)))
self._connect()
#TODO: add the most common index.
except Exception, e:
@@ -51,81 +52,112 @@ def _connect(self):
settings.get('mysql.host', 'localhost'),
settings.get('msyql.db', self.__database__))
self.engine = create_engine(dsn, pool_recycle=3600)
- self.metadata.create_all(self.engine)
+ Base.metadata.create_all(self.engine)
+ #self.metadata.create_all(self.engine)
except Exception, e:
logging.error('Could not connect to db "%s"' % repr(e))
raise e
- def put_announce(self, note, author, created,
- version=None, channel=None, platform=None,
- lang='en', locale='us',
- idle_time=0, start_time=None, end_time=None, **kw):
+
+ def parse_date(self, datestr):
+ if datestr is None:
+ return None
+ try:
+ return float(datestr)
+ except ValueError:
+ pass
+ try:
+ return float (parser.parse(datestr).strftime('%s'))
+ except ValueError:
+ pass
+ return None
+
+ def normalize_announce(self, data):
now = time()
+ data['start_time'] = self.parse_date(data.get('start_time', now))
+ data['end_time'] = self.parse_date(data.get('end_time'))
+ if data.get('channel') == 'all':
+ data['channel'] = None
+ if data.get('platform') == 'all':
+ data['platform'] = None
+ if not data.get('version'):
+ data['version'] = None
snip = {
- 'channel': channel,
- 'version': version,
- 'platform': platform,
- 'idle': idle_time,
- 'lang': lang,
- 'locale': locale,
- 'note': note,
- 'start_time': start_time if start_time else now,
- 'end_time': end_time,
- 'author': author,
- 'created': created if created else now,
+ 'id': data.get('id'),
+ 'channel': data.get('channel'),
+ 'version': data.get('version'),
+ 'platform': data.get('platform'),
+ 'idle_time': data.get('idle_time', 0),
+ 'lang': data.get('lang'),
+ 'locale': data.get('locale'),
+ 'note': json.dumps({
+ 'title': data.get('title'),
+ 'text': data.get('note')
+ }),
+ 'dest_url': data.get('dest_url'),
+ 'start_time': data.get('start_time', now),
+ 'end_time': data.get('end_time'),
+ 'author': data.get('author'),
+ 'created': data.get('created', now),
}
- self.insert(snip)
- query = ("insert into %s.%s " % (self.__database__,
- self.__tablename__) +
- "(start_time, end_time, idle_time, "
- "lang, locale, channel, platform, version, "
- "created, author, note) "
- "values (:start_time, :end_time, :idle_time, "
- ":lang, :locale, :channel, :platform, :version, "
- ":created, :author, :note);")
- self.engine.execute(text(query), **dict(snip))
+ if snip.get('id') is None:
+ snip['id'] = self._gen_key(snip)
+ return snip
+
+
+ def put_announce(self, data):
+ if data.get('note') is None:
+ raise StorageException('Nothing to do.')
+ snip = self.normalize_announce(data)
+ session = scoped_session(sessionmaker(bind=self.engine))()
+ import pdb; pdb.set_trace()
+ campaign = Campaign(**snip)
+ session.add(campaign)
+ session.commit()
return self
- def get_announce(self, channel=None, platform=None, version=None,
- lang='en', locale=None,
- start_time=None, end_time=None, author=None, created=None,
- idle_time=0, last_accessed=None, **kw):
+ def get_announce(self, data):
# Really shouldn't allow "global" variables, but I know full well
# that they're going to want them.
params = {}
sql =("select id, note from %s.%s where " % (self.__database__,
self.__tablename__) +
" start_time < unix_timestamp() and end_time > unix_timestamp() ")
- if last_accessed:
+ if data.get('last_accessed'):
sql += "and created > :last_accessed "
- params['last_accessed'] = int(last_accessed)
- if platform:
+ params['last_accessed'] = int(data.get('last_accessed'))
+ if data.get('platform'):
sql += "and coalesce(platform, :platform) = :platform "
- params['platform'] = platform
- if channel:
+ params['platform'] = data.get('platform')
+ if data.get('channel'):
sql += "and coalesce(channel, :channel) = :channel "
- params['channel'] = channel
- if lang:
+ params['channel'] = data.get('channel')
+ if data.get('lang'):
sql += "and coalesce(lang, :lang) = :lang "
- params['lang'] = lang
- if locale:
+ params['lang'] = data.get('lang')
+ if data.get('locale'):
sql += "and coalesce(locale, :locale) = :locale "
- params['locale'] = locale
- if idle_time:
+ params['locale'] = data.get('locale')
+ if data.get('idle_time'):
sql += "and coalesce(idle_time, :idle_time) = :idle_time "
- params['idle_time'] = idle_time
+ params['idle_time'] = data.get('idle_time')
sql += " order by id"
items = self.engine.execute(text(sql), **dict(params))
result = []
for item in items:
note = json.loads(item.note)
- note.update({'id': item.id})
+ note.update({'id': item.id,
+ 'url':
+ self.url_pattern % (
+ self.config.get('redir_host', 'localhost'),
+ self.config.get('redir_path', '/'),
+ item.id)})
result.append(note)
return result
def get_all_announce(self, limit=None):
result = []
- sql = 'select * from %s.%s ' % (self.__database__,
+ sql = 'select * from %s.%s order by created desc ' % (self.__database__,
self.__tablename__)
if limit:
sql += 'limit %d' % limit
Oops, something went wrong.

0 comments on commit 9badc17

Please sign in to comment.