Permalink
Browse files

Version 0.5

    * Made 'put' exception text clearer
    * changed refs to 'snippets' to 'announcements'
    * changed return exceptions to raise.
    * Added manifest file to include templates
    * Added JSON responses
    * Cleaned up/simpified login/auth routines.
    * Added clickable IDs
    * Cleaned up some variable names (more consistent)
    * Added manifest file to include templates
    * Added JSON responses
    * Cleaned up/simpified login/auth routines.
    * Added clickable IDs
    * Added more tests for API functions
    * Cleaned up some variable names (more consistent)
    * Fix to use setup.py develop to resolve missing templates
      (I owe rfkelly several of his favorite beverages.)
    * Added path to health check
    * Updated Makefile to add run and install
    * Adding some verbosity to solve remote install issue
    * Added logger calls
    * Added config options to dump stack on exceptions, drop to debugger.
    * Added quick self-verifier
    * Added sql health check routine
    * Changed "fetchall*" to more consistent "get_all"
    * Added redirect for "/" urls => "/author/"
    * Normalized loc & lang to lowercase in db
    * Fixed bad JSON in default addresses
    * Added unit tests
    * removed extra __init__
    * added nose to reqs.
  • Loading branch information...
1 parent f23a65b commit 104b175152e748140555fc3be83198265626098d @jrconlin committed Oct 26, 2012
View
@@ -0,0 +1,5 @@
+include README.rst
+include MANIFEST.in
+include campaign.ini
+recursive-include campaign/templates *.mako *.css
+
View
@@ -1,16 +1,21 @@
-APPNAME = geoip
+APPNAME = campaign
VE = virtualenv
PY = bin/python
PI = bin/pip
NO = bin/nosetests -s --with-xunit
+PS = bin/pserve
all: build
build:
$(VE) --no-site-packages .
+ bin/easy_install -U distribute
$(PI) install -r prod-reqs.txt
- $(PY) setup.py build
+ $(PY) setup.py develop
test:
$(NO) $(APPNAME)
+run:
+ $(PS) campaign-local.ini
+
View
@@ -19,6 +19,10 @@ db.db = /tmp/campaigns.sqlite
#db.password = snip
#db.db = campaign
+#dbg.traceback = False
+#dbg.break_unknown_exception = False
+#dbg.self_diag = False
+
beaker.session.cache_dir = %(here)s/data
beaker.session.key = campaign
beaker.session.secret = Secret.
View
@@ -3,13 +3,17 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""Main entry point
"""
+import logging
+
from pyramid.config import Configurator
from metlog.config import client_from_stream_config
from campaign.resources import Root
from campaign.storage.sql import Storage
from mozsvc.config import load_into_settings
from mozsvc.middlewares import _resolve_name
+logger = logging.getLogger('campaign')
+
def get_group(group_name, dictionary):
if group_name is None:
@@ -29,6 +33,27 @@ def configure_from_settings(object_name, settings):
cls = _resolve_name(config.pop('backend'))
return cls(**config)
+def self_diag(config):
+ import warnings
+ import sys
+ import os
+ bad = False
+ if sys.version_info[:3] < (2,5,0) or sys.version_info[:3] > (3,0,0):
+ warnings.warn('Please run this code under version '
+ '2.6 or 2.7 of python.');
+ bad |= True
+ templatePath = os.path.join(os.path.dirname(__file__), 'templates',
+ 'login.mako')
+ if not os.path.exists(templatePath):
+ warnings.warn(('Could not find required template. %s\n Your install ' %
+ templatePath) +
+ 'may be corrupt. Please reinstall.');
+ bad |= True
+ if not config.registry['storage'].health_check():
+ warnings.warn('Storage reported an error. Please check settings.');
+ bad |= True
+
+
def main(global_config, **settings):
load_into_settings(global_config['__file__'], settings)
@@ -44,6 +69,8 @@ def main(global_config, **settings):
open(global_config['__file__'], 'r'),
'metlog')
config.registry['metlog'] = metlog_client
+ if settings.get('dbg.self_diag', False):
+ self_diag(config)
return config.make_wsgi_app()
@@ -81,6 +81,10 @@ def normalize_announce(self, data):
# customize for each memory model
+ def health_check(self):
+ # Is the current memory model working?
+ return False;
+
def del_announce(self, keys):
pass
View
@@ -17,7 +17,7 @@ class Campaign(Base):
channel = Column('channel', String(24), index=True, nullable=True)
version = Column('version', Float, index=True, nullable=True)
platform = Column('platform', String(24), index=True, nullable=True)
- lang = Column('lang', String(24), index=True)
+ lang = Column('lang', String(24), index=True, nullable=True)
locale = Column('locale', String(24), index=True, nullable=True)
start_time = Column('start_time', Integer, index=True)
end_time = Column('end_time', Integer, index=True, nullable=True)
@@ -64,20 +64,42 @@ def _connect(self):
logging.error('Could not connect to db "%s"' % repr(e))
raise e
+ def health_check(self):
+ try:
+ healthy = True
+ with self.engine.begin() as conn:
+ conn.execute(("insert into %s (id, channel, platform, " %
+ self.__tablename__) +
+ "start_time, end_time, note, dest_url, author, created) " +
+ "values ('test', 'test', 'test', 0, 0, 'test', 'test', " +
+ "'test', 0)")
+ resp = conn.execute(("select id, note from %s where " %
+ self.__tablename__) + "id='test';")
+ if resp.rowcount == 0:
+ healthy = False
+ conn.execute("delete from %s where id='test';" %
+ self.__tablename__)
+ except Exception, e:
+ import warnings
+ warnings.warn(str(e))
+ return False
+ return healthy
+
def resolve(self, token):
if token is None:
return None
sql = 'select * from campaigns where id = :id'
items = self.engine.execute(text(sql), {'id': token})
- if items.rowcount == 0:
+ row = items.fetchone()
+ if items.rowcount == 0 or row is None:
return None
- result = dict(zip(items.keys(), items.fetchone()))
+ result = dict(zip(items.keys(), row))
return result
def put_announce(self, data):
if data.get('note') is None:
- raise StorageException('Nothing to do.')
+ raise StorageException('Incomplete record. Skipping.')
snip = self.normalize_announce(data)
campaign = Campaign(**snip)
self.session.add(campaign)
@@ -89,10 +111,19 @@ def get_announce(self, data):
# that they're going to want them.
params = {}
settings = self.config.get_settings()
- now = int(time.time())
+ # The window allows the db to cache the query for the length of the
+ # window. This is because the database won't cache a query if it
+ # differs from a previous one. The timestamp will cause the query to
+ # not be cached.
+ window = int(settings.get('db.query_window', 1))
+ if window == 0:
+ window = 1
+ now = int(time.time() / window )
sql =("select id, note from %s where " % self.__tablename__ +
- " coalesce(start_time, %s) < %s " % (now-1, now) +
- "and coalesce(end_time, %s) > %s " % (now+1, now))
+ " coalesce(round(start_time / %s), %s) < %s " % (window,
+ now-1, now) +
+ "and coalesce(round(end_time / %s), %s) > %s " % (window,
+ now+1, now))
if data.get('last_accessed'):
sql += "and created > :last_accessed "
params['last_accessed'] = int(data.get('last_accessed'))
@@ -108,10 +139,14 @@ def get_announce(self, data):
if data.get('locale'):
sql += "and coalesce(locale, :locale) = :locale "
params['locale'] = data.get('locale')
- if data.get('idle_time'):
- sql += "and coalesce(idle_time, :idle_time) = :idle_time "
- params['idle_time'] = data.get('idle_time')
+ if not data.get('idle_time'):
+ data['idle_time'] = 0
+ sql += "and coalesce(idle_time, 0) <= :idle_time "
+ params['idle_time'] = data.get('idle_time')
sql += " order by id"
+ if (settings.get('dbg.show_query', False)):
+ print sql;
+ print params;
items = self.engine.execute(text(sql), **dict(params))
result = []
for item in items:
@@ -139,5 +174,11 @@ def del_announce(self, keys):
#TODO: how do you safely do an "in (keys)" call?
sql = 'delete from %s where id = :key' % self.__tablename__
for key in keys:
- self.engine.execute(text(sql), {"key": key});
+ self.engine.execute(text(sql), {"key": key})
self.session.commit()
+
+ def purge(self):
+ sql = 'delete from %s;' % self.__tablename__
+ self.engine.execute(text(sql))
+ self.session.commit()
+
@@ -11,8 +11,9 @@
<head>
<title>Please log in</title>
<link rel="stylesheet" type="text/css" href="/style.css" />
+ <meta charset="utf-8" />
</head>
- <body>
+ <body data-test="login">
<hgroup>
<h2>Please Log in</h2>
</hgroup>
@@ -4,7 +4,7 @@
import json
config = pageargs.get('config', {})
- notes = pageargs.get('notes', [])
+ announcements = pageargs.get('announcements', [])
author = pageargs.get('author', 'UNKNOWN')
%>
@@ -86,7 +86,7 @@
time_format = '%Y %b %d - %H:%M:%S UTC'
%>
<!-- Wanna guess what ougth to be done as a rest call? hint: -->
-%for note in notes:
+%for note in announcements:
<%
dnote = dict(note);
if dnote.get('start_time'):
@@ -112,7 +112,7 @@
%>
<div class="record row">
<div class="delete"><input type="checkbox" value="${note.id}"></div>
-<div class="id">${dnote['id']}</div>
+ <div class="id"><a href="/redirect/${dnote['id']}">${dnote['id']}</a></div>
<div class="created">${strftime(time_format, localtime(dnote['created']))}</div>
<div class="start_time">${dnote['start_time']}</div>
<div class="end_time">${dnote['end_time']}</div>
@@ -2,6 +2,12 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-def main(global_config, **settings):
- print 'starting app...'
- pass
+class TConfig:
+
+ def __init__(self, data):
+ self.settings = data
+
+ def get_settings(self):
+ return self.settings
+
+
@@ -0,0 +1,99 @@
+import json
+import time
+import unittest2
+from pprint import pprint
+from campaign.storage.sql import Storage
+from campaign.tests import TConfig
+
+class TestStorage(unittest2.TestCase):
+
+ now = int(time.time())
+
+ test_announce = {
+ 'start_time': int(now - 300),
+ 'end_time': int(now + 3000),
+ 'lang': 'en',
+ 'locale': 'US',
+ 'note': 'Text Body',
+ 'title': 'Test',
+ 'dest_url': 'http://example.com'
+ }
+
+ def setUp(self):
+ self.storage = Storage(config = TConfig({'db.type': 'sqlite',
+ 'db.db': ':memory:'}))
+
+ def tearDown(self):
+ self.storage.purge()
+
+ def test_announcement(self):
+ self.storage.put_announce(self.test_announce)
+ items = self.storage.get_all_announce()
+ self.failUnless(len(items) > 0)
+ self.failUnless(self.test_announce['note'] in items[0].note)
+ self.failUnless(self.test_announce['title'] in items[0].note)
+ self.failUnless(self.test_announce['dest_url'] in items[0].dest_url)
+
+ def update_note(self, announce, note_text):
+ return announce.copy()
+
+ def test_search(self):
+ """ Yes, this test does a lot of things. That's because I need
+ to exercise the search algo using a lot of records. """
+ # really wish that update allowed chaining.
+ updates = [{'lang':None, 'locale':None, 'title':'Everyone'},
+ {'platform':'a', 'channel':'a', 'title':'p:a;c:a'},
+ {'platform':'b', 'channel':'a', 'title':'p:b;c:a'},
+ {'platform':'a', 'start_time': self.now + 1,
+ 'end_time': self.now + 3, 'title':'notyet'},
+ {'platform':'a', 'end_time': self.now-5, 'title':'tooold'},
+ {'platform':'a', 'idle_time': 10, 'title': 'idle:10'},
+ {'platform':'a', 'channel':'b', 'lang':'a', 'locale':'a',
+ 'idle_time': 10, 'title': 'full_rec'}
+ ]
+ # load the database
+ for update in updates:
+ test = self.test_announce.copy()
+ test.update(update)
+ self.storage.put_announce(test)
+ data = {'platform':'f', 'channel':'f', 'version':0}
+ announce = self.storage.get_announce(data)
+ self.assertEqual(len(announce), 1)
+ self.assertEqual(announce[0]['title'], 'Everyone')
+ data = {'platform':'a', 'channel':'a'}
+ announce = self.storage.get_announce(data)
+ # only Everyone and p:a;c:a should be returned.
+ print "P&C check:"
+ self.assertEqual(len(announce), 2)
+
+ data = {'platform':'a', 'channel':'a', 'idle_time': 15}
+ announce = self.storage.get_announce(data)
+ print "Idle Check:"
+ self.assertEqual(len(announce), 3)
+
+ data = {'platform':'a', 'channel':'b'}
+ announce = self.storage.get_announce(data)
+ print "P&C2 check:"
+ self.assertEqual(len(announce), 1)
+ # Store the unique record data for the resolve check.
+ resolve_rec = announce[0]
+
+ data = {'platform':'a', 'channel':'a'}
+ time.sleep(self.now + 2 - int(time.time()))
+ print "Wake check: %s " % (int(time.time()) - self.now)
+ announce = self.storage.get_announce(data)
+ self.assertEqual(len(announce), 3);
+
+ time.sleep(self.now + 4 - int(time.time()))
+ print "Expire check: %s " % (int(time.time()) - self.now)
+ data = {'platform':'a', 'channel':'a'}
+ announce = self.storage.get_announce(data)
+ self.assertEqual(len(announce), 2);
+
+ # Since we have an ID for a unique record, query it to make
+ # sure records resolve.
+ print "resolve check: %s" % resolve_rec['id']
+ rec = self.storage.resolve(resolve_rec['id'])
+ self.assertEqual('Everyone', json.loads(rec['note'])['title'])
+
+#TODO: continue tests
Oops, something went wrong.

0 comments on commit 104b175

Please sign in to comment.