Added a stats system in the background that will silently collect usa…

…ge statistics. fixes #3. fixes #4. fixes #8. Laid the groundwork for #5 and #6. #10 is implemented, but untested.

app.yaml

@@ -8,5 +8,9 @@ handlers:
   static_dir: static
 - url: /connected
   script: devlinks.py
+- url: /stats/.*
+  script: devlinks.py
+- url: /_ah/prospective_search
+  script: devlinks.py
 - url: /.*
   script: devlinks.py

channels.py

@@ -1,7 +1,7 @@
 from django.utils import simplejson
 from google.appengine.api import channel, memcache
 
-import models
+import models, stats
 
 class Channel():
   token = None
@@ -19,6 +19,7 @@ def __init__(self, address, generate=True):
          self.token = device.token
        else:
          self.token = channel.create_channel(self.address)
+         stats.record("channel_created", self.address)
          self.cached = False
          device.updateToken(self.token)
        memcache.set("token_%s" % self.address, self.token, time=7200)

devlinks.py

@@ -1,12 +1,15 @@
 #!/usr/bin/python2.4
 
 import os
-from google.appengine.ext import webapp
+from django.utils import simplejson
+from google.appengine.ext import webapp, db
 from google.appengine.ext.webapp import template
 from google.appengine.ext.webapp.util import run_wsgi_app
-from google.appengine.api import users
+from google.appengine.api import users, prospective_search
 
-import auth, models, channels
+import auth, models, channels, stats
+
+import logging
 
 class ConnectedPage(webapp.RequestHandler):
 
@@ -27,6 +30,7 @@ def post(self, name="Chrome"):
       for link in last_links:
         channel.queueLink(link)
       channel.send()
+      stats.record("user_connected", user.email())
       self.response.out.write(device.address)
 
 class MainPage(webapp.RequestHandler):
@@ -123,6 +127,40 @@ def post(self, json=False):
         link_data = models.getLink(link)
         link_data.markRead()
 
+class SubscribeHandler(webapp.RequestHandler):
+  def get(self):
+    path = os.path.join(os.path.dirname(__file__), 'subscribe.html')
+    self.response.out.write(template.render(path, {}))
+
+  def post(self):
+    event = self.request.POST['event']
+    datapoint = self.request.POST['datapoint']
+    subscription_id = models.StatsSubscription(event=event, datapoint=datapoint).put()
+    prospective_search.subscribe(stats.StatsRecord, 'event:%s' % event, subscription_id)
+    self.response.out.write("Subscribed the datapoint %s to %s events." % (datapoint, event))
+
+class StatsHandler(webapp.RequestHandler):
+  def post(self):
+    record = prospective_search.get_document(self.request)
+    subscriber_keys = map(db.Key, self.request.get_all('id'))
+    subscribers = db.get(subscriber_keys)
+    datapoints = []
+    stats_json = []
+    for subscriber_key, subscriber in zip(subscriber_keys, subscribers):
+      if not subscriber:
+        prospective_search.unsubscribe(stats.StatsRecord, subscriber_key)
+      else:
+        datapoints.append(models.getStats(subscriber.datapoint, record.timestamp.date()))
+    for datapoint in datapoints:
+      datapoint.increment()
+      day = record.timestamp.date()
+      date = "%s/%s/%s" % (day.month, day.day, day.year)
+      json = {'datapoint': datapoint.datapoint, 'count': datapoint.count, 'date': date}
+      stats_json.append(json)
+    db.put(datapoints)
+    logging.debug(simplejson.dumps(stats_json))
+
+
 application = webapp.WSGIApplication([
     ('/', MainPage),
     ('/links/add', AddLinkPage),
@@ -131,6 +169,8 @@ def post(self, json=False):
     ('/channels/connected/([^/]*)', ConnectedPage),
     ('/channels/get', TokenPage),
     ('/channels/get/(.*)', TokenPage),
+    ('/stats/subscribe', SubscribeHandler),
+    ('/_ah/prospective_search', StatsHandler)
     ], debug=True)
 
 

models.py

@@ -1,8 +1,10 @@
 from google.appengine.ext import db
 from google.appengine.api import memcache
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, date
+from datetime import date as datetime_date
+from django.utils import simplejson
 
-import logging
+import stats
 
 class UserDoesNotExistError(Exception):
   account = None
@@ -23,6 +25,10 @@ def updateLastSeen(self):
     self.last_seen = datetime.now()
 
   def save(self):
+    try:
+      self.key()
+    except db.NotSavedError:
+      stats.record("user_added", self.user.email())
     self.put()
     memcache.set("user_%s_data" % self.user.user_id(), self)
     return self
@@ -51,6 +57,10 @@ class DeviceData(db.Model):
   token_expiration = db.DateTimeProperty()
 
   def save(self):
+    try:
+      self.key()
+    except db.NotSavedError:
+      stats.record("device_added", self.address)
     if self.address == None:
       self.address = "%s/%s" % (self.user.user.email(), self.name)
     self.put()
@@ -88,13 +98,36 @@ class LinkData(db.Model):
   received = db.BooleanProperty(default=False)
 
   def save(self):
+    try:
+      self.key()
+    except db.NotSavedError:
+      stats.record("link_added", self.sender.user.user.email())
     self.put()
     return self
 
   def markRead(self):
     self.received = True
+    stats.record("link_opened", self.sender.user.user.email())
     self.save()
 
+class StatsData(db.Model):
+  datapoint = db.StringProperty()
+  count = db.IntegerProperty()
+  date = db.DateProperty()
+
+  def save(self):
+    self.put()
+    memcache.set("stats_%s_%s" % (self.datapoint, self.date), self)
+    return self
+
+  def increment(self):
+    self.count = self.count + 1
+    memcache.set("stats_%s_%s" % (self.datapoint, self.date), self)
+
+class StatsSubscription(db.Model):
+  event = db.StringProperty()
+  datapoint = db.StringProperty()
+
 def getUser(account):
   user = memcache.get("user_%s_data", account.user_id())
   if user == None:
@@ -127,3 +160,16 @@ def getUnreadLinks(device, count=1000):
 
 def getLinksByAccount(user, count=1000):
   return LinkData.all().filter("receiver IN", user.devices).order("-date").fetch(count)
+
+def getStats(datapoint, date=False):
+  if not date:
+    date = datetime_date.today()
+  stats = memcache.get("stats_%s_%s" % (datapoint, date))
+  if stats == None:
+    stats = StatsData.all().filter("datapoint =", datapoint).filter("date =", date).get()
+    if stats == None:
+      stats = StatsData(datapoint=datapoint, date=date, count=0)
+      stats.put()
+    else:
+      memcache.set("stats_%s_%s" % (datapoint, date), stats)
+  return stats

queue.yaml

@@ -0,0 +1,3 @@
+queue:
+- name: stats
+  rate: 10/s

stats.py

@@ -0,0 +1,18 @@
+from google.appengine.api import prospective_search
+from google.appengine.ext import db
+from datetime import datetime
+
+import logging
+
+class StatsRecord(db.Model):
+  event = db.StringProperty()
+  timestamp = db.DateTimeProperty()
+  value = db.StringProperty()
+
+def record(key, value, timestamp=False):
+  record = StatsRecord(event=key, value=value)
+  if not timestamp:
+    timestamp = datetime.now()
+  record.timestamp = timestamp
+  logging.info("Firing stats off. Event: %s" % key)
+  prospective_search.match(record, result_task_queue='stats')

subscribe.html

@@ -0,0 +1,12 @@
+<html>
+  <head>
+    <title>Add Subscription</title>
+  </head>
+  <body>
+    <form method="POST">
+      Event: <input type="text" name="event" /><br />
+      Datapoint: <input type="text" name="datapoint" /><br />
+      <input type="submit" value="Add Subscription" />
+    </form>
+  </body>
+</html>