Skip to content
Browse files

Initial go at GA reporter tool

  • Loading branch information...
1 parent 1a9a2bd commit 285bf706c1fbdd60e9df82a2fca8fda9378a3da6 @ebidel ebidel committed May 8, 2012
Showing with 215 additions and 32 deletions.
  1. +30 −20 metrics/api.py
  2. +4 −0 metrics/app.yaml
  3. +0 −1 metrics/index.yaml
  4. +7 −9 metrics/models.py
  5. +75 −0 metrics/reporter.py
  6. +18 −2 metrics/settings.py
  7. +81 −0 metrics/shardedcounter.py
View
50 metrics/api.py
@@ -9,6 +9,7 @@
import simplejson
import webapp2
+import models
import settings
@@ -22,33 +23,42 @@ def wrapped(self, *args, **kwargs):
return fn(self, *args, **kwargs)
return wrapped
- @__add_headers
- def install(self, post_body=None):
- if post_body is not None:
- self.response.out.write(post_body)
- else:
- json = {
- 'rpc': 'install'
- }
- self.response.out.write(simplejson.dumps(json))
+ # @__add_headers
+ # def install(self, post_body=None):
+ # if post_body is not None:
+ # self.response.out.write(post_body)
+ # else:
+ # json = {
+ # 'rpc': 'install'
+ # }
- @__add_headers
- def cmd(self, post_body=None):
- if post_body is not None:
- self.response.out.write(post_body)
- else:
- json = {
- 'rpc': 'cmd'
- }
- self.response.out.write(simplejson.dumps(json))
+ # @__add_headers
+ # def cmd(self, post_body=None):
+ # if post_body is not None:
+ # self.response.out.write(post_body)
+ # else:
+ # json = {
+ # 'rpc': 'cmd'
+ # }
+ @__add_headers
def get(self, method):
+ q = models.Report.all().filter('cmd', method)
if method == 'install':
- return self.install()
+ results = q.order("-height").fetch(limit=settings.MAX_FETCH_LIMIT)
+ #self.install()
elif method == 'cmd':
- return self.cmd()
+ #self.cmd()
+
+ self.response.out.write(simplejson.dumps(json))
def post(self, method):
+ # m = models.Message(
+ # cmd='init',
+ # version=1.0
+ # )
+ # m.put()
+
#TOOO: figure out how to verify this post request came from the yeoman cli.
if method == 'install':
return self.install(self.request.body)
View
4 metrics/app.yaml
@@ -24,6 +24,10 @@ handlers:
- url: /static
static_dir: static
+- url: /admin/.*
+ script: google.appengine.ext.admin.application
+ login: admin
+
- url: /api.*
script: api.app
View
1 metrics/index.yaml
@@ -9,4 +9,3 @@ indexes:
# manually, move them above the marker line. The index.yaml file is
# automatically uploaded to the admin console when you next deploy
# your application using appcfg.py.
-
View
16 metrics/models.py
@@ -8,18 +8,16 @@
#from google.appengine.api import memcache
from google.appengine.ext import db
+import shardedcounter
-class Command(db.Model):
- """Model for commands."""
- class Type(object):
- INSTALL = 1
- SCAFFOLD = 2
- MODEL = 3
-
- type = db.IntegerProperty(required=True)
- fetch_date = db.DateTimeProperty(auto_now=True, auto_now_add=True)
+class Report(db.Model):
+ """Model for report messages received by the API endpoints."""
+ cmd = db.StringProperty(required=True)
+ sub_cmd = db.StringProperty()
+ received_date = db.DateTimeProperty(auto_now_add=True)
+ version = db.FloatProperty(required=True)
# class Resource(db.Model):
# """Model for content resources."""
View
75 metrics/reporter.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+
+"""This module defines the stat reporter tool."""
+
+__author__ = 'ebidel@gmail.com (Eric Bidelman)'
+
+
+import random
+import time
+import urllib
+import urllib2
+
+import settings
+
+
+TRACKING_CODE = 'UA-31537568-1'
+
+
+class Analytics(object):
+
+ BASE_URL = 'http://www.google-analytics.com/collect/__utm.gif'
+
+ def __init__(self, tracking_code):
+ self.tracking_code = tracking_code
+
+ def send(self, path='/', recorded_at=None):
+ """Sends data to Google Analytics.
+
+ Args:
+ path: A string representing the url path of the pageview to record.
+ URL query parameters may be included. The format should map to the
+ the command that was issued:
+ yeoman init -> /init
+ yeoman add model -> /add/model
+ recorded_at: When the hit was recorded in seconds since the epoch.
+ If absent, now is used.
+ """
+ recorded_at = recorded_at or time.time()
+
+ params = {
+ 'v': '1', # GA API tracking version.
+ 'tid': self.tracking_code, # Tracking code ID.
+ 't': 'pageview', # Event type
+ 'cid': '%s%s' % (time.time(), random.random()), # Client ID
+ 'aip': '1', # Anonymize IP
+ 'qt': int((time.time() - recorded_at) * 1e3), # Queue Time. Delta (milliseconds) between now and when hit was recorded.
+ #'dt': ,
+ 'p': path, #urllib.quote_plus(path)
+ 'an': settings.APP['title'], # Application Name.
+ 'av': settings.APP['version'], # Application Version.
+ 'z': time.time() # Cache bust. Probably don't need, but be safe. Should be last param.
+ #'sc': ??,
+ #'cm*': ??,
+ #'cd*': ??,
+ }
+
+ #time.ctime(seconds) -> 'Tue May 8 20:37:35 2012'
+
+ encoded_params = urllib.urlencode(params)
+
+ url = '%s?%s' % (self.BASE_URL, encoded_params)
+ print url
+ #response = urllib2.urlopen(url)
+ #print response.code
+ #print response.read()
+
+
+def main():
+ ga = Analytics(TRACKING_CODE)
+ ga.send('/test/model') # Test his recorded now.
+ #ga.send('/add/model', recorded_at=time.time() - 120) # Test hit recorded 2 minutes ago.
+
+
+if __name__ == '__main__':
+ main()
View
20 metrics/settings.py
@@ -1,5 +1,18 @@
import os
+# Get version of Yeoman file VERSION file, otherwise use GAE app.yaml version
+# for this app.
+# TODO(ebidel): The version should be the same everywhere.
+def get_app_version():
+ version_str = []
+ try:
+ with open(os.path.join(os.path.dirname(__file__), '..', 'VERSION')) as f:
+ for line in f:
+ version_str.append(line.split('=')[1].split('\n')[0])
+ return '.'.join(version_str)
+ except:
+ return os.environ['CURRENT_VERSION_ID'].split('.')[0]
+
# Hack to get custom tags working django 1.3 + python27.
#INSTALLED_APPS = (
# 'nothing',
@@ -22,6 +35,9 @@
TEMPLATE_DEBUG = DEBUG
APP = {
- 'title': 'Yeoman Stats',
- 'version': os.environ['CURRENT_VERSION_ID'].split('.')[0]
+ 'title': 'Yeoman Insight',
+ 'version': get_app_version()
}
+
+
+#MAX_FETCH_LIMIT = 1000
View
81 metrics/shardedcounter.py
@@ -0,0 +1,81 @@
+# Copyright 2008 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from google.appengine.api import memcache
+from google.appengine.ext import db
+import random
+
+class GeneralCounterShardConfig(db.Model):
+ """Tracks the number of shards for each named counter."""
+ name = db.StringProperty(required=True)
+ num_shards = db.IntegerProperty(required=True, default=20)
+
+
+class GeneralCounterShard(db.Model):
+ """Shards for each named counter"""
+ name = db.StringProperty(required=True)
+ count = db.IntegerProperty(required=True, default=0)
+
+
+def get_count(name):
+ """Retrieve the value for a given sharded counter.
+
+ Parameters:
+ name - The name of the counter
+ """
+ total = memcache.get(name)
+ if total is None:
+ total = 0
+ for counter in GeneralCounterShard.all().filter('name = ', name):
+ total += counter.count
+ memcache.add(name, total, 60)
+ return total
+
+
+def increment(name):
+ """Increment the value for a given sharded counter.
+
+ Parameters:
+ name - The name of the counter
+ """
+ config = GeneralCounterShardConfig.get_or_insert(name, name=name)
+ def txn():
+ index = random.randint(0, config.num_shards - 1)
+ shard_name = name + str(index)
+ counter = GeneralCounterShard.get_by_key_name(shard_name)
+ if counter is None:
+ counter = GeneralCounterShard(key_name=shard_name, name=name)
+ counter.count += 1
+ counter.put()
+ db.run_in_transaction(txn)
+ # does nothing if the key does not exist
+ memcache.incr(name)
+
+
+def increase_shards(name, num):
+ """Increase the number of shards for a given sharded counter.
+ Will never decrease the number of shards.
+
+ Parameters:
+ name - The name of the counter
+ num - How many shards to use
+
+ """
+ config = GeneralCounterShardConfig.get_or_insert(name, name=name)
+ def txn():
+ if config.num_shards < num:
+ config.num_shards = num
+ config.put()
+ db.run_in_transaction(txn)

0 comments on commit 285bf70

Please sign in to comment.
Something went wrong with that request. Please try again.