add in a metrics lib and a sample post (bug 742785)

mozilla · Apr 11, 2012 · c2b6fd8 · c2b6fd8
1 parent 2fc8d45
commit c2b6fd8
Show file tree

Hide file tree

Showing 5 changed files with 150 additions and 0 deletions.
diff --git a/lib/metrics/__init__.py b/lib/metrics/__init__.py
@@ -0,0 +1,72 @@
+import json
+import urllib2
+import uuid
+
+from django.conf import settings
+
+from celeryutils import task
+import commonware.log
+
+log = commonware.log.getLogger('z.metrics')
+
+
+def send(action, data):
+    """
+    Logs some data and then sends it to the metrics cluster through a
+    delayed celery task.
+    """
+    uid = str(uuid.uuid4())
+    data = json.dumps(data)
+    # This is the most reliable call we can make.
+    log.info(u'%s|%s|%s' % (uid, action, data))
+    # Do this async and if it fails we can re-run it again from the log.
+    metrics.delay(uid, action, data)
+
+
+def send_request(action, request, data):
+    """
+    Passes to send, but pulls what we'd like out of the request
+    before doing so. Use this from Django views.
+    """
+    data['user-agent'] = request.META.get('HTTP_USER_AGENT')
+    send(action, data)
+
+
+@task
+def metrics(uid, action, data, **kw):
+    """
+    Actually sends the data to the server, done async in celery. If celery or
+    the http ping fails, then we can recreate this from the log.
+
+    Returns the status code or False if it failed to run.
+    """
+    destination = settings.METRICS_SERVER
+    # If no destination is set. Just ignore this request.
+    if not destination:
+        return
+
+    timeout = settings.METRICS_SERVER_TIMEOUT
+    namespace = settings.DOMAIN.replace('.', '_') + action
+
+    destination = '%s/%s/%s' % (destination, namespace, uid)
+    headers = {'Content-Type': 'application/json'}
+    request = urllib2.Request(destination, data, headers)
+
+    try:
+        response = urllib2.urlopen(request, timeout=timeout)
+    except urllib2.HTTPError, error:
+        # Will occur when a 3xx or greater code is returned
+        log.error('Posting to metrics failed: %s, uuid: %s'
+                  % (error.code, uid))
+        return error.code
+    except:
+        # Will occur when some other error occurs.
+        log.error('Posting to metrics failed uuid: %s' % uid, exc_info=True)
+        return
+
+    # Catches codes that are 2xx but not 200.
+    if response.status_code != 200:
+        log.error('Posting to metrics failed: %s, uuid: %s'
+                  % (response.status_code, uid))
+
+    return response.status_code
diff --git a/lib/metrics/test.py b/lib/metrics/test.py
@@ -0,0 +1,60 @@
+# -*- coding: utf8 -*-
+import json
+import urlparse
+
+import mock
+from nose.tools import eq_
+
+from django.conf import settings
+from lib.metrics import metrics, send, send_request
+
+import test_utils
+
+
+@mock.patch('lib.metrics.urllib2.urlopen')
+@mock.patch.object(settings, 'METRICS_SERVER', 'http://localhost')
+class TestMetrics(test_utils.TestCase):
+
+    def test_called(self, urlopen):
+        send('install', {})
+        eq_(urlopen.call_args[0][0].data, '{}')
+
+    def test_called_data(self, urlopen):
+        data = {'foo': 'bar'}
+        send('install', data)
+        eq_(urlopen.call_args[0][0].data, json.dumps(data))
+
+    def test_called_url(self, urlopen):
+        send('install', {})
+        url = urlopen.call_args[0][0].get_full_url()
+        eq_(urlparse.urlparse(url)[:2], ('http', 'localhost'))
+
+    def test_some_unicode(self, urlopen):
+        send('install', {'name': u'Вагиф Сәмәдоғлу'})
+
+    def test_send_request(self, urlopen):
+        request = mock.Mock()
+        request.META = {'HTTP_USER_AGENT': 'py'}
+        send_request('install', request, {})
+        eq_(urlopen.call_args[0][0].data, '{"user-agent": "py"}')
+
+    def get_response(self, code):
+        response = mock.Mock()
+        response.status_code = code
+        return response
+
+    def test_error(self, urlopen):
+        urlopen.return_value = self.get_response(403)
+        eq_(metrics('x', 'install', {}), 403)
+
+    def test_good(self, urlopen):
+        urlopen.return_value = self.get_response(200)
+        eq_(metrics('x', 'install', {}), 200)
+
+    def test_other(self, urlopen):
+        urlopen.return_value = self.get_response(206)
+        eq_(metrics('x', 'install', {}), 206)
+
+    def test_uid(self, urlopen):
+        metrics('x', 'install', {})
+        assert urlopen.call_args[0][0].get_full_url().endswith('x')
diff --git a/lib/settings_base.py b/lib/settings_base.py
@@ -1415,3 +1415,9 @@ def read_only_mode(env):
 TOTEM_BINARIES = {'thumbnailer': 'totem-video-thumbnailer',
                   'indexer': 'totem-video-indexer'}
 VIDEO_LIBRARIES = ['lib.video.totem', 'lib.video.ffmpeg']
+
+# This is the metrics REST server for receiving data, should be a URL
+# including the protocol.
+METRICS_SERVER = ''
+# And how long we'll give the server to respond.
+METRICS_SERVER_TIMEOUT = 10
diff --git a/mkt/detail/tests/test_views.py b/mkt/detail/tests/test_views.py
@@ -243,6 +243,14 @@ def test_record_logged_out(self):
         res = self.client.post(self.url)
         eq_(res.status_code, 302)
 
+    @mock.patch('mkt.detail.views.send_request')
+    def test_record_metrics(self, send_request):
+        res = self.client.post(self.url)
+        eq_(res.status_code, 200)
+        eq_(send_request.call_args[0][0], 'install')
+        eq_(send_request.call_args[0][2], {'app-domain': u'cbc.ca',
+                                           'app-id': self.addon.pk})
+
     def test_record_install(self):
         res = self.client.post(self.url)
         eq_(res.status_code, 200)

diff --git a/mkt/detail/views.py b/mkt/detail/views.py
@@ -6,6 +6,7 @@
 from addons.models import Addon
 from addons.decorators import addon_view_factory
 from amo.decorators import json_view, login_required, post_required, write
+from lib.metrics import send_request
 from mkt.webapps.models import Installed
 
 addon_view = addon_view_factory(qs=Addon.objects.valid)
@@ -37,5 +38,8 @@ def record(request, addon):
     if addon.is_webapp():
         installed, c = Installed.objects.safer_get_or_create(addon=addon,
             user=request.amo_user)
+        send_request('install', request, {
+                        'app-domain': addon.domain_from_url(addon.origin),
+                        'app-id': addon.pk})
         return {'addon': addon.pk,
                 'receipt': installed.receipt if installed else ''}