Permalink
Browse files

Code cleanup

* move all analyzer-only views into a separate file with a note on top
* nix the spam dashboard and fold the duplicates report into the
  analyzers dashboard
* rename a bunch of things to make names less long
* move some things in analytics/views.py into analytics/tools.py
* nix all the mobile templates for analytics stuff--it's going to be
  desktop only for now
  • Loading branch information...
1 parent 4812ea7 commit a86d3ce987e287b0865d02a894f2c850b0c78eca @willkg willkg committed Feb 6, 2014
View
249 fjord/analytics/analyzer_views.py
@@ -0,0 +1,249 @@
+# Note: These views are viewable only by people in the analyzers
+# group. They should all have the analyzer_required decorator.
+#
+# Also, because we have this weird thing with new users,
+# they should also have the check_new_user decorator.
+#
+# Thus each view should be structured like this:
+#
+# @check_new_user
+# @analyzer_required
+# def my_view(request):
+# ...
+
+from collections import defaultdict
+from datetime import datetime, timedelta
+
+from elasticutils.contrib.django import es_required_or_50x
+
+from django.shortcuts import render
+
+from fjord.analytics.forms import OccurrencesComparisonForm
+from fjord.base.util import analyzer_required, check_new_user
+from fjord.feedback.models import Product, ResponseMappingType
+
+
+@check_new_user
+@analyzer_required
+def analytics_dashboard(request):
+ """Main page for analytics related things"""
+ template = 'analytics/analyzer/dashboard.html'
+ return render(request, template)
+
+
+@check_new_user
+@analyzer_required
+def analytics_products(request):
+ """Products list view"""
+ template = 'analytics/analyzer/products.html'
+ products = Product.objects.all()
+ return render(request, template, {
+ 'products': products
+ })
+
+
+@check_new_user
+@analyzer_required
+@es_required_or_50x(error_template='analytics/es_down.html')
+def analytics_occurrences(request):
+ template = 'analytics/analyzer/occurrences.html'
+
+ first_facet_bi = None
+ first_params = {}
+ first_facet_total = 0
+
+ second_facet_bi = None
+ second_params = {}
+ second_facet_total = 0
+
+ if 'product' in request.GET:
+ form = OccurrencesComparisonForm(request.GET)
+ if form.is_valid():
+ cleaned = form.cleaned_data
+
+ # First item
+ first_resp_s = (ResponseMappingType.search()
+ .filter(product=cleaned['product'])
+ .filter(locale__startswith='en'))
+
+ first_params['product'] = cleaned['product']
+
+ if cleaned['first_version']:
+ first_resp_s = first_resp_s.filter(
+ version=cleaned['first_version'])
+ first_params['version'] = cleaned['first_version']
+ if cleaned['first_start_date']:
+ first_resp_s = first_resp_s.filter(
+ created__gte=cleaned['first_start_date'])
+ first_params['date_start'] = cleaned['first_start_date']
+ if cleaned['first_end_date']:
+ first_resp_s = first_resp_s.filter(
+ created__lte=cleaned['first_end_date'])
+ first_params['date_end'] = cleaned['first_end_date']
+ if cleaned['first_search_term']:
+ first_resp_s = first_resp_s.query(
+ description__text=cleaned['first_search_term'])
+ first_params['q'] = cleaned['first_search_term']
+
+ if ('date_start' not in first_params
+ and 'date_end' not in first_params):
+
+ # FIXME - If there's no start date, then we want
+ # "everything" so we use a hard-coded 2013-01-01 date
+ # here to hack that.
+ #
+ # Better way might be to change the dashboard to allow
+ # for an "infinite" range, but there's no other use
+ # case for that and the ranges are done in the ui--not
+ # in the backend.
+ first_params['date_start'] = '2013-01-01'
+
+ # Have to do raw because we want a size > 10.
+ first_resp_s = first_resp_s.facet_raw(
+ description_bigrams={
+ 'terms': {
+ 'field': 'description_bigrams',
+ 'size': '30',
+ },
+ 'facet_filter': first_resp_s._build_query()['filter']
+ }
+ )
+ first_resp_s = first_resp_s[0:0]
+
+ first_facet_total = first_resp_s.count()
+ first_facet = first_resp_s.facet_counts()
+
+ first_facet_bi = first_facet['description_bigrams']
+ first_facet_bi = sorted(
+ first_facet_bi, key=lambda item: -item['count'])
+
+ if (cleaned['second_version']
+ or cleaned['second_search_term']
+ or cleaned['second_start_date']):
+
+ second_resp_s = (ResponseMappingType.search()
+ .filter(product=cleaned['product'])
+ .filter(locale__startswith='en'))
+
+ second_params['product'] = cleaned['product']
+
+ if cleaned['second_version']:
+ second_resp_s = second_resp_s.filter(
+ version=cleaned['second_version'])
+ second_params['version'] = cleaned['second_version']
+ if cleaned['second_start_date']:
+ second_resp_s = second_resp_s.filter(
+ created__gte=cleaned['second_start_date'])
+ second_params['date_start'] = cleaned['second_start_date']
+ if cleaned['second_end_date']:
+ second_resp_s = second_resp_s.filter(
+ created__lte=cleaned['second_end_date'])
+ second_params['date_end'] = cleaned['second_end_date']
+ if form.cleaned_data['second_search_term']:
+ second_resp_s = second_resp_s.query(
+ description__text=cleaned['second_search_term'])
+ second_params['q'] = cleaned['second_search_term']
+
+ if ('date_start' not in second_params
+ and 'date_end' not in second_params):
+
+ # FIXME - If there's no start date, then we want
+ # "everything" so we use a hard-coded 2013-01-01 date
+ # here to hack that.
+ #
+ # Better way might be to change the dashboard to allow
+ # for an "infinite" range, but there's no other use
+ # case for that and the ranges are done in the ui--not
+ # in the backend.
+ second_params['date_start'] = '2013-01-01'
+
+ # Have to do raw because we want a size > 10.
+ second_resp_s = second_resp_s.facet_raw(
+ description_bigrams={
+ 'terms': {
+ 'field': 'description_bigrams',
+ 'size': '30',
+ },
+ 'facet_filter': second_resp_s._build_query()['filter']
+ }
+ )
+ second_resp_s = second_resp_s[0:0]
+
+ second_facet_total = second_resp_s.count()
+ second_facet = second_resp_s.facet_counts()
+
+ second_facet_bi = second_facet['description_bigrams']
+ second_facet_bi = sorted(
+ second_facet_bi, key=lambda item: -item['count'])
+
+ permalink = request.build_absolute_uri()
+
+ else:
+ permalink = ''
+ form = OccurrencesComparisonForm()
+
+ # FIXME - We have responses that have no product set. This ignores
+ # those. That's probably the right thing to do for the Occurrences Report
+ # but maybe not.
+ products = [prod for prod in ResponseMappingType.get_products() if prod]
+
+ return render(request, template, {
+ 'permalink': permalink,
+ 'form': form,
+ 'products': products,
+ 'first_facet_bi': first_facet_bi,
+ 'first_params': first_params,
+ 'first_facet_total': first_facet_total,
+ 'first_normalization': round(first_facet_total * 1.0 / 1000, 3),
+ 'second_facet_bi': second_facet_bi,
+ 'second_params': second_params,
+ 'second_facet_total': second_facet_total,
+ 'second_normalization': round(second_facet_total * 1.0 / 1000, 3),
+ 'render_time': datetime.now(),
+ })
+
+
+@check_new_user
+@analyzer_required
+@es_required_or_50x(error_template='analytics/es_down.html')
+def analytics_duplicates(request):
+ """Shows all duplicate descriptions over the last n days"""
+ template = 'analytics/analyzer/duplicates.html'
+
+ n = 14
+
+ responses = (ResponseMappingType.search()
+ .filter(created__gte=datetime.now() - timedelta(days=n))
+ .values_dict('description', 'happy', 'created', 'locale',
+ 'user_agent', 'id')
+ .order_by('created').all())
+
+ total_count = len(responses)
+
+ response_dupes = {}
+ for resp in responses:
+ response_dupes.setdefault(resp['description'], []).append(resp)
+
+ response_dupes = [
+ (key, val) for key, val in response_dupes.items()
+ if len(val) > 1
+ ]
+
+ # convert the dict into a list of tuples sorted by the number of
+ # responses per tuple largest number first
+ response_dupes = sorted(response_dupes, key=lambda item: len(item[1]) * -1)
+
+ # duplicate_count -> count
+ # i.e. "how many responses had 2 duplicates?"
+ summary_counts = defaultdict(int)
+ for desc, responses in response_dupes:
+ summary_counts[len(responses)] = summary_counts[len(responses)] + 1
+ summary_counts = sorted(summary_counts.items(), key=lambda item: item[0])
+
+ return render(request, template, {
+ 'n': 14,
+ 'response_dupes': response_dupes,
+ 'render_time': datetime.now(),
+ 'summary_counts': summary_counts,
+ 'total_count': total_count,
+ })
View
5 ...plates/analytics/analytics_dashboard.html → ...mplates/analytics/analyzer/dashboard.html
@@ -10,11 +10,14 @@
<div class="block">
<h2>Analytics things:</h2>
<p>
- <a href="{{ url('analytics_occurrences_comparison') }}">Occurrences Comparsion Report</a>
+ <a href="{{ url('analytics_occurrences') }}">Occurrences Comparison Report</a>
</p>
<p>
<a href="{{ url('analytics_products') }}">Manage Products</a>
</p>
+ <p>
+ <a href="{{ url('analytics_duplicates') }}">Spam: Duplicates</a>
+ </p>
</div>
{% endblock %}
</div>
View
4 .../templates/analytics/spam_duplicates.html → ...plates/analytics/analyzer/duplicates.html
@@ -1,7 +1,7 @@
-{% extends "analytics/spam_dashboard.html" %}
+{% extends "analytics/analyzer/dashboard.html" %}
{# Note: This is not l10n-ized since it's only available to analyzers for now. #}
-{% block content %}
+{% block content_middle %}
<div class="col full">
<div class="block feedback">
<h2>Spam: Duplicates over the last {{ n }} days ordered by most duplicate-y</h2>
View
4 ...ics/analytics_occurrences_comparison.html → ...lates/analytics/analyzer/occurrences.html
@@ -1,4 +1,4 @@
-{% extends "analytics/analytics_dashboard.html" %}
+{% extends "analytics/analyzer/dashboard.html" %}
{# Note: This is not l10n-ized since it's only available to analyzers for now. #}
{% block content_middle %}
@@ -9,7 +9,7 @@
Report rendered on {{ render_time|datetime('%a %b %d %Y at %H:%M:%S') }}
</p>
- <form id="occurrence-form" method="GET" action="{{ url('analytics_occurrences_comparison') }}">
+ <form id="occurrence-form" method="GET" action="{{ url('analytics_occurrences') }}">
{{ form.non_field_errors() }}
<ul class="container-group">
View
2 ...mplates/analytics/analytics_products.html → ...emplates/analytics/analyzer/products.html
@@ -1,4 +1,4 @@
-{% extends "analytics/analytics_dashboard.html" %}
+{% extends "analytics/analyzer/dashboard.html" %}
{# Note: This is not l10n-ized since it's only available to analyzers for now. #}
{% block content_middle %}
View
7 fjord/analytics/templates/analytics/mobile/analytics_dashboard.html
@@ -1,7 +0,0 @@
-{% extends "mobile/base.html" %}
-
-{% block content %}
- <p>
- Analytics Dashboard is not available on mobile devices, yet.
- </p>
-{% endblock %}
View
7 fjord/analytics/templates/analytics/mobile/analytics_products.html
@@ -1,7 +0,0 @@
-{% extends "mobile/base.html" %}
-
-{% block content %}
- <p>
- Product management is not available on mobile devices, yet.
- </p>
-{% endblock %}
View
7 fjord/analytics/templates/analytics/mobile/spam_dashboard.html
@@ -1,7 +0,0 @@
-{% extends "mobile/base.html" %}
-
-{% block content %}
- <p>
- Spam Dashboard is not available on mobile devices, yet.
- </p>
-{% endblock %}
View
37 fjord/analytics/templates/analytics/spam_dashboard.html
@@ -1,37 +0,0 @@
-{% extends "analytics/dashboard_base.html" %}
-{# Note: This is not l10n-ized since it's only available to analyzers for now. #}
-
-{% block body_id %}sdashboard{% endblock %}
-
-{% block content %}
-
-<div class="col">
-{% block content_leftside %}
- <div class="block">
- <h2>Reports</h2>
- <p>
- <a href="{{ url('spam_duplicates') }}">Spam: Duplicates</a>
- </p>
- </div>
-{% endblock %}
-</div>
-
-<div class="col wide">
-{% block content_middle %}
- <div class="block">
- <h2>
- Welcome Spamfighter {{ displayname(user) }}!
- </h2>
- <p>
- This dashboard holds spam-related reports for Analyzers.
- </p>
- </div>
-{% endblock %}
-</div>
-
-<div class="col">
-{% block content_rightside %}
-{% endblock %}
-</div>
-
-{% endblock %}
View
116 fjord/analytics/tests/test_analyzer_views.py
@@ -0,0 +1,116 @@
+import json
+import logging
+
+from nose.tools import eq_
+from pyelasticsearch.exceptions import Timeout
+from pyquery import PyQuery
+
+from django.contrib.auth.models import Group
+from django.http import QueryDict
+
+from fjord.analytics import views
+from fjord.analytics.tools import counts_to_options, zero_fill
+from fjord.base.tests import TestCase, LocalizingClient, profile, reverse, user
+from fjord.base.util import epoch_milliseconds
+from fjord.feedback.tests import response
+from fjord.search.tests import ElasticTestCase
+
+
+logger = logging.getLogger(__name__)
+
+
+
+
+class TestAnalyticsDashboardView(ElasticTestCase):
+ client_class = LocalizingClient
+
+ def test_permissions(self):
+ # Verifies that only analyzers can see the analytics dashboard
+ # link
+ resp = self.client.get(reverse('dashboard'))
+ eq_(200, resp.status_code)
+ assert 'adashboard' not in resp.content
+
+ # Verifies that only analyzers can see the analytics dashboard
+ resp = self.client.get(reverse('analytics_dashboard'))
+ eq_(403, resp.status_code)
+
+ # Verify analyzers can see analytics dashboard link
+ jane = user(email='jane@example.com', save=True)
+ profile(user=jane, save=True)
+ jane.groups.add(Group.objects.get(name='analyzers'))
+
+ self.client_login_user(jane)
+ resp = self.client.get(reverse('dashboard'))
+ eq_(200, resp.status_code)
+ assert 'adashboard' in resp.content
+
+ # Verify analyzers can see analytics dashboard
+ resp = self.client.get(reverse('analytics_dashboard'))
+ eq_(200, resp.status_code)
+
+
+class TestOccurrencesView(ElasticTestCase):
+ client_class = LocalizingClient
+
+ def setUp(self):
+ super(TestOccurrencesView, self).setUp()
+ # Set up some sample data
+ items = [
+ # happy, locale, description
+ (True, 'en-US', 'apple banana orange pear'),
+ (True, 'en-US', 'orange pear kiwi'),
+ (True, 'en-US', 'chocolate chocolate yum'),
+ (False, 'en-US', 'apple banana grapefruit'),
+
+ # This one doesn't create bigrams because there isn't enough words
+ (False, 'en-US', 'orange'),
+
+ # This one shouldn't show up
+ (False, 'es', 'apple banana'),
+ ]
+ for happy, locale, description in items:
+ response(
+ happy=happy, locale=locale, description=description, save=True)
+
+ self.refresh()
+
+ # Create analyzer and log analyzer in
+ jane = user(email='jane@example.com', save=True)
+ profile(user=jane, save=True)
+ jane.groups.add(Group.objects.get(name='analyzers'))
+
+ self.client_login_user(jane)
+
+ def test_occurrences(self):
+ url = reverse('analytics_occurrences')
+
+ # No results when you initially look at the page
+ resp = self.client.get(url)
+ eq_(200, resp.status_code)
+ assert 'id="results"' not in resp.content
+
+ # 'product' is a required field
+ resp = self.client.get(url, {'product': ''})
+ eq_(200, resp.status_code)
+ # FIXME - this test is too loose
+ assert 'This field is required' in resp.content
+
+ # At least a version, search term or start date is required
+ resp = self.client.get(url, {'product': 'Firefox'})
+ eq_(200, resp.status_code)
+ assert 'This field is required' not in resp.content
+ assert 'Must specify at least one' in resp.content
+
+ # Minimal required for results
+ resp = self.client.get(url, {
+ 'product': 'Firefox',
+ 'first_version': '17.0.0'}
+ )
+ eq_(200, resp.status_code)
+ assert 'This field is required' not in resp.content
+ assert 'Must speicfy at least one' not in resp.content
+ assert 'id="results"' in resp.content
+
+ # FIXME - when things are less prototypy, add tests for
+ # specific results
View
90 fjord/analytics/tests/test_tools.py
@@ -1,8 +1,15 @@
+from datetime import datetime
from unittest import TestCase
from nose.tools import eq_
-from fjord.analytics.tools import generate_query_parsed, to_tokens, unescape
+from fjord.analytics.tools import (
+ counts_to_options,
+ generate_query_parsed,
+ to_tokens,
+ unescape,
+ zero_fill)
+from fjord.base.util import epoch_milliseconds
class TestQueryParsed(TestCase):
@@ -157,3 +164,84 @@ def test_query_parsed_edge_cases(self):
generate_query_parsed('foo', u'foo\\\\bar'),
{'text': {'foo': u'foo\\bar'}}
)
+
+
+class TestCountsHelper(TestCase):
+ def setUp(self):
+ self.counts = [('apples', 5), ('bananas', 10), ('oranges', 6)]
+
+ def test_basic(self):
+ """Correct options should be set and values should be sorted.
+ """
+ options = counts_to_options(self.counts, 'fruit', 'Fruit')
+ eq_(options['name'], 'fruit')
+ eq_(options['display'], 'Fruit')
+
+ eq_(options['options'][0], {
+ 'name': 'bananas',
+ 'display': 'bananas',
+ 'value': 'bananas',
+ 'count': 10,
+ 'checked': False,
+ })
+ eq_(options['options'][1], {
+ 'name': 'oranges',
+ 'display': 'oranges',
+ 'value': 'oranges',
+ 'count': 6,
+ 'checked': False,
+ })
+ eq_(options['options'][2], {
+ 'name': 'apples',
+ 'display': 'apples',
+ 'value': 'apples',
+ 'count': 5,
+ 'checked': False,
+ })
+
+ def test_map_dict(self):
+ options = counts_to_options(self.counts, 'fruit', display_map={
+ 'apples': 'Apples',
+ 'bananas': 'Bananas',
+ 'oranges': 'Oranges',
+ })
+ # Note that options get sorted by count.
+ eq_(options['options'][0]['display'], 'Bananas')
+ eq_(options['options'][1]['display'], 'Oranges')
+ eq_(options['options'][2]['display'], 'Apples')
+
+ def test_map_func(self):
+ options = counts_to_options(self.counts, 'fruit',
+ value_map=lambda s: s.upper())
+ # Note that options get sorted by count.
+ eq_(options['options'][0]['value'], 'BANANAS')
+ eq_(options['options'][1]['value'], 'ORANGES')
+ eq_(options['options'][2]['value'], 'APPLES')
+
+ def test_checked(self):
+ options = counts_to_options(self.counts, 'fruit', checked='apples')
+ # Note that options get sorted by count.
+ assert not options['options'][0]['checked']
+ assert not options['options'][1]['checked']
+ assert options['options'][2]['checked']
+
+
+class TestZeroFillHelper(TestCase):
+ def test_zerofill(self):
+ start = datetime(2012, 1, 1)
+ end = datetime(2012, 1, 7)
+ data1 = {
+ epoch_milliseconds(datetime(2012, 1, 3)): 1,
+ epoch_milliseconds(datetime(2012, 1, 5)): 1,
+ }
+ data2 = {
+ epoch_milliseconds(datetime(2012, 1, 2)): 1,
+ epoch_milliseconds(datetime(2012, 1, 5)): 1,
+ epoch_milliseconds(datetime(2012, 1, 10)): 1,
+ }
+ zero_fill(start, end, [data1, data2])
+
+ for day in range(1, 8):
+ millis = epoch_milliseconds(datetime(2012, 1, day))
+ assert millis in data1, "Day %s was not zero filled." % day
+ assert millis in data2, "Day %s was not zero filled." % day
View
208 fjord/analytics/tests/test_views.py
@@ -10,97 +10,14 @@
from django.http import QueryDict
from fjord.analytics import views
-from fjord.analytics.views import counts_to_options, _zero_fill
-from fjord.base.tests import TestCase, LocalizingClient, profile, reverse, user
-from fjord.base.util import epoch_milliseconds
+from fjord.base.tests import LocalizingClient, profile, reverse, user
from fjord.feedback.tests import response
from fjord.search.tests import ElasticTestCase
logger = logging.getLogger(__name__)
-class TestCountsHelper(TestCase):
- def setUp(self):
- self.counts = [('apples', 5), ('bananas', 10), ('oranges', 6)]
-
- def test_basic(self):
- """Correct options should be set and values should be sorted.
- """
- options = counts_to_options(self.counts, 'fruit', 'Fruit')
- eq_(options['name'], 'fruit')
- eq_(options['display'], 'Fruit')
-
- eq_(options['options'][0], {
- 'name': 'bananas',
- 'display': 'bananas',
- 'value': 'bananas',
- 'count': 10,
- 'checked': False,
- })
- eq_(options['options'][1], {
- 'name': 'oranges',
- 'display': 'oranges',
- 'value': 'oranges',
- 'count': 6,
- 'checked': False,
- })
- eq_(options['options'][2], {
- 'name': 'apples',
- 'display': 'apples',
- 'value': 'apples',
- 'count': 5,
- 'checked': False,
- })
-
- def test_map_dict(self):
- options = counts_to_options(self.counts, 'fruit', display_map={
- 'apples': 'Apples',
- 'bananas': 'Bananas',
- 'oranges': 'Oranges',
- })
- # Note that options get sorted by count.
- eq_(options['options'][0]['display'], 'Bananas')
- eq_(options['options'][1]['display'], 'Oranges')
- eq_(options['options'][2]['display'], 'Apples')
-
- def test_map_func(self):
- options = counts_to_options(self.counts, 'fruit',
- value_map=lambda s: s.upper())
- # Note that options get sorted by count.
- eq_(options['options'][0]['value'], 'BANANAS')
- eq_(options['options'][1]['value'], 'ORANGES')
- eq_(options['options'][2]['value'], 'APPLES')
-
- def test_checked(self):
- options = counts_to_options(self.counts, 'fruit', checked='apples')
- # Note that options get sorted by count.
- assert not options['options'][0]['checked']
- assert not options['options'][1]['checked']
- assert options['options'][2]['checked']
-
-
-class TestZeroFillHelper(TestCase):
- def test_zerofill(self):
- start = datetime(2012, 1, 1)
- end = datetime(2012, 1, 7)
- data1 = {
- epoch_milliseconds(datetime(2012, 1, 3)): 1,
- epoch_milliseconds(datetime(2012, 1, 5)): 1,
- }
- data2 = {
- epoch_milliseconds(datetime(2012, 1, 2)): 1,
- epoch_milliseconds(datetime(2012, 1, 5)): 1,
- epoch_milliseconds(datetime(2012, 1, 10)): 1,
- }
- _zero_fill(start, end, [data1, data2])
-
- for day in range(1, 8):
- millis = epoch_milliseconds(datetime(2012, 1, day))
- assert millis in data1, "Day %s was not zero filled." % day
- assert millis in data2, "Day %s was not zero filled." % day
-
-
class TestDashboardView(ElasticTestCase):
client_class = LocalizingClient
@@ -454,126 +371,3 @@ def test_response_view_mobile(self):
eq_(200, r.status_code)
self.assertTemplateUsed(r, 'analytics/mobile/response.html')
assert str(resp.description) in r.content
-
-
-class TestAnalyticsDashboardView(ElasticTestCase):
- client_class = LocalizingClient
-
- def test_permissions(self):
- # Verifies that only analyzers can see the analytics dashboard
- # link
- resp = self.client.get(reverse('dashboard'))
- eq_(200, resp.status_code)
- assert 'adashboard' not in resp.content
-
- # Verifies that only analyzers can see the analytics dashboard
- resp = self.client.get(reverse('analytics_dashboard'))
- eq_(403, resp.status_code)
-
- # Verify analyzers can see analytics dashboard link
- jane = user(email='jane@example.com', save=True)
- profile(user=jane, save=True)
- jane.groups.add(Group.objects.get(name='analyzers'))
-
- self.client_login_user(jane)
- resp = self.client.get(reverse('dashboard'))
- eq_(200, resp.status_code)
- assert 'adashboard' in resp.content
-
- # Verify analyzers can see analytics dashboard
- resp = self.client.get(reverse('analytics_dashboard'))
- eq_(200, resp.status_code)
-
-
-class TestOccurrencesReportView(ElasticTestCase):
- client_class = LocalizingClient
-
- def setUp(self):
- super(TestOccurrencesReportView, self).setUp()
- # Set up some sample data
- items = [
- # happy, locale, description
- (True, 'en-US', 'apple banana orange pear'),
- (True, 'en-US', 'orange pear kiwi'),
- (True, 'en-US', 'chocolate chocolate yum'),
- (False, 'en-US', 'apple banana grapefruit'),
-
- # This one doesn't create bigrams because there isn't enough words
- (False, 'en-US', 'orange'),
-
- # This one shouldn't show up
- (False, 'es', 'apple banana'),
- ]
- for happy, locale, description in items:
- response(
- happy=happy, locale=locale, description=description, save=True)
-
- self.refresh()
-
- # Create analyzer and log analyzer in
- jane = user(email='jane@example.com', save=True)
- profile(user=jane, save=True)
- jane.groups.add(Group.objects.get(name='analyzers'))
-
- self.client_login_user(jane)
-
- def test_occurrence_report(self):
- url = reverse('analytics_occurrences_comparison')
-
- # No results when you initially look at the page
- resp = self.client.get(url)
- eq_(200, resp.status_code)
- assert 'id="results"' not in resp.content
-
- # 'product' is a required field
- resp = self.client.get(url, {'product': ''})
- eq_(200, resp.status_code)
- # FIXME - this test is too loose
- assert 'This field is required' in resp.content
-
- # At least a version, search term or start date is required
- resp = self.client.get(url, {'product': 'Firefox'})
- eq_(200, resp.status_code)
- assert 'This field is required' not in resp.content
- assert 'Must specify at least one' in resp.content
-
- # Minimal required for results
- resp = self.client.get(url, {
- 'product': 'Firefox',
- 'first_version': '17.0.0'}
- )
- eq_(200, resp.status_code)
- assert 'This field is required' not in resp.content
- assert 'Must speicfy at least one' not in resp.content
- assert 'id="results"' in resp.content
-
- # FIXME - when things are less prototypy, add tests for
- # specific results
-
-
-class TestSpamDashboardView(ElasticTestCase):
- client_class = LocalizingClient
-
- def test_permissions(self):
- # Verifies that only analyzers can see the spam dashboard link
- resp = self.client.get(reverse('dashboard'))
- eq_(200, resp.status_code)
- assert 'sdashboard' not in resp.content
-
- # Verifies that only analyzers can see the spam dashboard
- resp = self.client.get(reverse('spam_dashboard'))
- eq_(403, resp.status_code)
-
- # Verify analyzers can see spam dashboard link
- jane = user(email='jane@example.com', save=True)
- profile(user=jane, save=True)
- jane.groups.add(Group.objects.get(name='analyzers'))
-
- self.client_login_user(jane)
- resp = self.client.get(reverse('dashboard'))
- eq_(200, resp.status_code)
- assert 'sdashboard' in resp.content
-
- # Verify analyzers can see spam dashboard
- resp = self.client.get(reverse('spam_dashboard'))
- eq_(200, resp.status_code)
View
126 fjord/analytics/tools.py
@@ -1,5 +1,10 @@
+from math import floor
import json
+from django.template.defaultfilters import slugify
+
+from fjord.base.util import epoch_milliseconds
+
WHITESPACE = u' \t\r\n'
@@ -207,3 +212,124 @@ def default(self, value):
if hasattr(value, 'strftime'):
return value.isoformat()
return super(JSONDatetimeEncoder, self).default(value)
+
+
+def counts_to_options(counts, name, display=None, display_map=None,
+ value_map=None, checked=None):
+ """Generates a set of option blocks from a set of facet counts.
+
+ One options block represents a set of options to search for, as
+ well as the query parameter that can be used to search for that
+ opinion, and the friendly name to show the opinion block as.
+
+ For each option the keys mean:
+ - `name`: Used to name in the DOM.
+ - `display`: Shown to the user.
+ - `value`: The value to set the query parameter to in order to
+ search for this option.
+ - `count`: The facet count of this option.
+ - `checked`: Whether the checkbox should start checked.
+
+ :arg counts: A list of tuples of the form (count, item), like from
+ ES.
+ :arg name: The name of the search string that corresponds to this
+ block. Like "locale" or "platform".
+ :arg display: The human friendly title to represent this set of
+ options.
+ :arg display_map: Either a dictionary or a function to map items
+ to their display names. For a dictionary, the form is {item:
+ display}. For a function, the form is lambda item:
+ display_name.
+ :arg value_map: Like `display_map`, but for mapping the values
+ that get put into the query string for searching.
+ :arg checked: Which item should be marked as checked.
+ """
+ if display is None:
+ display = name
+
+ options = {
+ 'name': name,
+ 'display': display,
+ 'options': [],
+ }
+
+ # This is used in the loop below, to be a bit neater and so we can
+ # do it for both value and display generically.
+ def from_map(source, item):
+ """Look up an item from a source.
+
+ The source may be a dictionary, a function, or None, in which
+ case the item is returned unmodified.
+
+ """
+ if source is None:
+ return item
+ elif callable(source):
+ return source(item)
+ else:
+ return source[item]
+
+ # Built an option dict for every item.
+ for item, count in counts:
+ options['options'].append({
+ 'name': slugify(item),
+ 'display': from_map(display_map, item),
+ 'value': from_map(value_map, item),
+ 'count': count,
+ 'checked': checked == item,
+ })
+ options['options'].sort(key=lambda item: item['count'], reverse=True)
+ return options
+
+
+DAY_IN_MILLIS = 24 * 60 * 60 * 1000.0
+
+
+def zero_fill(start, end, data_sets, spacing=DAY_IN_MILLIS):
+ """Given one or more histogram dicts, zero fill them in a range.
+
+ The format of the dictionaries should be {milliseconds: numeric
+ value}. It is important that the time points in the dictionary are
+ equally spaced. If they are not, extra points will be added.
+
+ This method works with milliseconds because that is the format
+ elasticsearch and Javascript use.
+
+ :arg start: Datetime to start zero filling.
+ :arg end: Datetime to stop zero filling at.
+ :arg data_sets: A list of dictionaries to zero fill.
+ :arg spacing: Number of milliseconds between data points.
+ """
+ start_millis = epoch_milliseconds(start)
+ # Date ranges are inclusive on both ends.
+ end_millis = epoch_milliseconds(end) + spacing
+
+ # `timestamp` is a loop counter that iterates over the timestamps
+ # from start to end. It can't just be `timestamp = start`, because
+ # then the zeros being adding to the data might not be aligned
+ # with the data already in the graph, since we aren't counting by
+ # 24 hours, and the data could have a timezone offset.
+ #
+ # This block picks a time up to `spacing` time after `start` so
+ # that it lines up with the data. If there is no data, then we use
+ # `stamp = start`, because there is nothing to align with.
+
+ # start <= timestamp < start + spacing
+ days = [d for d in data_sets if d.keys()]
+ if days:
+ source = days[0]
+ timestamp = source.keys()[0]
+ d = floor((timestamp - start_millis) / spacing)
+ timestamp -= d * spacing
+ else:
+ # If there no data, it doesn't matter how it aligns.
+ timestamp = start_millis
+
+ # Iterate in the range `start` to `end`, starting from
+ # `timestamp`, increasing by `spacing` each time. This ensures
+ # there is a data point for each day.
+ while timestamp < end_millis:
+ for d in data_sets:
+ if timestamp not in d:
+ d[timestamp] = 0
+ timestamp += spacing
View
20 fjord/analytics/urls.py
@@ -11,17 +11,19 @@
url(r'^dashboard/response/(?P<responseid>\d+)/?$',
'response_view', name='response_view'),
+)
+
+# These are analyzer-group only views.
+urlpatterns += patterns(
+ 'fjord.analytics.analyzer_views',
+
# Analytics dashboard
- url(r'^analytics_dashboard/?$', 'analytics_dashboard',
+ url(r'^analytics/?$', 'analytics_dashboard',
name='analytics_dashboard'),
- url(r'^analytics_dashboard/occurrences_comparison/?$',
- 'analytics_occurrences_comparison',
- name='analytics_occurrences_comparison'),
- url(r'^analytics_dashboard/products/?$',
- 'analytics_products',
+ url(r'^analytics/occurrences/?$', 'analytics_occurrences',
+ name='analytics_occurrences'),
+ url(r'^analytics/products/?$', 'analytics_products',
name='analytics_products'),
- # Spam dashboard
- url(r'^spam_dashboard/?$', 'spam_dashboard', name='spam_dashboard'),
- url(r'^spam_dashboard/duplicates/?$', 'spam_duplicates', name='spam_duplicates'),
+ url(r'^analytics/duplicates/?$', 'analytics_duplicates', name='analytics_duplicates'),
)
View
384 fjord/analytics/views.py
@@ -1,149 +1,29 @@
+# Note: These views are public with the exception of the response_view
+# which has "secure" parts to it in the template.
+
import json
-from collections import defaultdict
from datetime import datetime, timedelta
-from math import floor
from django.http import HttpResponse
from django.shortcuts import get_object_or_404, render
-from django.template.defaultfilters import slugify
from elasticutils.contrib.django import F, es_required_or_50x
from funfactory.urlresolvers import reverse
from mobility.decorators import mobile_template
from tower import ugettext as _
-from fjord.analytics.forms import OccurrencesComparisonForm
-from fjord.analytics.tools import JSONDatetimeEncoder, generate_query_parsed
+from fjord.analytics.tools import (
+ JSONDatetimeEncoder,
+ generate_query_parsed,
+ counts_to_options,
+ zero_fill)
from fjord.base.helpers import locale_name
from fjord.base.util import (
- analyzer_required,
check_new_user,
smart_int,
smart_date,
- epoch_milliseconds,
Atom1FeedWithRelatedLinks)
-from fjord.feedback.models import Product, Response, ResponseMappingType
-
-
-def counts_to_options(counts, name, display=None, display_map=None,
- value_map=None, checked=None):
- """Generates a set of option blocks from a set of facet counts.
-
- One options block represents a set of options to search for, as
- well as the query parameter that can be used to search for that
- opinion, and the friendly name to show the opinion block as.
-
- For each option the keys mean:
- - `name`: Used to name in the DOM.
- - `display`: Shown to the user.
- - `value`: The value to set the query parameter to in order to
- search for this option.
- - `count`: The facet count of this option.
- - `checked`: Whether the checkbox should start checked.
-
- :arg counts: A list of tuples of the form (count, item), like from
- ES.
- :arg name: The name of the search string that corresponds to this
- block. Like "locale" or "platform".
- :arg display: The human friendly title to represent this set of
- options.
- :arg display_map: Either a dictionary or a function to map items
- to their display names. For a dictionary, the form is {item:
- display}. For a function, the form is lambda item:
- display_name.
- :arg value_map: Like `display_map`, but for mapping the values
- that get put into the query string for searching.
- :arg checked: Which item should be marked as checked.
- """
- if display is None:
- display = name
-
- options = {
- 'name': name,
- 'display': display,
- 'options': [],
- }
-
- # This is used in the loop below, to be a bit neater and so we can
- # do it for both value and display generically.
- def from_map(source, item):
- """Look up an item from a source.
-
- The source may be a dictionary, a function, or None, in which
- case the item is returned unmodified.
-
- """
- if source is None:
- return item
- elif callable(source):
- return source(item)
- else:
- return source[item]
-
- # Built an option dict for every item.
- for item, count in counts:
- options['options'].append({
- 'name': slugify(item),
- 'display': from_map(display_map, item),
- 'value': from_map(value_map, item),
- 'count': count,
- 'checked': checked == item,
- })
- options['options'].sort(key=lambda item: item['count'], reverse=True)
- return options
-
-
-DAY_IN_MILLIS = 24 * 60 * 60 * 1000.0
-
-
-def _zero_fill(start, end, data_sets, spacing=DAY_IN_MILLIS):
- """Given one or more histogram dicts, zero fill them in a range.
-
- The format of the dictionaries should be {milliseconds: numeric
- value}. It is important that the time points in the dictionary are
- equally spaced. If they are not, extra points will be added.
-
- This method works with milliseconds because that is the format
- elasticsearch and Javascript use.
-
- :arg start: Datetime to start zero filling.
- :arg end: Datetime to stop zero filling at.
- :arg data_sets: A list of dictionaries to zero fill.
- :arg spacing: Number of milliseconds between data points.
- """
- start_millis = epoch_milliseconds(start)
- # Date ranges are inclusive on both ends.
- end_millis = epoch_milliseconds(end) + spacing
-
- # `timestamp` is a loop counter that iterates over the timestamps
- # from start to end. It can't just be `timestamp = start`, because
- # then the zeros being adding to the data might not be aligned
- # with the data already in the graph, since we aren't counting by
- # 24 hours, and the data could have a timezone offset.
- #
- # This block picks a time up to `spacing` time after `start` so
- # that it lines up with the data. If there is no data, then we use
- # `stamp = start`, because there is nothing to align with.
-
- # start <= timestamp < start + spacing
- days = [d for d in data_sets if d.keys()]
- if days:
- source = days[0]
- timestamp = source.keys()[0]
- d = floor((timestamp - start_millis) / spacing)
- timestamp -= d * spacing
- else:
- # If there no data, it doesn't matter how it aligns.
- timestamp = start_millis
-
- # Iterate in the range `start` to `end`, starting from
- # `timestamp`, increasing by `spacing` each time. This ensures
- # there is a data point for each day.
- while timestamp < end_millis:
- for d in data_sets:
- if timestamp not in d:
- d[timestamp] = 0
- timestamp += spacing
+from fjord.feedback.models import Response, ResponseMappingType
@check_new_user
@@ -218,8 +98,9 @@ def generate_atom_feed(request, search):
feed.writeString('utf-8'), mimetype='application/atom+xml')
-def generate_dashboard_atom_url(request):
- """For a given request, generates the dashboard atom url"""
+def generate_dashboard_url(request, output_format='atom',
+ viewname='dashboard'):
+ """For a given request, generates the dashboard url for the given format"""
qd = request.GET.copy()
# Remove anything from the querystring that isn't good for a feed:
@@ -229,9 +110,9 @@ def generate_dashboard_atom_url(request):
'version', 'q'):
del qd[mem]
- qd['format'] = 'atom'
+ qd['format'] = output_format
- return reverse('dashboard') + '?' + qd.urlencode()
+ return reverse(viewname) + '?' + qd.urlencode()
@check_new_user
@@ -439,7 +320,7 @@ def empty_to_unknown(text):
happy_data = dict((p['time'], p['count']) for p in histograms['happy'])
sad_data = dict((p['time'], p['count']) for p in histograms['sad'])
- _zero_fill(search_date_start, search_date_end, [happy_data, sad_data])
+ zero_fill(search_date_start, search_date_end, [happy_data, sad_data])
histogram = [
{'label': _('Happy'), 'name': 'happy',
'data': sorted(happy_data.items())},
@@ -457,238 +338,5 @@ def empty_to_unknown(text):
'next_page': page + 1 if end < search_count else None,
'current_search': current_search,
'selected': selected,
- 'atom_url': generate_dashboard_atom_url(request),
- })
-
-
-@check_new_user
-@analyzer_required
-@es_required_or_50x(error_template='analytics/es_down.html')
-@mobile_template('analytics/{mobile/}analytics_dashboard.html')
-def analytics_dashboard(request, template):
- return render(request, template)
-
-
-@check_new_user
-@analyzer_required
-@mobile_template('analytics/{mobile/}analytics_products.html')
-def analytics_products(request, template):
- products = Product.objects.all()
- return render(request, template, {
- 'products': products
- })
-
-
-@check_new_user
-@analyzer_required
-@es_required_or_50x(error_template='analytics/es_down.html')
-def analytics_occurrences_comparison(request):
- template = 'analytics/analytics_occurrences_comparison.html'
-
- first_facet_bi = None
- first_params = {}
- first_facet_total = 0
-
- second_facet_bi = None
- second_params = {}
- second_facet_total = 0
-
- if 'product' in request.GET:
- form = OccurrencesComparisonForm(request.GET)
- if form.is_valid():
- cleaned = form.cleaned_data
-
- # First item
- first_resp_s = (ResponseMappingType.search()
- .filter(product=cleaned['product'])
- .filter(locale__startswith='en'))
-
- first_params['product'] = cleaned['product']
-
- if cleaned['first_version']:
- first_resp_s = first_resp_s.filter(
- version=cleaned['first_version'])
- first_params['version'] = cleaned['first_version']
- if cleaned['first_start_date']:
- first_resp_s = first_resp_s.filter(
- created__gte=cleaned['first_start_date'])
- first_params['date_start'] = cleaned['first_start_date']
- if cleaned['first_end_date']:
- first_resp_s = first_resp_s.filter(
- created__lte=cleaned['first_end_date'])
- first_params['date_end'] = cleaned['first_end_date']
- if cleaned['first_search_term']:
- first_resp_s = first_resp_s.query(
- description__text=cleaned['first_search_term'])
- first_params['q'] = cleaned['first_search_term']
-
- if ('date_start' not in first_params
- and 'date_end' not in first_params):
-
- # FIXME - If there's no start date, then we want
- # "everything" so we use a hard-coded 2013-01-01 date
- # here to hack that.
- #
- # Better way might be to change the dashboard to allow
- # for an "infinite" range, but there's no other use
- # case for that and the ranges are done in the ui--not
- # in the backend.
- first_params['date_start'] = '2013-01-01'
-
- # Have to do raw because we want a size > 10.
- first_resp_s = first_resp_s.facet_raw(
- description_bigrams={
- 'terms': {
- 'field': 'description_bigrams',
- 'size': '30',
- },
- 'facet_filter': first_resp_s._build_query()['filter']
- }
- )
- first_resp_s = first_resp_s[0:0]
-
- first_facet_total = first_resp_s.count()
- first_facet = first_resp_s.facet_counts()
-
- first_facet_bi = first_facet['description_bigrams']
- first_facet_bi = sorted(
- first_facet_bi, key=lambda item: -item['count'])
-
- if (cleaned['second_version']
- or cleaned['second_search_term']
- or cleaned['second_start_date']):
-
- second_resp_s = (ResponseMappingType.search()
- .filter(product=cleaned['product'])
- .filter(locale__startswith='en'))
-
- second_params['product'] = cleaned['product']
-
- if cleaned['second_version']:
- second_resp_s = second_resp_s.filter(
- version=cleaned['second_version'])
- second_params['version'] = cleaned['second_version']
- if cleaned['second_start_date']:
- second_resp_s = second_resp_s.filter(
- created__gte=cleaned['second_start_date'])
- second_params['date_start'] = cleaned['second_start_date']
- if cleaned['second_end_date']:
- second_resp_s = second_resp_s.filter(
- created__lte=cleaned['second_end_date'])
- second_params['date_end'] = cleaned['second_end_date']
- if form.cleaned_data['second_search_term']:
- second_resp_s = second_resp_s.query(
- description__text=cleaned['second_search_term'])
- second_params['q'] = cleaned['second_search_term']
-
- if ('date_start' not in second_params
- and 'date_end' not in second_params):
-
- # FIXME - If there's no start date, then we want
- # "everything" so we use a hard-coded 2013-01-01 date
- # here to hack that.
- #
- # Better way might be to change the dashboard to allow
- # for an "infinite" range, but there's no other use
- # case for that and the ranges are done in the ui--not
- # in the backend.
- second_params['date_start'] = '2013-01-01'
-
- # Have to do raw because we want a size > 10.
- second_resp_s = second_resp_s.facet_raw(
- description_bigrams={
- 'terms': {
- 'field': 'description_bigrams',
- 'size': '30',
- },
- 'facet_filter': second_resp_s._build_query()['filter']
- }
- )
- second_resp_s = second_resp_s[0:0]
-
- second_facet_total = second_resp_s.count()
- second_facet = second_resp_s.facet_counts()
-
- second_facet_bi = second_facet['description_bigrams']
- second_facet_bi = sorted(
- second_facet_bi, key=lambda item: -item['count'])
-
- permalink = request.build_absolute_uri()
-
- else:
- permalink = ''
- form = OccurrencesComparisonForm()
-
- # FIXME - We have responses that have no product set. This ignores
- # those. That's probably the right thing to do for the Occurrences Report
- # but maybe not.
- products = [prod for prod in ResponseMappingType.get_products() if prod]
-
- return render(request, template, {
- 'permalink': permalink,
- 'form': form,
- 'products': products,
- 'first_facet_bi': first_facet_bi,
- 'first_params': first_params,
- 'first_facet_total': first_facet_total,
- 'first_normalization': round(first_facet_total * 1.0 / 1000, 3),
- 'second_facet_bi': second_facet_bi,
- 'second_params': second_params,
- 'second_facet_total': second_facet_total,
- 'second_normalization': round(second_facet_total * 1.0 / 1000, 3),
- 'render_time': datetime.now(),
- })
-
-
-@check_new_user
-@analyzer_required
-@es_required_or_50x(error_template='analytics/es_down.html')
-@mobile_template('analytics/{mobile/}spam_dashboard.html')
-def spam_dashboard(request, template):
- return render(request, template)
-
-
-@check_new_user
-@analyzer_required
-@es_required_or_50x(error_template='analytics/es_down.html')
-def spam_duplicates(request):
- """Shows all duplicate descriptions over the last n days"""
- template = 'analytics/spam_duplicates.html'
-
- n = 14
-
- responses = (ResponseMappingType.search()
- .filter(created__gte=datetime.now() - timedelta(days=n))
- .values_dict('description', 'happy', 'created', 'locale',
- 'user_agent', 'id')
- .order_by('created').all())
-
- total_count = len(responses)
-
- response_dupes = {}
- for resp in responses:
- response_dupes.setdefault(resp['description'], []).append(resp)
-
- response_dupes = [
- (key, val) for key, val in response_dupes.items()
- if len(val) > 1
- ]
-
- # convert the dict into a list of tuples sorted by the number of
- # responses per tuple largest number first
- response_dupes = sorted(response_dupes, key=lambda item: len(item[1]) * -1)
-
- # duplicate_count -> count
- # i.e. "how many responses had 2 duplicates?"
- summary_counts = defaultdict(int)
- for desc, responses in response_dupes:
- summary_counts[len(responses)] = summary_counts[len(responses)] + 1
- summary_counts = sorted(summary_counts.items(), key=lambda item: item[0])
-
- return render(request, template, {
- 'n': 14,
- 'response_dupes': response_dupes,
- 'render_time': datetime.now(),
- 'summary_counts': summary_counts,
- 'total_count': total_count,
+ 'atom_url': generate_dashboard_url(request),
})
View
1 fjord/base/templates/base.html
@@ -35,7 +35,6 @@ <h1 class="title"><a href="/">
<li><a class="dashboard" href="{{ url('dashboard') }}"><span>{{ _('Dashboard') }}</span></a></li>
{% if user.is_authenticated() and user.has_perm('analytics.can_view_dashboard') %}
<li><a class="adashboard" href="{{ url('analytics_dashboard') }}"><span>{{ _('Analytics') }}</span></a></li>
- <li><a class="sdashboard" href="{{ url('spam_dashboard') }}"><span>{{ _('Damn Spam') }}</span></a></li>
{% endif %}
</ul>

0 comments on commit a86d3ce

Please sign in to comment.