bring back .csv for stats (bug 682102)

Author: Jeff Balogh

apps/stats/tests/test_views.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 import csv
 import json
-from datetime import datetime
 
 from nose.tools import eq_
 
@@ -294,21 +293,28 @@ def _test_cache_control(self):
             'Bad or no cache-control: %r' % response.get('cache-control', ''))
 
 
-class TestJSON(StatsTest, amo.tests.ESTestCase):
+class TestResponses(StatsTest, amo.tests.ESTestCase):
     es = True
 
     def setUp(self):
-        super(TestJSON, self).setUp()
+        super(TestResponses, self).setUp()
         self.index()
 
+    def csv_eq(self, response, expected):
+        # Drop the first 4 lines, which contain the header comment.
+        content = response.content.splitlines()[4:]
+        # Strip any extra spaces from the expected content.
+        expected = [line.strip() for line in expected.splitlines()]
+        self.assertListEqual(content, expected)
+
     def index(self):
         updates = UpdateCount.objects.values_list('id', flat=True)
         tasks.index_update_counts(list(updates))
         downloads = DownloadCount.objects.values_list('id', flat=True)
         tasks.index_download_counts(list(downloads))
         self.refresh('update_counts')
 
-    def test_usage(self):
+    def test_usage_json(self):
         r = self.get_view_response('stats.usage_series', group='day',
                                    format='json')
         eq_(r.status_code, 200)
@@ -317,7 +323,16 @@ def test_usage(self):
             {'count': 1000, 'date': '2009-06-01', 'end': '2009-06-01'},
         ])
 
-    def test_usage_by_app(self):
+    def test_usage_csv(self):
+        r = self.get_view_response('stats.usage_series', group='day',
+                                   format='csv')
+        eq_(r.status_code, 200)
+        self.csv_eq(r,
+                    """date,count
+                       2009-06-02,1500
+                       2009-06-01,1000""")
+
+    def test_usage_by_app_json(self):
         r = self.get_view_response('stats.apps_series', group='day',
                                    format='json')
         eq_(r.status_code, 200)
@@ -340,7 +355,15 @@ def test_usage_by_app(self):
             }
         ])
 
-    def test_usage_by_locale(self):
+    def test_usage_by_app_csv(self):
+        r = self.get_view_response('stats.apps_series', group='day',
+                                   format='csv')
+        eq_(r.status_code, 200)
+        self.csv_eq(r, """date,count,{ec8030f7-c20a-464f-9b0e-13a3a9e97384}
+                          2009-06-02,1500,{u'4.0': 1500}
+                          2009-06-01,1000,{u'4.0': 1000}""")
+
+    def test_usage_by_locale_json(self):
         r = self.get_view_response('stats.locales_series', group='day',
                                    format='json')
         eq_(r.status_code, 200)
@@ -365,7 +388,15 @@ def test_usage_by_locale(self):
             }
         ])
 
-    def test_usage_by_os(self):
+    def test_usage_by_locale_csv(self):
+        r = self.get_view_response('stats.locales_series', group='day',
+                                   format='csv')
+        eq_(r.status_code, 200)
+        self.csv_eq(r, """date,count,English (US) (en-us),Ελληνικά (el)
+                          2009-06-02,1500,300,400
+                          2009-06-01,1000,300,400""")
+
+    def test_usage_by_os_json(self):
         r = self.get_view_response('stats.os_series', group='day',
                                    format='json')
         eq_(r.status_code, 200)
@@ -390,7 +421,12 @@ def test_usage_by_os(self):
             }
         ])
 
-    def test_usage_by_version(self):
+    def test_usage_by_os_csv(self):
+        r = self.get_view_response('stats.os_series', group='day',
+                                   format='csv')
+        eq_(r.status_code, 200)
+
+    def test_usage_by_version_json(self):
         r = self.get_view_response('stats.versions_series', group='day',
                                    format='json')
         eq_(r.status_code, 200)
@@ -415,7 +451,15 @@ def test_usage_by_version(self):
             }
         ])
 
-    def test_usage_by_status(self):
+    def test_usage_by_version_csv(self):
+        r = self.get_view_response('stats.versions_series', group='day',
+                                   format='csv')
+        eq_(r.status_code, 200)
+        self.csv_eq(r, """date,count,2.0,1.0
+                          2009-06-02,1500,950,550
+                          2009-06-01,1000,800,200""")
+
+    def test_usage_by_status_json(self):
         r = self.get_view_response('stats.statuses_series', group='day',
                                    format='json')
         eq_(r.status_code, 200)
@@ -440,6 +484,14 @@ def test_usage_by_status(self):
             }
         ])
 
+    def test_usage_by_status_csv(self):
+        r = self.get_view_response('stats.statuses_series', group='day',
+                                   format='csv')
+        eq_(r.status_code, 200)
+        self.csv_eq(r, """date,count,userEnabled,userDisabled
+                          2009-06-02,1500,1370,130
+                          2009-06-01,1000,950,50""")
+
     def test_overview(self):
         r = self.get_view_response('stats.overview_series', group='day',
                                    format='json')
@@ -523,8 +575,7 @@ def test_overview(self):
                                      {'downloads': 0, 'updates': 0})
                 next_actual = next(actual)
 
-
-    def test_downloads(self):
+    def test_downloads_json(self):
         r = self.get_view_response('stats.downloads_series', group='day',
                                    format='json')
         eq_(r.status_code, 200)
@@ -539,7 +590,21 @@ def test_downloads(self):
             {"count": 10, "date": "2009-06-01", "end": "2009-06-01"},
         ])
 
-    def test_downloads_sources(self):
+    def test_downloads_csv(self):
+        r = self.get_view_response('stats.downloads_series', group='day',
+                                   format='csv')
+        eq_(r.status_code, 200)
+        self.csv_eq(r, """date,count
+                          2009-09-03,10
+                          2009-08-03,10
+                          2009-07-03,10
+                          2009-06-28,10
+                          2009-06-20,10
+                          2009-06-12,10
+                          2009-06-07,10
+                          2009-06-01,10""")
+
+    def test_downloads_sources_json(self):
         r = self.get_view_response('stats.sources_series', group='day',
                                    format='json')
         eq_(r.status_code, 200)
@@ -585,3 +650,17 @@ def test_downloads_sources(self):
              "data": {"api": 2, "search": 3}
             }
         ])
+
+    def test_downloads_sources_csv(self):
+        r = self.get_view_response('stats.sources_series', group='day',
+                                   format='csv')
+        eq_(r.status_code, 200)
+        self.csv_eq(r, """date,count,search,api
+                          2009-09-03,10,3,2
+                          2009-08-03,10,3,2
+                          2009-07-03,10,3,2
+                          2009-06-28,10,3,2
+                          2009-06-20,10,3,2
+                          2009-06-12,10,3,2
+                          2009-06-07,10,3,2
+                          2009-06-01,10,3,2""")

apps/stats/unicode_csv.py

@@ -31,60 +31,6 @@ def csv_prep(stats, field_list, precision='1'):
     return (stats, fields)
 
 
-def csv_dynamic_prep(stats, queryset, field_list, total_key, dynamic_key):
-    """Prepare dynamic stats for CSV output.
-
-    This is suitable for stats containing breakdown values.
-    All Decimal values will be rounded and converted to integers.
-
-    Returns a tuple containing a row generator and a list of field
-    names suitable for the CSV header.
-    """
-    if not queryset:
-        return ([], [])
-
-    # Summarize entire queryset to get all dynamic field names and
-    # determine if we need to calculate 'unknown' values.
-    totals = queryset.summary(**dict(field_list))
-
-    # Since there may be averages in play, round all decimals to integers
-    totals = list(decimal_to_int_gen([totals]))[0]
-    stats = decimal_to_int_gen(stats)
-
-    # Perform 'unknown' calculations if there is a difference between
-    # dynamic field total and grand total.
-    dyn_sum = totals[dynamic_key].sum_reduce()
-    if dyn_sum < totals[total_key]:
-        totals[dynamic_key]['unknown'] = totals[total_key] - dyn_sum
-        stats = unknown_gen(stats, total_key, dynamic_key)
-
-    # Flatten the nested dynamic dictionary, grab all keys and sort them
-    # by their values.
-    d = flatten_dict({dynamic_key: totals[dynamic_key]})
-    dyn_keys = sorted(d, key=d.__getitem__, reverse=True)
-
-    # Build the final list of field keys, replacing dynamic_key with all its
-    # breakdown fields in dyn_keys.
-    dyn_index = zip(*field_list)[0].index(dynamic_key)
-    fields = [k for k, v in field_list[:dyn_index]]
-    fields.extend(k for k in dyn_keys)
-    fields.extend(k for k, v in field_list[dyn_index + 1:])
-
-    # For CSV headers, we trim the "dynamic_key/" portion from the start of all
-    # the dyn_keys.
-    headings = [k for k, v in field_list[:dyn_index]]
-    headings.extend(k[len(dynamic_key) + 1:] for k in dyn_keys)
-    headings.extend(k for k, v in field_list[dyn_index + 1:])
-
-    # Almost done...
-    # For each row: flatten the dynamic field dictionary, and turn the row
-    # into a list of values.
-    stats = flatten_gen(stats, flatten_key=dynamic_key)
-    stats = values_gen(stats, fields, zero_val=0)
-
-    return (stats, headings)
-
-
 def flatten_dict(d, key=None):
     """Flatten a nested dictionary.