Merge pull request #3366 from diox/fix-compat-report-attempt

Add logging in compatibility_report, calculate totals only once

src/olympia/compat/cron.py

@@ -86,22 +86,22 @@ def compatibility_report(index=None):
                               'max': compat.max.version_int}
             doc['max_version'] = compat.max.version
 
-        total = sum(updates.values())
-        # Remember the total so we can show % of usage later.
-        compat_total, created = CompatTotals.objects.safer_get_or_create(
-            defaults={'total': total})
-        if not created:
-            compat_total.update(total=total)
-
-        # Figure out which add-ons are in the top 95%.
-        running_total = 0
-        for addon, count in sorted(updates.items(), key=lambda x: x[1],
-                                   reverse=True):
-            running_total += count
-            docs[addon]['top_95_all'] = running_total < (.95 * total)
+    total = sum(updates.values())
+    # Remember the total so we can show % of usage later.
+    compat_total, created = CompatTotals.objects.safer_get_or_create(
+        defaults={'total': total})
+    if not created:
+        compat_total.update(total=total)
+
+    # Figure out which add-ons are in the top 95%.
+    running_total = 0
+    for addon, count in sorted(updates.items(), key=lambda x: x[1],
+                               reverse=True):
+        running_total += count
+        docs[addon]['top_95_all'] = running_total < (.95 * total)
 
     # Mark the top 95% of add-ons compatible with the previous version for each
-    # app + version combo.
+    # version.
     for compat in FIREFOX_COMPAT:
         version = vint(compat['previous'])
         # Find all the docs that have a max_version compatible with version.
@@ -118,6 +118,9 @@ def compatibility_report(index=None):
 
     # Send it all to the index.
     for chunk in chunked(docs.values(), 150):
+        log.info('Indexing compat %s-%s. [%s]' % (
+            chunk[0]['id'], chunk[-1]['id'], len(chunk)))
+        # FIXME: use bulk indexing (see issue #3319)
         for doc in chunk:
             for index in indices:
                 AppCompat.index(doc, id=doc['id'], refresh=False, index=index)

src/olympia/zadmin/tests/test_views.py

@@ -16,14 +16,15 @@
 from olympia.amo.tests import formset, initial
 from olympia.access.models import Group, GroupUser
 from olympia.addons.models import Addon, CompatOverride, CompatOverrideRange
+from olympia.addons.utils import generate_addon_guid
 from olympia.amo.urlresolvers import reverse
 from olympia.amo.tests.test_helpers import get_image_path
 from olympia.amo.utils import urlparams
 from olympia.applications.models import AppVersion
 from olympia.bandwagon.models import FeaturedCollection, MonthlyPick
 from olympia.compat import FIREFOX_COMPAT
 from olympia.compat.cron import compatibility_report
-from olympia.compat.models import CompatReport
+from olympia.compat.models import CompatReport, CompatTotals
 from olympia.constants.base import VALIDATOR_SKELETON_RESULTS
 from olympia.devhub.models import ActivityLog
 from olympia.files.models import File, FileUpload
@@ -1424,19 +1425,16 @@ def setUp(self):
         self.url = reverse('zadmin.compat')
         self.client.login(username='admin@mozilla.com', password='password')
         self.app_version = FIREFOX_COMPAT[0]['main']
-        self.addon = self.populate(guid='xxx')
-        self.generate_reports(self.addon, good=0, bad=0, app=amo.FIREFOX,
-                              app_version=self.app_version)
 
     def update(self):
         compatibility_report()
         self.refresh()
 
-    def populate(self, **kw):
+    def populate(self):
         now = datetime.now()
-        name = 'Addon %s' % now
-        kw.update(guid=name)
-        addon = amo.tests.addon_factory(name=name, **kw)
+        guid = generate_addon_guid()
+        name = 'Addon %s' % guid
+        addon = amo.tests.addon_factory(name=name, guid=guid)
         UpdateCount.objects.create(addon=addon, count=10, date=now)
         return addon
 
@@ -1454,7 +1452,40 @@ def get_pq(self, **kw):
         assert response.status_code == 200
         return pq(response.content)('#compat-results')
 
+    def test_compat_totals(self):
+        assert not CompatTotals.objects.exists()
+
+        # Add second add-on, generate reports for both.
+        addon1 = self.populate()
+        addon2 = self.populate()
+        # count needs to be higher than 50 to test totals properly.
+        UpdateCount.objects.filter(addon=addon1).update(count=60)
+        self.generate_reports(addon1, good=1, bad=2, app=amo.FIREFOX,
+                              app_version=self.app_version)
+        self.generate_reports(addon2, good=3, bad=4, app=amo.FIREFOX,
+                              app_version=self.app_version)
+        assert CompatTotals.objects.count() == 1
+        assert CompatTotals.objects.get().total == 70
+
+    def test_compat_totals_already_exists(self):
+        CompatTotals.objects.create(total=42)
+
+        # Add second add-on, generate reports for both.
+        addon1 = self.populate()
+        addon2 = self.populate()
+        # count needs to be higher than 50 to test totals properly.
+        UpdateCount.objects.filter(addon=addon1).update(count=60)
+        self.generate_reports(addon1, good=1, bad=2, app=amo.FIREFOX,
+                              app_version=self.app_version)
+        self.generate_reports(addon2, good=3, bad=4, app=amo.FIREFOX,
+                              app_version=self.app_version)
+        assert CompatTotals.objects.count() == 1
+        assert CompatTotals.objects.get().total == 70
+
     def test_defaults(self):
+        addon = self.populate()
+        self.generate_reports(addon, good=0, bad=0, app=amo.FIREFOX,
+                              app_version=self.app_version)
         r = self.client.get(self.url)
         assert r.status_code == 200
         table = pq(r.content)('#compat-results')