Added "apps added" stats crons (bug 858250, 858251)

apps/stats/tasks.py

@@ -7,25 +7,29 @@
 from django.db import connection, transaction
 from django.db.models import Sum, Max
 
-from apiclient.discovery import build
 import commonware.log
+from apiclient.discovery import build
 from celeryutils import task
 from oauth2client.client import OAuth2Credentials
 
 import amo
 import amo.search
 from addons.models import Addon, AddonUser
 from bandwagon.models import Collection
-from mkt.monolith.models import MonolithRecord
-from mkt.webapps.models import Installed
-from stats.models import Contribution
+from constants.base import ADDON_PREMIUM_API, ADDON_WEBAPP_TYPES
+from lib.es.utils import get_indices
 from reviews.models import Review
+from stats.models import Contribution
 from users.models import UserProfile
 from versions.models import Version
-from lib.es.utils import get_indices
+
+from mkt.constants.regions import REGIONS_CHOICES_SLUG
+from mkt.monolith.models import MonolithRecord
+from mkt.webapps.models import Installed
+
+from . import search
 from .models import (AddonCollectionCount, CollectionCount, CollectionStats,
                      DownloadCount, ThemeUserCount, UpdateCount)
-from . import search
 
 
 log = commonware.log.getLogger('z.task')
@@ -109,7 +113,8 @@ def update_google_analytics(date, **kw):
     h = httplib2.Http()
     creds.authorize(h)
     service = build('analytics', 'v3', http=h)
-    domain = getattr(settings, 'GOOGLE_ANALYTICS_DOMAIN', None) or settings.DOMAIN
+    domain = getattr(settings,
+                     'GOOGLE_ANALYTICS_DOMAIN', None) or settings.DOMAIN
     profile_id = get_profile_id(service, domain)
     if profile_id is None:
         log.critical('Failed to update global stats: could not access a Google'
@@ -143,37 +148,36 @@ def update_google_analytics(date, **kw):
 
 @task
 def update_global_totals(job, date, **kw):
-    log.info("Updating global statistics totals (%s) for (%s)" %
-                   (job, date))
+    log.info('Updating global statistics totals (%s) for (%s)' % (job, date))
 
     jobs = _get_daily_jobs(date)
     jobs.update(_get_metrics_jobs(date))
 
     num = jobs[job]()
 
-    q = """REPLACE INTO
-                global_stats(`name`, `count`, `date`)
-            VALUES
-                (%s, %s, %s)"""
+    q = """REPLACE INTO global_stats (`name`, `count`, `date`)
+           VALUES (%s, %s, %s)"""
     p = [job, num or 0, date]
 
     try:
         cursor = connection.cursor()
         cursor.execute(q, p)
         transaction.commit_unless_managed()
     except Exception, e:
-        log.critical("Failed to update global stats: (%s): %s" % (p, e))
+        log.critical('Failed to update global stats: (%s): %s' % (p, e))
 
-    # monolith is only used for marketplace
+    # Monolith is only used for Marketplace.
     if job.startswith(('apps', 'mmo')):
         try:
-            value = json.dumps({'count': num or 0})
-            MonolithRecord(recorded=date, key=job, value=value,
-                           user_hash='none').save()
+            # Only record if num is greater than zero.
+            if num:
+                value = json.dumps({'count': num})
+                MonolithRecord(recorded=date, key=job, value=value,
+                               user_hash='none').save()
         except Exception as e:
-            log.critical("Update of monolith table failed: (%s): %s" % (p, e))
+            log.critical('Update of monolith table failed: (%s): %s' % (p, e))
 
-    log.debug("Committed global stats details: (%s) has (%s) for (%s)"
+    log.debug('Committed global stats details: (%s) has (%s) for (%s)'
               % tuple(p))
 
 
@@ -237,6 +241,8 @@ def _get_daily_jobs(date=None):
                 sum=Sum('count'))['sum']),
 
         # Marketplace stats
+        # TODO: Remove 'apps_count_new' once we fully migrate to the new
+        # 'apps_added_*' stats.
         'apps_count_new': (Addon.objects
                 .filter(created__range=(date, next_date),
                         type=amo.ADDON_WEBAPP).count),
@@ -262,6 +268,25 @@ def _get_daily_jobs(date=None):
             addon__type=amo.ADDON_WEBAPP).values('user').distinct().count,
     }
 
+    # Add various "Apps Added" for all the dimensions we need.
+    apps = Addon.objects.filter(created__range=(date, next_date),
+                                type=amo.ADDON_WEBAPP)
+    for region_slug, region in REGIONS_CHOICES_SLUG:
+        # Apps added by package type and region.
+        for package_type in ADDON_WEBAPP_TYPES.values():
+            stats.update({
+                'apps_added_%s_%s' % (region_slug, package_type):
+                apps.filter(is_packaged=package_type == 'packaged')
+                    .exclude(addonexcludedregion__region=region.id).count
+            })
+        # Apps added by premium type and region.
+        for premium_type, pt_name in ADDON_PREMIUM_API.items():
+            stats.update({
+                'apps_added_%s_%s' % (region_slug, pt_name):
+                apps.filter(premium_type=premium_type)
+                    .exclude(addonexcludedregion__region=region.id).count
+            })
+
     # If we're processing today's stats, we'll do some extras.  We don't do
     # these for re-processed stats because they change over time (eg. add-ons
     # move from sandbox -> public

apps/stats/tests/test_cron.py

@@ -9,6 +9,7 @@
 import amo.tests
 from addons.models import Addon, AddonUser
 from bandwagon.models import CollectionAddon, Collection
+from mkt.constants.regions import REGIONS_CHOICES_SLUG
 from mkt.webapps.models import Installed
 from reviews.models import Review
 from stats import cron, tasks
@@ -33,12 +34,12 @@ def test_stats_for_date(self):
 
     @mock.patch('stats.tasks.MonolithRecord')
     def test_mmo_user_total_count_updates_monolith(self, record):
-        date = datetime.date(2013, 3, 11)
+        UserProfile.objects.create(source=amo.LOGIN_SOURCE_MMO_BROWSERID)
         job = 'mmo_user_count_total'
 
-        tasks.update_global_totals(job, date)
+        tasks.update_global_totals(job, datetime.date.today())
         self.assertTrue(record.called)
-        eq_(record.call_args[1]['value'], '{"count": 0}')
+        eq_(record.call_args[1]['value'], '{"count": 1}')
 
     @mock.patch('stats.tasks.MonolithRecord')
     def test_addon_total_downloads_doesnot_update_monolith(self, record):
@@ -58,6 +59,32 @@ def test_app_new(self):
         Addon.objects.create(type=amo.ADDON_WEBAPP)
         eq_(tasks._get_daily_jobs()['apps_count_new'](), 1)
 
+    def test_app_added_counts(self):
+        app = Addon.objects.create(type=amo.ADDON_WEBAPP)
+        regions = dict(REGIONS_CHOICES_SLUG)
+
+        # Add a region exclusion.
+        excluded_region = regions['br']
+        app.addonexcludedregion.create(region=excluded_region.id)
+
+        jobs = tasks._get_daily_jobs()
+
+        # Check package type counts.
+        for region_slug in regions.keys():
+            expected_count = 0 if region_slug == excluded_region.slug else 1
+            count = jobs['apps_added_%s_hosted' % region_slug]()
+            eq_(count, expected_count,
+                'Incorrect count for region %s. Got %d, expected %d.' % (
+                    region_slug, count, expected_count))
+
+        # Check premium type counts.
+        for region_slug in regions.keys():
+            expected_count = 0 if region_slug == excluded_region.slug else 1
+            count = jobs['apps_added_%s_free' % region_slug]()
+            eq_(count, expected_count,
+                'Incorrect count for region %s. Got %d, expected %d.' % (
+                    region_slug, count, expected_count))
+
     def test_apps_installed(self):
         addon = Addon.objects.create(type=amo.ADDON_WEBAPP)
         user = UserProfile.objects.create(username='foo')