calculate regional popularity and hook it up to the adolescent flag (bug 763698)

Author: cvan

apps/addons/search.py

@@ -1,16 +1,22 @@
 import logging
 from operator import attrgetter
 
+from django.db.models import Count
+
 import elasticutils.contrib.django as elasticutils
 import pyes.exceptions as pyes
 
 import amo
 from .models import Addon
 from bandwagon.models import Collection
 from compat.models import AppCompat
+from stats.models import ClientData
 from users.models import UserProfile
 from versions.compare import version_int
 
+import mkt
+from mkt.webapps.models import Installed
+
 
 log = logging.getLogger('z.es')
 
@@ -54,6 +60,24 @@ def extract(addon):
         d['weekly_downloads'] = addon.persona.popularity
         # Boost on popularity.
         d['_boost'] = addon.persona.popularity ** .2
+    elif addon.type == amo.ADDON_WEBAPP:
+        installed_ids = list(Installed.objects.filter(addon=addon)
+                             .values_list('id', flat=True))
+        d['popularity'] = d['_boost'] = len(installed_ids)
+
+        # Calculate regional popularity for "mature regions"
+        # (installs + reviews/installs from that region).
+        installs = dict(ClientData.objects.filter(installed__in=installed_ids)
+                        .annotate(region_counts=Count('region'))
+                        .values_list('region', 'region_counts').distinct())
+        for region in mkt.regions.ALL_REGION_IDS:
+            cnt = installs.get(region, 0)
+            if cnt:
+                # Magic number (like all other scores up in this piece).
+                d['popularity_%s' % region] = d['popularity'] + cnt * 10
+            else:
+                d['popularity_%s' % region] = len(installed_ids)
+            d['_boost'] += cnt * 10
     else:
         # Boost by the number of users on a logarithmic scale. The maximum
         # boost (11,000,000 users for adblock) is about 5x.

mkt/browse/templates/browse/landing.html

@@ -33,7 +33,7 @@ <h2 class="sticky">{{ title if category else _('Featured') }}</h2>
       <div>
         <h2 class="see-all c">
           <a href="{{ request.path|urlparams(category=category.slug or None,
-                                             sort='downloads') }}">
+                                             sort='popularity') }}">
           <span>{{ _('By popularity') }}</span> <em>{{ _('See all') }}</em></a>
         </h2>
       </div>

mkt/constants/regions.py

@@ -136,4 +136,5 @@ class BR(REGION):
 REGIONS_DICT = dict(REGIONS_CHOICES)
 REGIONS_CHOICES_ID_DICT = dict(REGIONS_CHOICES_ID)
 
+ALL_REGION_IDS = sorted(REGIONS_CHOICES_ID_DICT.keys())
 REGION_IDS = sorted(REGIONS_CHOICES_ID_DICT.keys()[1:])

mkt/search/api.py

@@ -2,6 +2,8 @@
 
 import amo
 from amo.helpers import absolutify
+
+import mkt
 from mkt.api.base import MarketplaceResource
 from mkt.search.views import _get_query, _filter_search
 from mkt.search.forms import ApiSearchForm
@@ -29,8 +31,9 @@ def get_list(self, request=None, **kwargs):
             raise self.form_errors(form)
 
         # Search specific processing of the results.
-        qs = _get_query(request)
-        qs = _filter_search(qs, form.cleaned_data)
+        region = getattr(request, 'REGION', mkt.regions.WORLDWIDE)
+        qs = _get_query(region)
+        qs = _filter_search(qs, form.cleaned_data, region=region)
         res = amo.utils.paginate(request, qs)
 
         # Rehydrate the results as per tastypie.

mkt/search/forms.py

@@ -9,6 +9,7 @@
 
 SORT_CHOICES = [
     (None, _lazy(u'Relevance')),
+    ('popularity', _lazy(u'Popularity')),
     ('downloads', _lazy(u'Weekly Downloads')),
     ('rating', _lazy(u'Top Rated')),
     ('price', _lazy(u'Price')),

mkt/search/tests/test_views.py

@@ -2,6 +2,7 @@
 
 from django.conf import settings
 
+import mock
 from nose import SkipTest
 from nose.tools import eq_, nottest
 from pyquery import PyQuery as pq
@@ -13,11 +14,13 @@
 from amo.urlresolvers import reverse
 from amo.utils import urlparams
 from search.tests.test_views import TestAjaxSearch
+from stats.models import ClientData
+from users.models import UserProfile
 
 import mkt
 from mkt.search.forms import DEVICE_CHOICES_IDS
 from mkt.webapps.tests.test_views import PaidAppMixin
-from mkt.webapps.models import AddonExcludedRegion as AER, Webapp
+from mkt.webapps.models import AddonExcludedRegion as AER, Installed, Webapp
 
 
 class SearchBase(amo.tests.ESTestCase):
@@ -50,6 +53,7 @@ def check_results(self, params, expected):
         got = self.get_results(r)
         eq_(got, expected,
             'Got: %s. Expected: %s. Parameters: %s' % (got, expected, params))
+        return r
 
 
 class TestWebappSearch(PaidAppMixin, SearchBase):
@@ -239,7 +243,7 @@ def test_device_tablet(self):
 
     def test_results_sort_default(self):
         self._generate(3)
-        self.check_sort_links(None, 'Relevance', 'weekly_downloads')
+        self.check_sort_links(None, 'Relevance', 'popularity')
 
     def test_results_sort_unknown(self):
         self._generate(3)
@@ -267,7 +271,7 @@ def test_price_sort_visible_for_paid_search(self):
     def test_price_sort_visible_for_paid_browse(self):
         # 'Sort by Price' option should be removed if filtering by free apps.
         r = self.client.get(reverse('browse.apps'),
-                            {'price': 'free', 'sort': 'downloads'})
+                            {'price': 'free', 'sort': 'popularity'})
         eq_(r.status_code, 200)
         assert 'price' not in dict(r.context['sort_opts']), (
             'Unexpected price sort')
@@ -289,7 +293,7 @@ def test_redirect_free_price_sort(self):
             # `sort=price` should be changed to `sort=downloads` if
             # `price=free` is in querystring.
             r = self.client.get(url, {'price': 'free', 'sort': 'price'})
-            self.assert3xx(r, urlparams(url, price='free', sort='downloads'),
+            self.assert3xx(r, urlparams(url, price='free', sort='popularity'),
                            302)
 
     def test_region_exclusions(self):
@@ -300,6 +304,81 @@ def test_region_exclusions(self):
             self.check_results({'q': 'Steam', 'region': region},
                                [] if region == 'br' else [self.webapp.id])
 
+    @mock.patch.object(mkt.regions.BR, 'adolescent', True)
+    def test_adolescent_popularity(self):
+        self.skip_if_disabled(settings.REGION_STORES)
+
+        # Adolescent regions use global popularity.
+
+        # Webapp:   Global: 0, Regional: 0
+        # Unknown1: Global: 1, Regional: 1 + 10 * 1 = 11
+        # Unknown2: Global: 2, Regional: 0
+
+        user = UserProfile.objects.all()[0]
+        cd = ClientData.objects.create(region=mkt.regions.BR.id)
+
+        unknown1 = amo.tests.app_factory()
+        Installed.objects.create(addon=unknown1, user=user, client_data=cd)
+
+        unknown2 = amo.tests.app_factory()
+        Installed.objects.create(addon=unknown2, user=user)
+        Installed.objects.create(addon=unknown2, user=user)
+
+        self.reindex(Webapp)
+
+        r = self.check_results({'sort': 'popularity',
+                                'region': mkt.regions.BR.slug},
+                               [unknown2.id, unknown1.id, self.webapp.id])
+
+        # Check the actual popularity scores.
+        by_popularity = list(r.context['pager'].object_list
+                              .values_dict('popularity'))
+        eq_(by_popularity,
+            [{'id': unknown2.id, 'popularity': 2},
+             {'id': unknown1.id, 'popularity': 1},
+             {'id': self.webapp.id, 'popularity': 0}])
+
+    @mock.patch.object(mkt.regions.BR, 'adolescent', False)
+    def test_mature_popularity(self):
+        self.skip_if_disabled(settings.REGION_STORES)
+
+        # Mature regions use regional popularity.
+
+        # Webapp:   Global: 1, Regional: 1 * 2 + 10 * 1 = 12
+        # Unknown1: Global: 1, Regional: 1 * 2 + 10 * 2 = 22
+        # Unknown2: Global: 2, Regional: 2
+
+        region = mkt.regions.BR
+
+        user = UserProfile.objects.all()[0]
+        cd = ClientData.objects.create(region=region.id)
+
+        Installed.objects.get_or_create(addon=self.webapp, user=user)
+        Installed.objects.create(addon=self.webapp, user=user, client_data=cd)
+
+        unknown1 = amo.tests.app_factory()
+        Installed.objects.create(addon=unknown1, user=user, client_data=cd)
+        Installed.objects.create(addon=unknown1,
+            user=UserProfile.objects.create(), client_data=cd)
+
+        unknown2 = amo.tests.app_factory()
+        Installed.objects.create(addon=unknown2, user=user)
+        Installed.objects.create(addon=unknown2, user=user)
+
+        self.reindex(Webapp)
+
+        r = self.check_results({'sort': 'popularity',
+                                'region': region.slug},
+                               [unknown1.id, self.webapp.id, unknown2.id])
+
+        # Check the actual popularity scores.
+        by_popularity = list(r.context['pager'].object_list
+                              .values_dict('popularity_%s' % region.id))
+        eq_(by_popularity,
+            [{'id': unknown1.id, 'popularity_7': 22},
+             {'id': self.webapp.id, 'popularity_7': 12},
+             {'id': unknown2.id, 'popularity_7': 2}])
+
 
 class TestSuggestions(TestAjaxSearch):
 

mkt/search/views.py

@@ -29,6 +29,8 @@ def __init__(self, text, urlparams, selected=False, children=None):
 
 DEFAULT_FILTERS = ['cat', 'price', 'device', 'sort']
 DEFAULT_SORTING = {
+    'popularity': '-popularity',
+    # TODO: Should popularity replace downloads?
     'downloads': '-weekly_downloads',
     'rating': '-bayesian_rating',
     'created': '-created',
@@ -39,7 +41,7 @@ def __init__(self, text, urlparams, selected=False, children=None):
 
 
 def _filter_search(qs, query, filters=None, sorting=None,
-                   sorting_default='-weekly_downloads'):
+                   sorting_default='-popularity', region=None):
     """Filter an ES queryset based on a list of filters."""
     # Intersection of the form fields present and the filters we want to apply.
     filters = filters or DEFAULT_FILTERS
@@ -58,8 +60,24 @@ def _filter_search(qs, query, filters=None, sorting=None,
     if 'device' in show:
         qs = qs.filter(device=forms.DEVICE_CHOICES_IDS[query['device']])
     if 'sort' in show:
-        qs = qs.order_by(sorting[query['sort']])
+        sort_by = sorting[query['sort']]
+
+        # For "Adolescent" regions popularity is global installs + reviews.
+
+        if query['sort'] == 'popularity' and region and not region.adolescent:
+            # For "Mature" regions popularity becomes installs + reviews
+            # from only that region.
+            sort_by = '-popularity_%s' % region.id
+
+        qs = qs.order_by(sort_by)
     elif not query.get('q'):
+
+        if (sorting_default == 'popularity' and region and
+            not region.adolescent):
+            # For "Mature" regions popularity becomes installs + reviews
+            # from only that region.
+            sorting_default = '-popularity_%s' % region.id
+
         # Sort by a default if there was no query so results are predictable.
         qs = qs.order_by(sorting_default)
 
@@ -103,8 +121,7 @@ def sort_sidebar(query, form):
             for key, text in form.fields['sort'].choices]
 
 
-def _get_query(request):
-    region = getattr(request, 'REGION', mkt.regions.WORLDWIDE)
+def _get_query(region):
     return Webapp.from_search(region=region).facet('category')
 
 
@@ -116,16 +133,18 @@ def _app_search(request, category=None, browse=None):
     # Remove `sort=price` if `price=free`.
     if query.get('price') == 'free' and query.get('sort') == 'price':
         return {'redirect': amo.utils.urlparams(request.get_full_path(),
-                                                sort='downloads',
+                                                sort='popularity',
                                                 price='free')}
 
-    qs = _get_query(request)
+    region = getattr(request, 'REGION', mkt.regions.WORLDWIDE)
+
+    qs = _get_query(region)
     # On mobile, always only show mobile apps. Bug 767620
     if request.MOBILE:
         qs = qs.filter(uses_flash=False)
         query['device'] = 'mobile'
 
-    qs = _filter_search(qs, query)
+    qs = _filter_search(qs, query, region=region)
 
     # If we're mobile, leave no witnesses. (i.e.: hide "Applied Filters:
     # Mobile")

mkt/webapps/models.py

@@ -428,7 +428,7 @@ def from_search(cls, cat=None, region=None):
     @classmethod
     def popular(cls, cat=None, region=None):
         """Elastically grab the most popular apps."""
-        return cls.from_search(cat, region).order_by('-weekly_downloads')
+        return cls.from_search(cat, region).order_by('-popularity')
 
     @classmethod
     def latest(cls, cat=None, region=None):