Merge 56138ec into 17d5546

.coveragerc

@@ -1,4 +1,5 @@
 [report]
-omit = 
+omit =
     /home/travis/virtualenv*
     */test/*
+    */templates/*

pybossa/cache/helpers.py

@@ -202,7 +202,7 @@ def n_available_tasks_for_user(project, user_id=None, user_ip=None):
                project_id=:project_id AND user_id=:user_id)
                ; '''
     else:
-        user_pref_list = cached_users.get_user_preferences(user_id, map_to_country_codes=True)
+        user_pref_list = cached_users.get_user_preferences(user_id)
         user_filter_list = cached_users.get_user_filters(user_id)
 
         timeout = project_info.get("timeout", TIMEOUT)

pybossa/cache/task_browse_helpers.py

@@ -7,7 +7,7 @@
 from pybossa.cache import memoize, ONE_DAY
 from pybossa.core import db
 from pybossa.util import (convert_est_to_utc,
-    get_user_pref_db_clause, get_user_filter_db_clause)
+    get_user_pref_db_clause, get_user_filter_db_clause, map_locations)
 from flask import current_app
 import pybossa.app_settings as app_settings
 from functools import reduce
@@ -362,6 +362,8 @@ def validate_user_preferences(user_pref):
         raise ValueError('invalid locations user preference: {}'
                         .format(loc))
 
+    user_pref['locations'] = map_locations(loc)['locations']
+
 
 def _get_field_filters(filters):
     filters = filters if type(filters) is list else json.loads(filters)

pybossa/cache/users.py

@@ -18,7 +18,6 @@
 """Cache module for users."""
 from sqlalchemy.sql import text
 from sqlalchemy.exc import ProgrammingError
-from pybossa import app_settings
 from pybossa.core import db, timeouts
 from pybossa.cache import cache, memoize, delete_memoized, FIVE_MINUTES, \
     ONE_DAY, ONE_WEEK, memoize_with_l2_cache
@@ -30,7 +29,7 @@
 from pybossa.leaderboard.data import get_leaderboard as gl
 from pybossa.leaderboard.jobs import leaderboard as lb
 import json
-from pybossa.util import get_user_pref_db_clause, get_user_filter_db_clause
+from pybossa.util import get_user_pref_db_clause, get_user_filter_db_clause, map_locations
 from pybossa.data_access import data_access_levels
 from pybossa.util import get_taskrun_date_range_sql_clause_params
 from flask import current_app
@@ -343,9 +342,20 @@ def get_user_pref_metadata(name):
     upref_mdata.update(row[1] or {})
     if data_access_levels:
         upref_mdata['data_access'] = row[2] or []
+
+    if 'locations' in upref_mdata:
+        map_locations_upref_mdata(upref_mdata)
     return upref_mdata
 
 
+def map_locations_upref_mdata(upref_mdata):
+    mapped = map_locations(upref_mdata['locations'])
+
+    upref_mdata['country_codes'] = mapped['country_codes']
+    upref_mdata['country_names'] = mapped['country_names']
+    upref_mdata['locations'] = mapped['locations']
+
+
 def delete_user_pref_metadata(user):
     delete_memoized(get_user_pref_metadata, user.name)
     delete_memoized(get_user_by_id, user.id)
@@ -367,26 +377,12 @@ def delete_taskbrowse_bookmarks(user):
     delete_memoized(get_user_by_id, user.id)
 
 
-def get_user_preferences(user_id, map_to_country_codes=False):
+def get_user_preferences(user_id):
     user = get_user_by_id(user_id)
     user_pref = user.user_pref or {} if user else {}
-    if map_to_country_codes and 'locations' in user_pref:
-        new_locations_set = set()
-        for location in user_pref['locations']:
-            new_locations_set.add(location)
-            # location is a country code, fetch country name
-            if len(location) == 2:
-                mapped_loc = app_settings.upref_mdata.get_country_by_country_code(location)
-            # location is a country name, fetch country code
-            else:
-                mapped_loc = app_settings.upref_mdata.get_country_code_by_country(location)
-            if mapped_loc is not None:
-                new_locations_set.add(mapped_loc)
-            else:
-                current_app.logger.warning(f"Invalid country code '{location}' for user {user_id} in get_user_preferences")
-
-        user_pref['locations'] = list(new_locations_set)
     user_email = user.email_addr if user else None
+    if 'locations' in user_pref:
+        map_locations_upref_mdata(user_pref)
     return get_user_pref_db_clause(user_pref, user_email)
 
 

pybossa/forms/forms.py

@@ -785,6 +785,10 @@ class UserPrefMetadataForm(Form):
         lazy_gettext('Language(s)'), choices=[],default="")
     locations = Select2Field(
         lazy_gettext('Location(s)'), choices=[], default="")
+    country_codes = Select2Field(
+        lazy_gettext('Location - Country Code(s)'), choices=[], default="")
+    country_names = Select2Field(
+        lazy_gettext('Location - Country Name(s)'), choices=[], default="")
     work_hours_from = TimeField(
         lazy_gettext('Work Hours From'),
         [TimeFieldsValidator(["work_hours_to", "timezone"],
@@ -823,6 +827,8 @@ def set_upref_mdata_choices(self):
         upref_mdata_choices = app_settings.upref_mdata.get_upref_mdata_choices()
         self.languages.choices = upref_mdata_choices['languages']
         self.locations.choices = upref_mdata_choices['locations']
+        self.country_codes.choices = upref_mdata_choices['country_codes']
+        self.country_names.choices = upref_mdata_choices['country_names']
         self.timezone.choices = upref_mdata_choices['timezones']
         self.user_type.choices = upref_mdata_choices['user_types']
 

pybossa/sched.py

@@ -496,7 +496,7 @@ def locked_task_sql(project_id, user_id=None, limit=1, rand_within_priority=Fals
     '''
     filters = []
     if filter_user_prefs:
-        filters.append('AND ({}) AND ({})'.format(cached_users.get_user_preferences(user_id, map_to_country_codes=True), cached_users.get_user_filters(user_id)))
+        filters.append('AND ({}) AND ({})'.format(cached_users.get_user_preferences(user_id), cached_users.get_user_filters(user_id)))
     if task_type == 'gold':
         filters.append('AND task.calibration = 1')
     elif task_type == 'no_gold':

pybossa/util.py

@@ -35,6 +35,8 @@
 from math import ceil
 from tempfile import NamedTemporaryFile
 
+from pybossa import app_settings
+
 import dateutil.parser
 import dateutil.tz
 import simplejson
@@ -1413,3 +1415,29 @@ def delete_redis_keys(sentinel, pattern):
     if not keys_to_delete:
         return False
     return bool(sentinel.master.delete(*keys_to_delete))
+
+
+def map_locations(locations):
+    country_codes_set = set()
+    country_names_set = set()
+    for location in locations:
+        if len(location) == 2:
+            country_codes_set.add(location)
+            mapped_cn = app_settings.upref_mdata.get_country_name_by_country_code(location)
+            if mapped_cn is not None:
+                country_names_set.add(mapped_cn)
+            else:
+                current_app.logger.warning(f"Invalid country code '{location}' in map_locations")
+        else:
+            country_names_set.add(location)
+            mapped_cc = app_settings.upref_mdata.get_country_code_by_country_name(location)
+            if mapped_cc is not None:
+                country_codes_set.add(mapped_cc)
+            else:
+                current_app.logger.warning(f"Invalid country name '{location}' in map_locations")
+
+    return {
+        'locations': list(country_codes_set.union(country_names_set)),
+        'country_codes': list(country_codes_set),
+        'country_names': list(country_names_set)
+    }

pybossa/view/account.py

@@ -607,7 +607,9 @@ def _show_public_profile(user, form, can_update):
                     form=form,
                     can_update=can_update,
                     private_instance=bool(data_access_levels),
-                    upref_mdata_enabled=bool(app_settings.upref_mdata))
+                    upref_mdata_enabled=bool(app_settings.upref_mdata),
+                    country_name_to_country_code=app_settings.upref_mdata.country_name_to_country_code,
+                    country_code_to_country_name=app_settings.upref_mdata.country_code_to_country_name)
 
     return handle_content_type(response)
 
@@ -631,7 +633,9 @@ def _show_own_profile(user, form, current_user, can_update):
                     form=form,
                     can_update=can_update,
                     private_instance=bool(data_access_levels),
-                    upref_mdata_enabled=bool(app_settings.upref_mdata))
+                    upref_mdata_enabled=bool(app_settings.upref_mdata),
+                    country_name_to_country_code=app_settings.upref_mdata.country_name_to_country_code,
+                    country_code_to_country_name=app_settings.upref_mdata.country_code_to_country_name)
 
     response = make_response(handle_content_type(response))
     response.headers['Cache-Control'] = 'no-store'
@@ -1144,7 +1148,9 @@ def add_metadata(name):
                                form=form,
                                input_form=True,
                                can_update=can_update,
-                               upref_mdata_enabled=bool(app_settings.upref_mdata))
+                               upref_mdata_enabled=bool(app_settings.upref_mdata),
+                               country_name_to_country_code=app_settings.upref_mdata.country_name_to_country_code(),
+                               country_code_to_country_name=app_settings.upref_mdata.country_code_to_country_name())
 
     user_pref, metadata = get_user_pref_and_metadata(name, form)
     user.info['metadata'] = metadata
@@ -1288,6 +1294,8 @@ def get_user_data_as_form(user):
     return {
         'languages': user_pref.get('languages'),
         'locations': user_pref.get('locations'),
+        'country_codes': user_pref.get('country_codes'),
+        'country_names': user_pref.get('country_names'),
         'user_type': metadata.get('user_type'),
         'work_hours_from': metadata.get('work_hours_from'),
         'work_hours_to': metadata.get('work_hours_to'),
@@ -1312,7 +1320,15 @@ def get_user_pref_and_metadata(user_name, form):
                         profile=form.profile.data)
         if form.languages.data:
             user_pref['languages'] = form.languages.data
-        if form.locations.data:
+
+        if form.country_names.data and form.country_codes.data:
+            # combine the two arrays
+            user_pref['locations'] = [x for arr in (form.country_names.data, form.country_codes.data) if arr is not None for x in arr]
+        elif form.country_names.data:
+            user_pref['locations'] = form.country_names.data
+        elif form.country_codes.data:
+            user_pref['locations'] = form.country_codes.data
+        elif form.locations.data:
             user_pref['locations'] = form.locations.data
         return user_pref, metadata
 

test/test_auditlog.py

@@ -279,7 +279,10 @@ def test_project_create(self):
             assert log.user_id == 1, log.user_id
 
     @with_context
-    def test_project_create(self):
+    @patch('pybossa.view.account.app_settings.upref_mdata.country_name_to_country_code', new={})
+    @patch('pybossa.view.account.app_settings.upref_mdata.country_code_to_country_name', new={})
+    @patch('pybossa.cache.task_browse_helpers.app_settings.upref_mdata')
+    def test_project_create(self, upref_mdata):
         self.register()
         self.signin()
         self.new_project()

test/test_cache/test_cache_projects.py

@@ -908,7 +908,10 @@ def test_parse_tasks_browse_args_raise_exception(self):
         assert_raises(ValueError, parse_tasks_browse_args, args)
 
     @with_context
-    def test_task_browse_user_pref_args_no_upref_mdata_config(self):
+    @patch('pybossa.cache.task_browse_helpers.map_locations')
+    @patch('pybossa.cache.task_browse_helpers.app_settings.upref_mdata.get_valid_user_preferences')
+    @patch('pybossa.cache.task_browse_helpers.app_settings.upref_mdata')
+    def test_task_browse_user_pref_args_no_upref_mdata_config(self, upref_mdata, get_valid_user_preferences, map_locations):
         """Test task browse user preference without user_pref settings loaded under pybossa.core.upref_mdata_choices"""
         args = dict(
             task_id=12345, pcomplete_from=0,
@@ -937,14 +940,18 @@ def test_task_browse_user_pref_args_no_upref_mdata_config(self):
             filter_by_upref={'languages': ['English'], 'locations': ['Fiji']},
             in_progress='Yes')
 
-
+        get_valid_user_preferences.return_value = {}
+        map_locations.return_value = {
+            'locations': ['Fiji']
+        }
         pargs = parse_tasks_browse_args(args)
         assert pargs == valid_args, pargs
 
     @with_context
+    @patch('pybossa.cache.task_browse_helpers.map_locations')
     @patch('pybossa.cache.task_browse_helpers.app_settings.upref_mdata.get_valid_user_preferences')
     @patch('pybossa.cache.task_browse_helpers.app_settings.upref_mdata')
-    def test_task_browse_user_pref_args(self, upref_mdata, get_valid_user_preferences):
+    def test_task_browse_user_pref_args(self, upref_mdata, get_valid_user_preferences, map_locations):
         """Test task browse user preference works with valid user_pref settings"""
         get_valid_user_preferences.return_value = dict(languages=["en", "sp"],
                                     locations=["us", "uk"])
@@ -973,6 +980,10 @@ def test_task_browse_user_pref_args(self, upref_mdata, get_valid_user_preference
             display_columns=['task_id', 'priority'], display_info_columns=['co_id'],
             filter_by_upref={'languages': ['en'], 'locations': ['us']})
 
+        map_locations.return_value = {
+            'locations': ['us']
+        }
+
         pargs = parse_tasks_browse_args(args)
         assert pargs == valid_args, pargs
 

test/test_coowners.py

@@ -1,4 +1,5 @@
 import json
+from unittest.mock import patch
 from test.helper import web
 from test import with_context
 from test import db, Fixtures
@@ -141,7 +142,10 @@ def test_03_misc(self):
         assert "User is not a project owner" in str(res.data), res.data
 
     @with_context
-    def test_coowner_can(self):
+    @patch('pybossa.view.account.app_settings.upref_mdata.country_name_to_country_code', new={})
+    @patch('pybossa.view.account.app_settings.upref_mdata.country_code_to_country_name', new={})
+    @patch('pybossa.cache.task_browse_helpers.app_settings.upref_mdata')
+    def test_coowner_can(self, upref_mdata):
         """
         Coowner can access features
         """

test/test_forms.py

@@ -393,6 +393,8 @@ def setUp(self):
 
         self.upref_mdata_valid_choices = dict(languages=[("en", "en"), ("sp", "sp")],
                                     locations=[("us", "us"), ("uk", "uk")],
+                                    country_codes=[("us", "us"), ("uk", "uk")],
+                                    country_names=[("us", "us"), ("uk", "uk")],
                                     timezones=[("", ""), ("ACT", "Australia Central Time")],
                                     user_types=[("Researcher", "Researcher"), ("Analyst", "Analyst")])
 

test/test_privacy.py

@@ -176,7 +176,10 @@ def test_05_app_stats_index(self):
         self.signout()
 
     @with_context
-    def test_06_user_public_profile(self):
+    @patch('pybossa.view.account.app_settings.upref_mdata.country_name_to_country_code', new={})
+    @patch('pybossa.view.account.app_settings.upref_mdata.country_code_to_country_name', new={})
+    @patch('pybossa.cache.task_browse_helpers.app_settings.upref_mdata')
+    def test_06_user_public_profile(self, upref_mdata):
         '''Test PRIVACY user public profile privacy is respected'''
         # As Anonymous user
         url = '/account/%s' % self.name
@@ -200,7 +203,10 @@ def test_06_user_public_profile(self):
         self.signout()
 
     @with_request_context
-    def test_07_user_public_profile_json(self):
+    @patch('pybossa.view.account.app_settings.upref_mdata.country_name_to_country_code', new={})
+    @patch('pybossa.view.account.app_settings.upref_mdata.country_code_to_country_name', new={})
+    @patch('pybossa.cache.task_browse_helpers.app_settings.upref_mdata')
+    def test_07_user_public_profile_json(self, upref_mdata):
         '''Test PRIVACY user public profile privacy is respected for API access'''
         # As Anonymous user
         admin, user, owner = UserFactory.create_batch(3)
@@ -433,8 +439,11 @@ def test_05_app_stats_index(self):
         self.signout()
 
     @patch.dict(flask_app.config, {'ENFORCE_PRIVACY': True})
+    @patch('pybossa.view.account.app_settings.upref_mdata.country_name_to_country_code', new={})
+    @patch('pybossa.view.account.app_settings.upref_mdata.country_code_to_country_name', new={})
+    @patch('pybossa.cache.task_browse_helpers.app_settings.upref_mdata')
     @with_context
-    def test_06_user_public_profile(self):
+    def test_06_user_public_profile(self, upref_mdata):
         '''Test PRIVACY user public profile privacy is respected'''
         admin, user, owner = UserFactory.create_batch(3)
         # As Anonymou user