From da45780b33c689463c54f5597c089f59374f7deb Mon Sep 17 00:00:00 2001
From: Noor Syed <noorsyed1a@gmail.com>
Date: Tue, 20 Jun 2023 09:26:30 -0400
Subject: [PATCH] RDISCROWD-6042 backwards compatible turkey spelling (#850)

* support alternate turkey spelling

remove file

fix null error

* backwards compatible turkey spelling

* fix keyerror

* bug fix
---
 pybossa/cache/task_browse_helpers.py |  3 +-
 pybossa/util.py                      | 15 ++++--
 test/test_util.py                    | 68 +++++++++++++++++++++-------
 3 files changed, 64 insertions(+), 22 deletions(-)

diff --git a/pybossa/cache/task_browse_helpers.py b/pybossa/cache/task_browse_helpers.py
index df3ae5c18..d5a7aaad5 100644
--- a/pybossa/cache/task_browse_helpers.py
+++ b/pybossa/cache/task_browse_helpers.py
@@ -350,9 +350,10 @@ def validate_user_preferences(user_pref):
     valid_languages = valid_user_preferences.get('languages')
     valid_locations = valid_user_preferences.get('locations')
 
-    # Support old spelling of Türkiye
+    # Support alternate spellings of Türkiye
     if valid_locations:
         valid_locations.append('Turkey')
+        valid_locations.append('Turkiye')
 
     lang = user_pref.get('languages')
     loc = user_pref.get('locations')
diff --git a/pybossa/util.py b/pybossa/util.py
index b77294857..bc4a4f574 100644
--- a/pybossa/util.py
+++ b/pybossa/util.py
@@ -1010,13 +1010,19 @@ def get_user_pref_db_clause(user_pref, user_email=None):
     # expand user preferences as per sql format for jsonb datatype
     # single user preference with multiple value or
     # multiple user preferences with single/multiple values
-    _valid = ((k, v) for k, v in user_pref.items() if isinstance(v, list))
-    user_prefs = [{k: [item]} for k, pref_list in _valid
-                  for item in pref_list]
     assign_key = 'assign_user'
     location_key = 'locations'
     language_key = 'languages'
 
+    _valid = ((k, v) for k, v in user_pref.items() if isinstance(v, list))
+    turkey_spellings = {'Turkey', 'Türkiye', 'Turkiye'}
+    if location_key in user_pref and \
+       any(turkey_spelling in user_pref[location_key] for turkey_spelling in turkey_spellings):
+            user_pref[location_key] = list(set(user_pref[location_key]).union(turkey_spellings))
+
+    user_prefs = [{k: [item]} for k, pref_list in _valid
+                  for item in pref_list]
+
     if not user_prefs:
         user_pref_sql = '''(task.user_pref IS NULL OR task.user_pref = \'{}\' )'''
         if user_email:
@@ -1024,8 +1030,9 @@ def get_user_pref_db_clause(user_pref, user_email=None):
                     AND task.user_pref->\'{}\' IS NOT NULL AND task.user_pref @> :assign_user)
                     '''.format(location_key, language_key, assign_key)
     else:
-        sql = ('task.user_pref @> \'{}\''.format(json.dumps(up).lower())
+        sql = ('task.user_pref @> \'{}\''.format(json.dumps(up, ensure_ascii=False).lower())
                    for up in user_prefs)
+
         user_pref_sql = '''( (task.user_pref-> \'{}\' IS NULL AND task.user_pref-> \'{}\' IS NULL) OR ({}) )'''.format(location_key, language_key, ' OR '.join(sql))
         if user_email:
             email_sql = ''' AND (task.user_pref->\'{}\' IS NULL OR task.user_pref @> :assign_user)
diff --git a/test/test_util.py b/test/test_util.py
index 76a984226..58c1163d6 100644
--- a/test/test_util.py
+++ b/test/test_util.py
@@ -978,7 +978,7 @@ def test_process_tp_components(self):
         tp_code = """  <task-presenter auto-save-seconds="15" />
         <task-presenter :auto-save-seconds="15" v-bind:allow-save-work="true">
             <text-input id='_kp6zwx2rs' type='text' :validations='[]' pyb-answer='freeText' initial-value='nothing special'></text-input>
-            
+
             <div class="row">
               <div class="col-sm-3">
                 <div class="form-group">
@@ -989,11 +989,11 @@ def test_process_tp_components(self):
                 </div>
               </div>
             </div>
-        
+
             <radio-group-input pyb-answer='answer' name='userAnswer'
               :choices='{&quot;Chinese&quot;:&quot;Chinese&quot;,&quot;Korean&quot;:&quot;Korean&quot;,&quot;Japanese&quot;:&quot;Japanese&quot;}'
               initial-value='Chinese' :validations='["required"]'></radio-group-input>
-        
+
             <div id="_e9pm92ges">
               <div class="checkbox">
                       <label for="_mg59znxa7">
@@ -1001,7 +1001,7 @@ def test_process_tp_components(self):
                       </label>
                 </div>
             </div>
-        
+
             <multi-select-input
               pyb-answer='subjects'
               :choices='[&quot;Math&quot;,&quot;English&quot;,&quot;Social Study&quot;,&quot;Python&quot;]'
@@ -1060,31 +1060,31 @@ def test_process_table_component(self):
             <div slot="name" slot-scope="props">
                 <text-input :row="props.row" :initial-value="props.row.name" :validations='["required"]' pyb-table-answer="name"></text-input>
             </div>
-        
+
             <div slot="position" slot-scope="props">
                 <text-input :row="props.row" :initial-value="props.row.position" :validations='["required"]' pyb-table-answer="position"></text-input>
             </div>
-        
+
             <div slot="phoneNumber" slot-scope="props">
                 <text-input :row="props.row" :initial-value="props.row.phoneNumber" :validations='["required"]' pyb-table-answer="phoneNumber"></text-input>
             </div>
-        
+
             <div slot="emailAddress" slot-scope="props">
                 <text-input :row="props.row" :initial-value="props.row.emailAddress" :validations='["required"]' pyb-table-answer="emailAddress"></text-input>
             </div>
-        
+
             <div slot="physicalLocation" slot-scope="props">
                 <text-input :row="props.row" :initial-value="props.row.physicalLocation" :validations='["required"]' pyb-table-answer="physicalLocation"></text-input>
             </div>
-        
+
             <div slot="linkedIn" slot-scope="props">
                 <text-input :row="props.row" :initial-value="props.row.linkedIn" :validations='["required"]' pyb-table-answer="linkedIn"></text-input>
             </div>
-        
+
             <div slot="zoomInfo" slot-scope="props">
                 <text-input :row="props.row" :initial-value="props.row.zoomInfo" :validations='["required"]' pyb-table-answer="zoomInfo"></text-input initial-value="">
             </div>
-        
+
             <div slot="moreInfo" slot-scope="props">
                 <!--
                     Please enter you custom component in this area.
@@ -1092,7 +1092,7 @@ def test_process_table_component(self):
                  -->
                 <input-text-area cols="50" rows="4" :row="props.row" :initial-value="props.row.moreInfo" pyb-table-answer="moreInfo"></input-text-area>
             </div>
-        
+
         </table-element>
         """
         user_response = {"all_info":
@@ -1137,13 +1137,13 @@ def test_process_table_component_with_initial_data_from_task(self):
                   <label for="_hc921dja4">My test notes</label>
                   <input-text-area id='_hc921dja4' :validations='["required"]' rows='5' cols='100' pyb-answer='test notes'></input-text-area>
                 </div>
-                
+
                 <table-element
                   :key='task.id'
                    name='all_info'
                   :data='task.info.a.b'
                   :columns='["concept","actual_forecast","concept_period","hgrt_variable","hgrt_variable_type","hgrt_feedback","variable_notes","QC_Reason","QC_Notes"]'
-                  column-id='and_id'  
+                  column-id='and_id'
                     :options='{
                         "columnsClasses": {"QC_Notes":"min-width-class", "QC_Reason":"min-width-dropdown-class"},
                         "headings": {
@@ -1166,12 +1166,12 @@ def test_process_table_component_with_initial_data_from_task(self):
                           :choices="{&quot;bestpractices-provideqcnote&quot;:&quot;Best Practices - Provide QCNOTE&quot;,&quot;correct&quot;:&quot;Correct&quot;,&quot;datapointcanbeextracted-provideqcnote&quot;:&quot;Data point can be extracted - Provide QCNOTE&quot;,&quot;incorrectvariable&quot;:&quot;Incorrect Variable&quot;,&quot;incorrectwffeedback&quot;:&quot;Incorrect WF Feedback&quot;,&quot;incorrectwffeedback&amp;bestpractices-provideqcnote&quot;:&quot;Incorrect WF Feedback &amp; Best Practices - Provide QCNOTE&quot;,&quot;incorrectwffeedback&amp;incorrectrule-provideqcnote&quot;:&quot;Incorrect WF Feedback &amp; Incorrect Rule - Provide QCNOTE&quot;,&quot;n&#x2F;a-borgupdate&quot;:&quot;N&#x2F;A - Borg Update&quot;,&quot;n&#x2F;a-blocked&quot;:&quot;N&#x2F;A - Blocked&quot;,&quot;nocomparisonordirectionallanguage&quot;:&quot;No Comparison or Directional Language&quot;,&quot;nocurrencyparameter&quot;:&quot;No Currency Parameter&quot;,&quot;nomultiplierselected&quot;:&quot;No Multiplier Selected&quot;,&quot;noperiodreference&quot;:&quot;No Period Reference&quot;,&quot;nostartinganchor&quot;:&quot;No Starting Anchor&quot;,&quot;other-provideqcnote&quot;:&quot;Other - Provide QCNOTE&quot;}"
                       ></dropdown-input>
                     </div>
-                
+
                     <div slot="QC_Notes" slot-scope="props">
                      <!-- <text-input :row="props.row" pyb-table-answer="QC_Notes"></text-input> -->
-                      <input-text-area :initial-value="props.row.QC_Notes" initial-value="" cols="50" rows="4" pyb-table-answer="QC_Notes" :row="props.row"></input-text-area>  
+                      <input-text-area :initial-value="props.row.QC_Notes" initial-value="" cols="50" rows="4" pyb-table-answer="QC_Notes" :row="props.row"></input-text-area>
                     </div>
-                
+
                 </table-element>
                     <table-element
                       :key='task.id'
@@ -1823,3 +1823,37 @@ def test_can_update_user_info(self):
         assert set(disabled.keys()) == {'user_type'}
         assert set(hidden.keys()) == {'profile'}
         assert util.can_update_user_info(normal_user, normal_user2) == (False, None, None)
+
+class TestGetUserPrefDBClause(object):
+
+    def test_get_user_pref_db_clause(self):
+        user_pref = json.loads('{"locations": ["United States"]}')
+        user_pref_db_clause = util.get_user_pref_db_clause(user_pref=user_pref)
+        assert 'united states' in user_pref_db_clause and 'turkey' not in user_pref_db_clause
+
+    def test_get_user_pref_db_clause_old_turkey_spelling(self):
+        user_pref = json.loads('{"locations": ["United States", "Turkey"]}')
+        user_pref_db_clause = util.get_user_pref_db_clause(user_pref=user_pref)
+
+        assert 'united states' in user_pref_db_clause and \
+               'turkey' in user_pref_db_clause and \
+               'turkiye' in user_pref_db_clause and \
+               'türkiye' in user_pref_db_clause
+
+    def test_get_user_pref_db_clause_new_turkey_spelling(self):
+        user_pref = json.loads('{"locations": ["United States", "Turkiye"]}')
+        user_pref_db_clause = util.get_user_pref_db_clause(user_pref=user_pref)
+
+        assert 'united states' in user_pref_db_clause and \
+               'turkey' in user_pref_db_clause and \
+               'turkiye' in user_pref_db_clause and \
+               'türkiye' in user_pref_db_clause
+
+    def test_get_user_pref_db_clause_new_turkey_spelling_special_char(self):
+        user_pref = json.loads('{"locations": ["United States", "Türkiye"]}')
+        user_pref_db_clause = util.get_user_pref_db_clause(user_pref=user_pref)
+
+        assert 'united states' in user_pref_db_clause and \
+               'turkey' in user_pref_db_clause and \
+               'turkiye' in user_pref_db_clause and \
+               'türkiye' in user_pref_db_clause